You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zg...@apache.org on 2019/04/10 13:47:32 UTC
[hbase] branch hbase-native-client created (now 10f1283)
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a change to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git.
at 10f1283 HBASE-22024 Clean up BUCK related things for C++ native client
This branch includes the following new commits:
new 920676a HBASE-9977 Define C interface of HBase Client Asynchronous APIs
new 2e8bfb8 HBASE-10302 Fix rat check issues in hbase-native-client.
new 89dda63 HBASE-14087 Ensure correct ASF headers for docs/code
new d760ee9 HBASE-14852 Update build env
new d299213 HBASE-15078 Added ability to start/stop hbase local cluster for tests, global test_env for gtest, small changes to dockerfile and docker run.
new 5e604ca HBASE-14853 Add on protobuf to c++ chain
new de7a347 HBASE-15401 Add Zookeeper to third party
new 66e226d HBASE-14854 Read meta location from zk
new 881dd51 HBASE-15418 Clean up un-used warning in test util
new f68267e HBASE-15604 Add a good readme on the build.
new da7a30f HBASE-15649 Ignore buck-out when building docker image
new 36d79d9 HBASE-15045 Keep hbase-native-client/if and hbase-protocol in sync.
new 5c167a0 HBASE-14855 Connect to regionserver
new a7b33dd HBASE-15690 Add utility to get current username
new 9282789 HBASE-15620 Add on Call serialization
new c0fc766 HBASE-15696 Move region location cache serialization into serde
new 0f8c1f8 HBASE-15718 Add on TableName implementation and tests
new c87f191 HBASE-15724 Use explicit docker image
new 58e710c HBASE-15687 Allow decoding more than GetResponse from the server
new 5f84b28 HBASE-15730 Add on script to format all .h,.cc, and BUCK files.
new a28f85b HBASE-15731 Add on a connection pool
new 572c169 HBASE-15739 Add region location lookup from meta
new f21799d HBASE-15748 Don't link in static libunwind.
new 1fa8212 HBASE-15750 Add on meta deserialization
new 7593d6c HBASE-15655 Set up doxygen for documentation
new f8cb53b HBASE-15766 Show working puts
new 14e1b9c HBASE-15777 Fix needs header in client handler
new 33806c3 HBASE-15771 Document all the public classes
new 54b93bb HBASE-15761 Add on more server name tests
new b535ffa HBASE-15770 Stop using wangle's global executor
new 191935d HBASE-15792 Add on a test for locating a region
new f107bbb HBASE-15774 Fix Upgrade lock usage in connection pool.
new 7d2ef94 HBASE-15826 Clean up ASF license issues
new 32a5e54 HBASE-15823 Use call once for user util
new 4f651b7 HBASE-15821 Document TestUtil
new 54beb11 HBASE-15851 Makefile update for build env
new 6bb6431 HBASE-15822 Move to the latest docker base image
new 693849d HBASE-15926 Added ASF headers which were missed in last commit
new 26dd78c HBASE-15945 Patch for Cell
new cdf8a65 HBASE-16174 Hook cell test up, and fix broken cell test.
new f2d74a3 HBASE-16216 Clean up source code.
new 05f51ae HBASE-16740 start-docker.sh fails to run by complaining bzip2 error (Xiaobing Zhou)
new 0972e6b HBASE-15893 Get object (Sudeep Sunthankar)
new 84a20fa HBASE-17217 Docker should bind to a different port than 16010
new 13269de HBASE-16489 Configuration parsing (Sudeep Sunthankar)
new a9f76a2 HBASE-17218 [C++] Use Google Style guide and cpplint
new 21e728b HBASE-15902 Scan Object
new e0a0225 HBASE-17219 [C++] Reformat the code according to the style guidelines
new 365b603 HBASE-17218 [C++] Use Google Style guide and cpplint - ADDENDUM to add executable bit to the script
new 87d7316 HBASE-17220 [C++] Address major issues from cpplint
new 062c146 HBASE-17051 [C++] implement RPC client and connection management (Xiaobing Zhou)
new 5c96730 HBASE-15905 Makefile build env incorrectly links in tests (Priyadharshini karthikeyan and Sudeep Sunthankar)
new 563d036 HBASE-17283 [C++] Result class
new 2a72ee4 HBASE-17395 [C++] Use custom line wrapping in formatting
new e1ce4b9 HBASE-17395 [C++] Use custom line wrapping in formatting - ADDENDUM to remove an extra '=' sign.
new df9fa3d HBASE-17451 [C++] HBase Request and Response Converter
new 72f4f4c HBASE-17463 [C++] RpcClient should close the thread pool
new a8bc1f6 HBASE-17315 [C++] HBase Client and Table Implementation (Sudeep Sunthankar)
new cc8c8f9 HBASE-17219 [C++] Reformat the code according to the style guidelines - ADDENDUM after HBASE-17395
new 9a4a6cb HBASE-15705 Add on meta cache (Mikhail Antonov)
new 0c67147 HBASE-17481 [C++] cpplint should use line wrapping of 100
new 9321038 HBASE-17503 [C++] Configuration should be settable and used w/o XML files
new 851a408 HBASE-17485 [C++] Zookeeper quorum and znode lookup made configurable (Sudeep Sunthankar & enis)
new 719615a HBASE-17278 [C++] Cell Scanner and KeyValueCodec for encoding cells in RPC (Sudeep Sunthankar and enis)
new 608c192 HBASE-17585 [C++] Use KVCodec in the RPC request/response
new f58f72a HBASE-17612 [C++] Set client version info in RPC header
new c117c36 HBASE-17226 [C++] Filter and Query classes
new effe966 HBASE-17629 [C++] Timeouts and retry configuration for connections
new 670943a HBASE-17465 [C++] implement request retry mechanism over RPC (Xiaobing Zhou)
new ad85594 HBASE-17680 Run mini cluster through JNI in tests
new e71fff0 HBASE-17741 [C++] rename some files to use (dash) instead of _
new 4c5a089 HBASE-17728 Create separate build target for Configuration classes
new e9ab73f HBASE-17466 [C++] Test cleanup and minor improvements
new a2caa16 HBASE-17768 [C++] Makefile should recompile only the changed sources (Sudeep Sunthankar)
new 0458058 HBASE-17754 [C++] RawAsyncTable
new 37a6d70 Adding methods to run a stand-alone HBase instance
new 215a4a1 HBASE-17727 [C++] Make RespConverter work with RawAsyncTableImpl
new c26b5d9 HBASE-17771 [C++] Classes required for implementation of BatchCallerBuilder
new 59bfb21 HBASE-15894 [C++] Put and Mutation objects
new ab47d94 HBASE-16365 [C++] End to end Table::Put()
new eb8fe18 HBASE-17726 [C++] Move implementation from header to cc for request retry
new 569c6d0 HBASE-17800 [C++] handle exceptions in client RPC (Xiaobing Zhou and Enis Soztutar)
new c34838b HBASE-17576 [C++] Implement request retry mechanism over RPC for Multi calls. (Sudeep Sunthankar)
new e51f4a3 HBASE-17860 Implement secure native client connection
new 16beea2 HBASE-15602 Clean up using directives in cc files.
new 4545edd HBASE-15903 Delete Object
new 2710ab9 HBASE-17860 Implement secure native client connection - ADDENDUM to remove Dockerfile
new 8a2c245 HBASE-15903 Delete Object - addnedum makes PutGetDelete robust
new 0cb8452 HBASE-15903 Delete Object - addnedum moves misplaced sleep
new 395ea2c HBASE-18153 [C++] Add valgrind to the docker image
new c0f829e HBASE-18126 Increment class
new 39f4b71 HBASE-18126 Increment class - addendum creates TestToStringToInt64 in bytes-util-test
new f1614d8 HBASE-17907 [C++] End to end Scans from Client/Table
new 14c2d87 HBASE-18126 Increment class - addendum changes table name for test
new 348842d HBASE-18173 Append class
new fa7c7a9 HBASE-18173 Append class - Addendum to fix formatting
new bad90d0 HBASE-18174 Implement Table#checkAndPut()
new 62f042b HBASE-18188 [C++] Fix Handling do not retry exceptions
new aa3c216 HBASE-18178 [C++] Retrying meta location lookup and zookeeper connection
new a924ead HBASE-18210 Implement Table#checkAndDelete()
new 7d79613 HBASE-18236 [C++] Add batching and reporting to simple-client
new 448ec33 HBASE-18214 Replace the folly::AtomicHashMap usage in the RPC layer
new b6a3214 HBASE-18400 [C++] ConnectionId Equals/Hash should consider service_name (Xiaobing Zhou)
new f212486 HBASE-18400 [C++] ConnectionId Equals/Hash should consider service_name (Xiaobing Zhou) - ADDENDUM to reformat the code.
new 0b42c7a HBASE-18061 [C++] Fix retry logic in multi-get calls (Sudeep Sunthankar)
new 9918d09 HBASE-18338 [C++] Implement RpcTestServer (Xiaobing Zhou)
new 96d8879 HBASE-18407 [C++] make Configuration::Set/GetBool work for both true/false and 1/0 (Xiaobing Zhou)
new 4cfb6e9 HBASE-18086 Create native client which creates load on selected cluster
new 323a774 HBASE-18371 [C++] Update folly and wangle dependencies
new 16f45a0 HBASE-18338 [C++] Implement RpcTestServer - ADDENDUM to fix rsync for new proto files
new 913d1f1 HBASE-18466. [C++] Support handling exception in RpcTestServer
new f799780 HBASE-18459 [C++] Fix Segfault in location-cache
new cd6e21f HBASE-18536. [C++] Add fault injection infra
new 0368621 HBASE-18078. [C++] Harden RPC by handling various communication abnormalities
new 4bc225d HBASE-18564 [C++] Problems compiling with GCC
new efd0a46 HBASE-18537 [C++] Improvements to load-client
new f4041f3 HBASE-18565 [C++] Fix deadlock in AsyncScanRetryingCaller and other RPCs
new 725cc4c HBASE-18576. [C++] Add ping for RPC test
new 9fb0544 HBASE-18579 Enable core dump by default for docker
new 003df13 HBASE-18579 Enable core dump by default for docker -revert
new b665d67 HBASE-18603 buck complains about Build target path containing double slash
new 2509f52 HBASE-18578 [C++] Add pause for RPC test
new 874b001 HBASE-18204 [C++] Rpc connection close and reconnecting
new a4040d2 HBASE-18728 [C++] The library should be named libHBaseClient, instead of libHbaseClient
new 01bb387 HBASE-18507 [C++] Support for MultiPuts in AsyncBatchRpcRetryingCaller class (Sudeep Sunthankar)
new 0084543 HBASE-18720 [C++] Update wangle dependency to pick up the new release with Apache License v2
new 3751777 HBASE-18777 Create one table for subtests in multi-retry-test
new 0713a74 HBASE-18579 Enable core dump by default for docker
new 012b318 HBASE-18727 [C++] Don't link JNI/JVM stuff into libHbaseClient
new 4810f57 HBASE-18802 [C++] Remove hbase-macros.h
new 725bded HBASE-18725 [C++] Install header files as well as library
new f89c22e HBASE-18725 [C++] Install header files as well as library - ADDENDUM to only install .h files for if/
new 7b70670 HBASE-18861 [C++] Use boost::optional instead of std::experimental::optional
new 6984129 HBASE-18901 [C++] Provide CMAKE infrastructure
new 5785b60 HBASE-20009 Add license header to krb5.conf
new 5eaad7c HBASE-22038 fix buiding failures
new 10f1283 HBASE-22024 Clean up BUCK related things for C++ native client
The 137 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[hbase] 33/137: HBASE-15826 Clean up ASF license issues
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 7d2ef94b3cc9238c557f7a32c0a2d48653a36d06
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Fri May 13 14:53:47 2016 -0700
HBASE-15826 Clean up ASF license issues
---
Makefile | 17 +++++++++++++++++
bin/copy-protobuf.sh | 16 ++++++++++++++++
bin/format-code.sh | 16 ++++++++++++++++
bin/start-local-hbase.sh | 1 -
hbase.doxygen | 22 +++++++++++++++++++---
serde/server-name.h | 19 +++++++++++++++++++
6 files changed, 87 insertions(+), 4 deletions(-)
diff --git a/Makefile b/Makefile
index 90ec6ea..826233f 100644
--- a/Makefile
+++ b/Makefile
@@ -1,3 +1,20 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
build:
$(shell buck build core/... )
diff --git a/bin/copy-protobuf.sh b/bin/copy-protobuf.sh
index c9e70f2..36541cf 100755
--- a/bin/copy-protobuf.sh
+++ b/bin/copy-protobuf.sh
@@ -1,4 +1,20 @@
#!/usr/bin/env bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
set -euo pipefail
IFS=$'\n\t'
diff --git a/bin/format-code.sh b/bin/format-code.sh
index 24d6515..36a114d 100755
--- a/bin/format-code.sh
+++ b/bin/format-code.sh
@@ -1,4 +1,20 @@
#!/usr/bin/env bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
set -euo pipefail
IFS=$'\n\t'
diff --git a/bin/start-local-hbase.sh b/bin/start-local-hbase.sh
index 49637d6..12069c2 100755
--- a/bin/start-local-hbase.sh
+++ b/bin/start-local-hbase.sh
@@ -1,5 +1,4 @@
#!/usr/bin/env bash
-
##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
diff --git a/hbase.doxygen b/hbase.doxygen
index 22e2d21..fef404c 100644
--- a/hbase.doxygen
+++ b/hbase.doxygen
@@ -1,4 +1,20 @@
# Doxyfile 1.8.11
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#---------------------------------------------------------------------------
# Project related configuration options
@@ -103,14 +119,14 @@ WARN_LOGFILE =
#---------------------------------------------------------------------------
INPUT =
INPUT_ENCODING = UTF-8
-FILE_PATTERNS =
+FILE_PATTERNS =
RECURSIVE = YES
EXCLUDE = third-party/ buck-out/
EXCLUDE_SYMLINKS = NO
EXCLUDE_PATTERNS = /*-test.cc
EXCLUDE_SYMBOLS =
EXAMPLE_PATH =
-EXAMPLE_PATTERNS =
+EXAMPLE_PATTERNS =
EXAMPLE_RECURSIVE = NO
IMAGE_PATH =
INPUT_FILTER =
@@ -142,7 +158,7 @@ IGNORE_PREFIX =
# Configuration options related to the HTML output
#---------------------------------------------------------------------------
GENERATE_HTML = YES
-HTML_OUTPUT = html
+HTML_OUTPUT = html
HTML_FILE_EXTENSION = .html
HTML_HEADER =
HTML_FOOTER =
diff --git a/serde/server-name.h b/serde/server-name.h
index 9844465..6551fdc 100644
--- a/serde/server-name.h
+++ b/serde/server-name.h
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
#pragma once
#include "if/HBase.pb.h"
[hbase] 83/137: HBASE-17576 [C++] Implement request retry mechanism
over RPC for Multi calls. (Sudeep Sunthankar)
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit c34838bc673e0dcc9acc455b52e230dabdae357c
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Tue May 16 11:43:04 2017 -0700
HBASE-17576 [C++] Implement request retry mechanism over RPC for Multi calls. (Sudeep Sunthankar)
---
core/BUCK | 2 +
core/action.h | 13 +-
core/async-batch-rpc-retrying-caller.cc | 501 +++++++++++++++++++++++++++++++
core/async-batch-rpc-retrying-caller.h | 183 +++++++++++
core/async-rpc-retrying-caller-factory.h | 84 +++++-
core/async-rpc-retrying-caller.cc | 4 +-
core/async-rpc-retrying-test.cc | 4 +-
core/client-test.cc | 53 ++++
core/raw-async-table.cc | 22 +-
core/raw-async-table.h | 9 +-
core/request-converter.cc | 13 +-
core/simple-client.cc | 22 +-
core/table.cc | 18 +-
core/table.h | 3 +-
14 files changed, 897 insertions(+), 34 deletions(-)
diff --git a/core/BUCK b/core/BUCK
index 412ee3b..e9fc716 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -53,6 +53,7 @@ cxx_library(
"region-result.h",
"row.h",
"server-request.h",
+ "async-batch-rpc-retrying-caller.h",
],
srcs=[
"async-connection.cc",
@@ -77,6 +78,7 @@ cxx_library(
"zk-util.cc",
"multi-response.cc",
"region-result.cc",
+ "async-batch-rpc-retrying-caller.cc",
],
deps=[
"//exceptions:exceptions",
diff --git a/core/action.h b/core/action.h
index 3511683..21a0181 100644
--- a/core/action.h
+++ b/core/action.h
@@ -20,24 +20,23 @@
#pragma once
#include <memory>
-#include "core/row.h"
+#include "core/get.h"
-using hbase::Row;
namespace hbase {
class Action {
public:
- Action(std::shared_ptr<Row> action, int original_index)
+ Action(std::shared_ptr<hbase::Get> action, int32_t original_index)
: action_(action), original_index_(original_index) {}
~Action() {}
- int64_t original_index() const { return original_index_; }
+ int32_t original_index() const { return original_index_; }
- std::shared_ptr<Row> action() const { return action_; }
+ std::shared_ptr<hbase::Get> action() const { return action_; }
private:
- std::shared_ptr<Row> action_;
- int64_t original_index_;
+ std::shared_ptr<hbase::Get> action_;
+ int32_t original_index_;
int64_t nonce_ = -1;
int32_t replica_id_ = -1;
};
diff --git a/core/async-batch-rpc-retrying-caller.cc b/core/async-batch-rpc-retrying-caller.cc
new file mode 100644
index 0000000..f3be637
--- /dev/null
+++ b/core/async-batch-rpc-retrying-caller.cc
@@ -0,0 +1,501 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/async-batch-rpc-retrying-caller.h"
+#include <glog/logging.h>
+#include <limits>
+
+using folly::Future;
+using folly::Promise;
+using folly::Try;
+
+using folly::Future;
+using folly::Promise;
+using folly::Try;
+using hbase::Action;
+using hbase::LocationCache;
+using hbase::MultiResponse;
+using hbase::RegionLocation;
+using hbase::RegionRequest;
+using hbase::RequestConverter;
+using hbase::Result;
+using hbase::RpcClient;
+using hbase::ServerRequest;
+using hbase::pb::ServerName;
+using hbase::pb::TableName;
+using hbase::security::User;
+using std::chrono::nanoseconds;
+using std::chrono::milliseconds;
+using wangle::CPUThreadPoolExecutor;
+
+namespace hbase {
+
+AsyncBatchRpcRetryingCaller::AsyncBatchRpcRetryingCaller(
+ std::shared_ptr<AsyncConnection> conn, std::shared_ptr<folly::HHWheelTimer> retry_timer,
+ std::shared_ptr<TableName> table_name, const std::vector<hbase::Get> &actions,
+ nanoseconds pause_ns, int32_t max_attempts, nanoseconds operation_timeout_ns,
+ nanoseconds rpc_timeout_ns, int32_t start_log_errors_count)
+ : conn_(conn),
+ retry_timer_(retry_timer),
+ table_name_(table_name),
+ pause_ns_(pause_ns),
+ operation_timeout_ns_(operation_timeout_ns),
+ rpc_timeout_ns_(rpc_timeout_ns),
+ start_log_errors_count_(start_log_errors_count) {
+ CHECK(conn_ != nullptr);
+ CHECK(retry_timer_ != nullptr);
+ location_cache_ = conn_->region_locator();
+ rpc_client_ = conn_->rpc_client();
+ cpu_pool_ = conn_->cpu_executor();
+ CHECK(location_cache_ != nullptr);
+ CHECK(rpc_client_ != nullptr);
+ CHECK(cpu_pool_ != nullptr);
+
+ max_attempts_ = ConnectionUtils::Retries2Attempts(max_attempts);
+ uint32_t index = 0;
+ for (auto row : actions) {
+ actions_.push_back(std::make_shared<Action>(std::make_shared<hbase::Get>(row), index));
+ Promise<std::shared_ptr<Result>> prom{};
+ action2promises_.insert(
+ std::pair<uint64_t, Promise<std::shared_ptr<Result>>>(index, std::move(prom)));
+ action2futures_.push_back(action2promises_[index++].getFuture());
+ }
+}
+
+AsyncBatchRpcRetryingCaller::~AsyncBatchRpcRetryingCaller() {}
+
+Future<std::vector<Try<std::shared_ptr<Result>>>> AsyncBatchRpcRetryingCaller::Call() {
+ GroupAndSend(actions_, 1);
+ return collectAll(action2futures_);
+}
+
+int64_t AsyncBatchRpcRetryingCaller::RemainingTimeNs() {
+ return operation_timeout_ns_.count() - (TimeUtil::GetNowNanos() - start_ns_);
+}
+
+void AsyncBatchRpcRetryingCaller::LogException(int32_t tries,
+ std::shared_ptr<RegionRequest> region_request,
+ std::shared_ptr<std::exception> &error,
+ std::shared_ptr<ServerName> server_name) {
+ if (tries > start_log_errors_count_) {
+ std::string regions;
+ regions += region_request->region_location()->region_name() + ", ";
+ LOG(WARNING) << "Process batch for " << regions << " in " << table_name_->namespace_() << ":"
+ << table_name_->qualifier() << " from " << server_name->host_name()
+ << " failed, tries=" << tries << ":- " << error->what();
+ }
+}
+
+void AsyncBatchRpcRetryingCaller::LogException(
+ int32_t tries, std::vector<std::shared_ptr<RegionRequest>> ®ion_requests,
+ std::shared_ptr<std::exception> &error, std::shared_ptr<ServerName> server_name) {
+ if (tries > start_log_errors_count_) {
+ std::string regions;
+ for (const auto region_request : region_requests) {
+ regions += region_request->region_location()->region_name() + ", ";
+ }
+ LOG(WARNING) << "Process batch for " << regions << " in " << table_name_->namespace_() << ":"
+ << table_name_->qualifier() << " from " << server_name->host_name()
+ << " failed, tries=" << tries << error->what();
+ }
+}
+
+const std::string AsyncBatchRpcRetryingCaller::GetExtraContextForError(
+ std::shared_ptr<ServerName> server_name) {
+ return server_name ? server_name->ShortDebugString() : "";
+}
+
+// TODO HBASE-17800 pass folly ew instead of std::exception
+void AsyncBatchRpcRetryingCaller::AddError(const std::shared_ptr<Action> &action,
+ std::shared_ptr<std::exception> error,
+ std::shared_ptr<ServerName> server_name) {
+ folly::exception_wrapper ew;
+ ThrowableWithExtraContext twec(ew, TimeUtil::GetNowNanos(), GetExtraContextForError(server_name));
+ AddAction2Error(action->original_index(), twec);
+}
+
+void AsyncBatchRpcRetryingCaller::AddError(const std::vector<std::shared_ptr<Action>> &actions,
+ std::shared_ptr<std::exception> error,
+ std::shared_ptr<ServerName> server_name) {
+ for (const auto action : actions) {
+ AddError(action, error, server_name);
+ }
+}
+
+// TODO HBASE-17800 pass folly ew instead of std::exception
+void AsyncBatchRpcRetryingCaller::FailOne(const std::shared_ptr<Action> &action, int32_t tries,
+ std::shared_ptr<std::exception> error,
+ int64_t current_time, const std::string extras) {
+ auto action_index = action->original_index();
+ auto itr = action2promises_.find(action_index);
+ if (itr != action2promises_.end()) {
+ if (itr->second.isFulfilled()) {
+ return;
+ }
+ }
+ folly::exception_wrapper ew;
+ ThrowableWithExtraContext twec(ew, current_time, extras);
+ AddAction2Error(action_index, twec);
+ action2promises_[action_index].setException(
+ RetriesExhaustedException(tries - 1, action2errors_[action_index]));
+}
+
+void AsyncBatchRpcRetryingCaller::FailAll(const std::vector<std::shared_ptr<Action>> &actions,
+ int32_t tries, std::shared_ptr<std::exception> error,
+ std::shared_ptr<ServerName> server_name) {
+ for (const auto action : actions) {
+ FailOne(action, tries, error, TimeUtil::GetNowNanos(), GetExtraContextForError(server_name));
+ }
+}
+
+void AsyncBatchRpcRetryingCaller::FailAll(const std::vector<std::shared_ptr<Action>> &actions,
+ int32_t tries) {
+ for (const auto action : actions) {
+ auto action_index = action->original_index();
+ auto itr = action2promises_.find(action_index);
+ if (itr->second.isFulfilled()) {
+ return;
+ }
+ action2promises_[action_index].setException(
+ RetriesExhaustedException(tries - 1, action2errors_[action_index]));
+ }
+}
+
+void AsyncBatchRpcRetryingCaller::AddAction2Error(uint64_t action_index,
+ const ThrowableWithExtraContext &twec) {
+ auto erritr = action2errors_.find(action_index);
+ if (erritr != action2errors_.end()) {
+ erritr->second->push_back(twec);
+ } else {
+ action2errors_[action_index] = std::make_shared<std::vector<ThrowableWithExtraContext>>();
+ action2errors_[action_index]->push_back(twec);
+ }
+ return;
+}
+
+void AsyncBatchRpcRetryingCaller::OnError(const ActionsByRegion &actions_by_region, int32_t tries,
+ std::shared_ptr<std::exception> exc,
+ std::shared_ptr<ServerName> server_name) {
+ std::vector<std::shared_ptr<Action>> copied_actions;
+ std::vector<std::shared_ptr<RegionRequest>> region_requests;
+ for (const auto &action_by_region : actions_by_region) {
+ region_requests.push_back(action_by_region.second);
+ // Concurrent
+ for (const auto &action : action_by_region.second->actions()) {
+ copied_actions.push_back(action);
+ }
+ }
+ // TODO HBASE-17800 for exc check with DoNotRetryIOException
+ LogException(tries, region_requests, exc, server_name);
+ if (tries >= max_attempts_) {
+ FailAll(copied_actions, tries, exc, server_name);
+ return;
+ }
+ AddError(copied_actions, exc, server_name);
+ TryResubmit(copied_actions, tries);
+}
+
+void AsyncBatchRpcRetryingCaller::TryResubmit(std::vector<std::shared_ptr<Action>> actions,
+ int32_t tries) {
+ int64_t delay_ns;
+ if (operation_timeout_ns_.count() > 0) {
+ int64_t max_delay_ns = RemainingTimeNs() - ConnectionUtils::kSleepDeltaNs;
+ if (max_delay_ns <= 0) {
+ VLOG(8) << "Fail All from onError";
+ FailAll(actions, tries);
+ return;
+ }
+ delay_ns = std::min(max_delay_ns, ConnectionUtils::GetPauseTime(pause_ns_.count(), tries - 1));
+ } else {
+ delay_ns = ConnectionUtils::GetPauseTime(pause_ns_.count(), tries - 1);
+ }
+ // TODO This gives segfault @ present, when retried
+ // retry_timer_->scheduleTimeoutFn([&]() { GroupAndSend(actions, tries + 1); },
+ // milliseconds(TimeUtil::ToMillis(delay_ns)));
+}
+
+Future<std::vector<Try<std::shared_ptr<RegionLocation>>>>
+AsyncBatchRpcRetryingCaller::GetRegionLocations(const std::vector<std::shared_ptr<Action>> &actions,
+ int64_t locate_timeout_ns) {
+ auto locs = std::vector<Future<std::shared_ptr<RegionLocation>>>{};
+ for (auto const &action : actions) {
+ locs.push_back(location_cache_->LocateRegion(*table_name_, action->action()->row(),
+ RegionLocateType::kCurrent, locate_timeout_ns));
+ }
+
+ return collectAll(locs);
+}
+
+void AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<Action>> &actions,
+ int32_t tries) {
+ int64_t locate_timeout_ns;
+ if (operation_timeout_ns_.count() > 0) {
+ locate_timeout_ns = RemainingTimeNs();
+ if (locate_timeout_ns <= 0) {
+ FailAll(actions_, tries);
+ return;
+ }
+ } else {
+ locate_timeout_ns = -1L;
+ }
+
+ GetRegionLocations(actions, locate_timeout_ns)
+ .then([&](std::vector<Try<std::shared_ptr<RegionLocation>>> &loc) {
+ std::lock_guard<std::mutex> lock(multi_mutex_);
+ ActionsByServer actions_by_server;
+ std::vector<std::shared_ptr<Action>> locate_failed;
+
+ for (uint64_t i = 0; i < loc.size(); ++i) {
+ auto action = actions[i];
+ if (loc[i].hasValue()) {
+ auto region_loc = loc[i].value();
+ // Add it to actions_by_server;
+ // Concurrent
+ auto search =
+ actions_by_server.find(std::make_shared<ServerName>(region_loc->server_name()));
+ if (search != actions_by_server.end()) {
+ search->second->AddActionsByRegion(region_loc, action);
+ } else {
+ // Create new key
+ auto server_request = std::make_shared<ServerRequest>(region_loc);
+ server_request->AddActionsByRegion(region_loc, action);
+ auto server_name = std::make_shared<ServerName>(region_loc->server_name());
+ actions_by_server[server_name] = server_request;
+ }
+ locate_failed.push_back(action);
+ VLOG(8) << "row [" << action->action()->row() << "] of table["
+ << table_name_->namespace_() << ":" << table_name_->qualifier()
+ << " found in region [" << region_loc->region_name() << "]; host["
+ << region_loc->server_name().host_name() << "]; port["
+ << region_loc->server_name().port() << "];";
+ } else if (loc[i].hasException()) {
+ VLOG(8) << "Exception occured while locating region:- "
+ << loc[i].exception().getCopied()->what() << " for action index " << i;
+ // TODO Feedback needed, Java API only identifies DoNotRetryIOException
+ // We might receive runtime error from location-cache.cc too, we are treating both same
+ if (loc[i].exception().is_compatible_with<std::runtime_error>()) {
+ std::string extra = "";
+ FailOne(action, tries, nullptr, TimeUtil::GetNowNanos(),
+ loc[i].exception().what().toStdString());
+ return;
+ }
+ // TODO HBASE-17800 for exc check with DoNotRetryIOException
+ /*
+ else if (loc[i].exception().is_compatible_with<hbase::DoNotRetryIOException>()) {
+ int64_t current_time = 0;
+ std::string extra = "";
+ FailOne(action, tries, nullptr, TimeUtil::GetNowNanos(),
+ loc[i].exception().what().toStdString());
+ return;
+ }*/
+ AddError(action, std::make_shared<std::exception>(*loc[i].exception().getCopied()),
+ nullptr);
+ locate_failed.push_back(action);
+ }
+ }
+
+ if (!actions_by_server.empty()) {
+ Send(actions_by_server, tries);
+ }
+
+ if (!locate_failed.empty()) {
+ TryResubmit(locate_failed, tries);
+ }
+ })
+ .onError([&](const folly::exception_wrapper &ew) {
+ std::lock_guard<std::mutex> lock(multi_mutex_);
+ auto exc = ew.getCopied();
+ VLOG(8) << "GetRegionLocations() exception: " << ew.what().toStdString();
+ });
+ return;
+}
+
+Future<std::vector<Try<std::unique_ptr<Response>>>> AsyncBatchRpcRetryingCaller::GetMultiResponse(
+ const ActionsByServer &actions_by_server) {
+ // Concurrent.
+ auto multi_calls = std::vector<Future<std::unique_ptr<hbase::Response>>>{};
+ auto user = User::defaultUser();
+ for (const auto &action_by_server : actions_by_server) {
+ std::unique_ptr<Request> multi_req =
+ RequestConverter::ToMultiRequest(action_by_server.second->actions_by_region());
+ auto host = action_by_server.first->host_name();
+ int port = action_by_server.first->port();
+ multi_calls.push_back(
+ rpc_client_->AsyncCall(host, port, std::move(multi_req), user, "ClientService"));
+ }
+ return collectAll(multi_calls);
+}
+
+void AsyncBatchRpcRetryingCaller::Send(ActionsByServer &actions_by_server, int32_t tries) {
+ int64_t remaining_ns;
+ if (operation_timeout_ns_.count() > 0) {
+ remaining_ns = RemainingTimeNs();
+ if (remaining_ns <= 0) {
+ std::vector<std::shared_ptr<Action>> failed_actions;
+ for (const auto &action_by_server : actions_by_server) {
+ // Concurrent
+ for (auto &value : action_by_server.second->actions_by_region()) {
+ // Concurrent
+ for (const auto &failed_action : value.second->actions()) {
+ failed_actions.push_back(failed_action);
+ }
+ }
+ }
+ FailAll(failed_actions, tries);
+ return;
+ }
+ } else {
+ remaining_ns = std::numeric_limits<int64_t>::max();
+ }
+
+ std::vector<std::shared_ptr<Request>> multi_reqv;
+ for (const auto &action_by_server : actions_by_server)
+ multi_reqv.push_back(
+ std::move(RequestConverter::ToMultiRequest(action_by_server.second->actions_by_region())));
+
+ GetMultiResponse(actions_by_server)
+ .then([=](const std::vector<Try<std::unique_ptr<hbase::Response>>> &completed_responses) {
+ std::lock_guard<std::mutex> lock(multi_mutex_);
+ for (uint64_t num = 0; num < completed_responses.size(); ++num) {
+ if (completed_responses[num].hasValue()) {
+ auto multi_response =
+ ResponseConverter::GetResults(multi_reqv[num], *completed_responses[num].value());
+ for (const auto &action_by_server : actions_by_server) {
+ OnComplete(action_by_server.second->actions_by_region(), tries,
+ action_by_server.first, std::move(multi_response));
+ }
+ } else if (completed_responses[num].hasException()) {
+ VLOG(8) << "Received exception: "
+ << completed_responses[num].exception().getCopied()->what()
+ << " from server for action index " << num;
+ // TODO: we should call OnError here as well.
+ }
+ }
+ })
+ .onError([=](const folly::exception_wrapper &ew) {
+ auto exc = ew.getCopied();
+ VLOG(8) << "GetMultiResponse() exception: " << ew.what().toStdString();
+ std::lock_guard<std::mutex> lock(multi_mutex_);
+ for (const auto &action_by_server : actions_by_server) {
+ OnError(action_by_server.second->actions_by_region(), tries,
+ std::make_shared<std::exception>(*exc), action_by_server.first);
+ }
+ });
+ return;
+}
+
+void AsyncBatchRpcRetryingCaller::OnComplete(
+ const ActionsByRegion &actions_by_region, int32_t tries,
+ const std::shared_ptr<ServerName> server_name,
+ const std::unique_ptr<hbase::MultiResponse> multi_response) {
+ std::vector<std::shared_ptr<Action>> failed_actions;
+ for (const auto &action_by_region : actions_by_region) {
+ auto region_result_itr = multi_response->RegionResults().find(action_by_region.first);
+ if (region_result_itr == multi_response->RegionResults().end()) {
+ VLOG(8) << "Region " << action_by_region.first << " not found in MultiResults.";
+ // TODO Feedback needed Should we throw from here or continue for next action_by_region ?
+ // Throwing at present as this looks like an inconsistency
+ // Concurrent
+ auto exc = std::make_shared<std::runtime_error>("Invalid search for region " +
+ action_by_region.first + " in multi results");
+ FailAll(action_by_region.second->actions(), tries, exc, server_name);
+ return;
+ // std::runtime_error(
+ // "Invalid search for region " + action_by_region.first + " in multi results");
+ }
+ if (region_result_itr != multi_response->RegionResults().end()) {
+ // Concurrent
+ for (const auto &action : action_by_region.second->actions()) {
+ OnComplete(action, action_by_region.second, tries, server_name, region_result_itr->second,
+ failed_actions);
+ }
+ } else {
+ auto region_exc = multi_response->RegionException(action_by_region.first);
+ std::shared_ptr<std::exception> pexc;
+ if (region_exc == nullptr) {
+ VLOG(8) << "Server sent us neither results nor exceptions for " << action_by_region.first;
+ pexc = std::make_shared<std::exception>(std::runtime_error("Invalid response"));
+ // TODO: raise this exception to the application
+ } else {
+ // TODO HBASE-17800 for exc check with DoNotRetryIOException
+ LogException(tries, action_by_region.second, region_exc, server_name);
+ location_cache_->UpdateCachedLocation(*action_by_region.second->region_location(),
+ *region_exc);
+ std::string row_name;
+ if (tries >= max_attempts_) {
+ // Concurrent
+ FailAll(action_by_region.second->actions(), tries, region_exc, server_name);
+ return;
+ }
+ // Concurrent
+ AddError(action_by_region.second->actions(), region_exc, server_name);
+ for (const auto &action : action_by_region.second->actions()) {
+ failed_actions.push_back(action);
+ }
+ }
+ }
+ }
+ if (!failed_actions.empty()) {
+ TryResubmit(failed_actions, tries);
+ }
+ return;
+}
+
+void AsyncBatchRpcRetryingCaller::OnComplete(const std::shared_ptr<Action> &action,
+ const std::shared_ptr<RegionRequest> ®ion_request,
+ int32_t tries,
+ const std::shared_ptr<ServerName> &server_name,
+ const std::shared_ptr<RegionResult> ®ion_result,
+ std::vector<std::shared_ptr<Action>> &failed_actions) {
+ std::string err_msg;
+ try {
+ auto result_or_exc = region_result->ResultOrException(action->original_index());
+ auto result = std::get<0>(*result_or_exc);
+ auto exc = std::get<1>(*result_or_exc);
+ std::shared_ptr<std::exception> pexc;
+ if (exc != nullptr) {
+ LogException(tries, region_request, exc, server_name);
+ if (tries >= max_attempts_) {
+ FailOne(action, tries, exc, TimeUtil::GetNowNanos(), GetExtraContextForError(server_name));
+ } else {
+ failed_actions.push_back(action);
+ }
+ } else if (result != nullptr) {
+ action2promises_[action->original_index()].setValue(std::move(result));
+ } else {
+ VLOG(8) << "Server " << server_name->ShortDebugString()
+ << " sent us neither results nor exceptions for request @ index "
+ << action->original_index() << ", row " << action->action()->row() << " of "
+ << region_request->region_location()->region_name();
+ err_msg = "Invalid response";
+ AddError(action, std::make_shared<std::runtime_error>(err_msg), server_name);
+ failed_actions.push_back(action);
+ }
+ } catch (const std::out_of_range &oor) {
+ // TODO Feedback needed. Should we retry for he specific index again ?
+ // This should never occur, so we are throwing a std::runtime_error from here
+ VLOG(8) << "No ResultOrException found @ index " << action->original_index() << ", row "
+ << action->action()->row() << " of "
+ << region_request->region_location()->region_name();
+ throw std::runtime_error("ResultOrException not present @ index " +
+ std::to_string(action->original_index()));
+ }
+ return;
+}
+} /* namespace hbase */
diff --git a/core/async-batch-rpc-retrying-caller.h b/core/async-batch-rpc-retrying-caller.h
new file mode 100644
index 0000000..6803a0e
--- /dev/null
+++ b/core/async-batch-rpc-retrying-caller.h
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <folly/Format.h>
+#include <folly/futures/Future.h>
+#include <folly/futures/Promise.h>
+#include <folly/futures/Try.h>
+#include <folly/io/IOBuf.h>
+#include <folly/io/async/HHWheelTimer.h>
+#include <wangle/concurrent/CPUThreadPoolExecutor.h>
+
+#include <algorithm>
+#include <chrono>
+#include <functional>
+#include <map>
+#include <memory>
+#include <mutex>
+#include <stdexcept>
+#include <string>
+#include <tuple>
+#include <type_traits>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "connection/rpc-client.h"
+#include "core/action.h"
+#include "core/async-connection.h"
+#include "core/location-cache.h"
+#include "core/multi-response.h"
+#include "core/region-location.h"
+#include "core/region-request.h"
+#include "core/region-result.h"
+#include "core/request-converter.h"
+#include "core/response-converter.h"
+#include "core/result.h"
+#include "core/row.h"
+#include "core/server-request.h"
+#include "exceptions/exception.h"
+#include "if/Client.pb.h"
+#include "if/HBase.pb.h"
+#include "security/user.h"
+#include "utils/connection-util.h"
+#include "utils/sys-util.h"
+#include "utils/time-util.h"
+
+namespace hbase {
+/* Equals function for ServerName */
+struct ServerNameEquals {
+ bool operator()(const std::shared_ptr<ServerName> &lhs,
+ const std::shared_ptr<ServerName> &rhs) const {
+ return (lhs->start_code() == rhs->start_code() && lhs->host_name() == rhs->host_name() &&
+ lhs->port() == rhs->port());
+ }
+};
+
+struct ServerNameHash {
+ /** hash */
+ std::size_t operator()(const std::shared_ptr<ServerName> &sn) const {
+ std::size_t h = 0;
+ boost::hash_combine(h, sn->start_code());
+ boost::hash_combine(h, sn->host_name());
+ boost::hash_combine(h, sn->port());
+ return h;
+ }
+};
+
+class AsyncBatchRpcRetryingCaller {
+ public:
+ using ActionsByServer =
+ std::unordered_map<std::shared_ptr<ServerName>, std::shared_ptr<ServerRequest>,
+ ServerNameHash, ServerNameEquals>;
+ using ActionsByRegion = ServerRequest::ActionsByRegion;
+
+ AsyncBatchRpcRetryingCaller(std::shared_ptr<AsyncConnection> conn,
+ std::shared_ptr<folly::HHWheelTimer> retry_timer,
+ std::shared_ptr<pb::TableName> table_name,
+ const std::vector<hbase::Get> &actions, nanoseconds pause_ns,
+ int32_t max_attempts, nanoseconds operation_timeout_ns,
+ nanoseconds rpc_timeout_ns, int32_t start_log_errors_count);
+
+ ~AsyncBatchRpcRetryingCaller();
+
+ folly::Future<std::vector<folly::Try<std::shared_ptr<Result>>>> Call();
+
+ private:
+ int64_t RemainingTimeNs();
+
+ void LogException(int32_t tries, std::shared_ptr<RegionRequest> region_request,
+ std::shared_ptr<std::exception> &error,
+ std::shared_ptr<ServerName> server_name);
+
+ void LogException(int32_t tries, std::vector<std::shared_ptr<RegionRequest>> ®ion_requests,
+ std::shared_ptr<std::exception> &error,
+ std::shared_ptr<ServerName> server_name);
+
+ const std::string GetExtraContextForError(std::shared_ptr<ServerName> server_name);
+
+ void AddError(const std::shared_ptr<Action> &action, std::shared_ptr<std::exception> error,
+ std::shared_ptr<ServerName> server_name);
+
+ void AddError(const std::vector<std::shared_ptr<Action>> &actions,
+ std::shared_ptr<std::exception> error, std::shared_ptr<ServerName> server_name);
+
+ void FailOne(const std::shared_ptr<Action> &action, int32_t tries,
+ std::shared_ptr<std::exception> error, int64_t current_time,
+ const std::string extras);
+
+ void FailAll(const std::vector<std::shared_ptr<Action>> &actions, int32_t tries,
+ std::shared_ptr<std::exception> error, std::shared_ptr<ServerName> server_name);
+
+ void FailAll(const std::vector<std::shared_ptr<Action>> &actions, int32_t tries);
+
+ void AddAction2Error(uint64_t action_index, const ThrowableWithExtraContext &twec);
+
+ void OnError(const ActionsByRegion &actions_by_region, int32_t tries,
+ std::shared_ptr<std::exception> exc, std::shared_ptr<ServerName> server_name);
+
+ void TryResubmit(std::vector<std::shared_ptr<Action>> actions, int32_t tries);
+
+ folly::Future<std::vector<folly::Try<std::shared_ptr<RegionLocation>>>> GetRegionLocations(
+ const std::vector<std::shared_ptr<Action>> &actions, int64_t locate_timeout_ns);
+
+ void GroupAndSend(const std::vector<std::shared_ptr<Action>> &actions, int32_t tries);
+
+ folly::Future<std::vector<folly::Try<std::unique_ptr<Response>>>> GetMultiResponse(
+ const ActionsByServer &actions_by_server);
+
+ void Send(ActionsByServer &actions_by_server, int32_t tries);
+
+ void OnComplete(const ActionsByRegion &actions_by_region, int32_t tries,
+ const std::shared_ptr<ServerName> server_name,
+ const std::unique_ptr<MultiResponse> multi_results);
+
+ void OnComplete(const std::shared_ptr<Action> &action,
+ const std::shared_ptr<RegionRequest> ®ion_request, int32_t tries,
+ const std::shared_ptr<ServerName> &server_name,
+ const std::shared_ptr<RegionResult> ®ion_result,
+ std::vector<std::shared_ptr<Action>> &failed_actions);
+
+ private:
+ std::shared_ptr<folly::HHWheelTimer> retry_timer_;
+ std::shared_ptr<hbase::AsyncConnection> conn_;
+ std::shared_ptr<pb::TableName> table_name_;
+ std::vector<std::shared_ptr<Action>> actions_;
+ nanoseconds pause_ns_;
+ int32_t max_attempts_ = 0;
+ nanoseconds operation_timeout_ns_;
+ nanoseconds rpc_timeout_ns_;
+ int32_t start_log_errors_count_ = 0;
+
+ int64_t start_ns_ = TimeUtil::GetNowNanos();
+ int32_t tries_ = 1;
+ std::map<uint64_t, folly::Promise<std::shared_ptr<Result>>> action2promises_;
+ std::vector<folly::Future<std::shared_ptr<Result>>> action2futures_;
+ std::map<uint64_t, std::shared_ptr<std::vector<ThrowableWithExtraContext>>> action2errors_;
+
+ std::shared_ptr<AsyncRegionLocator> location_cache_ = nullptr;
+ std::shared_ptr<RpcClient> rpc_client_ = nullptr;
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_pool_ = nullptr;
+
+ std::mutex multi_mutex_;
+};
+
+} /* namespace hbase */
diff --git a/core/async-rpc-retrying-caller-factory.h b/core/async-rpc-retrying-caller-factory.h
index 5a80a06..f1ffdac 100644
--- a/core/async-rpc-retrying-caller-factory.h
+++ b/core/async-rpc-retrying-caller-factory.h
@@ -19,20 +19,19 @@
#pragma once
#include <folly/Logging.h>
-#include <folly/io/IOBuf.h>
#include <folly/io/async/EventBase.h>
#include <chrono>
#include <memory>
#include <string>
+#include <vector>
#include "connection/rpc-client.h"
+#include "core/async-batch-rpc-retrying-caller.h"
#include "core/async-rpc-retrying-caller.h"
+#include "core/row.h"
#include "if/Client.pb.h"
#include "if/HBase.pb.h"
-using hbase::pb::TableName;
-using std::chrono::nanoseconds;
-
namespace hbase {
class AsyncConnection;
@@ -58,7 +57,7 @@ class SingleRequestCallerBuilder
typedef SingleRequestCallerBuilder<RESP> GenenericThisType;
typedef std::shared_ptr<GenenericThisType> SharedThisPtr;
- SharedThisPtr table(std::shared_ptr<TableName> table_name) {
+ SharedThisPtr table(std::shared_ptr<pb::TableName> table_name) {
table_name_ = table_name;
return shared_this();
}
@@ -119,7 +118,7 @@ class SingleRequestCallerBuilder
private:
std::shared_ptr<AsyncConnection> conn_;
std::shared_ptr<folly::HHWheelTimer> retry_timer_;
- std::shared_ptr<TableName> table_name_;
+ std::shared_ptr<pb::TableName> table_name_;
nanoseconds rpc_timeout_nanos_;
nanoseconds operation_timeout_nanos_;
nanoseconds pause_;
@@ -130,6 +129,75 @@ class SingleRequestCallerBuilder
Callable<RESP> callable_;
}; // end of SingleRequestCallerBuilder
+class BatchCallerBuilder : public std::enable_shared_from_this<BatchCallerBuilder> {
+ public:
+ explicit BatchCallerBuilder(std::shared_ptr<AsyncConnection> conn,
+ std::shared_ptr<folly::HHWheelTimer> retry_timer)
+ : conn_(conn), retry_timer_(retry_timer) {}
+
+ virtual ~BatchCallerBuilder() = default;
+
+ typedef std::shared_ptr<BatchCallerBuilder> SharedThisPtr;
+
+ SharedThisPtr table(std::shared_ptr<pb::TableName> table_name) {
+ table_name_ = table_name;
+ return shared_this();
+ }
+
+ SharedThisPtr actions(std::shared_ptr<std::vector<hbase::Get>> actions) {
+ actions_ = actions;
+ return shared_this();
+ }
+
+ SharedThisPtr operation_timeout(nanoseconds operation_timeout_nanos) {
+ operation_timeout_nanos_ = operation_timeout_nanos;
+ return shared_this();
+ }
+
+ SharedThisPtr rpc_timeout(nanoseconds rpc_timeout_nanos) {
+ rpc_timeout_nanos_ = rpc_timeout_nanos;
+ return shared_this();
+ }
+
+ SharedThisPtr pause(nanoseconds pause_ns) {
+ pause_ns_ = pause_ns;
+ return shared_this();
+ }
+
+ SharedThisPtr max_attempts(int32_t max_attempts) {
+ max_attempts_ = max_attempts;
+ return shared_this();
+ }
+
+ SharedThisPtr start_log_errors_count(int32_t start_log_errors_count) {
+ start_log_errors_count_ = start_log_errors_count;
+ return shared_this();
+ }
+
+ folly::Future<std::vector<folly::Try<std::shared_ptr<Result>>>> Call() { return Build()->Call(); }
+
+ std::shared_ptr<AsyncBatchRpcRetryingCaller> Build() {
+ return std::make_shared<AsyncBatchRpcRetryingCaller>(
+ conn_, retry_timer_, table_name_, *actions_, pause_ns_, max_attempts_,
+ operation_timeout_nanos_, rpc_timeout_nanos_, start_log_errors_count_);
+ }
+
+ private:
+ SharedThisPtr shared_this() {
+ return std::enable_shared_from_this<BatchCallerBuilder>::shared_from_this();
+ }
+
+ private:
+ std::shared_ptr<AsyncConnection> conn_;
+ std::shared_ptr<folly::HHWheelTimer> retry_timer_;
+ std::shared_ptr<hbase::pb::TableName> table_name_ = nullptr;
+ std::shared_ptr<std::vector<hbase::Get>> actions_ = nullptr;
+ nanoseconds pause_ns_;
+ int32_t max_attempts_ = 0;
+ nanoseconds operation_timeout_nanos_;
+ nanoseconds rpc_timeout_nanos_;
+ int32_t start_log_errors_count_ = 0;
+};
class AsyncRpcRetryingCallerFactory {
private:
std::shared_ptr<AsyncConnection> conn_;
@@ -146,6 +214,10 @@ class AsyncRpcRetryingCallerFactory {
std::shared_ptr<SingleRequestCallerBuilder<RESP>> Single() {
return std::make_shared<SingleRequestCallerBuilder<RESP>>(conn_, retry_timer_);
}
+
+ std::shared_ptr<BatchCallerBuilder> Batch() {
+ return std::make_shared<BatchCallerBuilder>(conn_, retry_timer_);
+ }
};
} // namespace hbase
diff --git a/core/async-rpc-retrying-caller.cc b/core/async-rpc-retrying-caller.cc
index 7e211f7..f8b237b 100644
--- a/core/async-rpc-retrying-caller.cc
+++ b/core/async-rpc-retrying-caller.cc
@@ -143,9 +143,7 @@ void AsyncSingleRequestRpcRetryingCaller<RESP>::OnError(
*/
conn_->retry_executor()->add([&]() {
retry_timer_->scheduleTimeoutFn(
- [this]() {
- conn_->cpu_executor()->add([&]() { LocateThenCall(); });
- },
+ [this]() { conn_->cpu_executor()->add([&]() { LocateThenCall(); }); },
milliseconds(TimeUtil::ToMillis(delay_ns)));
});
}
diff --git a/core/async-rpc-retrying-test.cc b/core/async-rpc-retrying-test.cc
index ff28e79..487c34c 100644
--- a/core/async-rpc-retrying-test.cc
+++ b/core/async-rpc-retrying-test.cc
@@ -159,9 +159,9 @@ class MockFailingAsyncRegionLocator : public AsyncRegionLocatorBase {
uint32_t num_fails_ = 0;
public:
- explicit MockFailingAsyncRegionLocator(uint32_t num_fails)
+ explicit MockFailingAsyncRegionLocator(uint32_t num_fails)
: AsyncRegionLocatorBase(), num_fails_(num_fails) {}
- explicit MockFailingAsyncRegionLocator(std::shared_ptr<RegionLocation> region_location)
+ explicit MockFailingAsyncRegionLocator(std::shared_ptr<RegionLocation> region_location)
: AsyncRegionLocatorBase(region_location) {}
virtual ~MockFailingAsyncRegionLocator() {}
folly::Future<std::shared_ptr<hbase::RegionLocation>> LocateRegion(
diff --git a/core/client-test.cc b/core/client-test.cc
index 274168f..1c6ec4a 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -234,3 +234,56 @@ TEST_F(ClientTest, PutsWithTimestamp) {
table->Close();
client.Close();
}
+
+TEST_F(ClientTest, MultiGets) {
+ // Using TestUtil to populate test data
+ ClientTest::test_util->CreateTable("t", "d");
+
+ // Create TableName and Row to be fetched from HBase
+ auto tn = folly::to<hbase::pb::TableName>("t");
+
+ // Create a client
+ hbase::Client client(*ClientTest::test_util->conf());
+
+ // Get connection to HBase Table
+ auto table = client.Table(tn);
+ ASSERT_TRUE(table) << "Unable to get connection to Table.";
+
+ uint64_t num_rows = 10000;
+ // Perform Puts
+ for (uint64_t i = 0; i < num_rows; i++) {
+ table->Put(Put{"test" + std::to_string(i)}.AddColumn("d", std::to_string(i),
+ "value" + std::to_string(i)));
+ }
+
+ // Perform the Gets
+ std::vector<hbase::Get> gets;
+ for (uint64_t i = 0; i < num_rows; ++i) {
+ auto row = "test" + std::to_string(i);
+ hbase::Get get(row);
+ gets.push_back(get);
+ }
+ gets.push_back(hbase::Get("test2"));
+ gets.push_back(hbase::Get("testextra"));
+
+ auto results = table->Get(gets);
+
+ // Test the values, should be same as in put executed on hbase shell
+ ASSERT_TRUE(!results.empty()) << "Result vector shouldn't be empty.";
+
+ uint32_t i = 0;
+ for (; i < num_rows; ++i) {
+ ASSERT_TRUE(!results[i]->IsEmpty()) << "Result for Get " << gets[i].row()
+ << " must not be empty";
+ EXPECT_EQ("test" + std::to_string(i), results[i]->Row());
+ EXPECT_EQ("value" + std::to_string(i), *results[i]->Value("d", std::to_string(i)).get());
+ }
+ // We are inserting test2 twice so the below test should pass
+ ASSERT_TRUE(!results[i]->IsEmpty()) << "Result for Get " << gets[i].row() << " must not be empty";
+
+ ++i;
+ ASSERT_TRUE(results[i]->IsEmpty()) << "Result for Get " << gets[i].row() << " must be empty";
+
+ table->Close();
+ client.Close();
+}
diff --git a/core/raw-async-table.cc b/core/raw-async-table.cc
index 2bc9f36..9e0d4a3 100644
--- a/core/raw-async-table.cc
+++ b/core/raw-async-table.cc
@@ -91,4 +91,24 @@ Future<Unit> RawAsyncTable::Put(const hbase::Put& put) {
return caller->Call().then([caller](const auto r) { return r; });
}
-} /* namespace hbase */
+Future<std::vector<Try<std::shared_ptr<Result>>>> RawAsyncTable::Get(
+ const std::vector<hbase::Get>& gets) {
+ return this->Batch(gets);
+}
+
+Future<std::vector<Try<std::shared_ptr<Result>>>> RawAsyncTable::Batch(
+ const std::vector<hbase::Get>& gets) {
+ auto caller = connection_->caller_factory()
+ ->Batch()
+ ->table(table_name_)
+ ->actions(std::make_shared<std::vector<hbase::Get>>(gets))
+ ->rpc_timeout(connection_conf_->read_rpc_timeout())
+ ->operation_timeout(connection_conf_->operation_timeout())
+ ->pause(connection_conf_->pause())
+ ->max_attempts(connection_conf_->max_retries())
+ ->start_log_errors_count(connection_conf_->start_log_errors_count())
+ ->Build();
+
+ return caller->Call().then([caller](auto r) { return r; });
+}
+} // namespace hbase
diff --git a/core/raw-async-table.h b/core/raw-async-table.h
index 978a2b8..e26d46e 100644
--- a/core/raw-async-table.h
+++ b/core/raw-async-table.h
@@ -20,11 +20,11 @@
#include <folly/futures/Future.h>
#include <folly/futures/Unit.h>
-
#include <chrono>
#include <memory>
#include <string>
-
+#include <vector>
+#include "core/async-batch-rpc-retrying-caller.h"
#include "core/async-connection.h"
#include "core/async-rpc-retrying-caller-factory.h"
#include "core/async-rpc-retrying-caller.h"
@@ -34,6 +34,7 @@
#include "core/result.h"
using folly::Future;
+using folly::Try;
using folly::Unit;
using hbase::pb::TableName;
using std::chrono::nanoseconds;
@@ -59,6 +60,9 @@ class RawAsyncTable {
Future<Unit> Put(const hbase::Put& put);
void Close() {}
+ Future<std::vector<Try<std::shared_ptr<Result>>>> Get(const std::vector<hbase::Get>& gets);
+ Future<std::vector<Try<std::shared_ptr<Result>>>> Batch(const std::vector<hbase::Get>& gets);
+
private:
/* Data */
std::shared_ptr<AsyncConnection> connection_;
@@ -78,5 +82,4 @@ class RawAsyncTable {
std::shared_ptr<SingleRequestCallerBuilder<RESP>> CreateCallerBuilder(std::string row,
nanoseconds rpc_timeout);
};
-
} // namespace hbase
diff --git a/core/request-converter.cc b/core/request-converter.cc
index 4c12ee7..c90e1ab 100644
--- a/core/request-converter.cc
+++ b/core/request-converter.cc
@@ -50,7 +50,6 @@ std::unique_ptr<Request> RequestConverter::ToGetRequest(const Get &get,
auto pb_msg = std::static_pointer_cast<GetRequest>(pb_req->req_msg());
RequestConverter::SetRegion(region_name, pb_msg->mutable_region());
pb_msg->set_allocated_get((RequestConverter::ToGet(get)).release());
-
return pb_req;
}
@@ -114,12 +113,12 @@ std::unique_ptr<Request> RequestConverter::ToMultiRequest(
int action_num = 0;
for (const auto ®ion_action : action_by_region.second->actions()) {
auto pb_action = pb_region_action->add_action();
- auto action = region_action->action();
- if (auto pget = std::dynamic_pointer_cast<Get>(action)) {
- auto pb_get = RequestConverter::ToGet(*pget.get());
- pb_action->set_allocated_get(pb_get.release());
- pb_action->set_index(action_num);
- }
+ auto pget = region_action->action();
+ // We store only hbase::Get in hbase::Action as of now. It will be changed later on.
+ CHECK(pget) << "Unexpected. action can't be null";
+ auto pb_get = RequestConverter::ToGet(*pget);
+ pb_action->set_allocated_get(pb_get.release());
+ pb_action->set_index(action_num);
action_num++;
}
}
diff --git a/core/simple-client.cc b/core/simple-client.cc
index b417353..3a7d62b 100644
--- a/core/simple-client.cc
+++ b/core/simple-client.cc
@@ -26,6 +26,7 @@
#include <iostream>
#include <thread>
+#include "connection/rpc-client.h"
#include "core/client.h"
#include "core/get.h"
#include "core/put.h"
@@ -75,14 +76,16 @@ int main(int argc, char *argv[]) {
conf->SetInt("hbase.client.cpu.thread.pool.size", FLAGS_threads);
auto row = FLAGS_row;
+
auto tn = std::make_shared<TableName>(folly::to<TableName>(FLAGS_table));
auto num_puts = FLAGS_num_rows;
auto client = std::make_unique<Client>(*conf);
auto table = client->Table(*tn);
- // Do the Put requests
auto start_ns = TimeUtil::GetNowNanos();
+
+ // Do the Put requests
for (uint64_t i = 0; i < num_puts; i++) {
table->Put(*MakePut(Row(FLAGS_row, i)));
}
@@ -102,6 +105,23 @@ int main(int argc, char *argv[]) {
LOG(INFO) << "Successfully sent " << num_puts << " Get requests in "
<< TimeUtil::ElapsedMillis(start_ns) << " ms.";
+ // Do the Multi-Gets
+ std::vector<hbase::Get> gets;
+ for (uint64_t i = 0; i < num_puts; ++i) {
+ hbase::Get get(Row(FLAGS_row, i));
+ gets.push_back(get);
+ }
+
+ start_ns = TimeUtil::GetNowNanos();
+ auto results = table->Get(gets);
+
+ if (FLAGS_display_results) {
+ for (const auto &result : results) LOG(INFO) << result->DebugString();
+ }
+
+ LOG(INFO) << "Successfully sent " << gets.size() << " Multi-Get requests in "
+ << TimeUtil::ElapsedMillis(start_ns) << " ms.";
+
table->Close();
client->Close();
diff --git a/core/table.cc b/core/table.cc
index 8ace4af..a2f31d9 100644
--- a/core/table.cc
+++ b/core/table.cc
@@ -19,7 +19,6 @@
#include "core/table.h"
-#include <folly/futures/Future.h>
#include <chrono>
#include <limits>
#include <utility>
@@ -33,7 +32,6 @@
#include "serde/server-name.h"
#include "utils/time-util.h"
-using folly::Future;
using hbase::pb::TableName;
using hbase::security::User;
using std::chrono::milliseconds;
@@ -69,4 +67,20 @@ std::shared_ptr<RegionLocation> Table::GetRegionLocation(const std::string &row)
return async_connection_->region_locator()->LocateRegion(*table_name_, row).get();
}
+std::vector<std::shared_ptr<hbase::Result>> Table::Get(const std::vector<hbase::Get> &gets) {
+ auto tresults = async_table_->Get(gets).get(operation_timeout());
+ std::vector<std::shared_ptr<hbase::Result>> results{};
+ uint32_t num = 0;
+ for (auto tresult : tresults) {
+ if (tresult.hasValue()) {
+ results.push_back(tresult.value());
+ } else if (tresult.hasException()) {
+ LOG(ERROR) << "Caught exception:- " << tresult.exception().getCopied()->what() << " for "
+ << gets[num++].row();
+ throw tresult.exception().getCopied();
+ }
+ }
+ return results;
+}
+
} /* namespace hbase */
diff --git a/core/table.h b/core/table.h
index cbb95b7..142baae 100644
--- a/core/table.h
+++ b/core/table.h
@@ -54,8 +54,7 @@ class Table {
*/
std::shared_ptr<hbase::Result> Get(const hbase::Get &get);
- // TODO: next jira
- // std::vector<std::unique_ptr<hbase::Result>> Get(const std::vector<hbase::Get> &gets);
+ std::vector<std::shared_ptr<hbase::Result>> Get(const std::vector<hbase::Get> &gets);
/**
* @brief - Puts some data in the table.
[hbase] 50/137: HBASE-17220 [C++] Address major issues from cpplint
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 87d7316fba7ed82e5edcdeebfe41ae7cfab8a44f
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Fri Dec 16 16:28:34 2016 -0800
HBASE-17220 [C++] Address major issues from cpplint
---
bin/cpplint.sh | 4 +++-
connection/client-dispatcher.cc | 2 ++
connection/client-dispatcher.h | 1 +
connection/client-handler.h | 4 +++-
connection/connection-factory.h | 4 +++-
connection/connection-pool.cc | 3 +++
connection/connection-pool.h | 4 +++-
connection/pipeline.h | 2 ++
connection/response.h | 2 ++
connection/service.h | 2 ++
core/cell-test.cc | 16 ++++++++--------
core/cell.cc | 6 +++---
core/cell.h | 16 ++++++++++------
core/client.h | 1 +
core/configuration.cc | 4 +++-
core/configuration.h | 7 ++++---
core/get-test.cc | 9 +++++----
core/get.cc | 10 +++++-----
core/get.h | 13 +++++++------
core/hbase_configuration_loader.cc | 6 ++++--
core/location-cache.cc | 2 ++
core/meta-utils.cc | 1 +
core/meta-utils.h | 1 +
core/region-location.h | 1 +
core/scan-test.cc | 9 +++++----
core/scan.cc | 17 ++++++++---------
core/scan.h | 19 ++++++++++---------
core/time_range-test.cc | 2 +-
core/time_range.cc | 14 +++++++-------
core/time_range.h | 21 ++++++++++++---------
serde/region-info.h | 2 ++
serde/rpc.cc | 3 ++-
serde/rpc.h | 2 +-
serde/server-name.h | 8 ++++++--
serde/table-name.h | 1 +
serde/zk.cc | 2 ++
test-util/test-util.cc | 6 +++---
37 files changed, 139 insertions(+), 88 deletions(-)
diff --git a/bin/cpplint.sh b/bin/cpplint.sh
index 52ca581..169c382 100755
--- a/bin/cpplint.sh
+++ b/bin/cpplint.sh
@@ -24,4 +24,6 @@ OUTPUT=build/cpplint.py
wget -nc $CPPLINT_LOC -O $OUTPUT
# Execute the script
-find core connection serde utils test-util -name "*.h" -or -name "*.cc" | xargs -P8 python $OUTPUT
+# Exclude the following rules: build/header_guard (We use #pragma once instead)
+# readability/todo (TODOs are generic)
+find core connection serde utils test-util -name "*.h" -or -name "*.cc"| xargs -P8 python $OUTPUT --filter=-build/header_guard,-readability/todo
diff --git a/connection/client-dispatcher.cc b/connection/client-dispatcher.cc
index 655d765..0a03396 100644
--- a/connection/client-dispatcher.cc
+++ b/connection/client-dispatcher.cc
@@ -18,6 +18,8 @@
*/
#include "connection/client-dispatcher.h"
+#include <utility>
+
using namespace folly;
using namespace hbase;
using namespace wangle;
diff --git a/connection/client-dispatcher.h b/connection/client-dispatcher.h
index 5528a44..67e604e 100644
--- a/connection/client-dispatcher.h
+++ b/connection/client-dispatcher.h
@@ -24,6 +24,7 @@
#include <wangle/service/ClientDispatcher.h>
#include <atomic>
+#include <memory>
#include "connection/pipeline.h"
#include "connection/request.h"
diff --git a/connection/client-handler.h b/connection/client-handler.h
index 269b737..7306c9a 100644
--- a/connection/client-handler.h
+++ b/connection/client-handler.h
@@ -22,8 +22,10 @@
#include <wangle/channel/Handler.h>
#include <atomic>
+#include <memory>
#include <mutex>
#include <string>
+#include <utility>
#include "serde/rpc.h"
@@ -56,7 +58,7 @@ class ClientHandler : public wangle::Handler<std::unique_ptr<folly::IOBuf>,
* Create the handler
* @param user_name the user name of the user running this process.
*/
- ClientHandler(std::string user_name);
+ explicit ClientHandler(std::string user_name);
/**
* Get bytes from the wire.
diff --git a/connection/connection-factory.h b/connection/connection-factory.h
index 4627952..17d9941 100644
--- a/connection/connection-factory.h
+++ b/connection/connection-factory.h
@@ -20,6 +20,7 @@
#include <wangle/service/Service.h>
+#include <memory>
#include <string>
#include "connection/pipeline.h"
@@ -39,7 +40,8 @@ class ConnectionFactory {
* Constructor.
* There should only be one ConnectionFactory per client.
*/
- ConnectionFactory(std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool);
+ explicit ConnectionFactory(
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool);
/** Default Desctructor */
virtual ~ConnectionFactory() = default;
diff --git a/connection/connection-pool.cc b/connection/connection-pool.cc
index 2cb8e81..c216d0b 100644
--- a/connection/connection-pool.cc
+++ b/connection/connection-pool.cc
@@ -22,6 +22,9 @@
#include <folly/SocketAddress.h>
#include <wangle/service/Service.h>
+#include <memory>
+#include <utility>
+
using std::mutex;
using std::unique_ptr;
using std::shared_ptr;
diff --git a/connection/connection-pool.h b/connection/connection-pool.h
index c89744e..2624336 100644
--- a/connection/connection-pool.h
+++ b/connection/connection-pool.h
@@ -20,6 +20,7 @@
#include <folly/SharedMutex.h>
#include <boost/functional/hash.hpp>
+#include <memory>
#include <mutex>
#include <unordered_map>
@@ -60,7 +61,8 @@ struct ServerNameHash {
class ConnectionPool {
public:
/** Create connection pool wit default connection factory */
- ConnectionPool(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor);
+ explicit ConnectionPool(
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor);
/**
* Desctructor.
diff --git a/connection/pipeline.h b/connection/pipeline.h
index b969e2b..c4f3bbb 100644
--- a/connection/pipeline.h
+++ b/connection/pipeline.h
@@ -21,6 +21,8 @@
#include <folly/io/IOBufQueue.h>
#include <wangle/service/Service.h>
+#include <memory>
+
#include "connection/request.h"
#include "connection/response.h"
#include "utils/user-util.h"
diff --git a/connection/response.h b/connection/response.h
index 6458c6b..f032d09 100644
--- a/connection/response.h
+++ b/connection/response.h
@@ -19,6 +19,8 @@
#pragma once
#include <cstdint>
+#include <memory>
+#include <utility>
// Forward
namespace google {
diff --git a/connection/service.h b/connection/service.h
index 226f936..0d2e258 100644
--- a/connection/service.h
+++ b/connection/service.h
@@ -20,6 +20,8 @@
#include <wangle/service/Service.h>
+#include <memory>
+
#include "connection/request.h"
#include "connection/response.h"
diff --git a/core/cell-test.cc b/core/cell-test.cc
index 1d5c0eb..53574e8 100644
--- a/core/cell-test.cc
+++ b/core/cell-test.cc
@@ -30,7 +30,7 @@ TEST(CellTest, CellFailureTest) {
std::string family = "family";
std::string column = "column";
std::string value = "value";
- long timestamp = std::numeric_limits<long>::max();
+ int64_t timestamp = std::numeric_limits<int64_t>::max();
std::string tags = "";
std::unique_ptr<Cell> cell(
new Cell(row, family, column, timestamp, value, cell_type));
@@ -49,7 +49,7 @@ TEST(CellTest, CellSuceessTest) {
std::string family = "family-value";
std::string column = "column-value";
std::string value = "value-value";
- long timestamp = std::numeric_limits<long>::max();
+ int64_t timestamp = std::numeric_limits<int64_t>::max();
CellType cell_type = CellType::PUT;
const std::unique_ptr<Cell> cell(
new Cell(row, family, column, timestamp, value, cell_type));
@@ -70,7 +70,7 @@ TEST(CellTest, MultipleCellsTest) {
std::string family = "family-value";
std::string column = "column-value";
std::string value = "value-value";
- long timestamp = std::numeric_limits<long>::max();
+ int64_t timestamp = std::numeric_limits<int64_t>::max();
row += std::to_string(i);
value += std::to_string(i);
CellType cell_type = CellType::PUT;
@@ -88,7 +88,7 @@ TEST(CellTest, MultipleCellsTest) {
EXPECT_EQ("family-value", cell->Family());
EXPECT_EQ("column-value", cell->Qualifier());
EXPECT_EQ(value, cell->Value());
- EXPECT_EQ(std::numeric_limits<long>::max(), cell->Timestamp());
+ EXPECT_EQ(std::numeric_limits<int64_t>::max(), cell->Timestamp());
EXPECT_EQ(CellType::PUT, cell->Type());
i += 1;
}
@@ -103,7 +103,7 @@ TEST(CellTest, CellRowTest) {
std::string family = "D";
std::string column = "";
std::string value = "";
- long timestamp = std::numeric_limits<long>::max();
+ int64_t timestamp = std::numeric_limits<int64_t>::max();
CellType cell_type = CellType::PUT;
std::unique_ptr<Cell> cell(
new Cell(row, family, column, timestamp, value, cell_type));
@@ -122,7 +122,7 @@ TEST(CellTest, CellRowFamilyTest) {
std::string family = "only-family";
std::string column = "";
std::string value = "";
- long timestamp = std::numeric_limits<long>::max();
+ int64_t timestamp = std::numeric_limits<int64_t>::max();
CellType cell_type = CellType::PUT;
const std::unique_ptr<Cell> cell(
new Cell(row, family, column, timestamp, value, cell_type));
@@ -141,7 +141,7 @@ TEST(CellTest, CellRowFamilyValueTest) {
std::string family = "only-family";
std::string column = "";
std::string value = "only-value";
- long timestamp = std::numeric_limits<long>::max();
+ int64_t timestamp = std::numeric_limits<int64_t>::max();
CellType cell_type = CellType::PUT;
const std::unique_ptr<Cell> cell(
new Cell(row, family, column, timestamp, value, cell_type));
@@ -160,7 +160,7 @@ TEST(CellTest, CellRowFamilyColumnValueTest) {
std::string family = "only-family";
std::string column = "only-column";
std::string value = "only-value";
- long timestamp = std::numeric_limits<long>::max();
+ int64_t timestamp = std::numeric_limits<int64_t>::max();
CellType cell_type = CellType::PUT;
std::unique_ptr<Cell> cell(
new Cell(row, family, column, timestamp, value, cell_type));
diff --git a/core/cell.cc b/core/cell.cc
index 37ebd13..3102e53 100644
--- a/core/cell.cc
+++ b/core/cell.cc
@@ -23,7 +23,7 @@
namespace hbase {
Cell::Cell(const std::string &row, const std::string &family,
- const std::string &qualifier, const long ×tamp,
+ const std::string &qualifier, const int64_t timestamp,
const std::string &value, const hbase::CellType &cell_type)
: row_(row),
family_(family),
@@ -50,12 +50,12 @@ const std::string &Cell::Family() const { return family_; }
const std::string &Cell::Qualifier() const { return qualifier_; }
-unsigned long Cell::Timestamp() const { return timestamp_; }
+int64_t Cell::Timestamp() const { return timestamp_; }
const std::string &Cell::Value() const { return value_; }
hbase::CellType Cell::Type() const { return cell_type_; }
-long Cell::SequenceId() const { return sequence_id_; }
+int64_t Cell::SequenceId() const { return sequence_id_; }
} /* namespace hbase */
diff --git a/core/cell.h b/core/cell.h
index 16ed280..f55655c 100644
--- a/core/cell.h
+++ b/core/cell.h
@@ -19,6 +19,7 @@
#pragma once
+#include <cstdint>
#include <string>
namespace hbase {
@@ -36,25 +37,28 @@ enum CellType {
class Cell {
public:
Cell(const std::string &row, const std::string &family,
- const std::string &qualifier, const long ×tamp,
+ const std::string &qualifier, const int64_t timestamp,
const std::string &value, const hbase::CellType &cell_type);
virtual ~Cell();
const std::string &Row() const;
const std::string &Family() const;
const std::string &Qualifier() const;
- unsigned long Timestamp() const;
+ int64_t Timestamp() const;
const std::string &Value() const;
CellType Type() const;
- long SequenceId() const;
+ int64_t SequenceId() const;
private:
std::string row_;
std::string family_;
std::string qualifier_;
- unsigned long timestamp_;
+ // Since java does not have unsigned, we are also using signed numerics here
+ // so that we won't have surprises when large uint64's are treated as
+ // negative values in the java server side
+ int64_t timestamp_;
hbase::CellType cell_type_;
std::string value_;
- long sequence_id_;
+ int64_t sequence_id_;
};
-} /* namespace hbase */
+} // namespace hbase
diff --git a/core/client.h b/core/client.h
index 296a8a0..0ba1276 100644
--- a/core/client.h
+++ b/core/client.h
@@ -24,6 +24,7 @@
#include <wangle/concurrent/CPUThreadPoolExecutor.h>
#include <wangle/concurrent/IOThreadPoolExecutor.h>
+#include <memory>
#include <string>
#include "core/location-cache.h"
diff --git a/core/configuration.cc b/core/configuration.cc
index c074cf6..134f05f 100644
--- a/core/configuration.cc
+++ b/core/configuration.cc
@@ -17,9 +17,11 @@
*
*/
-#include "configuration.h"
+#include "core/configuration.h"
+#include <memory>
#include <stdexcept>
+#include <utility>
#include <glog/logging.h>
#include <boost/lexical_cast.hpp>
diff --git a/core/configuration.h b/core/configuration.h
index d313cf6..f09bbac 100644
--- a/core/configuration.h
+++ b/core/configuration.h
@@ -88,8 +88,9 @@ class Configuration {
struct ConfigData {
std::string value;
bool final;
- ConfigData() : final(false){};
- ConfigData(const std::string &value) : value(value), final(false) {}
+ ConfigData() : final(false) {}
+ explicit ConfigData(const std::string &value)
+ : value(value), final(false) {}
void operator=(const std::string &new_value) {
value = new_value;
final = false;
@@ -105,7 +106,7 @@ class Configuration {
* @brief Create Configuration object using config_map;
* @param config_map - Map consisting of properties.
*/
- Configuration(ConfigMap &config_map);
+ explicit Configuration(ConfigMap &config_map);
// Property map filled all the properties loaded by ConfigurationLoader
ConfigMap hb_property_;
diff --git a/core/get-test.cc b/core/get-test.cc
index 53c14e3..cef99b9 100644
--- a/core/get-test.cc
+++ b/core/get-test.cc
@@ -121,7 +121,8 @@ void GetMethods(Get &get, const std::string &row) {
// Test initial values
EXPECT_EQ(0, get.Timerange().MinTimeStamp());
- EXPECT_EQ(std::numeric_limits<long>::max(), get.Timerange().MaxTimeStamp());
+ EXPECT_EQ(std::numeric_limits<int64_t>::max(),
+ get.Timerange().MaxTimeStamp());
// Set & Test new values using TimeRange and TimeStamp
get.SetTimeRange(1000, 2000);
@@ -135,12 +136,12 @@ void GetMethods(Get &get, const std::string &row) {
ASSERT_THROW(get.SetTimeRange(-1000, 2000), std::runtime_error);
ASSERT_THROW(get.SetTimeRange(1000, -2000), std::runtime_error);
ASSERT_THROW(get.SetTimeRange(1000, 200), std::runtime_error);
- ASSERT_THROW(get.SetTimeStamp(std::numeric_limits<long>::max()),
+ ASSERT_THROW(get.SetTimeStamp(std::numeric_limits<int64_t>::max()),
std::runtime_error);
// Test some exceptions
ASSERT_THROW(get.SetMaxVersions(0), std::runtime_error);
- ASSERT_THROW(get.SetMaxVersions(std::numeric_limits<unsigned int>::max() + 1),
+ ASSERT_THROW(get.SetMaxVersions(std::numeric_limits<uint32_t>::max() + 1),
std::runtime_error);
}
@@ -218,7 +219,7 @@ TEST(Get, MultiGet) {
}
TEST(Get, Exception) {
- std::string row(std::numeric_limits<short>::max() + 1, 'X');
+ std::string row(std::numeric_limits<int16_t>::max() + 1, 'X');
ASSERT_THROW(Get tmp = Get(row), std::runtime_error);
ASSERT_THROW(Get tmp = Get(""), std::runtime_error);
}
diff --git a/core/get.cc b/core/get.cc
index a70da37..6ba4e86 100644
--- a/core/get.cc
+++ b/core/get.cc
@@ -96,7 +96,7 @@ const FamilyMap &Get::Family() const { return family_map_; }
int Get::MaxVersions() const { return max_versions_; }
-Get &Get::SetMaxVersions(uint32_t max_versions) {
+Get &Get::SetMaxVersions(int32_t max_versions) {
if (0 == max_versions)
throw std::runtime_error("max_versions must be positive");
@@ -111,12 +111,12 @@ Get &Get::SetCacheBlocks(bool cache_blocks) {
return *this;
}
-Get &Get::SetTimeRange(long min_timestamp, long max_timestamp) {
+Get &Get::SetTimeRange(int64_t min_timestamp, int64_t max_timestamp) {
tr_.reset(new TimeRange(min_timestamp, max_timestamp));
return *this;
}
-Get &Get::SetTimeStamp(long timestamp) {
+Get &Get::SetTimeStamp(int64_t timestamp) {
tr_.reset(new TimeRange(timestamp, timestamp + 1));
return *this;
}
@@ -124,7 +124,7 @@ Get &Get::SetTimeStamp(long timestamp) {
const TimeRange &Get::Timerange() const { return *tr_; }
void Get::CheckRow(const std::string &row) {
- const int kMaxRowLength = std::numeric_limits<short>::max();
+ const int kMaxRowLength = std::numeric_limits<int16_t>::max();
int row_length = row.size();
if (0 == row_length) {
throw std::runtime_error("Row length can't be 0");
@@ -135,4 +135,4 @@ void Get::CheckRow(const std::string &row) {
std::to_string(kMaxRowLength));
}
}
-}
+} // namespace hbase
diff --git a/core/get.h b/core/get.h
index 90c42ee..f79c633 100644
--- a/core/get.h
+++ b/core/get.h
@@ -40,7 +40,7 @@ class Get {
/**
* Constructors
*/
- Get(const std::string& row);
+ explicit Get(const std::string& row);
Get(const Get& cget);
Get& operator=(const Get& cget);
@@ -56,7 +56,7 @@ class Get {
* is 1.
* @param max_versions max_versons to set
*/
- Get& SetMaxVersions(uint32_t max_versions = 1);
+ Get& SetMaxVersions(int32_t max_versions = 1);
/**
* @brief Returns whether blocks should be cached for this Get operation.
@@ -86,13 +86,13 @@ class Get {
* @param minStamp the minimum timestamp, inclusive
* @param maxStamp the maximum timestamp, exclusive
*/
- Get& SetTimeRange(long min_timestamp, long max_timestamp);
+ Get& SetTimeRange(int64_t min_timestamp, int64_t max_timestamp);
/**
* @brief Get versions of columns with the specified timestamp.
* @param The timestamp to be set
*/
- Get& SetTimeStamp(long timestamp);
+ Get& SetTimeStamp(int64_t timestamp);
/**
* @brief Get all columns from the specified family.
@@ -131,7 +131,7 @@ class Get {
private:
std::string row_ = "";
- uint32_t max_versions_ = 1;
+ int32_t max_versions_ = 1;
bool cache_blocks_ = true;
bool check_existence_only_ = false;
FamilyMap family_map_;
@@ -146,4 +146,5 @@ class Get {
*/
void CheckRow(const std::string& row);
};
-}
+
+} // namespace hbase
diff --git a/core/hbase_configuration_loader.cc b/core/hbase_configuration_loader.cc
index d3aff8c..20d0449 100644
--- a/core/hbase_configuration_loader.cc
+++ b/core/hbase_configuration_loader.cc
@@ -127,7 +127,8 @@ optional<Configuration> HBaseConfigurationLoader::LoadDefaultResources() {
}
}
if (success) {
- return std::experimental::make_optional<Configuration>(conf_property);
+ return std::experimental::make_optional<Configuration>(
+ Configuration(conf_property));
} else {
return optional<Configuration>();
}
@@ -151,7 +152,8 @@ optional<Configuration> HBaseConfigurationLoader::LoadResources(
}
}
if (success) {
- return std::experimental::make_optional<Configuration>(conf_property);
+ return std::experimental::make_optional<Configuration>(
+ Configuration(conf_property));
} else {
return optional<Configuration>();
}
diff --git a/core/location-cache.cc b/core/location-cache.cc
index 4359795..a0ca5ca 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -23,6 +23,8 @@
#include <wangle/concurrent/CPUThreadPoolExecutor.h>
#include <wangle/concurrent/IOThreadPoolExecutor.h>
+#include <utility>
+
#include "connection/response.h"
#include "if/Client.pb.h"
#include "if/ZooKeeper.pb.h"
diff --git a/core/meta-utils.cc b/core/meta-utils.cc
index 23d2041..e1946d6 100644
--- a/core/meta-utils.cc
+++ b/core/meta-utils.cc
@@ -20,6 +20,7 @@
#include "core/meta-utils.h"
#include <folly/Conv.h>
+#include <memory>
#include "connection/request.h"
#include "connection/response.h"
diff --git a/core/meta-utils.h b/core/meta-utils.h
index 303a273..33da1a8 100644
--- a/core/meta-utils.h
+++ b/core/meta-utils.h
@@ -18,6 +18,7 @@
*/
#pragma once
+#include <memory>
#include <string>
#include "connection/request.h"
diff --git a/core/region-location.h b/core/region-location.h
index b3c124d..f0295b1 100644
--- a/core/region-location.h
+++ b/core/region-location.h
@@ -19,6 +19,7 @@
#pragma once
#include <memory>
+#include <string>
#include "connection/service.h"
#include "if/HBase.pb.h"
diff --git a/core/scan-test.cc b/core/scan-test.cc
index b6a6f3e..0a4001a 100644
--- a/core/scan-test.cc
+++ b/core/scan-test.cc
@@ -154,7 +154,8 @@ void ScanMethods(Scan &scan) {
// Test initial values
EXPECT_EQ(0, scan.Timerange().MinTimeStamp());
- EXPECT_EQ(std::numeric_limits<long>::max(), scan.Timerange().MaxTimeStamp());
+ EXPECT_EQ(std::numeric_limits<int64_t>::max(),
+ scan.Timerange().MaxTimeStamp());
// Set & Test new values using TimeRange and TimeStamp
scan.SetTimeRange(1000, 2000);
@@ -168,7 +169,7 @@ void ScanMethods(Scan &scan) {
ASSERT_THROW(scan.SetTimeRange(-1000, 2000), std::runtime_error);
ASSERT_THROW(scan.SetTimeRange(1000, -2000), std::runtime_error);
ASSERT_THROW(scan.SetTimeRange(1000, 200), std::runtime_error);
- ASSERT_THROW(scan.SetTimeStamp(std::numeric_limits<long>::max()),
+ ASSERT_THROW(scan.SetTimeStamp(std::numeric_limits<int64_t>::max()),
std::runtime_error);
}
@@ -179,7 +180,7 @@ TEST(Scan, Object) {
// Resetting TimeRange values so that the copy construction and assignment
// operator tests pass.
- scan.SetTimeRange(0, std::numeric_limits<long>::max());
+ scan.SetTimeRange(0, std::numeric_limits<int64_t>::max());
Scan scancp(scan);
ScanMethods(scancp);
CheckFamiliesAfterCopy(scancp);
@@ -228,7 +229,7 @@ TEST(Scan, FromGet) {
}
TEST(Scan, Exception) {
- std::string row(std::numeric_limits<short>::max() + 1, 'X');
+ std::string row(std::numeric_limits<int16_t>::max() + 1, 'X');
ASSERT_THROW(Scan tmp(row), std::runtime_error);
ASSERT_THROW(Scan tmp(""), std::runtime_error);
}
diff --git a/core/scan.cc b/core/scan.cc
index 2dd8ef2..036acfb 100644
--- a/core/scan.cc
+++ b/core/scan.cc
@@ -113,14 +113,14 @@ void Scan::SetReversed(bool reversed) { reversed_ = reversed; }
bool Scan::IsReversed() const { return reversed_; }
-void Scan::SetStartRow(std::string &start_row) {
+void Scan::SetStartRow(const std::string &start_row) {
CheckRow(start_row);
start_row_ = start_row;
}
const std::string &Scan::StartRow() const { return start_row_; }
-void Scan::SetStopRow(std::string &stop_row) {
+void Scan::SetStopRow(const std::string &stop_row) {
CheckRow(stop_row);
stop_row_ = stop_row;
}
@@ -167,18 +167,18 @@ Scan &Scan::SetMaxVersions(uint32_t max_versions) {
int Scan::MaxVersions() const { return max_versions_; }
-void Scan::SetMaxResultSize(long max_result_size) {
+void Scan::SetMaxResultSize(int64_t max_result_size) {
max_result_size_ = max_result_size;
}
-long Scan::MaxResultSize() const { return max_result_size_; }
+int64_t Scan::MaxResultSize() const { return max_result_size_; }
-Scan &Scan::SetTimeRange(long min_stamp, long max_stamp) {
+Scan &Scan::SetTimeRange(int64_t min_stamp, int64_t max_stamp) {
tr_.reset(new TimeRange(min_stamp, max_stamp));
return *this;
}
-Scan &Scan::SetTimeStamp(long timestamp) {
+Scan &Scan::SetTimeStamp(int64_t timestamp) {
tr_.reset(new TimeRange(timestamp, timestamp + 1));
return *this;
}
@@ -186,7 +186,7 @@ Scan &Scan::SetTimeStamp(long timestamp) {
const TimeRange &Scan::Timerange() const { return *tr_; }
void Scan::CheckRow(const std::string &row) {
- const int kMaxRowLength = std::numeric_limits<short>::max();
+ const int32_t kMaxRowLength = std::numeric_limits<int16_t>::max();
int row_length = row.size();
if (0 == row_length) {
throw std::runtime_error("Row length can't be 0");
@@ -201,5 +201,4 @@ void Scan::CheckRow(const std::string &row) {
bool Scan::HasFamilies() const { return !family_map_.empty(); }
const FamilyMap &Scan::Family() const { return family_map_; }
-}
-/* namespace hbase */
+} // namespace hbase
diff --git a/core/scan.h b/core/scan.h
index 99ed26a..e2e7f1a 100644
--- a/core/scan.h
+++ b/core/scan.h
@@ -19,6 +19,7 @@
#pragma once
+#include <cstdint>
#include <map>
#include <memory>
#include <string>
@@ -52,7 +53,7 @@ class Scan {
* the Scanner will start from the next closest row after the specified row.
* @param start_row - row to start scanner at or after
*/
- Scan(const std::string &start_row);
+ explicit Scan(const std::string &start_row);
/**
* @brief Create a Scan operation for the range of rows specified.
@@ -65,7 +66,7 @@ class Scan {
* @brief Builds a scan object with the same specs as get.
* @param get - get to model scan after
*/
- Scan(const Get &get);
+ explicit Scan(const Get &get);
/**
* @brief Get all columns from the specified family.Overrides previous calls
@@ -102,7 +103,7 @@ class Scan {
* @throws std::runtime_error if start_row length is 0 or greater than
* MAX_ROW_LENGTH
*/
- void SetStartRow(std::string &start_row);
+ void SetStartRow(const std::string &start_row);
/**
* @brief returns start_row of the Scan.
@@ -116,7 +117,7 @@ class Scan {
* @throws std::runtime_error if stop_row length is 0 or greater than
* MAX_ROW_LENGTH
*/
- void SetStopRow(std::string &stop_row);
+ void SetStopRow(const std::string &stop_row);
/**
* @brief returns stop_row of the Scan.
@@ -218,12 +219,12 @@ class Scan {
* configured value will be used instead. (Defaults to unlimited).
* @param The maximum result size in bytes.
*/
- void SetMaxResultSize(long max_result_size);
+ void SetMaxResultSize(int64_t max_result_size);
/**
* @brief the maximum result size in bytes.
*/
- long MaxResultSize() const;
+ int64_t MaxResultSize() const;
/**
* @brief Get versions of columns only within the specified timestamp range,
@@ -233,7 +234,7 @@ class Scan {
* @param min_stamp - minimum timestamp value, inclusive.
* @param max_stamp - maximum timestamp value, exclusive.
*/
- Scan &SetTimeRange(long min_stamp, long max_stamp);
+ Scan &SetTimeRange(int64_t min_stamp, int64_t max_stamp);
/**
* @brief Get versions of columns with the specified timestamp. Note, default
@@ -242,7 +243,7 @@ class Scan {
* beyond the defaut.
* @param timestamp - version timestamp
*/
- Scan &SetTimeStamp(long timestamp);
+ Scan &SetTimeStamp(int64_t timestamp);
/**
* @brief Return Timerange
@@ -264,7 +265,7 @@ class Scan {
std::string stop_row_ = "";
uint32_t max_versions_ = 1;
int caching_ = -1;
- long max_result_size_ = -1;
+ int64_t max_result_size_ = -1;
bool cache_blocks_ = true;
bool load_column_families_on_demand_ = false;
bool reversed_ = false;
diff --git a/core/time_range-test.cc b/core/time_range-test.cc
index e066c57..49cb359 100644
--- a/core/time_range-test.cc
+++ b/core/time_range-test.cc
@@ -29,7 +29,7 @@ TEST(TimeRange, DefaultObject) {
ASSERT_NO_THROW(timerange_def = new TimeRange());
EXPECT_EQ(0, timerange_def->MinTimeStamp());
- EXPECT_EQ(std::numeric_limits<long>::max(), timerange_def->MaxTimeStamp());
+ EXPECT_EQ(std::numeric_limits<int64_t>::max(), timerange_def->MaxTimeStamp());
EXPECT_NE(1000, timerange_def->MinTimeStamp());
EXPECT_NE(2000, timerange_def->MaxTimeStamp());
delete timerange_def;
diff --git a/core/time_range.cc b/core/time_range.cc
index 2accc9a..8c37b29 100644
--- a/core/time_range.cc
+++ b/core/time_range.cc
@@ -26,7 +26,7 @@ namespace hbase {
TimeRange::TimeRange()
: min_timestamp_(0L),
- max_timestamp_(std::numeric_limits<long>::max()),
+ max_timestamp_(std::numeric_limits<int64_t>::max()),
all_time_(true) {}
TimeRange::TimeRange(const TimeRange &tr) {
@@ -44,13 +44,13 @@ TimeRange &TimeRange::operator=(const TimeRange &tr) {
TimeRange::~TimeRange() {}
-TimeRange::TimeRange(long min_timestamp) {
+TimeRange::TimeRange(int64_t min_timestamp) {
this->min_timestamp_ = min_timestamp;
- this->max_timestamp_ = std::numeric_limits<long>::max();
+ this->max_timestamp_ = std::numeric_limits<int64_t>::max();
this->all_time_ = false;
}
-TimeRange::TimeRange(long min_timestamp, long max_timestamp) {
+TimeRange::TimeRange(int64_t min_timestamp, int64_t max_timestamp) {
if (min_timestamp < 0 || max_timestamp < 0) {
throw std::runtime_error("Timestamp cannot be negative. min_timestamp: " +
std::to_string(min_timestamp) +
@@ -68,9 +68,9 @@ TimeRange::TimeRange(long min_timestamp, long max_timestamp) {
this->all_time_ = false;
}
-long TimeRange::MinTimeStamp() const { return this->min_timestamp_; }
+int64_t TimeRange::MinTimeStamp() const { return this->min_timestamp_; }
-long TimeRange::MaxTimeStamp() const { return this->max_timestamp_; }
+int64_t TimeRange::MaxTimeStamp() const { return this->max_timestamp_; }
bool TimeRange::IsAllTime() const { return this->all_time_; }
-}
+} // namespace hbase
diff --git a/core/time_range.h b/core/time_range.h
index 171f3ed..d645ecd 100644
--- a/core/time_range.h
+++ b/core/time_range.h
@@ -19,21 +19,24 @@
#pragma once
+#include <cstdint>
+
namespace hbase {
class TimeRange {
public:
/**
- * @brief Default constructor. Represents interval [0, Long.MAX_VALUE)
+ * @brief Default constructor. Represents interval [0,
+ * std::numeric_limits<int64_t>::max())
* (allTime)
*/
TimeRange();
TimeRange(const TimeRange &tr);
TimeRange &operator=(const TimeRange &tr);
/**
- * @brief Represents interval [minStamp, Long.MAX_VALUE)
+ * @brief Represents interval [minStamp, std::numeric_limits<int64_t>::max())
* @param minStamp the minimum timestamp value, inclusive
*/
- TimeRange(long min_timestamp);
+ explicit TimeRange(int64_t min_timestamp);
/**
* @brief Represents interval [minStamp, maxStamp)
* @param minStamp the minimum timestamp, inclusive
@@ -41,15 +44,15 @@ class TimeRange {
* @throws std::runtime_error if min_timestamp < 0 or max_timestamp < 0 or
* max_timestamp < min_timestamp
*/
- TimeRange(long min_timestamp, long max_timestamp);
- long MinTimeStamp() const;
- long MaxTimeStamp() const;
+ TimeRange(int64_t min_timestamp, int64_t max_timestamp);
+ int64_t MinTimeStamp() const;
+ int64_t MaxTimeStamp() const;
bool IsAllTime() const;
~TimeRange();
private:
- long min_timestamp_;
- long max_timestamp_;
+ int64_t min_timestamp_;
+ int64_t max_timestamp_;
bool all_time_;
};
-}
+} // namespace hbase
diff --git a/serde/region-info.h b/serde/region-info.h
index 1b0a3fb..1f08298 100644
--- a/serde/region-info.h
+++ b/serde/region-info.h
@@ -24,6 +24,8 @@
#include <folly/Conv.h>
#include <boost/algorithm/string/predicate.hpp>
+#include <string>
+
namespace hbase {
namespace pb {
template <class String>
diff --git a/serde/rpc.cc b/serde/rpc.cc
index b573738..d863d50 100644
--- a/serde/rpc.cc
+++ b/serde/rpc.cc
@@ -20,12 +20,13 @@
#include "serde/rpc.h"
#include <folly/Logging.h>
-#include <folly/Logging.h>
#include <folly/io/Cursor.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
#include <google/protobuf/message.h>
+#include <utility>
+
#include "if/HBase.pb.h"
#include "if/RPC.pb.h"
diff --git a/serde/rpc.h b/serde/rpc.h
index 873d01c..05ceee9 100644
--- a/serde/rpc.h
+++ b/serde/rpc.h
@@ -114,4 +114,4 @@ class RpcSerde {
/* data */
uint8_t auth_type_;
};
-}
+} // namespace hbase
diff --git a/serde/server-name.h b/serde/server-name.h
index dda9bf2..41e3c77 100644
--- a/serde/server-name.h
+++ b/serde/server-name.h
@@ -21,6 +21,9 @@
#include <folly/Conv.h>
#include <folly/String.h>
+
+#include <string>
+
#include "if/HBase.pb.h"
namespace hbase {
@@ -39,5 +42,6 @@ void parseTo(String in, ServerName &out) {
// Now keep everything after the : (delim + 1) to the end.
out.set_port(folly::to<int>(s.substr(delim + 1)));
}
-}
-}
+
+} // namespace pb
+} // namespace hbase
diff --git a/serde/table-name.h b/serde/table-name.h
index 707ba44..b8b7776 100644
--- a/serde/table-name.h
+++ b/serde/table-name.h
@@ -20,6 +20,7 @@
#include <memory>
#include <string>
+#include <vector>
#include <folly/Conv.h>
#include <folly/String.h>
diff --git a/serde/zk.cc b/serde/zk.cc
index 63c96ed..69b55fa 100644
--- a/serde/zk.cc
+++ b/serde/zk.cc
@@ -23,6 +23,8 @@
#include <folly/io/IOBuf.h>
#include <google/protobuf/message.h>
+#include <string>
+
using hbase::ZkDeserializer;
using std::runtime_error;
using folly::IOBuf;
diff --git a/test-util/test-util.cc b/test-util/test-util.cc
index 88ce7c8..e355bdf 100644
--- a/test-util/test-util.cc
+++ b/test-util/test-util.cc
@@ -45,16 +45,16 @@ TestUtil::TestUtil() : temp_dir_(TestUtil::RandString()) {
auto p = temp_dir_.path().string();
auto cmd = std::string{"bin/start-local-hbase.sh " + p};
auto res_code = std::system(cmd.c_str());
- CHECK(res_code == 0);
+ CHECK_EQ(res_code, 0);
}
TestUtil::~TestUtil() {
auto res_code = std::system("bin/stop-local-hbase.sh");
- CHECK(res_code == 0);
+ CHECK_EQ(res_code, 0);
}
void TestUtil::RunShellCmd(const std::string &command) {
auto cmd_string = folly::sformat("echo \"{}\" | ../bin/hbase shell", command);
auto res_code = std::system(cmd_string.c_str());
- CHECK(res_code == 0);
+ CHECK_EQ(res_code, 0);
}
[hbase] 37/137: HBASE-15822 Move to the latest docker base image
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 6bb6431c9127296b4bf0470e2da6f898486b7eaf
Author: Elliott Clark <el...@fb.com>
AuthorDate: Tue May 31 10:42:45 2016 -0700
HBASE-15822 Move to the latest docker base image
---
Dockerfile | 2 +-
if/RPC.proto | 1 +
if/WAL.proto | 5 +++--
3 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index be0cbc6..ca87523 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM pjameson/buck-folly-watchman:20160425
+FROM pjameson/buck-folly-watchman:20160511
ARG CC=/usr/bin/gcc-5
ARG CXX=/usr/bin/g++-5
diff --git a/if/RPC.proto b/if/RPC.proto
index 59bb03d..8413d25 100644
--- a/if/RPC.proto
+++ b/if/RPC.proto
@@ -125,6 +125,7 @@ message RequestHeader {
// 0 is NORMAL priority. 200 is HIGH. If no priority, treat it as NORMAL.
// See HConstants.
optional uint32 priority = 6;
+ optional uint32 timeout = 7;
}
message ResponseHeader {
diff --git a/if/WAL.proto b/if/WAL.proto
index cb9bd8f..2061b22 100644
--- a/if/WAL.proto
+++ b/if/WAL.proto
@@ -44,9 +44,9 @@ message WALKey {
required uint64 log_sequence_number = 3;
required uint64 write_time = 4;
/*
- This parameter is deprecated in favor of clusters which
+ This parameter is deprecated in favor of clusters which
contains the list of clusters that have consumed the change.
- It is retained so that the log created by earlier releases (0.94)
+ It is retained so that the log created by earlier releases (0.94)
can be read by the newer releases.
*/
optional UUID cluster_id = 5 [deprecated=true];
@@ -132,6 +132,7 @@ message StoreDescriptor {
required bytes family_name = 1;
required string store_home_dir = 2; //relative to region dir
repeated string store_file = 3; // relative to store dir
+ optional uint64 store_file_size_bytes = 4; // size of store file
}
/**
[hbase] 29/137: HBASE-15761 Add on more server name tests
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 54b93bbd38e7db9ffa2cd0be72f862eaadb7b304
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Wed May 4 12:04:18 2016 -0700
HBASE-15761 Add on more server name tests
---
serde/server-name-test.cc | 18 ++++++++++++++++++
serde/server-name.h | 4 +++-
2 files changed, 21 insertions(+), 1 deletion(-)
diff --git a/serde/server-name-test.cc b/serde/server-name-test.cc
index 35dcbc1..2281fa2 100644
--- a/serde/server-name-test.cc
+++ b/serde/server-name-test.cc
@@ -30,3 +30,21 @@ TEST(TestServerName, TestMakeServerName) {
ASSERT_EQ("test", sn.host_name());
ASSERT_EQ(123, sn.port());
}
+
+TEST(TestServerName, TestIps) {
+ auto sn = folly::to<ServerName>("127.0.0.1:999");
+ ASSERT_EQ("127.0.0.1", sn.host_name());
+ ASSERT_EQ(999, sn.port());
+}
+
+TEST(TestServerName, TestThrow) {
+ ASSERT_ANY_THROW(folly::to<ServerName>("Ther's no colon here"));
+}
+
+TEST(TestServerName, TestIPV6) {
+ auto sn = folly::to<ServerName>("[::::1]:123");
+
+ ASSERT_EQ("[::::1]", sn.host_name());
+ ASSERT_EQ(123, sn.port());
+
+}
diff --git a/serde/server-name.h b/serde/server-name.h
index bdba087..9844465 100644
--- a/serde/server-name.h
+++ b/serde/server-name.h
@@ -12,7 +12,9 @@ template <class String> void parseTo(String in, ServerName &out) {
std::string s = folly::to<std::string>(in);
auto delim = s.rfind(":");
- DCHECK(delim != std::string::npos);
+ if (delim == std::string::npos) {
+ throw std::runtime_error("Couldn't parse server name");
+ }
out.set_host_name(s.substr(0, delim));
// Now keep everything after the : (delim + 1) to the end.
out.set_port(folly::to<int>(s.substr(delim + 1)));
[hbase] 25/137: HBASE-15655 Set up doxygen for documentation
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 7593d6cc118c2a58f4b6793ef229ae2185958c43
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Wed May 4 18:19:33 2016 -0700
HBASE-15655 Set up doxygen for documentation
Summary:
Add on doxygen config.
It outputs to the docs folder.
Added on a makefile
Test Plan:
doxygen hbase.doxygen
Differential Revision: https://reviews.facebook.net/D57663
---
Dockerfile | 2 +-
Makefile | 20 ++++
hbase.doxygen | 319 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 340 insertions(+), 1 deletion(-)
diff --git a/Dockerfile b/Dockerfile
index 8e33e74..be0cbc6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -24,7 +24,7 @@ ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -p
ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
-RUN apt-get install -y vim maven inetutils-ping python-pip && \
+RUN apt-get install -y vim maven inetutils-ping python-pip doxygen graphviz && \
pip install yapf && \
apt-get -qq clean && \
apt-get -y -qq autoremove && \
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..90ec6ea
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,20 @@
+build:
+ $(shell buck build core/... )
+
+check:
+ $(shell buck test --all --no-results-cache )
+
+doc:
+ $(shell doxygen hbase.doxygen > /dev/null )
+
+clean:
+ $(shell rm -rf docs )
+
+help:
+ @echo "Available targets:"
+ @echo ""
+ @echo " build : will build everything."
+ @echo " clean : will remove the docs folder"
+ @echo " check : will test everything."
+
+all: build doc check
diff --git a/hbase.doxygen b/hbase.doxygen
new file mode 100644
index 0000000..22e2d21
--- /dev/null
+++ b/hbase.doxygen
@@ -0,0 +1,319 @@
+# Doxyfile 1.8.11
+
+#---------------------------------------------------------------------------
+# Project related configuration options
+#---------------------------------------------------------------------------
+DOXYFILE_ENCODING = UTF-8
+PROJECT_NAME = "HBase Native Client"
+PROJECT_NUMBER =
+PROJECT_BRIEF =
+PROJECT_LOGO = ../hbase-server/src/main/resources/hbase-webapps/static/hbase_logo_small.png
+OUTPUT_DIRECTORY = docs
+CREATE_SUBDIRS = YES
+ALLOW_UNICODE_NAMES = NO
+OUTPUT_LANGUAGE = English
+BRIEF_MEMBER_DESC = YES
+REPEAT_BRIEF = YES
+ABBREVIATE_BRIEF =
+ALWAYS_DETAILED_SEC = NO
+INLINE_INHERITED_MEMB = NO
+FULL_PATH_NAMES = NO
+STRIP_FROM_PATH =
+STRIP_FROM_INC_PATH =
+SHORT_NAMES = NO
+JAVADOC_AUTOBRIEF = NO
+QT_AUTOBRIEF = NO
+MULTILINE_CPP_IS_BRIEF = NO
+INHERIT_DOCS = YES
+SEPARATE_MEMBER_PAGES = NO
+TAB_SIZE = 4
+ALIASES =
+TCL_SUBST =
+OPTIMIZE_OUTPUT_FOR_C = NO
+OPTIMIZE_OUTPUT_JAVA = NO
+OPTIMIZE_FOR_FORTRAN = NO
+OPTIMIZE_OUTPUT_VHDL = NO
+EXTENSION_MAPPING =
+MARKDOWN_SUPPORT = YES
+AUTOLINK_SUPPORT = YES
+BUILTIN_STL_SUPPORT = NO
+CPP_CLI_SUPPORT = NO
+SIP_SUPPORT = NO
+IDL_PROPERTY_SUPPORT = YES
+DISTRIBUTE_GROUP_DOC = NO
+GROUP_NESTED_COMPOUNDS = NO
+SUBGROUPING = YES
+INLINE_GROUPED_CLASSES = NO
+INLINE_SIMPLE_STRUCTS = NO
+TYPEDEF_HIDES_STRUCT = NO
+LOOKUP_CACHE_SIZE = 0
+#---------------------------------------------------------------------------
+# Build related configuration options
+#---------------------------------------------------------------------------
+EXTRACT_ALL = NO
+EXTRACT_PRIVATE = NO
+EXTRACT_PACKAGE = NO
+EXTRACT_STATIC = NO
+EXTRACT_LOCAL_CLASSES = YES
+EXTRACT_LOCAL_METHODS = NO
+EXTRACT_ANON_NSPACES = NO
+HIDE_UNDOC_MEMBERS = NO
+HIDE_UNDOC_CLASSES = NO
+HIDE_FRIEND_COMPOUNDS = NO
+HIDE_IN_BODY_DOCS = NO
+INTERNAL_DOCS = NO
+CASE_SENSE_NAMES = YES
+HIDE_SCOPE_NAMES = NO
+HIDE_COMPOUND_REFERENCE= NO
+SHOW_INCLUDE_FILES = YES
+SHOW_GROUPED_MEMB_INC = NO
+FORCE_LOCAL_INCLUDES = NO
+INLINE_INFO = YES
+SORT_MEMBER_DOCS = YES
+SORT_BRIEF_DOCS = NO
+SORT_MEMBERS_CTORS_1ST = NO
+SORT_GROUP_NAMES = NO
+SORT_BY_SCOPE_NAME = NO
+STRICT_PROTO_MATCHING = NO
+GENERATE_TODOLIST = YES
+GENERATE_TESTLIST = YES
+GENERATE_BUGLIST = YES
+GENERATE_DEPRECATEDLIST= YES
+ENABLED_SECTIONS =
+MAX_INITIALIZER_LINES = 30
+SHOW_USED_FILES = YES
+SHOW_FILES = YES
+SHOW_NAMESPACES = YES
+FILE_VERSION_FILTER =
+LAYOUT_FILE =
+CITE_BIB_FILES =
+#---------------------------------------------------------------------------
+# Configuration options related to warning and progress messages
+#---------------------------------------------------------------------------
+QUIET = NO
+WARNINGS = YES
+WARN_IF_UNDOCUMENTED = YES
+WARN_IF_DOC_ERROR = YES
+WARN_NO_PARAMDOC = NO
+WARN_AS_ERROR = NO
+WARN_FORMAT = "$file:$line: $text"
+WARN_LOGFILE =
+#---------------------------------------------------------------------------
+# Configuration options related to the input files
+#---------------------------------------------------------------------------
+INPUT =
+INPUT_ENCODING = UTF-8
+FILE_PATTERNS =
+RECURSIVE = YES
+EXCLUDE = third-party/ buck-out/
+EXCLUDE_SYMLINKS = NO
+EXCLUDE_PATTERNS = /*-test.cc
+EXCLUDE_SYMBOLS =
+EXAMPLE_PATH =
+EXAMPLE_PATTERNS =
+EXAMPLE_RECURSIVE = NO
+IMAGE_PATH =
+INPUT_FILTER =
+FILTER_PATTERNS =
+FILTER_SOURCE_FILES = NO
+FILTER_SOURCE_PATTERNS =
+USE_MDFILE_AS_MAINPAGE =
+#---------------------------------------------------------------------------
+# Configuration options related to source browsing
+#---------------------------------------------------------------------------
+SOURCE_BROWSER = NO
+INLINE_SOURCES = NO
+STRIP_CODE_COMMENTS = YES
+REFERENCED_BY_RELATION = NO
+REFERENCES_RELATION = NO
+REFERENCES_LINK_SOURCE = YES
+SOURCE_TOOLTIPS = YES
+USE_HTAGS = NO
+VERBATIM_HEADERS = YES
+CLANG_ASSISTED_PARSING = NO
+CLANG_OPTIONS =
+#---------------------------------------------------------------------------
+# Configuration options related to the alphabetical class index
+#---------------------------------------------------------------------------
+ALPHABETICAL_INDEX = YES
+COLS_IN_ALPHA_INDEX = 5
+IGNORE_PREFIX =
+#---------------------------------------------------------------------------
+# Configuration options related to the HTML output
+#---------------------------------------------------------------------------
+GENERATE_HTML = YES
+HTML_OUTPUT = html
+HTML_FILE_EXTENSION = .html
+HTML_HEADER =
+HTML_FOOTER =
+HTML_STYLESHEET =
+HTML_EXTRA_STYLESHEET =
+HTML_EXTRA_FILES =
+HTML_COLORSTYLE_HUE = 220
+HTML_COLORSTYLE_SAT = 100
+HTML_COLORSTYLE_GAMMA = 80
+HTML_TIMESTAMP = NO
+HTML_DYNAMIC_SECTIONS = NO
+HTML_INDEX_NUM_ENTRIES = 100
+GENERATE_DOCSET = NO
+DOCSET_FEEDNAME = "Doxygen generated docs"
+DOCSET_BUNDLE_ID = org.doxygen.Project
+DOCSET_PUBLISHER_ID = org.doxygen.Publisher
+DOCSET_PUBLISHER_NAME = Publisher
+GENERATE_HTMLHELP = NO
+CHM_FILE =
+HHC_LOCATION =
+GENERATE_CHI = NO
+CHM_INDEX_ENCODING =
+BINARY_TOC = NO
+TOC_EXPAND = NO
+GENERATE_QHP = NO
+QCH_FILE =
+QHP_NAMESPACE = org.doxygen.Project
+QHP_VIRTUAL_FOLDER = doc
+QHP_CUST_FILTER_NAME =
+QHP_CUST_FILTER_ATTRS =
+QHP_SECT_FILTER_ATTRS =
+QHG_LOCATION =
+GENERATE_ECLIPSEHELP = NO
+ECLIPSE_DOC_ID = org.doxygen.Project
+DISABLE_INDEX = NO
+GENERATE_TREEVIEW = NO
+ENUM_VALUES_PER_LINE = 4
+TREEVIEW_WIDTH = 250
+EXT_LINKS_IN_WINDOW = NO
+FORMULA_FONTSIZE = 10
+FORMULA_TRANSPARENT = YES
+USE_MATHJAX = NO
+MATHJAX_FORMAT = HTML-CSS
+MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest
+MATHJAX_EXTENSIONS =
+MATHJAX_CODEFILE =
+SEARCHENGINE = YES
+SERVER_BASED_SEARCH = NO
+EXTERNAL_SEARCH = NO
+SEARCHENGINE_URL =
+SEARCHDATA_FILE = searchdata.xml
+EXTERNAL_SEARCH_ID =
+EXTRA_SEARCH_MAPPINGS =
+#---------------------------------------------------------------------------
+# Configuration options related to the LaTeX output
+#---------------------------------------------------------------------------
+GENERATE_LATEX = YES
+LATEX_OUTPUT = latex
+LATEX_CMD_NAME = latex
+MAKEINDEX_CMD_NAME = makeindex
+COMPACT_LATEX = NO
+PAPER_TYPE = a4
+EXTRA_PACKAGES =
+LATEX_HEADER =
+LATEX_FOOTER =
+LATEX_EXTRA_STYLESHEET =
+LATEX_EXTRA_FILES =
+PDF_HYPERLINKS = YES
+USE_PDFLATEX = YES
+LATEX_BATCHMODE = NO
+LATEX_HIDE_INDICES = NO
+LATEX_SOURCE_CODE = NO
+LATEX_BIB_STYLE = plain
+LATEX_TIMESTAMP = NO
+#---------------------------------------------------------------------------
+# Configuration options related to the RTF output
+#---------------------------------------------------------------------------
+GENERATE_RTF = NO
+RTF_OUTPUT = rtf
+COMPACT_RTF = NO
+RTF_HYPERLINKS = NO
+RTF_STYLESHEET_FILE =
+RTF_EXTENSIONS_FILE =
+RTF_SOURCE_CODE = NO
+#---------------------------------------------------------------------------
+# Configuration options related to the man page output
+#---------------------------------------------------------------------------
+GENERATE_MAN = NO
+MAN_OUTPUT = man
+MAN_EXTENSION = .3
+MAN_SUBDIR =
+MAN_LINKS = NO
+#---------------------------------------------------------------------------
+# Configuration options related to the XML output
+#---------------------------------------------------------------------------
+GENERATE_XML = NO
+XML_OUTPUT = xml
+XML_PROGRAMLISTING = YES
+#---------------------------------------------------------------------------
+# Configuration options related to the DOCBOOK output
+#---------------------------------------------------------------------------
+GENERATE_DOCBOOK = NO
+DOCBOOK_OUTPUT = docbook
+DOCBOOK_PROGRAMLISTING = NO
+#---------------------------------------------------------------------------
+# Configuration options for the AutoGen Definitions output
+#---------------------------------------------------------------------------
+GENERATE_AUTOGEN_DEF = NO
+#---------------------------------------------------------------------------
+# Configuration options related to the Perl module output
+#---------------------------------------------------------------------------
+GENERATE_PERLMOD = NO
+PERLMOD_LATEX = NO
+PERLMOD_PRETTY = YES
+PERLMOD_MAKEVAR_PREFIX =
+#---------------------------------------------------------------------------
+# Configuration options related to the preprocessor
+#---------------------------------------------------------------------------
+ENABLE_PREPROCESSING = YES
+MACRO_EXPANSION = NO
+EXPAND_ONLY_PREDEF = NO
+SEARCH_INCLUDES = YES
+INCLUDE_PATH =
+INCLUDE_FILE_PATTERNS =
+PREDEFINED =
+EXPAND_AS_DEFINED =
+SKIP_FUNCTION_MACROS = YES
+#---------------------------------------------------------------------------
+# Configuration options related to external references
+#---------------------------------------------------------------------------
+TAGFILES =
+GENERATE_TAGFILE =
+ALLEXTERNALS = NO
+EXTERNAL_GROUPS = YES
+EXTERNAL_PAGES = YES
+PERL_PATH = /usr/bin/perl
+#---------------------------------------------------------------------------
+# Configuration options related to the dot tool
+#---------------------------------------------------------------------------
+CLASS_DIAGRAMS = YES
+MSCGEN_PATH =
+DIA_PATH =
+HIDE_UNDOC_RELATIONS = YES
+HAVE_DOT = YES
+DOT_NUM_THREADS = 0
+DOT_FONTNAME = Helvetica
+DOT_FONTSIZE = 10
+DOT_FONTPATH =
+CLASS_GRAPH = YES
+COLLABORATION_GRAPH = YES
+GROUP_GRAPHS = YES
+UML_LOOK = NO
+UML_LIMIT_NUM_FIELDS = 10
+TEMPLATE_RELATIONS = NO
+INCLUDE_GRAPH = YES
+INCLUDED_BY_GRAPH = YES
+CALL_GRAPH = NO
+CALLER_GRAPH = NO
+GRAPHICAL_HIERARCHY = YES
+DIRECTORY_GRAPH = YES
+DOT_IMAGE_FORMAT = png
+INTERACTIVE_SVG = NO
+DOT_PATH =
+DOTFILE_DIRS =
+MSCFILE_DIRS =
+DIAFILE_DIRS =
+PLANTUML_JAR_PATH =
+PLANTUML_INCLUDE_PATH =
+DOT_GRAPH_MAX_NODES = 50
+MAX_DOT_GRAPH_DEPTH = 0
+DOT_TRANSPARENT = NO
+DOT_MULTI_TARGETS = NO
+GENERATE_LEGEND = YES
+DOT_CLEANUP = YES
[hbase] 133/137: HBASE-18861 [C++] Use boost::optional instead of
std::experimental::optional
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 7b70670135af5b90922d2984b7d85b0aa9b6a20b
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Fri Sep 22 11:23:59 2017 -0700
HBASE-18861 [C++] Use boost::optional instead of std::experimental::optional
---
include/hbase/client/configuration.h | 4 +-
include/hbase/client/hbase-configuration-loader.h | 2 -
include/hbase/utils/optional.h | 7 +++-
src/hbase/client/client-test.cc | 5 ++-
src/hbase/client/configuration.cc | 46 +++++++++++------------
src/hbase/client/hbase-configuration-loader.cc | 8 ++--
src/hbase/client/hbase-configuration-test.cc | 36 +++++++++---------
src/hbase/client/result-test.cc | 5 ++-
8 files changed, 58 insertions(+), 55 deletions(-)
diff --git a/include/hbase/client/configuration.h b/include/hbase/client/configuration.h
index d70941c..06f9c79 100644
--- a/include/hbase/client/configuration.h
+++ b/include/hbase/client/configuration.h
@@ -24,12 +24,12 @@
#include <string>
#include <vector>
-#include <experimental/optional>
+#include "hbase/utils/optional.h"
namespace hbase {
template <class T>
-using optional = std::experimental::optional<T>;
+using optional = boost::optional<T>;
class Configuration {
public:
diff --git a/include/hbase/client/hbase-configuration-loader.h b/include/hbase/client/hbase-configuration-loader.h
index 8792aa2..d46217d 100644
--- a/include/hbase/client/hbase-configuration-loader.h
+++ b/include/hbase/client/hbase-configuration-loader.h
@@ -23,8 +23,6 @@
#include <string>
#include <vector>
-#include <boost/optional.hpp>
-
#include "hbase/client/configuration.h"
#include "hbase/utils/optional.h"
diff --git a/include/hbase/utils/optional.h b/include/hbase/utils/optional.h
index a05eab5..c43fd56 100644
--- a/include/hbase/utils/optional.h
+++ b/include/hbase/utils/optional.h
@@ -19,7 +19,8 @@
#pragma once
-#include <experimental/optional>
+#include <boost/optional.hpp>
+#include <boost/none.hpp>
namespace hbase {
@@ -27,6 +28,8 @@ namespace hbase {
* An optional value that may or may not be present.
*/
template <class T>
-using optional = std::experimental::optional<T>;
+using optional = boost::optional<T>;
+
+const boost::none_t none = boost::none;
} /* namespace hbase */
diff --git a/src/hbase/client/client-test.cc b/src/hbase/client/client-test.cc
index 7141047..0773d3d 100644
--- a/src/hbase/client/client-test.cc
+++ b/src/hbase/client/client-test.cc
@@ -34,15 +34,16 @@
#include "hbase/serde/table-name.h"
#include "hbase/test-util/test-util.h"
#include "hbase/utils/bytes-util.h"
+#include "hbase/utils/optional.h"
using hbase::Cell;
using hbase::Configuration;
using hbase::Get;
using hbase::RetriesExhaustedException;
+using hbase::none;
using hbase::Put;
using hbase::Table;
using hbase::TestUtil;
-using std::experimental::nullopt;
class ClientTest : public ::testing::Test {
public:
@@ -202,7 +203,7 @@ TEST_F(ClientTest, PutGetDelete) {
result = table->Get(get);
ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty.";
ASSERT_FALSE(result->Value("d", "1")) << "Column 1 should be gone";
- ASSERT_TRUE(result->Value("d", "extra") != nullopt) << "Column extra should have value";
+ ASSERT_TRUE(result->Value("d", "extra") != none) << "Column extra should have value";
EXPECT_EQ(valExt, *(result->Value("d", "ext"))) << "Column ext should have value";
// delete all cells from "extra" column
diff --git a/src/hbase/client/configuration.cc b/src/hbase/client/configuration.cc
index d829edc..385ea6a 100644
--- a/src/hbase/client/configuration.cc
+++ b/src/hbase/client/configuration.cc
@@ -23,9 +23,9 @@
#include <stdexcept>
#include <utility>
-#include <glog/logging.h>
#include <boost/format.hpp>
#include <boost/lexical_cast.hpp>
+#include <glog/logging.h>
namespace hbase {
@@ -94,23 +94,23 @@ optional<std::string> Configuration::GetEnv(const std::string &key) const {
if ("user.name" == key) {
#ifdef HAVE_GETLOGIN
- return std::experimental::make_optional(getlogin());
+ return getlogin();
#else
DLOG(WARNING) << "Client user.name not implemented";
- return optional<std::string>();
+ return none;
#endif
}
if ("user.dir" == key) {
#ifdef HAVE_GETCWD
if (getcwd(buf, sizeof(buf))) {
- return std::experimental::make_optional(buf);
+ return buf;
} else {
- return optional<std::string>();
+ return none;
}
#else
DLOG(WARNING) << "Client user.dir not implemented";
- return optional<std::string>();
+ return none;
#endif
}
@@ -118,33 +118,33 @@ optional<std::string> Configuration::GetEnv(const std::string &key) const {
#if defined(HAVE_GETUID) && defined(HAVE_GETPWUID_R)
uid = getuid();
if (!getpwuid_r(uid, &pw, buf, sizeof(buf), &pwp)) {
- return std::experimental::make_optional(buf);
+ return buf;
} else {
- return optional<std::string>();
+ return none;
}
#else
DLOG(WARNING) << "Client user.home not implemented";
- return optional<std::string>();
+ return none;
#endif
}
- return optional<std::string>();
+ return none;
}
optional<std::string> Configuration::GetProperty(const std::string &key) const {
auto found = hb_property_.find(key);
if (found != hb_property_.end()) {
- return std::experimental::make_optional(found->second.value);
+ return found->second.value;
} else {
- return optional<std::string>();
+ return none;
}
}
optional<std::string> Configuration::Get(const std::string &key) const {
optional<std::string> raw = GetProperty(key);
if (raw) {
- return std::experimental::make_optional(SubstituteVars(*raw));
+ return SubstituteVars(*raw);
} else {
- return optional<std::string>();
+ return none;
}
}
@@ -156,12 +156,12 @@ optional<int32_t> Configuration::GetInt(const std::string &key) const {
optional<std::string> raw = Get(key);
if (raw) {
try {
- return std::experimental::make_optional(boost::lexical_cast<int32_t>(*raw));
+ return boost::lexical_cast<int32_t>(*raw);
} catch (const boost::bad_lexical_cast &blex) {
throw std::runtime_error(blex.what());
}
}
- return optional<int32_t>();
+ return none;
}
int32_t Configuration::GetInt(const std::string &key, int32_t default_value) const {
@@ -172,12 +172,12 @@ optional<int64_t> Configuration::GetLong(const std::string &key) const {
optional<std::string> raw = Get(key);
if (raw) {
try {
- return std::experimental::make_optional(boost::lexical_cast<int64_t>(*raw));
+ return boost::lexical_cast<int64_t>(*raw);
} catch (const boost::bad_lexical_cast &blex) {
throw std::runtime_error(blex.what());
}
}
- return optional<int64_t>();
+ return none;
}
int64_t Configuration::GetLong(const std::string &key, int64_t default_value) const {
@@ -188,12 +188,12 @@ optional<double> Configuration::GetDouble(const std::string &key) const {
optional<std::string> raw = Get(key);
if (raw) {
try {
- return std::experimental::make_optional(boost::lexical_cast<double>(*raw));
+ return boost::lexical_cast<double>(*raw);
} catch (const boost::bad_lexical_cast &blex) {
throw std::runtime_error(blex.what());
}
}
- return optional<double>();
+ return none;
}
double Configuration::GetDouble(const std::string &key, double default_value) const {
@@ -204,9 +204,9 @@ optional<bool> Configuration::GetBool(const std::string &key) const {
optional<std::string> raw = Get(key);
if (raw) {
if (!strcasecmp((*raw).c_str(), "true") || !strcasecmp((*raw).c_str(), "1")) {
- return std::experimental::make_optional(true);
+ return true;
} else if (!strcasecmp((*raw).c_str(), "false") || !strcasecmp((*raw).c_str(), "0")) {
- return std::experimental::make_optional(false);
+ return false;
} else {
boost::format what("Unexpected value \"%s\" found being converted to bool for key \"%s\"");
what % (*raw);
@@ -214,7 +214,7 @@ optional<bool> Configuration::GetBool(const std::string &key) const {
throw std::runtime_error(what.str());
}
}
- return optional<bool>();
+ return none;
}
bool Configuration::GetBool(const std::string &key, bool default_value) const {
diff --git a/src/hbase/client/hbase-configuration-loader.cc b/src/hbase/client/hbase-configuration-loader.cc
index 98ef20e..2233794 100644
--- a/src/hbase/client/hbase-configuration-loader.cc
+++ b/src/hbase/client/hbase-configuration-loader.cc
@@ -125,9 +125,9 @@ optional<Configuration> HBaseConfigurationLoader::LoadDefaultResources() {
}
}
if (success) {
- return std::experimental::make_optional<Configuration>(Configuration(conf_property));
+ return Configuration(conf_property);
} else {
- return optional<Configuration>();
+ return none;
}
}
@@ -149,9 +149,9 @@ optional<Configuration> HBaseConfigurationLoader::LoadResources(
}
}
if (success) {
- return std::experimental::make_optional<Configuration>(Configuration(conf_property));
+ return Configuration(conf_property);
} else {
- return optional<Configuration>();
+ return none;
}
}
diff --git a/src/hbase/client/hbase-configuration-test.cc b/src/hbase/client/hbase-configuration-test.cc
index 173c4df..afa9c06 100644
--- a/src/hbase/client/hbase-configuration-test.cc
+++ b/src/hbase/client/hbase-configuration-test.cc
@@ -25,9 +25,9 @@
#include <boost/filesystem.hpp>
#include "hbase/client/configuration.h"
#include "hbase/client/hbase-configuration-loader.h"
+#include "hbase/utils/optional.h"
using namespace hbase;
-using std::experimental::nullopt;
const std::string kDefHBaseConfPath("./build/test-data/hbase-configuration-test/conf/");
const std::string kHBaseConfPath("./build/test-data/hbase-configuration-test/custom-conf/");
@@ -150,7 +150,7 @@ TEST(Configuration, LoadConfFromDefaultLocation) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
+ ASSERT_TRUE(conf != none) << "No configuration object present.";
EXPECT_STREQ((*conf).Get("custom-prop", "Set this value").c_str(), "custom-value");
EXPECT_STREQ((*conf).Get("default-prop", "Set this value").c_str(), "default-value");
}
@@ -167,7 +167,7 @@ TEST(Configuration, LoadConfFromCustomLocation) {
HBaseConfigurationLoader loader;
std::vector<std::string> resources{kHBaseSiteXml};
hbase::optional<Configuration> conf = loader.LoadResources(kHBaseConfPath, resources);
- ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
+ ASSERT_TRUE(conf != none) << "No configuration object present.";
EXPECT_STREQ((*conf).Get("custom-prop", "").c_str(), "custom-value");
EXPECT_STRNE((*conf).Get("custom-prop", "").c_str(), "some-value");
}
@@ -188,7 +188,7 @@ TEST(Configuration, LoadConfFromMultipleLocatons) {
std::string conf_paths = kDefHBaseConfPath + ":" + kHBaseConfPath;
std::vector<std::string> resources{kHBaseDefaultXml, kHBaseSiteXml};
hbase::optional<Configuration> conf = loader.LoadResources(conf_paths, resources);
- ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
+ ASSERT_TRUE(conf != none) << "No configuration object present.";
EXPECT_STREQ((*conf).Get("default-prop", "From hbase-default.xml").c_str(), "default-value");
EXPECT_STREQ((*conf).Get("custom-prop", "").c_str(), "custom-value");
EXPECT_STRNE((*conf).Get("custom-prop", "").c_str(), "some-value");
@@ -207,7 +207,7 @@ TEST(Configuration, DefaultValues) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
+ ASSERT_TRUE(conf != none) << "No configuration object present.";
EXPECT_STREQ((*conf).Get("default-prop", "Set this value.").c_str(), "default-value");
EXPECT_STREQ((*conf).Get("custom-prop", "Set this value.").c_str(), "custom-value");
}
@@ -219,7 +219,7 @@ TEST(Configuration, FinalValues) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
+ ASSERT_TRUE(conf != none) << "No configuration object present.";
EXPECT_STREQ((*conf).Get("hbase.rootdir", "").c_str(), "/root/hbase-docker/apps/hbase/data");
EXPECT_STREQ((*conf).Get("hbase.zookeeper.property.datadir", "").c_str(),
"/root/hbase-docker/zookeeper");
@@ -241,7 +241,7 @@ TEST(Configuration, EnvVars) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
+ ASSERT_TRUE(conf != none) << "No configuration object present.";
EXPECT_STREQ((*conf).Get("hbase-client.user.name", "").c_str(), "${user.name}");
EXPECT_STRNE((*conf).Get("hbase-client.user.name", "root").c_str(), "test-user");
}
@@ -253,7 +253,7 @@ TEST(Configuration, SelfRef) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
+ ASSERT_TRUE(conf != none) << "No configuration object present.";
EXPECT_STREQ((*conf).Get("selfRef", "${selfRef}").c_str(), "${selfRef}");
}
@@ -264,7 +264,7 @@ TEST(Configuration, VarExpansion) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
+ ASSERT_TRUE(conf != none) << "No configuration object present.";
EXPECT_STREQ((*conf).Get("foo.substs", "foo-value").c_str(),
"bar-value,bar-value,bar-value,bar-value,bar-value,bar-value,"
"bar-value,bar-value,bar-value,bar-value,");
@@ -278,7 +278,7 @@ TEST(Configuration, VarExpansionException) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
+ ASSERT_TRUE(conf != none) << "No configuration object present.";
ASSERT_THROW((*conf).Get("foo.substs.exception", "foo-value").c_str(), std::runtime_error);
}
@@ -289,7 +289,7 @@ TEST(Configuration, GetInt) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
+ ASSERT_TRUE(conf != none) << "No configuration object present.";
EXPECT_EQ(16000, (*conf).GetInt("int", 0));
EXPECT_EQ(2147483646, (*conf).GetInt("int.largevalue", 0));
}
@@ -301,7 +301,7 @@ TEST(Configuration, GetLong) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
+ ASSERT_TRUE(conf != none) << "No configuration object present.";
EXPECT_EQ(2147483850, (*conf).GetLong("long", 0));
EXPECT_EQ(9223372036854775807, (*conf).GetLong("long.largevalue", 0));
}
@@ -313,7 +313,7 @@ TEST(Configuration, GetDouble) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
+ ASSERT_TRUE(conf != none) << "No configuration object present.";
EXPECT_DOUBLE_EQ(17.9769e+100, (*conf).GetDouble("double", 0.0));
EXPECT_DOUBLE_EQ(170.769e+200, (*conf).GetDouble("double.largevalue", 0.0));
}
@@ -325,7 +325,7 @@ TEST(Configuration, GetBool) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
+ ASSERT_TRUE(conf != none) << "No configuration object present.";
EXPECT_EQ(true, (*conf).GetBool("bool.true", true));
EXPECT_EQ(false, (*conf).GetBool("bool.false", false));
}
@@ -337,7 +337,7 @@ TEST(Configuration, GetIntException) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
+ ASSERT_TRUE(conf != none) << "No configuration object present.";
ASSERT_THROW((*conf).GetInt("int.exception", 0), std::runtime_error);
}
@@ -348,7 +348,7 @@ TEST(Configuration, GetLongException) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
+ ASSERT_TRUE(conf != none) << "No configuration object present.";
ASSERT_THROW((*conf).GetLong("long.exception", 0), std::runtime_error);
}
@@ -359,7 +359,7 @@ TEST(Configuration, GetDoubleException) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
+ ASSERT_TRUE(conf != none) << "No configuration object present.";
ASSERT_THROW((*conf).GetDouble("double.exception", 0), std::runtime_error);
}
@@ -370,6 +370,6 @@ TEST(Configuration, GetBoolException) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
+ ASSERT_TRUE(conf != none) << "No configuration object present.";
ASSERT_THROW((*conf).GetBool("bool.exception", false), std::runtime_error);
}
diff --git a/src/hbase/client/result-test.cc b/src/hbase/client/result-test.cc
index 684c08d..3857d7b 100644
--- a/src/hbase/client/result-test.cc
+++ b/src/hbase/client/result-test.cc
@@ -26,11 +26,12 @@
#include "hbase/client/cell.h"
#include "hbase/client/result.h"
+#include "hbase/utils/optional.h"
using hbase::Cell;
using hbase::CellType;
+using hbase::none;
using hbase::Result;
-using std::experimental::nullopt;
void PopulateCells(std::vector<std::shared_ptr<Cell> > &cells) {
// Populate some Results
@@ -112,7 +113,7 @@ TEST(Result, FilledResult) {
// Value will be nullptr as no such family and qualifier is present
ASSERT_FALSE(result.Value("family-4", "qualifier"));
// Value will be present as family and qualifier is present
- ASSERT_TRUE(result.Value("family-4", "column-4") != nullopt);
+ ASSERT_TRUE(result.Value("family-4", "column-4") != none);
// Value should be present and match.
EXPECT_EQ(latest_cell->Value(), (*result.ColumnLatestCell("family-4", "column-4")).Value());
EXPECT_EQ("value-5", (*result.ColumnLatestCell("family-5", "column-5")).Value());
[hbase] 76/137: Adding methods to run a stand-alone HBase instance
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 37a6d7058dcb807609dc9f0f5d97e8584bafb86e
Author: Sudeep Sunthankar <su...@hashmapinc.com>
AuthorDate: Fri Mar 24 15:58:58 2017 +1100
Adding methods to run a stand-alone HBase instance
Signed-off-by: Enis Soztutar <en...@apache.org>
---
test-util/test-util.cc | 22 +++++++++++++++++++++-
test-util/test-util.h | 4 ++++
2 files changed, 25 insertions(+), 1 deletion(-)
diff --git a/test-util/test-util.cc b/test-util/test-util.cc
index dfe4260..54d2c83 100644
--- a/test-util/test-util.cc
+++ b/test-util/test-util.cc
@@ -46,7 +46,9 @@ std::string TestUtil::RandString(int len) {
TestUtil::TestUtil() : temp_dir_(TestUtil::RandString()) {}
-TestUtil::~TestUtil() { StopMiniCluster(); }
+TestUtil::~TestUtil() {
+ if (mini_) StopMiniCluster();
+}
void TestUtil::StartMiniCluster(int32_t num_region_servers) {
mini_ = std::make_unique<MiniCluster>();
@@ -69,3 +71,21 @@ void TestUtil::TablePut(const std::string &table, const std::string &row, const
const std::string &column, const std::string &value) {
mini_->TablePut(table, row, family, column, value);
}
+
+void TestUtil::StartStandAloneInstance() {
+ auto p = temp_dir_.path().string();
+ auto cmd = std::string{"bin/start-local-hbase.sh " + p};
+ auto res_code = std::system(cmd.c_str());
+ CHECK_EQ(res_code, 0);
+}
+
+void TestUtil::StopStandAloneInstance() {
+ auto res_code = std::system("bin/stop-local-hbase.sh");
+ CHECK_EQ(res_code, 0);
+}
+
+void TestUtil::RunShellCmd(const std::string &command) {
+ auto cmd_string = folly::sformat("echo \"{}\" | ../bin/hbase shell", command);
+ auto res_code = std::system(cmd_string.c_str());
+ CHECK_EQ(res_code, 0);
+}
diff --git a/test-util/test-util.h b/test-util/test-util.h
index 611404d..b5714ee 100644
--- a/test-util/test-util.h
+++ b/test-util/test-util.h
@@ -64,6 +64,10 @@ class TestUtil {
void TablePut(const std::string &table, const std::string &row, const std::string &family,
const std::string &column, const std::string &value);
+ void StartStandAloneInstance();
+ void StopStandAloneInstance();
+ void RunShellCmd(const std::string &);
+
private:
std::unique_ptr<MiniCluster> mini_;
folly::test::TemporaryDirectory temp_dir_;
[hbase] 45/137: HBASE-16489 Configuration parsing (Sudeep
Sunthankar)
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 13269dead7192e8960c2c58954b9bd537df1dfc3
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Wed Nov 30 14:57:18 2016 -0800
HBASE-16489 Configuration parsing (Sudeep Sunthankar)
---
core/BUCK | 10 +
core/configuration.cc | 231 ++++++++++++++++++++++
core/configuration.h | 191 ++++++++++++++++++
core/hbase_configuration-test.cc | 388 +++++++++++++++++++++++++++++++++++++
core/hbase_configuration_loader.cc | 214 ++++++++++++++++++++
core/hbase_configuration_loader.h | 146 ++++++++++++++
6 files changed, 1180 insertions(+)
diff --git a/core/BUCK b/core/BUCK
index 7e9044a..9dbc82e 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -29,6 +29,8 @@ cxx_library(
"meta-utils.h",
"get.h",
"time_range.h",
+ "configuration.h",
+ "hbase_configuration_loader.h",
],
srcs=[
"cell.cc",
@@ -37,6 +39,8 @@ cxx_library(
"meta-utils.cc",
"get.cc",
"time_range.cc",
+ "configuration.cc",
+ "hbase_configuration_loader.cc",
],
deps=[
"//connection:connection",
@@ -75,6 +79,12 @@ cxx_test(name="time_range-test",
],
deps=[":core", ],
run_test_separately=True, )
+cxx_test(name="hbase_configuration-test",
+ srcs=[
+ "hbase_configuration-test.cc",
+ ],
+ deps=[":core", ],
+ run_test_separately=True, )
cxx_binary(name="simple-client",
srcs=["simple-client.cc", ],
deps=[":core", "//connection:connection"], )
diff --git a/core/configuration.cc b/core/configuration.cc
new file mode 100644
index 0000000..9816cff
--- /dev/null
+++ b/core/configuration.cc
@@ -0,0 +1,231 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "configuration.h"
+
+#include <stdexcept>
+
+#include <boost/lexical_cast.hpp>
+#include <glog/logging.h>
+
+namespace hbase {
+
+Configuration::Configuration(ConfigMap &config_map)
+ : hb_property_(std::move(config_map)) {}
+
+Configuration::~Configuration() {}
+
+size_t Configuration::IsSubVariable(const std::string &expr,
+ std::string &sub_variable) const {
+ size_t start_pos = expr.find("${");
+ if (std::string::npos != start_pos) {
+ size_t pos_next = expr.find("}", start_pos + 1);
+ if (std::string::npos != pos_next) {
+ sub_variable = expr.substr(start_pos + 2, pos_next - (start_pos + 2));
+ }
+ }
+ return start_pos;
+}
+
+std::string Configuration::SubstituteVars(const std::string &expr) const {
+ if (0 == expr.size()) return expr;
+
+ std::string eval(expr);
+ std::string value_to_be_replaced("");
+ std::string var("");
+ for (int i = 0; i < kMaxSubsts; i++) {
+ var = "";
+ size_t start_pos = IsSubVariable(eval, var);
+ if (start_pos != std::string::npos) {
+ // We are blindly checking for environment property at first.
+ // If we don't get any value from GetEnv, check in hbase-site.xml.
+ value_to_be_replaced =
+ GetEnv(var).value_or(GetProperty(var).value_or(""));
+
+ // we haven't found any value yet so we are returning eval
+ if (0 == value_to_be_replaced.size()) {
+ return eval;
+ }
+
+ // return original expression if there is a loop
+ if (value_to_be_replaced == expr) {
+ return expr;
+ }
+
+ eval.replace(start_pos, var.size() + 3, value_to_be_replaced);
+
+ } else {
+ // No further expansion required.
+ return eval;
+ }
+ }
+ // We reached here if the loop is exhausted
+ // If MAX_SUBSTS is exhausted, check if more variable substitution is reqd.
+ // If any-more substitutions are reqd, throw an error.
+ var = "";
+ if (IsSubVariable(eval, var) != std::string::npos) {
+ throw std::runtime_error("Variable substitution depth too large: " +
+ std::to_string(kMaxSubsts) + " " + expr);
+ } else {
+ return eval;
+ }
+}
+
+optional<std::string> Configuration::GetEnv(const std::string &key) const {
+ char buf[2048];
+
+ if ("user.name" == key) {
+#ifdef HAVE_GETLOGIN
+ return std::experimental::make_optional(getlogin());
+#else
+ DLOG(WARNING) << "Client user.name not implemented";
+ return optional<std::string>();
+#endif
+ }
+
+ if ("user.dir" == key) {
+#ifdef HAVE_GETCWD
+ if (getcwd(buf, sizeof(buf))) {
+ return std::experimental::make_optional(buf);
+ } else {
+ return optional<std::string>();
+ }
+#else
+ DLOG(WARNING) << "Client user.dir not implemented";
+ return optional<std::string>();
+#endif
+ }
+
+ if ("user.home" == key) {
+#if defined(HAVE_GETUID) && defined(HAVE_GETPWUID_R)
+ uid = getuid();
+ if (!getpwuid_r(uid, &pw, buf, sizeof(buf), &pwp)) {
+ return std::experimental::make_optional(buf);
+ } else {
+ return optional<std::string>();
+ }
+#else
+ DLOG(WARNING) << "Client user.home not implemented";
+ return optional<std::string>();
+#endif
+ }
+ return optional<std::string>();
+}
+
+optional<std::string> Configuration::GetProperty(const std::string &key) const {
+
+ auto found = hb_property_.find(key);
+ if (found != hb_property_.end()) {
+ return std::experimental::make_optional(found->second.value);
+ } else {
+ return optional<std::string>();
+ }
+}
+
+optional<std::string> Configuration::Get(const std::string &key) const {
+ optional<std::string> raw = GetProperty(key);
+ if (raw) {
+ return std::experimental::make_optional(SubstituteVars(*raw));
+ } else {
+ return optional<std::string>();
+ }
+}
+
+std::string Configuration::Get(const std::string &key,
+ const std::string &default_value) const {
+ return Get(key).value_or(default_value);
+}
+
+optional<int32_t> Configuration::GetInt(const std::string &key) const {
+ optional<std::string> raw = Get(key);
+ if (raw) {
+ try {
+ return std::experimental::make_optional(
+ boost::lexical_cast<int32_t>(*raw));
+ }
+ catch (const boost::bad_lexical_cast &blex) {
+ throw std::runtime_error(blex.what());
+ }
+ }
+ return optional<int32_t>();
+}
+
+int32_t Configuration::GetInt(const std::string &key,
+ int32_t default_value) const {
+ return GetInt(key).value_or(default_value);
+}
+
+optional<int64_t> Configuration::GetLong(const std::string &key) const {
+ optional<std::string> raw = Get(key);
+ if (raw) {
+ try {
+ return std::experimental::make_optional(
+ boost::lexical_cast<int64_t>(*raw));
+ }
+ catch (const boost::bad_lexical_cast &blex) {
+ throw std::runtime_error(blex.what());
+ }
+ }
+ return optional<int64_t>();
+}
+
+int64_t Configuration::GetLong(const std::string &key,
+ int64_t default_value) const {
+ return GetLong(key).value_or(default_value);
+}
+
+optional<double> Configuration::GetDouble(const std::string &key) const {
+ optional<std::string> raw = Get(key);
+ if (raw) {
+ try {
+ return std::experimental::make_optional(
+ boost::lexical_cast<double>(*raw));
+ }
+ catch (const boost::bad_lexical_cast &blex) {
+ throw std::runtime_error(blex.what());
+ }
+ }
+ return optional<double>();
+}
+
+double Configuration::GetDouble(const std::string &key,
+ double default_value) const {
+ return GetDouble(key).value_or(default_value);
+}
+
+optional<bool> Configuration::GetBool(const std::string &key) const {
+ optional<std::string> raw = Get(key);
+ if (raw) {
+ if (!strcasecmp((*raw).c_str(), "true")) {
+ return std::experimental::make_optional(true);
+ } else if (!strcasecmp((*raw).c_str(), "false")) {
+ return std::experimental::make_optional(false);
+ } else {
+ throw std::runtime_error(
+ "Unexpected value found while conversion to bool.");
+ }
+ }
+ return optional<bool>();
+}
+
+bool Configuration::GetBool(const std::string &key, bool default_value) const {
+ return GetBool(key).value_or(default_value);
+}
+
+} /* namespace hbase */
diff --git a/core/configuration.h b/core/configuration.h
new file mode 100644
index 0000000..44d6cd6
--- /dev/null
+++ b/core/configuration.h
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+/* Some pieces of code have been added from HDFS-8707 */
+#pragma once
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include <experimental/optional>
+
+namespace hbase {
+
+template <class T>
+using optional = std::experimental::optional<T>;
+
+class Configuration {
+ public:
+ ~Configuration();
+
+ /**
+ * @brief Returns value identified by key in ConfigMap else default value if
+ * property is absent.
+ * @param key Property whose value is to be fetched. SubstituteVars will be
+ * called for any variable expansion.
+ */
+ std::string Get(const std::string &key,
+ const std::string &default_value) const;
+
+ /**
+ * @brief Returns int32_t identified by key in ConfigMap else default value if
+ * property is absent.
+ * @param key Property whose value is to be fetched. Internally Get(key) will
+ * be called.
+ * @throws std::runtime_error if conversion to int32_t fails
+ */
+ int32_t GetInt(const std::string &key, int32_t default_value) const;
+
+ /**
+ * @brief Returns int64_t identified by key in ConfigMap else default value if
+ * property is absent.
+ * @param key Property whose value is to be fetched. Internally Get(key) will
+ * be called and
+ * @throws std::runtime_error if conversion to int64_t fails
+ */
+ int64_t GetLong(const std::string &key, int64_t default_value) const;
+
+ /**
+ * @brief Returns double identified by key in ConfigMap else default value if
+ * property is absent.
+ * @param key Property whose value is to be fetched. Internally Get(key) will
+ * be called.
+ * @throws std::runtime_error if conversion to double fails
+ */
+ double GetDouble(const std::string &key, double default_value) const;
+
+ /**
+ * @brief Returns bool identified by key in ConfigMap else default value if
+ * property is absent.
+ * @param key Property whose value is to be fetched. Internally Get(key) will
+ * be called.
+ * @throws std::runtime_error if conversion to bool fails
+ */
+ bool GetBool(const std::string &key, bool default_value) const;
+
+ private:
+ friend class HBaseConfigurationLoader;
+
+ /* Transparent data holder for property values */
+ /* Same as Jira HDFS-8707 */
+ struct ConfigData {
+ std::string value;
+ bool final;
+ ConfigData() : final(false) {};
+ ConfigData(const std::string &value) : value(value), final(false) {}
+ void operator=(const std::string &new_value) {
+ value = new_value;
+ final = false;
+ }
+ };
+
+ /**
+ * @brief Map which hold configuration properties.
+ */
+ using ConfigMap = std::map<std::string, ConfigData>;
+
+ /**
+ * @brief Create Configuration object using config_map;
+ * @param config_map - Map consisting of properties.
+ */
+ Configuration(ConfigMap &config_map);
+
+ // Property map filled all the properties loaded by ConfigurationLoader
+ ConfigMap hb_property_;
+
+ // Variable expansion depth
+ const int kMaxSubsts = 20;
+
+ /**
+ * @brief returns value for a property identified by key in ConfigMap.
+ * SubstituteVars will be called for any variable expansion.
+ * @param key Key whose value is to be fetched.
+ */
+ optional<std::string> Get(const std::string &key) const;
+
+ /**
+ * @brief returns optional int32_t value identified by the key in ConfigMap.
+ * Get(key) is called to get the string value which is then converted to
+ * int32_t using boost::lexical_cast.
+ * @param key Key whose value is to be fetched.
+ * @throws std::runtime_error if conversion to int32_t fails
+ */
+ optional<int32_t> GetInt(const std::string &key) const;
+
+ /**
+ * @brief returns optional int64_t value identified by the key in ConfigMap.
+ * Get(key) is called internally to get the string value which is then
+ * converted to int64_t using boost::lexical_cast.
+ * @param key Key whose value is to be fetched.
+ * @throws std::runtime_error if conversion to int64_t fails
+ */
+ optional<int64_t> GetLong(const std::string &key) const;
+
+ /**
+ * @brief returns optional double value identified by the key in ConfigMap.
+ * Get(key) is called to get the string value which is then converted to
+ * double using boost::lexical_cast.
+ * @param key Key whose value is to be fetched.
+ * @throws std::runtime_error if conversion to double fails
+ */
+ optional<double> GetDouble(const std::string &key) const;
+
+ /**
+ * @brief returns optional bool for a property identified by key in ConfigMap.
+ * Get(key) is called to get the string value which is then converted to bool
+ * by checking the validity.
+ * @param key Key whose value is to be fetched. Get(key) is called to get the
+ * string value which is then converted to bool.
+ * @throws std::runtime_error if conversion to bool fails
+ */
+ optional<bool> GetBool(const std::string &key) const;
+
+ /**
+ * @brief This method will perform any variable expansion if present.
+ * @param expression expression string to perform substitutions on.
+ * @throws std::runtime_error if MAX_SUBSTS is exhausted and any more
+ * substitutions are reqd to be performed.
+ */
+ std::string SubstituteVars(const std::string &expr) const;
+
+ /**
+ * @brief This method will check if variable expansion has to be performed or
+ * not. Expression will be checked for presence of "${" and "}" to perform
+ * variable expansion.
+ * @param expr Expression on which will be checked for validity.
+ * @param sub_variable Extracted variable from expr which will be checked
+ * against environment value or ConfigMap values.
+ */
+ size_t IsSubVariable(const std::string &expr,
+ std::string &sub_variable) const;
+
+ /**
+ * @brief This method will fetch value for key from environment if present.
+ * @param key key to be fetched from environment.
+ */
+ optional<std::string> GetEnv(const std::string &key) const;
+
+ /**
+ * @brief This method will fetch value for key from ConfigMap if present.
+ * @param key key to be fetched from environment.
+ */
+ optional<std::string> GetProperty(const std::string &key) const;
+};
+} /* namespace hbase */
diff --git a/core/hbase_configuration-test.cc b/core/hbase_configuration-test.cc
new file mode 100644
index 0000000..167f506
--- /dev/null
+++ b/core/hbase_configuration-test.cc
@@ -0,0 +1,388 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <fstream>
+#include <iostream>
+
+#include <boost/filesystem.hpp>
+#include <gtest/gtest.h>
+#include <glog/logging.h>
+#include "core/hbase_configuration_loader.h"
+#include "core/configuration.h"
+
+using namespace hbase;
+
+const std::string kDefHBaseConfPath("./build/test-data/hbase-configuration-test/conf/");
+const std::string kHBaseConfPath("./build/test-data/hbase-configuration-test/custom-conf/");
+
+const std::string kHBaseDefaultXml("hbase-default.xml");
+const std::string kHBaseSiteXml("hbase-site.xml");
+
+const std::string kHBaseDefaultXmlData(
+ "<?xml version=\"1.0\"?>\n<?xml-stylesheet type=\"text/xsl\" "
+ "href=\"configuration.xsl\"?>\n<!--\n/**\n *\n * Licensed to the Apache "
+ "Software Foundation (ASF) under one\n * or more contributor license "
+ "agreements. See the NOTICE file\n * distributed with this work for "
+ "additional information\n * regarding copyright ownership. The ASF "
+ "licenses this file\n * to you under the Apache License, Version 2.0 "
+ "(the\n * \"License\"); you may not use this file except in compliance\n * "
+ "with the License. You may obtain a copy of the License at\n *\n * "
+ "http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by "
+ "applicable law or agreed to in writing, software\n * distributed under "
+ "the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES "
+ "OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License "
+ "for the specific language governing permissions and\n * limitations under "
+ "the License.\n "
+ "*/\n-->\n<configuration>\n\n<property>\n<name>hbase.rootdir</"
+ "name>\n<value>/root/hbase-docker/apps/hbase/data</value>\n<final>true</"
+ "final>\n</"
+ "property>\n\n<property>\n<name>hbase.zookeeper.property.datadir</"
+ "name>\n<value>This value will be "
+ "overwritten</value>\n<final>false</final>\n</"
+ "property>\n\n<property>\n<name>default-prop</name>\n<value>default-value</"
+ "value>\n</property>\n\n</configuration>");
+const std::string kHBaseSiteXmlData(
+ "<?xml version=\"1.0\"?>\n<?xml-stylesheet type=\"text/xsl\" "
+ "href=\"configuration.xsl\"?>\n<!--\n/**\n *\n * Licensed to the Apache "
+ "Software Foundation (ASF) under one\n * or more contributor license "
+ "agreements. See the NOTICE file\n * distributed with this work for "
+ "additional information\n * regarding copyright ownership. The ASF "
+ "licenses this file\n * to you under the Apache License, Version 2.0 "
+ "(the\n * \"License\"); you may not use this file except in compliance\n * "
+ "with the License. You may obtain a copy of the License at\n *\n * "
+ "http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by "
+ "applicable law or agreed to in writing, software\n * distributed under "
+ "the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES "
+ "OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License "
+ "for the specific language governing permissions and\n * limitations under "
+ "the License.\n "
+ "*/\n-->\n<configuration>\n\n<property>\n<name>hbase.rootdir</"
+ "name>\n<value>This value will not be be "
+ "overwritten</value>\n</"
+ "property>\n\n<property>\n<name>hbase.zookeeper.property.datadir</"
+ "name>\n<value>/root/hbase-docker/zookeeper</value>\n</"
+ "property>\n\n<property>\n<name>hbase-client.user.name</"
+ "name>\n<value>${user.name}</value>\n</"
+ "property>\n\n<property>\n<name>hbase-client.user.dir</"
+ "name>\n<value>${user.dir}</value>\n</"
+ "property>\n\n<property>\n<name>hbase-client.user.home</"
+ "name>\n<value>${user.home}</value>\n</"
+ "property>\n\n<property>\n<name>selfRef</name>\n<value>${selfRef}</"
+ "value>\n</property>\n\n<property>\n<name>foo.substs</"
+ "name>\n<value>${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${"
+ "bar},${bar},</value>\n</"
+ "property>\n\n<property>\n<name>foo.substs.exception</"
+ "name>\n<value>${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${"
+ "bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar},${bar}"
+ ",${bar},${bar}</value>\n</property>\n\n<property>\n<name>bar</"
+ "name>\n<value>bar-value</value>\n</"
+ "property>\n\n<property>\n<name>custom-prop</name>\n<value>custom-value</"
+ "value>\n</property>\n\n<property>\n<name>int</name>\n<value>16000</"
+ "value>\n</property>\n\n<property>\n<name>int.largevalue</"
+ "name>\n<value>2147483646</value>\n</"
+ "property>\n\n<property>\n<name>int.exception</name>\n<value>2147483648</"
+ "value>\n</property>\n\n<property>\n<name>long</name>\n<value>2147483850</"
+ "value>\n</property>\n\n<property>\n<name>long.largevalue</"
+ "name>\n<value>9223372036854775807</value>\n</"
+ "property>\n\n<property>\n<name>long.exception</"
+ "name>\n<value>9223372036854775810</value>\n</"
+ "property>\n\n<property>\n<name>double</name>\n<value>17.9769e+100</"
+ "value>\n</property>\n\n<property>\n<name>double.largevalue</"
+ "name>\n<value>170.769e+200</value>\n</"
+ "property>\n\n<property>\n<name>double.exception</"
+ "name>\n<value>1.79769e+310</value>\n</"
+ "property>\n\n<property>\n<name>bool.true</name>\n<value>true</value>\n</"
+ "property>\n\n<property>\n<name>bool.false</name>\n<value>false</value>\n</"
+ "property>\n\n<property>\n<name>bool.exception</name>\n<value>unknown "
+ "bool</value>\n</property>\n\n</configuration>");
+
+void WriteDataToFile(const std::string &file, const std::string &xml_data) {
+ std::ofstream hbase_conf;
+ hbase_conf.open(file.c_str());
+ hbase_conf << xml_data;
+ hbase_conf.close();
+}
+
+void CreateHBaseConf(const std::string &dir, const std::string &file,
+ const std::string xml_data) {
+ // Directory will be created if not present
+ if (!boost::filesystem::exists(dir)) {
+ boost::filesystem::create_directories(dir);
+ }
+ // Remove temp file always
+ boost::filesystem::remove((dir + file).c_str());
+ WriteDataToFile((dir + file), xml_data);
+}
+
+void CreateHBaseConfWithEnv() {
+
+ CreateHBaseConf(kDefHBaseConfPath, kHBaseDefaultXml, kHBaseDefaultXmlData);
+ CreateHBaseConf(kDefHBaseConfPath, kHBaseSiteXml, kHBaseSiteXmlData);
+ setenv("HBASE_CONF", kDefHBaseConfPath.c_str(), 1);
+}
+
+/*
+ * Config will be loaded from $HBASE_CONF. We set it @ kDefHBaseConfPath
+ * Config values will be loaded from hbase-default.xml and hbase-site.xml
+ * present in the above path.
+ */
+TEST(Configuration, LoadConfFromDefaultLocation) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+ CreateHBaseConf(kDefHBaseConfPath, kHBaseDefaultXml, kHBaseDefaultXmlData);
+ CreateHBaseConf(kDefHBaseConfPath, kHBaseSiteXml, kHBaseSiteXmlData);
+ setenv("HBASE_CONF", kDefHBaseConfPath.c_str(), 0);
+
+ HBaseConfigurationLoader loader;
+ hbase::optional<Configuration> conf = loader.LoadDefaultResources();
+ ASSERT_TRUE(conf) << "No configuration object present.";
+ EXPECT_STREQ((*conf).Get("custom-prop", "Set this value").c_str(),
+ "custom-value");
+ EXPECT_STREQ((*conf).Get("default-prop", "Set this value").c_str(),
+ "default-value");
+}
+
+/*
+ * Config will be loaded from hbase-site.xml defined at
+ * kHBaseConfPath
+ */
+TEST(Configuration, LoadConfFromCustomLocation) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+ CreateHBaseConf(kHBaseConfPath, kHBaseSiteXml, kHBaseSiteXmlData);
+
+ HBaseConfigurationLoader loader;
+ std::vector<std::string> resources{kHBaseSiteXml};
+ hbase::optional<Configuration> conf =
+ loader.LoadResources(kHBaseConfPath, resources);
+ ASSERT_TRUE(conf) << "No configuration object present.";
+ EXPECT_STREQ((*conf).Get("custom-prop", "").c_str(), "custom-value");
+ EXPECT_STRNE((*conf).Get("custom-prop", "").c_str(), "some-value");
+}
+
+/*
+ * Config will be loaded from hbase-defualt.xml and hbase-site.xml @
+ * kDefHBaseConfPath and kHBaseConfPath respectively.
+ */
+TEST(Configuration, LoadConfFromMultipleLocatons) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+ CreateHBaseConf(kDefHBaseConfPath, kHBaseDefaultXml, kHBaseDefaultXmlData);
+ CreateHBaseConf(kDefHBaseConfPath, kHBaseSiteXml, kHBaseSiteXmlData);
+ CreateHBaseConf(kHBaseConfPath, kHBaseDefaultXml, kHBaseDefaultXmlData);
+ CreateHBaseConf(kHBaseConfPath, kHBaseSiteXml, kHBaseSiteXmlData);
+
+ HBaseConfigurationLoader loader;
+ std::string conf_paths = kDefHBaseConfPath + ":" + kHBaseConfPath;
+ std::vector<std::string> resources{kHBaseDefaultXml, kHBaseSiteXml};
+ hbase::optional<Configuration> conf =
+ loader.LoadResources(conf_paths, resources);
+ ASSERT_TRUE(conf) << "No configuration object present.";
+ EXPECT_STREQ((*conf).Get("default-prop", "From hbase-default.xml").c_str(),
+ "default-value");
+ EXPECT_STREQ((*conf).Get("custom-prop", "").c_str(), "custom-value");
+ EXPECT_STRNE((*conf).Get("custom-prop", "").c_str(), "some-value");
+}
+
+/*
+ * Config will be loaded from hbase-defualt.xml and hbase-site.xml @
+ * $HBASE_CONF.
+ * We set HBASE_CONF to kDefHBaseConfPath
+ * Below tests load the conf files in the same way unless specified.
+ */
+TEST(Configuration, DefaultValues) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+ CreateHBaseConfWithEnv();
+
+ HBaseConfigurationLoader loader;
+ hbase::optional<Configuration> conf = loader.LoadDefaultResources();
+ ASSERT_TRUE(conf) << "No configuration object present.";
+ EXPECT_STREQ((*conf).Get("default-prop", "Set this value.").c_str(),
+ "default-value");
+ EXPECT_STREQ((*conf).Get("custom-prop", "Set this value.").c_str(),
+ "custom-value");
+}
+
+TEST(Configuration, FinalValues) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+ CreateHBaseConfWithEnv();
+
+ HBaseConfigurationLoader loader;
+ hbase::optional<Configuration> conf = loader.LoadDefaultResources();
+ ASSERT_TRUE(conf) << "No configuration object present.";
+ EXPECT_STREQ((*conf).Get("hbase.rootdir", "").c_str(),
+ "/root/hbase-docker/apps/hbase/data");
+ EXPECT_STREQ((*conf).Get("hbase.zookeeper.property.datadir", "").c_str(),
+ "/root/hbase-docker/zookeeper");
+ EXPECT_STRNE((*conf).Get("hbase.rootdir", "").c_str(),
+ "This value will not be be overwritten");
+ EXPECT_STRNE((*conf).Get("hbase.zookeeper.property.datadir", "").c_str(),
+ "This value will be overwritten");
+}
+
+/*
+ * Config will be loaded from HBASE_CONF which we set in
+ * CreateHBaseConfWithEnv().
+ * Config values will be loaded from hbase-default.xml and hbase-site.xml in the
+ * above path.
+ */
+TEST(Configuration, EnvVars) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+ CreateHBaseConfWithEnv();
+
+ HBaseConfigurationLoader loader;
+ hbase::optional<Configuration> conf = loader.LoadDefaultResources();
+ ASSERT_TRUE(conf) << "No configuration object present.";
+ EXPECT_STREQ((*conf).Get("hbase-client.user.name", "").c_str(),
+ "${user.name}");
+ EXPECT_STRNE((*conf).Get("hbase-client.user.name", "root").c_str(),
+ "test-user");
+}
+
+TEST(Configuration, SelfRef) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+ CreateHBaseConfWithEnv();
+
+ HBaseConfigurationLoader loader;
+ hbase::optional<Configuration> conf = loader.LoadDefaultResources();
+ ASSERT_TRUE(conf) << "No configuration object present.";
+ EXPECT_STREQ((*conf).Get("selfRef", "${selfRef}").c_str(), "${selfRef}");
+}
+
+TEST(Configuration, VarExpansion) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+ CreateHBaseConfWithEnv();
+
+ HBaseConfigurationLoader loader;
+ hbase::optional<Configuration> conf = loader.LoadDefaultResources();
+ ASSERT_TRUE(conf) << "No configuration object present.";
+ EXPECT_STREQ((*conf).Get("foo.substs", "foo-value").c_str(),
+ "bar-value,bar-value,bar-value,bar-value,bar-value,bar-value,"
+ "bar-value,bar-value,bar-value,bar-value,");
+ EXPECT_STRNE((*conf).Get("foo.substs", "foo-value").c_str(), "bar-value");
+}
+
+TEST(Configuration, VarExpansionException) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+ CreateHBaseConfWithEnv();
+
+ HBaseConfigurationLoader loader;
+ hbase::optional<Configuration> conf = loader.LoadDefaultResources();
+ ASSERT_TRUE(conf) << "No configuration object present.";
+ ASSERT_THROW((*conf).Get("foo.substs.exception", "foo-value").c_str(),
+ std::runtime_error);
+}
+
+TEST(Configuration, GetInt) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+ CreateHBaseConfWithEnv();
+
+ HBaseConfigurationLoader loader;
+ hbase::optional<Configuration> conf = loader.LoadDefaultResources();
+ ASSERT_TRUE(conf) << "No configuration object present.";
+ EXPECT_EQ(16000, (*conf).GetInt("int", 0));
+ EXPECT_EQ(2147483646, (*conf).GetInt("int.largevalue", 0));
+}
+
+TEST(Configuration, GetLong) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+ CreateHBaseConfWithEnv();
+
+ HBaseConfigurationLoader loader;
+ hbase::optional<Configuration> conf = loader.LoadDefaultResources();
+ ASSERT_TRUE(conf) << "No configuration object present.";
+ EXPECT_EQ(2147483850, (*conf).GetLong("long", 0));
+ EXPECT_EQ(9223372036854775807, (*conf).GetLong("long.largevalue", 0));
+}
+
+TEST(Configuration, GetDouble) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+ CreateHBaseConfWithEnv();
+
+ HBaseConfigurationLoader loader;
+ hbase::optional<Configuration> conf = loader.LoadDefaultResources();
+ ASSERT_TRUE(conf) << "No configuration object present.";
+ EXPECT_DOUBLE_EQ(17.9769e+100, (*conf).GetDouble("double", 0.0));
+ EXPECT_DOUBLE_EQ(170.769e+200, (*conf).GetDouble("double.largevalue", 0.0));
+}
+
+TEST(Configuration, GetBool) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+ CreateHBaseConfWithEnv();
+
+ HBaseConfigurationLoader loader;
+ hbase::optional<Configuration> conf = loader.LoadDefaultResources();
+ ASSERT_TRUE(conf) << "No configuration object present.";
+ EXPECT_EQ(true, (*conf).GetBool("bool.true", true));
+ EXPECT_EQ(false, (*conf).GetBool("bool.false", false));
+}
+
+TEST(Configuration, GetIntException) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+ CreateHBaseConfWithEnv();
+
+ HBaseConfigurationLoader loader;
+ hbase::optional<Configuration> conf = loader.LoadDefaultResources();
+ ASSERT_TRUE(conf) << "No configuration object present.";
+ ASSERT_THROW((*conf).GetInt("int.exception", 0), std::runtime_error);
+}
+
+TEST(Configuration, GetLongException) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+ CreateHBaseConfWithEnv();
+
+ HBaseConfigurationLoader loader;
+ hbase::optional<Configuration> conf = loader.LoadDefaultResources();
+ ASSERT_TRUE(conf) << "No configuration object present.";
+ ASSERT_THROW((*conf).GetLong("long.exception", 0), std::runtime_error);
+}
+
+TEST(Configuration, GetDoubleException) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+ CreateHBaseConfWithEnv();
+
+ HBaseConfigurationLoader loader;
+ hbase::optional<Configuration> conf = loader.LoadDefaultResources();
+ ASSERT_TRUE(conf) << "No configuration object present.";
+ ASSERT_THROW((*conf).GetDouble("double.exception", 0), std::runtime_error);
+}
+
+TEST(Configuration, GetBoolException) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+ CreateHBaseConfWithEnv();
+
+ HBaseConfigurationLoader loader;
+ hbase::optional<Configuration> conf = loader.LoadDefaultResources();
+ ASSERT_TRUE(conf) << "No configuration object present.";
+ ASSERT_THROW((*conf).GetBool("bool.exception", false), std::runtime_error);
+}
diff --git a/core/hbase_configuration_loader.cc b/core/hbase_configuration_loader.cc
new file mode 100644
index 0000000..43f733a
--- /dev/null
+++ b/core/hbase_configuration_loader.cc
@@ -0,0 +1,214 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/hbase_configuration_loader.h"
+
+#include <boost/foreach.hpp>
+#include <boost/property_tree/ptree.hpp>
+#include <boost/property_tree/xml_parser.hpp>
+#include <glog/logging.h>
+
+namespace hbase {
+
+bool is_valid_bool(const std::string &raw) {
+ if (raw.empty()) {
+ return false;
+ }
+
+ if (!strcasecmp(raw.c_str(), "true")) {
+ return true;
+ }
+ if (!strcasecmp(raw.c_str(), "false")) {
+ return true;
+ }
+ return false;
+}
+
+bool str_to_bool(const std::string &raw) {
+ if (!strcasecmp(raw.c_str(), "true")) {
+ return true;
+ }
+ return false;
+}
+
+HBaseConfigurationLoader::HBaseConfigurationLoader() {}
+
+HBaseConfigurationLoader::~HBaseConfigurationLoader() {}
+
+void HBaseConfigurationLoader::SetDefaultSearchPath() {
+ /*
+ * Try (in order, taking the first valid one):
+ * $HBASE_CONF_DIR
+ * /etc/hbase/conf
+ *
+ */
+ const char *hadoop_conf_dir_env = getenv("HBASE_CONF");
+ if (hadoop_conf_dir_env) {
+ AddToSearchPath(hadoop_conf_dir_env);
+ } else {
+ AddToSearchPath(kHBaseDefauktConfPath);
+ }
+}
+
+void HBaseConfigurationLoader::ClearSearchPath() { search_paths_.clear(); }
+
+void HBaseConfigurationLoader::SetSearchPath(const std::string &search_path) {
+ search_paths_.clear();
+
+ std::vector<std::string> paths;
+ std::string::size_type start = 0;
+ std::string::size_type end = search_path.find(kSearchPathSeparator);
+
+ while (end != std::string::npos) {
+ paths.push_back(search_path.substr(start, end - start));
+ start = ++end;
+ end = search_path.find(kSearchPathSeparator, start);
+ }
+ paths.push_back(search_path.substr(start, search_path.length()));
+
+ for (auto path : paths) {
+ AddToSearchPath(path);
+ }
+}
+
+void HBaseConfigurationLoader::AddToSearchPath(const std::string &search_path) {
+ if (search_path.empty()) return;
+
+ std::string path_to_add(search_path);
+ if (search_path.back() != kFileSeparator) {
+ path_to_add += kFileSeparator;
+ }
+ if (std::find(search_paths_.begin(), search_paths_.end(), path_to_add) ==
+ search_paths_.end())
+ search_paths_.push_back(path_to_add);
+}
+
+void HBaseConfigurationLoader::AddDefaultResources() {
+ resources_.push_back(kHBaseDefaultXml);
+ resources_.push_back(kHBaseSiteXml);
+}
+
+void HBaseConfigurationLoader::AddResources(const std::string &filename) {
+ if (std::find(resources_.begin(), resources_.end(), filename) ==
+ resources_.end())
+ resources_.push_back(filename);
+}
+
+optional<Configuration> HBaseConfigurationLoader::LoadDefaultResources() {
+ SetDefaultSearchPath();
+ AddDefaultResources();
+ ConfigMap conf_property;
+ bool success = false;
+ for (auto dir : search_paths_) {
+ for (auto file : resources_) {
+ std::string config_file = dir + file;
+ std::ifstream stream(config_file);
+ if (stream.is_open()) {
+ success |= LoadProperties(config_file, conf_property);
+ } else {
+ DLOG(WARNING) << "Unable to open file[" << config_file << "]";
+ }
+ }
+ }
+ if (success) {
+ return std::experimental::make_optional<Configuration>(conf_property);
+ } else {
+ return optional<Configuration>();
+ }
+}
+
+optional<Configuration> HBaseConfigurationLoader::LoadResources(
+ const std::string &search_path, const std::vector<std::string> &resources) {
+ SetSearchPath(search_path);
+ for (const auto &resource : resources) AddResources(resource);
+ ConfigMap conf_property;
+ bool success = false;
+ for (auto dir : search_paths_) {
+ for (auto file : resources_) {
+ std::string config_file = dir + file;
+ std::ifstream stream(config_file);
+ if (stream.is_open()) {
+ success |= LoadProperties(config_file, conf_property);
+ } else {
+ DLOG(WARNING) << "Unable to open file[" << config_file << "]";
+ }
+ }
+ }
+ if (success) {
+ return std::experimental::make_optional<Configuration>(conf_property);
+ } else {
+ return optional<Configuration>();
+ }
+}
+
+bool HBaseConfigurationLoader::LoadProperties(const std::string &file,
+ ConfigMap &property_map) {
+ // Create empty property tree object
+ using boost::property_tree::ptree;
+ ptree pt;
+ try {
+ // Load XML file and put contents in a property tree.
+ // If read fails, throw exception.
+ read_xml(file, pt);
+
+ // If configuration key is not found exception is thrown
+ std::string configuration = pt.get<std::string>("configuration");
+
+ // Iterate over configuration section.
+ // Store all found properties in ConfigMap
+ BOOST_FOREACH(ptree::value_type & v, pt.get_child("configuration")) {
+ if ("property" == v.first) {
+ std::string name_node = v.second.get<std::string>("name");
+ std::string value_node = v.second.get<std::string>("value");
+ if ((name_node.size() > 0) && (value_node.size() > 0)) {
+ boost::optional<std::string> final_node =
+ v.second.get_optional<std::string>("final");
+ UpdateMapWithValue(property_map, name_node, value_node, final_node);
+ }
+ }
+ }
+ }
+ catch (std::exception &ex) {
+ DLOG(WARNING) << "Exception in parsing file [" << file << "]:[" << ex.what()
+ << "]";
+ return false;
+ }
+ return true;
+}
+
+bool HBaseConfigurationLoader::UpdateMapWithValue(
+ ConfigMap &map, const std::string &key, const std::string &value,
+ boost::optional<std::string> final_text) {
+ auto map_value = map.find(key);
+ if (map_value != map.end() && map_value->second.final) {
+ return false;
+ }
+
+ bool final_value = false;
+ if (nullptr != final_text.get_ptr()) {
+ if (is_valid_bool(final_text.get())) {
+ final_value = str_to_bool(final_text.get());
+ }
+ }
+
+ map[key].value = value;
+ map[key].final = final_value;
+ return true;
+}
+} /* namespace hbase */
diff --git a/core/hbase_configuration_loader.h b/core/hbase_configuration_loader.h
new file mode 100644
index 0000000..54d90a5
--- /dev/null
+++ b/core/hbase_configuration_loader.h
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include <boost/optional.hpp>
+#include <experimental/optional>
+
+#include "core/configuration.h"
+
+namespace hbase {
+
+template <class T>
+using optional = std::experimental::optional<T>;
+
+class HBaseConfigurationLoader {
+
+ public:
+ HBaseConfigurationLoader();
+ ~HBaseConfigurationLoader();
+
+ /**
+ * @brief Creates a Configuration object based on default resources loaded
+ * from default search paths. Default search path will be either $HBASE_CONF
+ * is set or /etc/hbase/conf. Default resources are hbase-default.xml and
+ * hbase-site.xml.SetDefaultSearchPath() and AddDefaultResources() are used
+ * for the same.
+ * Values are loaded in from hbase-default.xml first and then from
+ * hbase-site.xml.
+ * Properties in hbase-site.xml will override the ones in hbase-default.xml
+ * unless marked as final
+ */
+ optional<Configuration> LoadDefaultResources();
+
+ /*
+ * @brief Creates a Configuration object based on resources loaded from search
+ * paths. Search paths are defined in search_path. Values are loaded from
+ * resources and will be overridden unless marked as final
+ * @param search_path - ':' search paths to load resources.
+ * @param resources - list of resources used to load configuration properties.
+ */
+ optional<Configuration> LoadResources(
+ const std::string &search_path,
+ const std::vector<std::string> &resources);
+
+ private:
+ using ConfigMap = Configuration::ConfigMap;
+ const std::string kHBaseDefaultXml = "hbase-default.xml";
+ const std::string kHBaseSiteXml = "hbase-site.xml";
+ const std::string kHBaseDefauktConfPath = "/etc/hbase/conf";
+
+ // Adds FILE_SEPARATOR to the search path
+ const char kFileSeparator = '/';
+
+ // Separator using which multiple search paths can be defined.
+ const char kSearchPathSeparator = ':';
+
+ /**
+ * List of paths which will be looked up for loading properties.
+ */
+ std::vector<std::string> search_paths_;
+
+ /**
+ * List of files which will be looked up in search_paths_ to load properties.
+ */
+ std::vector<std::string> resources_;
+
+ /**
+ * @brief This method sets the search path to the default search path (i.e.
+ * "$HBASE_CONF" or "/etc/hbase/conf" if HBASE_CONF is absent)
+ */
+ void SetDefaultSearchPath();
+
+ /**
+ * @brief Clears out the set search path(s)
+ */
+ void ClearSearchPath();
+
+ /**
+ * @brief Sets the search path to ":"-delimited paths, clearing already
+ * defined values
+ * @param search_path Single path or ":"-delimited separated paths
+ */
+ void SetSearchPath(const std::string &search_path);
+
+ /**
+ * @brief Adds an element to the search path if not already present.
+ * @param search_path Path that will be added to load config values
+ */
+ void AddToSearchPath(const std::string &search_path);
+
+ /**
+ * @brief This method will add default resources i.e. hbase-default.xml and
+ * hbase-site.xml to the default search path.
+ */
+ void AddDefaultResources();
+
+ /**
+ * @brief Adds resources to list for loading config values.
+ * @param filename to be added to resources.
+ */
+ void AddResources(const std::string &filename);
+
+ /**
+ * @brief Loads properties in file identified by file in a map identified by
+ * property_map
+ * @param file XML file which defines HBase configuration properties.
+ * @param property_map Property map representing HBase configuration as key
+ * value pairs.
+ * @throws Boost ptree exception if some parsing issue occurs.
+ */
+ bool LoadProperties(const std::string &file, ConfigMap &property_map);
+
+ /**
+ * @brief This method will create a map of the name and properties.
+ * @param map Property map to hold configuration properties.
+ * @param key value of name node.
+ * @param value value of value node.
+ * @param final_text value of final node true or false if present
+ */
+ bool UpdateMapWithValue(ConfigMap &map, const std::string &key,
+ const std::string &value,
+ boost::optional<std::string> final_text);
+};
+
+} /* namespace hbase */
[hbase] 78/137: HBASE-17771 [C++] Classes required for
implementation of BatchCallerBuilder
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit c26b5d9e9085057d735696e555bd18efc7c46489
Author: Sudeep Sunthankar <su...@hashmapinc.com>
AuthorDate: Wed Mar 29 16:55:21 2017 +1100
HBASE-17771 [C++] Classes required for implementation of BatchCallerBuilder
Signed-off-by: Enis Soztutar <en...@apache.org>
---
connection/request.cc | 4 ++
connection/request.h | 2 +
core/BUCK | 8 +++
core/{response-converter.h => action.h} | 34 ++++-----
core/get-test.cc | 5 +-
core/get.cc | 15 +---
core/get.h | 18 +----
core/multi-response.cc | 80 +++++++++++++++++++++
core/multi-response.h | 81 +++++++++++++++++++++
core/raw-async-table.cc | 2 +-
core/{response-converter.h => region-request.h} | 41 +++++------
core/region-result.cc | 54 ++++++++++++++
core/{response-converter.h => region-result.h} | 42 +++++------
core/request-converter.cc | 82 ++++++++++++++-------
core/request-converter.h | 12 ++++
core/response-converter.cc | 94 ++++++++++++++++++++++++-
core/response-converter.h | 7 ++
core/row.h | 62 ++++++++++++++++
core/server-request.h | 59 ++++++++++++++++
19 files changed, 573 insertions(+), 129 deletions(-)
diff --git a/connection/request.cc b/connection/request.cc
index 189130e..80883cc 100644
--- a/connection/request.cc
+++ b/connection/request.cc
@@ -39,3 +39,7 @@ std::unique_ptr<Request> Request::scan() {
return std::make_unique<Request>(std::make_shared<hbase::pb::ScanRequest>(),
std::make_shared<hbase::pb::ScanResponse>(), "Scan");
}
+std::unique_ptr<Request> Request::multi() {
+ return std::make_unique<Request>(std::make_shared<hbase::pb::MultiRequest>(),
+ std::make_shared<hbase::pb::MultiResponse>(), "Multi");
+}
diff --git a/connection/request.h b/connection/request.h
index 91c684d..520b380 100644
--- a/connection/request.h
+++ b/connection/request.h
@@ -39,6 +39,8 @@ class Request {
static std::unique_ptr<Request> mutate();
/** Create a request object for a scan */
static std::unique_ptr<Request> scan();
+ /** Create a request object for a multi */
+ static std::unique_ptr<Request> multi();
/**
* This should be private. Do not use this.
diff --git a/core/BUCK b/core/BUCK
index 2d77f2d..7483980 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -45,6 +45,12 @@ cxx_library(
"async-rpc-retrying-caller.h",
"hbase-rpc-controller.h",
"zk-util.h",
+ "action.h",
+ "multi-response.h",
+ "region-request.h",
+ "region-result.h",
+ "row.h",
+ "server-request.h",
],
srcs=[
"async-connection.cc",
@@ -62,6 +68,8 @@ cxx_library(
"response-converter.cc",
"table.cc",
"zk-util.cc",
+ "multi-response.cc",
+ "region-result.cc",
],
deps=[
"//exceptions:exceptions",
diff --git a/core/response-converter.h b/core/action.h
similarity index 50%
copy from core/response-converter.h
copy to core/action.h
index 743c14b..3511683 100644
--- a/core/response-converter.h
+++ b/core/action.h
@@ -20,36 +20,26 @@
#pragma once
#include <memory>
-#include <vector>
-#include "connection/response.h"
-#include "core/result.h"
-#include "if/Client.pb.h"
-#include "serde/cell-scanner.h"
+#include "core/row.h"
+using hbase::Row;
namespace hbase {
-/**
- * ResponseConverter class
- * This class converts a PB Response to corresponding Result or other objects.
- */
-class ResponseConverter {
+class Action {
public:
- ~ResponseConverter();
-
- static std::shared_ptr<Result> ToResult(const hbase::pb::Result& result,
- const std::unique_ptr<CellScanner>& cell_scanner);
+ Action(std::shared_ptr<Row> action, int original_index)
+ : action_(action), original_index_(original_index) {}
+ ~Action() {}
- /**
- * @brief Returns a Result object created by PB Message in passed Response object.
- * @param resp - Response object having the PB message.
- */
- static std::shared_ptr<hbase::Result> FromGetResponse(const Response& resp);
+ int64_t original_index() const { return original_index_; }
- static std::vector<std::shared_ptr<Result>> FromScanResponse(const Response& resp);
+ std::shared_ptr<Row> action() const { return action_; }
private:
- // Constructor not required. We have all static methods to extract response from PB messages.
- ResponseConverter();
+ std::shared_ptr<Row> action_;
+ int64_t original_index_;
+ int64_t nonce_ = -1;
+ int32_t replica_id_ = -1;
};
} /* namespace hbase */
diff --git a/core/get-test.cc b/core/get-test.cc
index 07d0003..6ee2715 100644
--- a/core/get-test.cc
+++ b/core/get-test.cc
@@ -21,7 +21,8 @@
#include <glog/logging.h>
#include <gtest/gtest.h>
-using namespace hbase;
+
+using hbase::Get;
const int NUMBER_OF_GETS = 5;
void CheckFamilies(Get &get) {
@@ -102,7 +103,7 @@ void CheckFamiliesAfterCopy(Get &get) {
}
void GetMethods(Get &get, const std::string &row) {
- EXPECT_EQ(row, get.Row());
+ EXPECT_EQ(row, get.row());
CheckFamilies(get);
EXPECT_EQ(true, get.CacheBlocks());
diff --git a/core/get.cc b/core/get.cc
index 5c5f446..afeb429 100644
--- a/core/get.cc
+++ b/core/get.cc
@@ -26,7 +26,7 @@ namespace hbase {
Get::~Get() {}
-Get::Get(const std::string &row) : row_(row) { CheckRow(row_); }
+Get::Get(const std::string &row) : Row(row) {}
Get::Get(const Get &get) {
row_ = get.row_;
@@ -78,8 +78,6 @@ Get &Get::AddColumn(const std::string &family, const std::string &qualifier) {
return *this;
}
-const std::string &Get::Row() const { return row_; }
-
hbase::pb::Consistency Get::Consistency() const { return consistency_; }
Get &Get::SetConsistency(hbase::pb::Consistency consistency) {
@@ -119,15 +117,4 @@ Get &Get::SetTimeStamp(int64_t timestamp) {
const TimeRange &Get::Timerange() const { return *tr_; }
-void Get::CheckRow(const std::string &row) {
- const int kMaxRowLength = std::numeric_limits<int16_t>::max();
- int row_length = row.size();
- if (0 == row_length) {
- throw std::runtime_error("Row length can't be 0");
- }
- if (row_length > kMaxRowLength) {
- throw std::runtime_error("Length of " + row + " is greater than max row size: " +
- std::to_string(kMaxRowLength));
- }
-}
} // namespace hbase
diff --git a/core/get.h b/core/get.h
index 5492f21..e0be4e7 100644
--- a/core/get.h
+++ b/core/get.h
@@ -25,9 +25,11 @@
#include <string>
#include <vector>
#include "core/query.h"
+#include "core/row.h"
#include "core/time-range.h"
#include "if/Client.pb.h"
+using hbase::Row;
namespace hbase {
/**
@@ -36,7 +38,7 @@ namespace hbase {
*/
using FamilyMap = std::map<std::string, std::vector<std::string>>;
-class Get : public Query {
+class Get : public Row, public Query {
public:
/**
* Constructors
@@ -110,11 +112,6 @@ class Get : public Query {
Get& AddColumn(const std::string& family, const std::string& qualifier);
/**
- * @brief Returns the row for this Get operation
- */
- const std::string& Row() const;
-
- /**
* @brief Returns true if family map (FamilyMap) is non empty false otherwise
*/
bool HasFamilies() const;
@@ -131,21 +128,12 @@ class Get : public Query {
Get& SetConsistency(hbase::pb::Consistency consistency);
private:
- std::string row_ = "";
int32_t max_versions_ = 1;
bool cache_blocks_ = true;
bool check_existence_only_ = false;
FamilyMap family_map_;
hbase::pb::Consistency consistency_ = hbase::pb::Consistency::STRONG;
std::unique_ptr<TimeRange> tr_ = std::make_unique<TimeRange>();
-
- /**
- * @brief Checks if the row for this Get operation is proper or not
- * @param row Row to check
- * @throws std::runtime_error if row is empty or greater than
- * MAX_ROW_LENGTH(i.e. std::numeric_limits<short>::max())
- */
- void CheckRow(const std::string& row);
};
} // namespace hbase
diff --git a/core/multi-response.cc b/core/multi-response.cc
new file mode 100644
index 0000000..562f3b6
--- /dev/null
+++ b/core/multi-response.cc
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/multi-response.h"
+#include "core/region-result.h"
+
+namespace hbase {
+
+MultiResponse::MultiResponse() {}
+
+int MultiResponse::Size() const {
+ int size = 0;
+ for (const auto& result : results_) {
+ size += result.second->ResultOrExceptionSize();
+ }
+ return size;
+}
+
+void MultiResponse::AddRegionResult(const std::string& region_name, int32_t original_index,
+ std::shared_ptr<Result> result,
+ std::shared_ptr<std::exception> exc) {
+ bool region_found = false;
+ for (auto itr = results_.begin(); itr != results_.end(); ++itr) {
+ if (itr->first == region_name) {
+ region_found = true;
+ itr->second->AddResultOrException(original_index, result, exc);
+ break;
+ }
+ }
+ if (!region_found) {
+ auto region_result = std::make_shared<RegionResult>();
+ region_result->AddResultOrException(original_index, result, exc);
+ results_[region_name] = region_result;
+ }
+}
+
+void MultiResponse::AddRegionException(const std::string& region_name,
+ std::shared_ptr<std::exception> exception) {
+ exceptions_[region_name] = exception;
+}
+
+std::shared_ptr<std::exception> MultiResponse::RegionException(
+ const std::string& region_name) const {
+ auto find = exceptions_.at(region_name);
+ return find;
+}
+
+const std::map<std::string, std::shared_ptr<std::exception> >& MultiResponse::RegionExceptions()
+ const {
+ return exceptions_;
+}
+
+void MultiResponse::AddStatistic(const std::string& region_name,
+ std::shared_ptr<RegionLoadStats> stat) {
+ results_[region_name]->set_stat(stat);
+}
+
+const std::map<std::string, std::shared_ptr<RegionResult> >& MultiResponse::RegionResults() const {
+ return results_;
+}
+
+MultiResponse::~MultiResponse() {}
+
+} /* namespace hbase */
diff --git a/core/multi-response.h b/core/multi-response.h
new file mode 100644
index 0000000..cebd2b7
--- /dev/null
+++ b/core/multi-response.h
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <core/region-result.h>
+#include <exception>
+#include <map>
+#include <memory>
+#include <string>
+
+#include "core/result.h"
+#include "if/Client.pb.h"
+
+using hbase::RegionResult;
+using hbase::Result;
+using hbase::pb::RegionLoadStats;
+
+namespace hbase {
+
+class MultiResponse {
+ public:
+ MultiResponse();
+ /**
+ * @brief Returns Number of pairs in this container
+ */
+ int Size() const;
+
+ /**
+ * Add the pair to the container, grouped by the regionName
+ *
+ * @param regionName
+ * @param originalIndex the original index of the Action (request).
+ * @param resOrEx the result or error; will be empty for successful Put and Delete actions.
+ */
+ void AddRegionResult(const std::string& region_name, int32_t original_index,
+ std::shared_ptr<Result> result, std::shared_ptr<std::exception> exc);
+
+ void AddRegionException(const std::string& region_name,
+ std::shared_ptr<std::exception> exception);
+
+ /**
+ * @return the exception for the region, if any. Null otherwise.
+ */
+ std::shared_ptr<std::exception> RegionException(const std::string& region_name) const;
+
+ const std::map<std::string, std::shared_ptr<std::exception>>& RegionExceptions() const;
+
+ void AddStatistic(const std::string& region_name, std::shared_ptr<RegionLoadStats> stat);
+
+ const std::map<std::string, std::shared_ptr<RegionResult>>& RegionResults() const;
+
+ ~MultiResponse();
+
+ private:
+ // map of regionName to map of Results by the original index for that Result
+ std::map<std::string, std::shared_ptr<hbase::RegionResult>> results_;
+ /**
+ * The server can send us a failure for the region itself, instead of individual failure.
+ * It's a part of the protobuf definition.
+ */
+ std::map<std::string, std::shared_ptr<std::exception>> exceptions_;
+};
+
+} /* namespace hbase */
diff --git a/core/raw-async-table.cc b/core/raw-async-table.cc
index 88a3382..9a680ed 100644
--- a/core/raw-async-table.cc
+++ b/core/raw-async-table.cc
@@ -56,7 +56,7 @@ folly::Future<RESP> RawAsyncTable::Call(
Future<std::shared_ptr<Result>> RawAsyncTable::Get(const hbase::Get& get) {
auto caller =
- CreateCallerBuilder<std::shared_ptr<Result>>(get.Row(), connection_conf_->read_rpc_timeout())
+ CreateCallerBuilder<std::shared_ptr<Result>>(get.row(), connection_conf_->read_rpc_timeout())
->action([=, &get](std::shared_ptr<hbase::HBaseRpcController> controller,
std::shared_ptr<hbase::RegionLocation> loc,
std::shared_ptr<hbase::RpcClient> rpc_client)
diff --git a/core/response-converter.h b/core/region-request.h
similarity index 50%
copy from core/response-converter.h
copy to core/region-request.h
index 743c14b..6f29d44 100644
--- a/core/response-converter.h
+++ b/core/region-request.h
@@ -18,38 +18,31 @@
*/
#pragma once
-
#include <memory>
+#include <queue>
#include <vector>
-#include "connection/response.h"
-#include "core/result.h"
-#include "if/Client.pb.h"
-#include "serde/cell-scanner.h"
+#include "core/action.h"
+#include "core/region-location.h"
+using hbase::Action;
namespace hbase {
-/**
- * ResponseConverter class
- * This class converts a PB Response to corresponding Result or other objects.
- */
-class ResponseConverter {
+class RegionRequest {
public:
- ~ResponseConverter();
-
- static std::shared_ptr<Result> ToResult(const hbase::pb::Result& result,
- const std::unique_ptr<CellScanner>& cell_scanner);
-
- /**
- * @brief Returns a Result object created by PB Message in passed Response object.
- * @param resp - Response object having the PB message.
- */
- static std::shared_ptr<hbase::Result> FromGetResponse(const Response& resp);
-
- static std::vector<std::shared_ptr<Result>> FromScanResponse(const Response& resp);
+ // Concurrent
+ using ActionList = std::vector<std::shared_ptr<Action>>;
+ explicit RegionRequest(const std::shared_ptr<hbase::RegionLocation> ®ion_loc)
+ : region_loc_(region_loc) {}
+ ~RegionRequest() {}
+ void AddAction(std::shared_ptr<Action> action) {
+ actions_.push_back(action);
+ }
+ std::shared_ptr<hbase::RegionLocation> region_location() const { return region_loc_; }
+ const ActionList &actions() const { return actions_; }
private:
- // Constructor not required. We have all static methods to extract response from PB messages.
- ResponseConverter();
+ std::shared_ptr<hbase::RegionLocation> region_loc_;
+ ActionList actions_;
};
} /* namespace hbase */
diff --git a/core/region-result.cc b/core/region-result.cc
new file mode 100644
index 0000000..d9ab942
--- /dev/null
+++ b/core/region-result.cc
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/region-result.h"
+#include <glog/logging.h>
+#include <stdexcept>
+
+using hbase::Result;
+using hbase::pb::RegionLoadStats;
+
+namespace hbase {
+
+RegionResult::RegionResult() {}
+
+RegionResult::~RegionResult() {}
+
+void RegionResult::AddResultOrException(int32_t index, std::shared_ptr<hbase::Result> result,
+ std::shared_ptr<std::exception> exc) {
+ auto index_found = result_or_excption_.find(index);
+ if (index_found == result_or_excption_.end()) {
+ result_or_excption_[index] = std::make_tuple(result ? result : nullptr, exc ? exc : nullptr);
+ } else {
+ throw std::runtime_error("Index " + std::to_string(index) +
+ " already set with ResultOrException");
+ }
+}
+
+void RegionResult::set_stat(std::shared_ptr<RegionLoadStats> stat) { stat_ = stat; }
+
+int RegionResult::ResultOrExceptionSize() const { return result_or_excption_.size(); }
+
+std::shared_ptr<ResultOrExceptionTuple> RegionResult::ResultOrException(int32_t index) const {
+ return std::make_shared<ResultOrExceptionTuple>(result_or_excption_.at(index));
+}
+
+const std::shared_ptr<RegionLoadStats>& RegionResult::stat() const { return stat_; }
+
+} /* namespace hbase */
diff --git a/core/response-converter.h b/core/region-result.h
similarity index 53%
copy from core/response-converter.h
copy to core/region-result.h
index 743c14b..9b7ca03 100644
--- a/core/response-converter.h
+++ b/core/region-result.h
@@ -19,37 +19,37 @@
#pragma once
+#include <map>
#include <memory>
-#include <vector>
-#include "connection/response.h"
+#include <string>
+#include <tuple>
#include "core/result.h"
#include "if/Client.pb.h"
-#include "serde/cell-scanner.h"
-namespace hbase {
+using hbase::Result;
+using hbase::pb::RegionLoadStats;
-/**
- * ResponseConverter class
- * This class converts a PB Response to corresponding Result or other objects.
- */
-class ResponseConverter {
+namespace hbase {
+using ResultOrExceptionTuple =
+ std::tuple<std::shared_ptr<hbase::Result>, std::shared_ptr<std::exception>>;
+class RegionResult {
public:
- ~ResponseConverter();
+ RegionResult();
+ void AddResultOrException(int32_t index, std::shared_ptr<hbase::Result> result,
+ std::shared_ptr<std::exception> exc);
- static std::shared_ptr<Result> ToResult(const hbase::pb::Result& result,
- const std::unique_ptr<CellScanner>& cell_scanner);
+ void set_stat(std::shared_ptr<RegionLoadStats> stat);
- /**
- * @brief Returns a Result object created by PB Message in passed Response object.
- * @param resp - Response object having the PB message.
- */
- static std::shared_ptr<hbase::Result> FromGetResponse(const Response& resp);
+ int ResultOrExceptionSize() const;
- static std::vector<std::shared_ptr<Result>> FromScanResponse(const Response& resp);
+ std::shared_ptr<ResultOrExceptionTuple> ResultOrException(int32_t index) const;
+
+ const std::shared_ptr<RegionLoadStats>& stat() const;
+
+ ~RegionResult();
private:
- // Constructor not required. We have all static methods to extract response from PB messages.
- ResponseConverter();
+ std::map<int, ResultOrExceptionTuple> result_or_excption_;
+ std::shared_ptr<RegionLoadStats> stat_;
};
-
} /* namespace hbase */
diff --git a/core/request-converter.cc b/core/request-converter.cc
index 227e04a..ff92b5c 100644
--- a/core/request-converter.cc
+++ b/core/request-converter.cc
@@ -23,6 +23,7 @@
using hbase::Request;
using hbase::pb::GetRequest;
+using hbase::pb::RegionAction;
using hbase::pb::RegionSpecifier;
using hbase::pb::RegionSpecifier_RegionSpecifierType;
using hbase::pb::ScanRequest;
@@ -43,35 +44,9 @@ void RequestConverter::SetRegion(const std::string ®ion_name,
std::unique_ptr<Request> RequestConverter::ToGetRequest(const Get &get,
const std::string ®ion_name) {
auto pb_req = Request::get();
-
auto pb_msg = std::static_pointer_cast<GetRequest>(pb_req->req_msg());
RequestConverter::SetRegion(region_name, pb_msg->mutable_region());
-
- auto pb_get = pb_msg->mutable_get();
- pb_get->set_max_versions(get.MaxVersions());
- pb_get->set_cache_blocks(get.CacheBlocks());
- pb_get->set_consistency(get.Consistency());
-
- if (!get.Timerange().IsAllTime()) {
- hbase::pb::TimeRange *pb_time_range = pb_get->mutable_time_range();
- pb_time_range->set_from(get.Timerange().MinTimeStamp());
- pb_time_range->set_to(get.Timerange().MaxTimeStamp());
- }
- pb_get->set_row(get.Row());
- if (get.HasFamilies()) {
- for (const auto &family : get.Family()) {
- auto column = pb_get->add_column();
- column->set_family(family.first);
- for (const auto &qualifier : family.second) {
- column->add_qualifier(qualifier);
- }
- }
- }
-
- if (get.filter() != nullptr) {
- pb_get->set_allocated_filter(Filter::ToProto(*(get.filter())).release());
- }
-
+ pb_msg->set_allocated_get((RequestConverter::ToGet(get)).release());
return pb_req;
}
@@ -123,4 +98,57 @@ std::unique_ptr<Request> RequestConverter::ToScanRequest(const Scan &scan,
return pb_req;
}
+
+std::unique_ptr<Request> RequestConverter::ToMultiRequest(
+ const ActionsByRegion &actions_by_region) {
+ auto pb_req = Request::multi();
+ auto pb_msg = std::static_pointer_cast<hbase::pb::MultiRequest>(pb_req->req_msg());
+
+ for (const auto &action_by_region : actions_by_region) {
+ auto pb_region_action = pb_msg->add_regionaction();
+ RequestConverter::SetRegion(action_by_region.first, pb_region_action->mutable_region());
+ int action_num = 0;
+ for (const auto ®ion_action : action_by_region.second->actions()) {
+ auto pb_action = pb_region_action->add_action();
+ auto action = region_action->action();
+ if (auto pget = std::dynamic_pointer_cast<Get>(action)) {
+ auto pb_get = RequestConverter::ToGet(*pget.get());
+ pb_action->set_allocated_get(pb_get.release());
+ pb_action->set_index(action_num);
+ }
+ action_num++;
+ }
+ }
+
+ VLOG(3) << "Multi Req:-" << pb_req->req_msg()->ShortDebugString();
+ return pb_req;
+}
+
+std::unique_ptr<hbase::pb::Get> RequestConverter::ToGet(const Get &get) {
+ auto pb_get = std::make_unique<hbase::pb::Get>();
+ pb_get->set_max_versions(get.MaxVersions());
+ pb_get->set_cache_blocks(get.CacheBlocks());
+ pb_get->set_consistency(get.Consistency());
+
+ if (!get.Timerange().IsAllTime()) {
+ hbase::pb::TimeRange *pb_time_range = pb_get->mutable_time_range();
+ pb_time_range->set_from(get.Timerange().MinTimeStamp());
+ pb_time_range->set_to(get.Timerange().MaxTimeStamp());
+ }
+ pb_get->set_row(get.row());
+ if (get.HasFamilies()) {
+ for (const auto &family : get.Family()) {
+ auto column = pb_get->add_column();
+ column->set_family(family.first);
+ for (const auto &qualifier : family.second) {
+ column->add_qualifier(qualifier);
+ }
+ }
+ }
+
+ if (get.filter() != nullptr) {
+ pb_get->set_allocated_filter(Filter::ToProto(*(get.filter())).release());
+ }
+ return pb_get;
+}
} /* namespace hbase */
diff --git a/core/request-converter.h b/core/request-converter.h
index 57f08cc..003afaa 100644
--- a/core/request-converter.h
+++ b/core/request-converter.h
@@ -21,14 +21,23 @@
#include <memory>
#include <string>
+#include <vector>
#include "connection/request.h"
+#include "core/action.h"
#include "core/get.h"
+#include "core/region-request.h"
#include "core/scan.h"
+#include "core/server-request.h"
#include "if/HBase.pb.h"
using hbase::pb::RegionSpecifier;
+using hbase::pb::RegionAction;
+using hbase::pb::ServerName;
+using hbase::ServerRequest;
+
namespace hbase {
+using ActionsByRegion = ServerRequest::ActionsByRegion;
/**
* RequestConverter class
* This class converts a Client side Get, Scan, Mutate operation to corresponding PB message.
@@ -53,6 +62,8 @@ class RequestConverter {
*/
static std::unique_ptr<Request> ToScanRequest(const Scan &scan, const std::string ®ion_name);
+ static std::unique_ptr<Request> ToMultiRequest(const ActionsByRegion ®ion_requests);
+
private:
// Constructor not required. We have all static methods to create PB requests.
RequestConverter();
@@ -64,6 +75,7 @@ class RequestConverter {
* Request.
*/
static void SetRegion(const std::string ®ion_name, RegionSpecifier *region_specifier);
+ static std::unique_ptr<hbase::pb::Get> ToGet(const Get &get);
};
} /* namespace hbase */
diff --git a/core/response-converter.cc b/core/response-converter.cc
index b2fff34..7729257 100644
--- a/core/response-converter.cc
+++ b/core/response-converter.cc
@@ -19,13 +19,18 @@
#include "core/response-converter.h"
+#include <glog/logging.h>
+#include <stdexcept>
#include <string>
+#include <utility>
#include <vector>
-
#include "core/cell.h"
+#include "core/multi-response.h"
+#include "exceptions/exception.h"
using hbase::pb::GetResponse;
using hbase::pb::ScanResponse;
+using hbase::pb::RegionLoadStats;
namespace hbase {
@@ -37,6 +42,7 @@ ResponseConverter::~ResponseConverter() {}
// go inside folly::Future's, making the move semantics extremely tricky.
std::shared_ptr<Result> ResponseConverter::FromGetResponse(const Response& resp) {
auto get_resp = std::static_pointer_cast<GetResponse>(resp.resp_msg());
+ VLOG(3) << "FromGetResponse:" << get_resp->ShortDebugString();
return ToResult(get_resp->result(), resp.cell_scanner());
}
@@ -52,16 +58,24 @@ std::shared_ptr<Result> ResponseConverter::ToResult(
// iterate over the cells coming from rpc codec
if (cell_scanner != nullptr) {
- while (cell_scanner->Advance()) {
+ int cells_read = 0;
+ while (cells_read != result.associated_cell_count()) {
+ if (cell_scanner->Advance()) {
vcells.push_back(cell_scanner->Current());
+ cells_read += 1;
+ } else {
+ LOG(ERROR)<< "CellScanner::Advance() returned false unexpectedly. Cells Read:- "
+ << cells_read << "; Expected Cell Count:- " << result.associated_cell_count();
+ std::runtime_error("CellScanner::Advance() returned false unexpectedly");
+ }
}
- // TODO: check associated cell count?
}
return std::make_shared<Result>(vcells, result.exists(), result.stale(), result.partial());
}
std::vector<std::shared_ptr<Result>> ResponseConverter::FromScanResponse(const Response& resp) {
auto scan_resp = std::static_pointer_cast<ScanResponse>(resp.resp_msg());
+ VLOG(3) << "FromScanResponse:" << scan_resp->ShortDebugString();
int num_results = resp.cell_scanner() != nullptr ? scan_resp->cells_per_result_size()
: scan_resp->results_size();
@@ -94,4 +108,78 @@ std::vector<std::shared_ptr<Result>> ResponseConverter::FromScanResponse(const R
return results;
}
+
+std::unique_ptr<hbase::MultiResponse> ResponseConverter::GetResults(std::shared_ptr<Request> req,
+ const Response& resp) {
+ auto multi_req = std::static_pointer_cast < hbase::pb::MultiRequest > (req->req_msg());
+ auto multi_resp = std::static_pointer_cast < hbase::pb::MultiResponse > (resp.resp_msg());
+ VLOG(3) << "GetResults:" << multi_resp->ShortDebugString();
+ int req_region_action_count = multi_req->regionaction_size();
+ int res_region_action_count = multi_resp->regionactionresult_size();
+ if (req_region_action_count != res_region_action_count) {
+ throw std::runtime_error(
+ "Request mutation count=" + std::to_string(req_region_action_count)
+ + " does not match response mutation result count="
+ + std::to_string(res_region_action_count));
+ }
+ auto multi_response = std::make_unique<hbase::MultiResponse>();
+ for (int32_t num = 0; num < res_region_action_count; num++) {
+ hbase::pb::RegionAction actions = multi_req->regionaction(num);
+ hbase::pb::RegionActionResult action_result = multi_resp->regionactionresult(num);
+ hbase::pb::RegionSpecifier rs = actions.region();
+ if (rs.has_type() && rs.type() != hbase::pb::RegionSpecifier::REGION_NAME) {
+ throw std::runtime_error("We support only encoded types for protobuf multi response.");
+ }
+
+ auto region_name = rs.value();
+ if (action_result.has_exception()) {
+ if (action_result.exception().has_value()) {
+ auto exc = std::make_shared < hbase::IOException > (action_result.exception().value());
+ VLOG(8) << "Store Region Exception:- " << exc->what();
+ multi_response->AddRegionException(region_name, exc);
+ }
+ continue;
+ }
+
+ if (actions.action_size() != action_result.resultorexception_size()) {
+ throw std::runtime_error(
+ "actions.action_size=" + std::to_string(actions.action_size())
+ + ", action_result.resultorexception_size="
+ + std::to_string(action_result.resultorexception_size()) + " for region "
+ + actions.region().value());
+ }
+
+ for (hbase::pb::ResultOrException roe : action_result.resultorexception()) {
+ std::shared_ptr < Result > result;
+ std::shared_ptr < std::exception > exc;
+ if (roe.has_exception()) {
+ if (roe.exception().has_value()) {
+ exc = std::make_shared < hbase::IOException > (roe.exception().value());
+ VLOG(8) << "Store ResultOrException:- " << exc->what();
+ }
+ } else if (roe.has_result()) {
+ result = ToResult(roe.result(), resp.cell_scanner());
+ } else if (roe.has_service_result()) {
+ // TODO Not processing Coprocessor Service Result;
+ } else {
+ // Sometimes, the response is just "it was processed". Generally, this occurs for things
+ // like mutateRows where either we get back 'processed' (or not) and optionally some
+ // statistics about the regions we touched.
+ std::vector < std::shared_ptr < Cell >> empty_cells;
+ result = std::make_shared < Result
+ > (empty_cells, multi_resp->processed() ? true : false, false, false);
+ }
+ multi_response->AddRegionResult(region_name, roe.index(), std::move(result), exc);
+ }
+ }
+
+ if (multi_resp->has_regionstatistics()) {
+ hbase::pb::MultiRegionLoadStats stats = multi_resp->regionstatistics();
+ for (int i = 0; i < stats.region_size(); i++) {
+ multi_response->AddStatistic(stats.region(i).value(),
+ std::make_shared < RegionLoadStats > (stats.stat(i)));
+ }
+ }
+ return multi_response;
+}
} /* namespace hbase */
diff --git a/core/response-converter.h b/core/response-converter.h
index 743c14b..a5095fd 100644
--- a/core/response-converter.h
+++ b/core/response-converter.h
@@ -21,11 +21,15 @@
#include <memory>
#include <vector>
+#include "connection/request.h"
#include "connection/response.h"
+#include "core/multi-response.h"
#include "core/result.h"
#include "if/Client.pb.h"
#include "serde/cell-scanner.h"
+using hbase::Request;
+using hbase::Response;
namespace hbase {
/**
@@ -47,6 +51,9 @@ class ResponseConverter {
static std::vector<std::shared_ptr<Result>> FromScanResponse(const Response& resp);
+ static std::unique_ptr<hbase::MultiResponse> GetResults(std::shared_ptr<Request> req,
+ const Response& resp);
+
private:
// Constructor not required. We have all static methods to extract response from PB messages.
ResponseConverter();
diff --git a/core/row.h b/core/row.h
new file mode 100644
index 0000000..2c7bdd1
--- /dev/null
+++ b/core/row.h
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <limits>
+#include <stdexcept>
+#include <string>
+
+#pragma once
+
+namespace hbase {
+
+class Row {
+ public:
+ Row() {}
+ explicit Row(const std::string &row) : row_(row) { CheckRow(row_); }
+
+ /**
+ * @brief Returns the row for the Row interface.
+ */
+ const std::string &row() const { return row_; }
+ virtual ~Row() {}
+
+ private:
+ /**
+ * @brief Checks if the row for this Get operation is proper or not
+ * @param row Row to check
+ * @throws std::runtime_error if row is empty or greater than
+ * MAX_ROW_LENGTH(i.e. std::numeric_limits<short>::max())
+ */
+ void CheckRow(const std::string &row) {
+ const int16_t kMaxRowLength = std::numeric_limits<int16_t>::max();
+ size_t row_length = row.size();
+ if (0 == row_length) {
+ throw std::runtime_error("Row length can't be 0");
+ }
+ if (row_length > kMaxRowLength) {
+ throw std::runtime_error("Length of " + row + " is greater than max row size: " +
+ std::to_string(kMaxRowLength));
+ }
+ }
+
+ protected:
+ std::string row_ = "";
+};
+
+} /* namespace hbase */
diff --git a/core/server-request.h b/core/server-request.h
new file mode 100644
index 0000000..827b2e7
--- /dev/null
+++ b/core/server-request.h
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <map>
+#include <memory>
+#include <stdexcept>
+#include <string>
+#include "core/action.h"
+#include "core/region-location.h"
+#include "core/region-request.h"
+
+using hbase::Action;
+using hbase::RegionRequest;
+
+namespace hbase {
+
+class ServerRequest {
+ public:
+ // Concurrent
+ using ActionsByRegion = std::map<std::string, std::shared_ptr<RegionRequest>>;
+
+ explicit ServerRequest(std::shared_ptr<RegionLocation> region_location) {
+ auto region_name = region_location->region_name();
+ auto region_request = std::make_shared<RegionRequest>(region_location);
+ actions_by_region_[region_name] = region_request;
+ }
+ ~ServerRequest() {}
+
+ void AddActionsByRegion(std::shared_ptr<RegionLocation> region_location,
+ std::shared_ptr<Action> action) {
+ auto region_name = region_location->region_name();
+ auto itr = actions_by_region_.at(region_name);
+ itr->AddAction(action);
+ }
+
+ const ActionsByRegion &actions_by_region() const { return actions_by_region_; }
+
+ private:
+ ActionsByRegion actions_by_region_;
+};
+} /* namespace hbase */
[hbase] 22/137: HBASE-15739 Add region location lookup from meta
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 572c1699e969d3cdf2900a425ae1a9ccc48d2db3
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Sat Apr 30 17:12:11 2016 -0700
HBASE-15739 Add region location lookup from meta
Summary:
Add the ability for location-cache to read from meta.
There's still no cache. Or any parsing.
Test Plan: Needs tests.
Differential Revision: https://reviews.facebook.net/D57489
---
connection/BUCK | 1 +
connection/client-dispatcher.cc | 2 +
connection/client-dispatcher.h | 3 +-
connection/client-handler.cc | 5 +-
connection/client-handler.h | 2 +-
connection/connection-factory.cc | 2 +-
connection/connection-factory.h | 1 +
connection/connection-pool.cc | 14 +++---
connection/pipeline.cc | 2 +
connection/pipeline.h | 1 +
connection/response.h | 6 ++-
core/BUCK | 3 +-
core/location-cache.cc | 37 +++++++++++++--
core/location-cache.h | 19 +++++++-
core/meta-utils.cc | 54 +++++++++++++++++++++-
core/meta-utils.h | 9 +++-
.../connection-factory.h => core/region-location.h | 26 ++++++-----
core/simple-client.cc | 48 ++-----------------
core/table-name-test.cc | 28 ++++++-----
core/table-name.cc | 34 --------------
core/table-name.h | 35 +++++++-------
serde/BUCK | 1 +
utils/BUCK | 7 +--
utils/user-util.cc | 2 +
utils/user-util.h | 3 +-
25 files changed, 201 insertions(+), 144 deletions(-)
diff --git a/connection/BUCK b/connection/BUCK
index 96f2136..39b8be5 100644
--- a/connection/BUCK
+++ b/connection/BUCK
@@ -43,6 +43,7 @@ cxx_library(name="connection",
"//third-party:folly",
"//third-party:wangle",
],
+ compiler_flags=['-Weffc++'],
visibility=['//core/...', ], )
cxx_test(name="connection-pool-test",
srcs=["connection-pool-test.cc", ],
diff --git a/connection/client-dispatcher.cc b/connection/client-dispatcher.cc
index eea0a17..817adc1 100644
--- a/connection/client-dispatcher.cc
+++ b/connection/client-dispatcher.cc
@@ -22,6 +22,8 @@ using namespace folly;
using namespace hbase;
using namespace wangle;
+ClientDispatcher::ClientDispatcher() : requests_(), current_call_id_(9) {}
+
void ClientDispatcher::read(Context *ctx, Response in) {
auto call_id = in.call_id();
auto search = requests_.find(call_id);
diff --git a/connection/client-dispatcher.h b/connection/client-dispatcher.h
index 877e877..826fc6a 100644
--- a/connection/client-dispatcher.h
+++ b/connection/client-dispatcher.h
@@ -30,6 +30,7 @@ class ClientDispatcher
: public wangle::ClientDispatcherBase<SerializePipeline,
std::unique_ptr<Request>, Response> {
public:
+ ClientDispatcher();
void read(Context *ctx, Response in) override;
folly::Future<Response> operator()(std::unique_ptr<Request> arg) override;
folly::Future<folly::Unit> close(Context *ctx) override;
@@ -45,6 +46,6 @@ private:
//
// uint32_t has a max of 4Billion so 10 more or less is
// not a big deal.
- uint32_t current_call_id_ = 10;
+ uint32_t current_call_id_;
};
} // namespace hbase
diff --git a/connection/client-handler.cc b/connection/client-handler.cc
index abcf5c1..3180f4e 100644
--- a/connection/client-handler.cc
+++ b/connection/client-handler.cc
@@ -34,8 +34,11 @@ using namespace folly;
using namespace wangle;
using hbase::pb::ResponseHeader;
using hbase::pb::GetResponse;
+using google::protobuf::Message;
-ClientHandler::ClientHandler(std::string user_name) : user_name_(user_name) {}
+ClientHandler::ClientHandler(std::string user_name)
+ : user_name_(user_name), need_send_header_(true), ser_(), deser_(),
+ resp_msgs_() {}
void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
if (LIKELY(buf != nullptr)) {
diff --git a/connection/client-handler.h b/connection/client-handler.h
index 41bb883..68513de 100644
--- a/connection/client-handler.h
+++ b/connection/client-handler.h
@@ -47,7 +47,7 @@ public:
std::unique_ptr<Request> r) override;
private:
- bool need_send_header_ = true;
+ bool need_send_header_;
std::string user_name_;
ClientSerializer ser_;
ClientDeserializer deser_;
diff --git a/connection/connection-factory.cc b/connection/connection-factory.cc
index b546269..9102d60 100644
--- a/connection/connection-factory.cc
+++ b/connection/connection-factory.cc
@@ -40,7 +40,7 @@ using namespace folly;
using namespace hbase;
using namespace wangle;
-ConnectionFactory::ConnectionFactory() {
+ConnectionFactory::ConnectionFactory() : bootstrap_() {
bootstrap_.group(std::make_shared<wangle::IOThreadPoolExecutor>(1));
bootstrap_.pipelineFactory(std::make_shared<RpcPipelineFactory>());
}
diff --git a/connection/connection-factory.h b/connection/connection-factory.h
index 5a45316..fc4e161 100644
--- a/connection/connection-factory.h
+++ b/connection/connection-factory.h
@@ -31,6 +31,7 @@ namespace hbase {
class ConnectionFactory {
public:
ConnectionFactory();
+ virtual ~ConnectionFactory() = default;
virtual std::shared_ptr<HBaseService> make_connection(const std::string &host,
int port);
diff --git a/connection/connection-pool.cc b/connection/connection-pool.cc
index a967df2..72c1306 100644
--- a/connection/connection-pool.cc
+++ b/connection/connection-pool.cc
@@ -35,14 +35,14 @@ class RemoveServiceFilter
public:
RemoveServiceFilter(std::shared_ptr<HBaseService> service, ServerName sn,
- ConnectionPool *cp)
+ ConnectionPool &cp)
: ServiceFilter<unique_ptr<Request>, Response>(service), sn_(sn),
cp_(cp) {}
folly::Future<folly::Unit> close() override {
if (!released.exchange(true)) {
return this->service_->close().then(
- [this]() { this->cp_->close(this->sn_); });
+ [this]() { this->cp_.close(this->sn_); });
} else {
return folly::makeFuture();
}
@@ -57,12 +57,14 @@ public:
private:
std::atomic<bool> released{false};
hbase::pb::ServerName sn_;
- ConnectionPool *cp_;
+ ConnectionPool &cp_;
};
-ConnectionPool::ConnectionPool() : cf_(std::make_shared<ConnectionFactory>()) {}
+ConnectionPool::ConnectionPool()
+ : cf_(std::make_shared<ConnectionFactory>()), connections_(), map_mutex_() {
+}
ConnectionPool::ConnectionPool(std::shared_ptr<ConnectionFactory> cf)
- : cf_(cf) {}
+ : cf_(cf), connections_(), map_mutex_() {}
std::shared_ptr<HBaseService> ConnectionPool::get(const ServerName &sn) {
SharedMutexWritePriority::UpgradeHolder holder(map_mutex_);
@@ -70,7 +72,7 @@ std::shared_ptr<HBaseService> ConnectionPool::get(const ServerName &sn) {
if (found == connections_.end() || found->second == nullptr) {
SharedMutexWritePriority::WriteHolder holder(std::move(holder));
auto new_con = cf_->make_connection(sn.host_name(), sn.port());
- auto wrapped = std::make_shared<RemoveServiceFilter>(new_con, sn, this);
+ auto wrapped = std::make_shared<RemoveServiceFilter>(new_con, sn, *this);
connections_[sn] = wrapped;
return new_con;
}
diff --git a/connection/pipeline.cc b/connection/pipeline.cc
index b9f5e0b..39cce8b 100644
--- a/connection/pipeline.cc
+++ b/connection/pipeline.cc
@@ -30,6 +30,8 @@ using namespace folly;
using namespace hbase;
using namespace wangle;
+RpcPipelineFactory::RpcPipelineFactory() : user_util_() {}
+
SerializePipeline::Ptr
RpcPipelineFactory::newPipeline(std::shared_ptr<AsyncTransportWrapper> sock) {
auto pipeline = SerializePipeline::create();
diff --git a/connection/pipeline.h b/connection/pipeline.h
index 6c4f4ff..e90cb93 100644
--- a/connection/pipeline.h
+++ b/connection/pipeline.h
@@ -31,6 +31,7 @@ using SerializePipeline =
class RpcPipelineFactory : public wangle::PipelineFactory<SerializePipeline> {
public:
+ RpcPipelineFactory();
SerializePipeline::Ptr
newPipeline(std::shared_ptr<folly::AsyncTransportWrapper> sock) override;
diff --git a/connection/response.h b/connection/response.h
index a7f7939..d2eb19a 100644
--- a/connection/response.h
+++ b/connection/response.h
@@ -31,10 +31,12 @@ namespace hbase {
class Response {
public:
- Response() : call_id_(0) {}
+ Response() : call_id_(0), response_(nullptr) {}
uint32_t call_id() { return call_id_; }
void set_call_id(uint32_t call_id) { call_id_ = call_id; }
- std::shared_ptr<google::protobuf::Message> response() { return response_; }
+ std::shared_ptr<google::protobuf::Message> response() const {
+ return response_;
+ }
void set_response(std::shared_ptr<google::protobuf::Message> response) {
response_ = std::move(response);
}
diff --git a/core/BUCK b/core/BUCK
index e555ba4..447248b 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -22,6 +22,7 @@ cxx_library(
"client.h",
"connection.h",
"hbase_macros.h",
+ "region-location.h",
"location-cache.h",
"table-name.h",
# TODO: move this out of exported
@@ -32,7 +33,6 @@ cxx_library(
"client.cc",
"location-cache.cc",
"meta-utils.cc",
- "table-name.cc",
],
deps=[
"//connection:connection",
@@ -42,6 +42,7 @@ cxx_library(
"//third-party:wangle",
"//third-party:zookeeper_mt",
],
+ compiler_flags=['-Weffc++'],
visibility=[
'PUBLIC',
], )
diff --git a/core/location-cache.cc b/core/location-cache.cc
index c81deba..539051a 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -21,20 +21,31 @@
#include <folly/Logging.h>
#include <folly/io/IOBuf.h>
+#include "connection/response.h"
+#include "if/Client.pb.h"
#include "if/ZooKeeper.pb.h"
#include "serde/zk-deserializer.h"
using namespace std;
using namespace folly;
-using namespace hbase::pb;
-using namespace hbase;
+
+using hbase::Response;
+using hbase::LocationCache;
+using hbase::RegionLocation;
+using hbase::HBaseService;
+using hbase::pb::ScanResponse;
+using hbase::pb::TableName;
+using hbase::pb::ServerName;
+using hbase::pb::MetaRegionServer;
+using hbase::pb::RegionInfo;
// TODO(eclark): make this configurable on client creation
static const char META_ZNODE_NAME[] = "/hbase/meta-region-server";
LocationCache::LocationCache(string quorum_spec,
shared_ptr<folly::Executor> executor)
- : quorum_spec_(quorum_spec), executor_(executor), meta_promise_(nullptr) {
+ : quorum_spec_(quorum_spec), executor_(executor), meta_promise_(nullptr),
+ meta_lock_(), cp_(), meta_util_() {
zk_ = zookeeper_init(quorum_spec.c_str(), nullptr, 1000, 0, 0, 0);
}
@@ -89,3 +100,23 @@ ServerName LocationCache::ReadMetaLocation() {
}
return mrs.server();
}
+
+Future<RegionLocation> LocationCache::locateFromMeta(const TableName &tn,
+ const string &row) {
+ return this->LocateMeta()
+ .then([&](ServerName sn) { return this->cp_.get(sn); })
+ .then([&](std::shared_ptr<HBaseService> service) {
+ return (*service)(std::move(meta_util_.make_meta_request(tn, row)));
+ })
+ .then([&](Response resp) {
+ // take the protobuf response and make it into
+ // a region location.
+ return this->parse_response(std::move(resp));
+ });
+}
+
+RegionLocation LocationCache::parse_response(const Response &resp) {
+ auto resp_msg = static_pointer_cast<ScanResponse>(resp.response());
+ LOG(ERROR) << "resp_msg = " << resp_msg->DebugString();
+ return RegionLocation{RegionInfo{}, ServerName{}, nullptr};
+}
diff --git a/core/location-cache.h b/core/location-cache.h
index ce97c43..cfd6838 100644
--- a/core/location-cache.h
+++ b/core/location-cache.h
@@ -27,9 +27,19 @@
#include <mutex>
#include <string>
-#include "if/HBase.pb.h"
+#include "connection/connection-pool.h"
+#include "core/meta-utils.h"
+#include "core/table-name.h"
+#include "core/region-location.h"
namespace hbase {
+
+class Request;
+class Response;
+namespace pb {
+class ServerName;
+}
+
class LocationCache {
public:
explicit LocationCache(std::string quorum_spec,
@@ -38,6 +48,9 @@ public:
// Meta Related Methods.
// These are only public until testing is complete
folly::Future<hbase::pb::ServerName> LocateMeta();
+ folly::Future<RegionLocation> locateFromMeta(const hbase::pb::TableName &tn,
+ const std::string &row);
+ RegionLocation parse_response(const Response &resp);
void InvalidateMeta();
private:
@@ -48,7 +61,11 @@ private:
std::shared_ptr<folly::Executor> executor_;
std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> meta_promise_;
std::mutex meta_lock_;
+ ConnectionPool cp_;
+ MetaUtil meta_util_;
+
+ // TODO: migrate this to a smart pointer with a deleter.
zhandle_t *zk_;
};
} // namespace hbase
diff --git a/core/meta-utils.cc b/core/meta-utils.cc
index 7c7d45f..d2fdd88 100644
--- a/core/meta-utils.cc
+++ b/core/meta-utils.cc
@@ -21,11 +21,61 @@
#include <folly/Conv.h>
+#include "connection/request.h"
+#include "connection/response.h"
#include "core/table-name.h"
+#include "if/Client.pb.h"
-using namespace hbase;
+using hbase::pb::TableName;
+using hbase::MetaUtil;
+using hbase::Request;
+using hbase::Response;
+using hbase::pb::ScanRequest;
+using hbase::pb::RegionSpecifier_RegionSpecifierType;
+
+static const std::string META_REGION = "1588230740";
std::string MetaUtil::region_lookup_rowkey(const TableName &tn,
- const std::string &row) {
+ const std::string &row) const {
return folly::to<std::string>(tn, ",", row, ",", "999999999999999999");
}
+
+std::unique_ptr<Request>
+MetaUtil::make_meta_request(const TableName tn, const std::string &row) const {
+ auto request = Request::scan();
+ auto msg = std::static_pointer_cast<ScanRequest>(request->req_msg());
+
+ msg->set_number_of_rows(1);
+ msg->set_close_scanner(true);
+
+ // Set the region this scan goes to
+ auto region = msg->mutable_region();
+ region->set_value(META_REGION);
+ region->set_type(RegionSpecifier_RegionSpecifierType::
+ RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
+
+ auto scan = msg->mutable_scan();
+ // We don't care about before, just now.
+ scan->set_max_versions(1);
+ // Meta should be cached at all times.
+ scan->set_cache_blocks(true);
+ // We only want one row right now.
+ //
+ // TODO(eclark): Figure out if we should get more.
+ scan->set_caching(1);
+ // Close the scan after we have data.
+ scan->set_small(true);
+ // We know where to start but not where to end.
+ scan->set_reversed(true);
+ // Give me everything or nothing.
+ scan->set_allow_partial_results(false);
+
+ // Set the columns that we need
+ auto info_col = scan->add_column();
+ info_col->set_family("info");
+ info_col->add_qualifier("server");
+ info_col->add_qualifier("regioninfo");
+
+ scan->set_start_row(region_lookup_rowkey(tn, row));
+ return request;
+}
diff --git a/core/meta-utils.h b/core/meta-utils.h
index 391d0ea..e007d02 100644
--- a/core/meta-utils.h
+++ b/core/meta-utils.h
@@ -20,11 +20,16 @@
#include <string>
+#include "connection/Request.h"
+#include "core/table-name.h"
+
namespace hbase {
-class TableName;
class MetaUtil {
public:
- std::string region_lookup_rowkey(const TableName &tn, const std::string &row);
+ std::string region_lookup_rowkey(const hbase::pb::TableName &tn,
+ const std::string &row) const;
+ std::unique_ptr<Request> make_meta_request(const hbase::pb::TableName tn,
+ const std::string &row) const;
};
} // namespace hbase
diff --git a/connection/connection-factory.h b/core/region-location.h
similarity index 63%
copy from connection/connection-factory.h
copy to core/region-location.h
index 5a45316..a46b8e2 100644
--- a/connection/connection-factory.h
+++ b/core/region-location.h
@@ -18,24 +18,28 @@
*/
#pragma once
-#include <wangle/service/Service.h>
+#include <memory>
-#include <string>
-
-#include "connection/pipeline.h"
-#include "connection/request.h"
-#include "connection/response.h"
#include "connection/service.h"
+#include "if/HBase.pb.h"
+
namespace hbase {
-class ConnectionFactory {
+
+class RegionLocation {
public:
- ConnectionFactory();
+ RegionLocation(hbase::pb::RegionInfo ri, hbase::pb::ServerName sn,
+ std::shared_ptr<HBaseService> service)
+ : ri_(ri), sn_(sn), service_(service) {}
- virtual std::shared_ptr<HBaseService> make_connection(const std::string &host,
- int port);
+ const hbase::pb::RegionInfo& region_info() { return ri_; }
+ const hbase::pb::ServerName& server_name() { return sn_; }
+ std::shared_ptr<HBaseService> service() { return service_; }
private:
- wangle::ClientBootstrap<SerializePipeline> bootstrap_;
+ hbase::pb::RegionInfo ri_;
+ hbase::pb::ServerName sn_;
+ std::shared_ptr<HBaseService> service_;
};
+
} // namespace hbase
diff --git a/core/simple-client.cc b/core/simple-client.cc
index 11dcd68..ab614e4 100644
--- a/core/simple-client.cc
+++ b/core/simple-client.cc
@@ -27,6 +27,7 @@
#include "connection/connection-pool.h"
#include "core/client.h"
+#include "core/table-name.h"
#include "if/Client.pb.h"
#include "if/ZooKeeper.pb.h"
@@ -38,13 +39,14 @@ using hbase::Request;
using hbase::HBaseService;
using hbase::LocationCache;
using hbase::ConnectionPool;
+using hbase::TableNameUtil;
using hbase::pb::ServerName;
using hbase::pb::RegionSpecifier_RegionSpecifierType;
using hbase::pb::GetRequest;
using hbase::pb::GetResponse;
// TODO(eclark): remove the need for this.
-DEFINE_string(region, "1588230740", "What region to send a get to");
+DEFINE_string(table, "t", "What region to send a get");
DEFINE_string(row, "test", "What row to get");
DEFINE_string(zookeeper, "localhost:2181", "What zk quorum to talk to");
@@ -59,48 +61,8 @@ int main(int argc, char *argv[]) {
auto cpu_ex = wangle::getCPUExecutor();
LocationCache cache{FLAGS_zookeeper, cpu_ex};
auto result =
- cache.LocateMeta()
- .then([&cp = cp](ServerName sn) { return cp.get(sn); })
- .then([](shared_ptr<HBaseService> con) {
- // Send the request
- auto r = Request::get();
- // This is a get request so make that
- auto req_msg = static_pointer_cast<GetRequest>(r->req_msg());
- // Set what region
- req_msg->mutable_region()->set_value(FLAGS_region);
- // It's always this.
- req_msg->mutable_region()->set_type(
- RegionSpecifier_RegionSpecifierType::
- RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
-
- // What row.
- req_msg->mutable_get()->set_row(FLAGS_row);
-
- return (*con)(std::move(r));
- })
- .then([](Response resp) {
- return static_pointer_cast<GetResponse>(resp.response());
- })
- .via(cpu_ex.get())
- .then([](shared_ptr<GetResponse> get_resp) {
- cout << "GetResponse has_result = " << get_resp->has_result()
- << '\n';
- if (get_resp->has_result()) {
- auto &r = get_resp->result();
- cout << "Result cell_size = " << r.cell_size() << endl;
- for (auto &cell : r.cell()) {
- cout << "\trow = " << cell.row()
- << " family = " << cell.family()
- << " qualifier = " << cell.qualifier()
- << " timestamp = " << cell.timestamp()
- << " value = " << cell.value() << endl;
- }
- return 0;
- }
-
- return 1;
- })
+ cache.locateFromMeta(TableNameUtil::create(FLAGS_table), FLAGS_row)
.get(milliseconds(5000));
- return result;
+ return 0;
}
diff --git a/core/table-name-test.cc b/core/table-name-test.cc
index 8aff375..7bad3f1 100644
--- a/core/table-name-test.cc
+++ b/core/table-name-test.cc
@@ -25,26 +25,30 @@
#include "core/table-name.h"
using namespace hbase;
+using hbase::pb::TableName;
TEST(TestTableName, TestToStringNoDefault) {
- TableName tn{"TestTableName"};
+ TableName tn;
+ tn.set_qualifier("TestTableName");
std::string result = folly::to<std::string>(tn);
ASSERT_EQ(result.find("default"), std::string::npos);
ASSERT_EQ("TestTableName", result);
}
+
+TEST(TestTableName, TestToStringNoDefaltWhenSet) {
+ TableName tn;
+ tn.set_namespace_("default");
+ tn.set_qualifier("TestTableName");
+ std::string result = folly::to<std::string>(tn);
+ ASSERT_EQ(result.find("default"), std::string::npos);
+ ASSERT_EQ("TestTableName", result);
+}
+
TEST(TestTableName, TestToStringIncludeNS) {
- TableName tn{"hbase", "acl"};
+ TableName tn;
+ tn.set_namespace_("hbase");
+ tn.set_qualifier("acl");
std::string result = folly::to<std::string>(tn);
ASSERT_EQ(result.find("hbase"), 0);
ASSERT_EQ("hbase:acl", result);
}
-TEST(TestTableName, TestIsDefault) {
- TableName default_t1{"in_default"};
- ASSERT_TRUE(default_t1.is_default_name_space());
-
- TableName default_t2{"default", "in_also"};
- ASSERT_TRUE(default_t2.is_default_name_space());
-
- TableName non_default{"testing", "hmm"};
- ASSERT_FALSE(non_default.is_default_name_space());
-}
diff --git a/core/table-name.cc b/core/table-name.cc
deleted file mode 100644
index 33087fd..0000000
--- a/core/table-name.cc
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#include "core/table-name.h"
-
-#include <folly/Conv.h>
-
-using namespace hbase;
-
-TableName::TableName(std::string table_name)
- : name_space_("default"), table_(table_name) {}
-TableName::TableName(std::string name_space, std::string table_name)
- : name_space_(name_space), table_(table_name) {}
-bool TableName::is_default_name_space() const {
- return name_space_.length() == 0 || name_space_ == "default";
-}
-bool TableName::operator==(const TableName &other) const {
- return name_space_ == other.name_space_ && table_ == other.table_;
-}
diff --git a/core/table-name.h b/core/table-name.h
index 816f7da..1612667 100644
--- a/core/table-name.h
+++ b/core/table-name.h
@@ -21,33 +21,30 @@
#include <memory>
#include <string>
+#include "if/HBase.pb.h"
#include <folly/Conv.h>
namespace hbase {
-
-// This is the core class of a HBase client.
-class TableName {
-public:
- explicit TableName(std::string table_name);
- explicit TableName(std::string name_space, std::string table_name);
-
- std::string name_space() const { return name_space_; };
- std::string table() const { return table_; };
- bool is_default_name_space() const;
- bool operator==(const TableName &other) const;
-
-private:
- std::string name_space_;
- std::string table_;
-};
+namespace pb {
// Provide folly::to<std::string>(TableName);
template <class String> void toAppend(const TableName &in, String *result) {
- if (in.is_default_name_space()) {
- folly::toAppend(in.table(), result);
+ if (!in.has_namespace_() || in.namespace_() == "default") {
+ folly::toAppend(in.qualifier(), result);
} else {
- folly::toAppend(in.name_space(), ':', in.table(), result);
+ folly::toAppend(in.namespace_(), ':', in.qualifier(), result);
}
}
+} // namespace pb
+
+class TableNameUtil {
+public:
+ static ::hbase::pb::TableName create(std::string table_name) {
+ ::hbase::pb::TableName tn;
+ tn.set_namespace_("default");
+ tn.set_qualifier(table_name);
+ return tn;
+ }
+};
} // namespace hbase
diff --git a/serde/BUCK b/serde/BUCK
index 0014c0b..539a221 100644
--- a/serde/BUCK
+++ b/serde/BUCK
@@ -34,6 +34,7 @@ cxx_library(name="serde",
":client-serializer-test",
":client-deserializer-test",
],
+ compiler_flags=['-Weffc++'],
visibility=[
'PUBLIC',
], )
diff --git a/utils/BUCK b/utils/BUCK
index ad12f7d..2ab6a89 100644
--- a/utils/BUCK
+++ b/utils/BUCK
@@ -25,12 +25,13 @@ cxx_library(name="utils",
deps=[
'//third-party:folly',
],
+ tests=[
+ ":user-util-test"
+ ],
visibility=[
'PUBLIC',
],
- tests=[
- ":user-util-test"
- ], )
+ compiler_flags=['-Weffc++'], )
cxx_test(name="user-util-test",
srcs=[
"user-util-test.cc",
diff --git a/utils/user-util.cc b/utils/user-util.cc
index b801537..3d963b3 100644
--- a/utils/user-util.cc
+++ b/utils/user-util.cc
@@ -27,6 +27,8 @@
using namespace hbase;
using namespace std;
+UserUtil::UserUtil() : init_{false}, user_name_{"drwho"}, m_() {}
+
string UserUtil::user_name() {
if (!init_) {
compute_user_name();
diff --git a/utils/user-util.h b/utils/user-util.h
index 9ca28e3..da95d78 100644
--- a/utils/user-util.h
+++ b/utils/user-util.h
@@ -26,11 +26,12 @@
namespace hbase {
class UserUtil {
public:
+ UserUtil();
std::string user_name();
private:
void compute_user_name();
- std::atomic<bool> init_{false};
+ std::atomic<bool> init_;
std::string user_name_;
std::mutex m_;
};
[hbase] 11/137: HBASE-15649 Ignore buck-out when building docker
image
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit da7a30f4dc8a06a9233f6879d53273194cca513c
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Wed Apr 13 17:02:38 2016 -0700
HBASE-15649 Ignore buck-out when building docker image
---
.dockerignore | 2 ++
1 file changed, 2 insertions(+)
diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..2cfffb7
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,2 @@
+buck-out
+.buckd
[hbase] 53/137: HBASE-17283 [C++] Result class
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 563d036d6b3e540a94f921c5e93e9f70002fb0f6
Author: Sudeep Sunthankar <su...@hashmapinc.com>
AuthorDate: Fri Dec 30 10:27:29 2016 +1100
HBASE-17283 [C++] Result class
Signed-off-by: Enis Soztutar <en...@apache.org>
---
core/BUCK | 7 ++
core/result-test.cc | 301 ++++++++++++++++++++++++++++++++++++++++++++++++++++
core/result.cc | 118 ++++++++++++++++++++
core/result.h | 131 +++++++++++++++++++++++
4 files changed, 557 insertions(+)
diff --git a/core/BUCK b/core/BUCK
index 20e4736..a5ea5c0 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -32,6 +32,7 @@ cxx_library(
"configuration.h",
"hbase_configuration_loader.h",
"scan.h",
+ "result.h",
],
srcs=[
"cell.cc",
@@ -43,6 +44,7 @@ cxx_library(
"configuration.cc",
"hbase_configuration_loader.cc",
"scan.cc",
+ "result.cc",
],
deps=[
"//connection:connection",
@@ -87,6 +89,11 @@ cxx_test(
srcs=["scan-test.cc",],
deps=[":core",],
run_test_separately=True,)
+cxx_test(
+ name="result-test",
+ srcs=["result-test.cc",],
+ deps=[":core",],
+ run_test_separately=True,)
cxx_binary(
name="simple-client",
srcs=["simple-client.cc",],
diff --git a/core/result-test.cc b/core/result-test.cc
new file mode 100644
index 0000000..134e80f
--- /dev/null
+++ b/core/result-test.cc
@@ -0,0 +1,301 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/result.h"
+
+#include <limits>
+#include <memory>
+#include <vector>
+#include <string>
+#include <gtest/gtest.h>
+
+#include "core/cell.h"
+using namespace hbase;
+
+void PopulateCells(std::vector<std::shared_ptr<Cell> > &cells) {
+ // Populate some Results
+ // We assume that for a single Cell, the corresponding row, families and
+ // qualifiers are present.
+ // We have also considered different versions in the test for the same row.
+ std::string row = "row";
+ for (int i = 0; i < 10; i++) {
+ std::string family = "family-" + std::to_string(i);
+ std::string column = "column-" + std::to_string(i);
+ std::string value = "value-" + std::to_string(i);
+
+ switch (i) {
+ case 5: {
+ cells.push_back(std::make_shared<Cell>(
+ row, family, column, 1482113040506, "value-5", CellType::PUT));
+ cells.push_back(std::make_shared<Cell>(
+ row, family, column, 1482111803856, "value-X", CellType::PUT));
+ break;
+ }
+ case 8: {
+ cells.push_back(std::make_shared<Cell>(
+ row, family, column, 1482113040506, "value-8", CellType::PUT));
+ cells.push_back(std::make_shared<Cell>(
+ row, family, column, 1482111803856, "value-X", CellType::PUT));
+ cells.push_back(std::make_shared<Cell>(
+ row, family, column, 1482110969958, "value-Y", CellType::PUT));
+ break;
+ }
+ case 9: {
+ cells.push_back(std::make_shared<Cell>(
+ row, family, column, 1482113040506, "value-9", CellType::PUT));
+ cells.push_back(std::make_shared<Cell>(
+ row, family, column, 1482111803856, "value-X", CellType::PUT));
+ cells.push_back(std::make_shared<Cell>(
+ row, family, column, 1482110969958, "value-Y", CellType::PUT));
+ cells.push_back(std::make_shared<Cell>(
+ row, family, column, 1482110876075, "value-Z", CellType::PUT));
+ break;
+ }
+ default: {
+ cells.push_back(std::make_shared<Cell>(row, family, column,
+ std::numeric_limits<long>::max(),
+ value, CellType::PUT));
+ }
+ }
+ }
+ return;
+}
+
+TEST(Result, EmptyResult) {
+ std::vector<std::shared_ptr<Cell> > cells;
+ Result result(cells, true, false, false);
+ EXPECT_EQ(true, result.IsEmpty());
+ EXPECT_EQ(0, result.Size());
+}
+
+TEST(Result, FilledResult) {
+ std::vector<std::shared_ptr<Cell> > cells;
+ PopulateCells(cells);
+
+ Result result(cells, true, false, false);
+
+ EXPECT_EQ(false, result.IsEmpty());
+ EXPECT_EQ(16, result.Size());
+
+ // Get Latest Cell for the given family and qualifier.
+ auto latest_cell(result.ColumnLatestCell("family", "column"));
+ // Nothing of the above family/qualifier combo is present so it should be
+ // nullptr
+ ASSERT_FALSE(latest_cell.get());
+
+ // Try to get the latest cell for the given family and qualifier.
+ latest_cell = result.ColumnLatestCell("family-4", "column-4");
+ // Now shouldn't be a nullptr
+ ASSERT_TRUE(latest_cell.get());
+ // And Value must match too
+ EXPECT_EQ("value-4", latest_cell->Value());
+
+ // Value will be nullptr as no such family and qualifier is present
+ ASSERT_FALSE(result.Value("family-4", "qualifier"));
+ // Value will be present as family and qualifier is present
+ ASSERT_TRUE(result.Value("family-4", "column-4"));
+ // Value should be present and match.
+ EXPECT_EQ(latest_cell->Value(),
+ (*result.ColumnLatestCell("family-4", "column-4")).Value());
+ EXPECT_EQ("value-5",
+ (*result.ColumnLatestCell("family-5", "column-5")).Value());
+ EXPECT_EQ("value-8",
+ (*result.ColumnLatestCell("family-8", "column-8")).Value());
+ EXPECT_EQ("value-7", *result.Value("family-7", "column-7"));
+
+ // Get cells for the given family and qualifier
+ auto column_cells = result.ColumnCells("family", "column");
+ // Size should be 0
+ EXPECT_EQ(0, column_cells.size());
+
+ // Size shouldn't be 0 and Row() and Value() must match
+ column_cells = result.ColumnCells("family-0", "column-0");
+ EXPECT_EQ(1, column_cells.size());
+ EXPECT_EQ("row", column_cells[0]->Row());
+ EXPECT_EQ("row", result.Row());
+
+ // Size shouldn't be 0 and Row() and Value() must match
+ column_cells = result.ColumnCells("family-5", "column-5");
+ EXPECT_EQ(2, column_cells.size());
+ EXPECT_EQ("row", column_cells[0]->Row());
+ EXPECT_EQ("row", column_cells[1]->Row());
+ EXPECT_EQ("value-5", column_cells[0]->Value());
+ EXPECT_EQ("value-X", column_cells[1]->Value());
+ EXPECT_EQ("row", result.Row());
+
+ // Size shouldn't be 0 and Row() and Value() must match
+ column_cells = result.ColumnCells("family-8", "column-8");
+ EXPECT_EQ(3, column_cells.size());
+ EXPECT_EQ("row", column_cells[0]->Row());
+ EXPECT_EQ("row", column_cells[1]->Row());
+ EXPECT_EQ("row", column_cells[2]->Row());
+ EXPECT_EQ("value-8", column_cells[0]->Value());
+ EXPECT_EQ("value-X", column_cells[1]->Value());
+ EXPECT_EQ("value-Y", column_cells[2]->Value());
+ EXPECT_EQ("row", result.Row());
+
+ // Size shouldn't be 0 and Row() and Value() must match
+ column_cells = result.ColumnCells("family-9", "column-9");
+ EXPECT_EQ(4, column_cells.size());
+ EXPECT_EQ("row", column_cells[0]->Row());
+ EXPECT_EQ("row", column_cells[1]->Row());
+ EXPECT_EQ("row", column_cells[2]->Row());
+ EXPECT_EQ("row", column_cells[3]->Row());
+ EXPECT_EQ("value-9", column_cells[0]->Value());
+ EXPECT_EQ("value-X", column_cells[1]->Value());
+ EXPECT_EQ("value-Y", column_cells[2]->Value());
+ EXPECT_EQ("value-Z", column_cells[3]->Value());
+ EXPECT_EQ("row", result.Row());
+
+ // Test all the Cell values
+ const auto &result_cells = result.Cells();
+ int i = 0, j = 0;
+ for (const auto &cell : result_cells) {
+ std::string row = "row";
+ std::string family = "family-" + std::to_string(i);
+ std::string column = "column-" + std::to_string(i);
+ std::string value = "value-" + std::to_string(i);
+ switch (j) {
+ case 6:
+ case 10:
+ case 13: {
+ EXPECT_EQ("value-X", cell->Value());
+ ++j;
+ continue;
+ }
+ case 11:
+ case 14: {
+ EXPECT_EQ("value-Y", cell->Value());
+ ++j;
+ continue;
+ }
+ case 15: {
+ EXPECT_EQ("value-Z", cell->Value());
+ ++j;
+ continue;
+ }
+ }
+ EXPECT_EQ(row, cell->Row());
+ EXPECT_EQ(family, cell->Family());
+ EXPECT_EQ(column, cell->Qualifier());
+ EXPECT_EQ(value, cell->Value());
+ ++i;
+ ++j;
+ }
+
+ auto result_map_tmp = result.Map();
+ result_map_tmp["testf"]["testq"][1] = "value";
+ EXPECT_EQ(11, result_map_tmp.size());
+
+ auto result_map = result.Map();
+ EXPECT_EQ(10, result_map.size());
+
+ i = 0;
+ for (auto family_map : result_map) {
+ std::string family = "family-" + std::to_string(i);
+ std::string qualifier = "column-" + std::to_string(i);
+ std::string value = "value-" + std::to_string(i);
+ EXPECT_EQ(family, family_map.first);
+ for (auto qualifier_map : family_map.second) {
+ EXPECT_EQ(qualifier, qualifier_map.first);
+ j = 0;
+ for (auto version_map : qualifier_map.second) {
+ switch (i) {
+ case 5: {
+ if (1 == j) {
+ EXPECT_EQ(1482111803856, version_map.first);
+ EXPECT_EQ("value-X", version_map.second);
+ } else if (0 == j) {
+ EXPECT_EQ(1482113040506, version_map.first);
+ EXPECT_EQ("value-5", version_map.second);
+ }
+ break;
+ }
+ case 8: {
+ if (2 == j) {
+ EXPECT_EQ(1482110969958, version_map.first);
+ EXPECT_EQ("value-Y", version_map.second);
+ } else if (1 == j) {
+ EXPECT_EQ(1482111803856, version_map.first);
+ EXPECT_EQ("value-X", version_map.second);
+ } else if (0 == j) {
+ EXPECT_EQ(1482113040506, version_map.first);
+ EXPECT_EQ("value-8", version_map.second);
+ }
+ break;
+ }
+ case 9: {
+ if (3 == j) {
+ EXPECT_EQ(1482110876075, version_map.first);
+ EXPECT_EQ("value-Z", version_map.second);
+ } else if (2 == j) {
+ EXPECT_EQ(1482110969958, version_map.first);
+ EXPECT_EQ("value-Y", version_map.second);
+ } else if (1 == j) {
+ EXPECT_EQ(1482111803856, version_map.first);
+ EXPECT_EQ("value-X", version_map.second);
+ } else if (0 == j) {
+ EXPECT_EQ(1482113040506, version_map.first);
+ EXPECT_EQ("value-9", version_map.second);
+ }
+ break;
+ }
+ default: {
+ EXPECT_EQ(std::numeric_limits<long>::max(), version_map.first);
+ EXPECT_EQ(value, version_map.second);
+ }
+ }
+ ++j;
+ }
+ }
+ ++i;
+ }
+
+ auto family_map = result.FamilyMap("family-0");
+ EXPECT_EQ(1, family_map.size());
+ i = 0;
+ for (auto qual_val_map : family_map) {
+ EXPECT_EQ("column-0", qual_val_map.first);
+ EXPECT_EQ("value-0", qual_val_map.second);
+ }
+
+ family_map = result.FamilyMap("family-1");
+ EXPECT_EQ(1, family_map.size());
+ i = 0;
+ for (auto qual_val_map : family_map) {
+ EXPECT_EQ("column-1", qual_val_map.first);
+ EXPECT_EQ("value-1", qual_val_map.second);
+ }
+
+ family_map = result.FamilyMap("family-5");
+ EXPECT_EQ(1, family_map.size());
+ i = 0;
+ for (auto qual_val_map : family_map) {
+ EXPECT_EQ("column-5", qual_val_map.first);
+ EXPECT_EQ("value-5", qual_val_map.second);
+ }
+
+ family_map = result.FamilyMap("family-9");
+ EXPECT_EQ(1, family_map.size());
+ i = 0;
+ for (auto qual_val_map : family_map) {
+ EXPECT_EQ("column-9", qual_val_map.first);
+ EXPECT_EQ("value-9", qual_val_map.second);
+ }
+}
diff --git a/core/result.cc b/core/result.cc
new file mode 100644
index 0000000..9842915
--- /dev/null
+++ b/core/result.cc
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/result.h"
+
+namespace hbase {
+
+Result::~Result() {}
+
+Result::Result(const std::vector<std::shared_ptr<Cell> > &cells, bool exists,
+ bool stale, bool partial)
+ : exists_(exists), stale_(stale), partial_(partial) {
+ for (const auto &cell : cells) {
+ cells_.push_back(cell);
+ // We create the map when cells are added. unlike java where map is created
+ // when result.getMap() is called
+ result_map_[cell->Family()][cell->Qualifier()][cell->Timestamp()] =
+ cell->Value();
+ }
+ row_ = (cells_.size() == 0 ? "" : cells_[0]->Row());
+}
+
+Result::Result(const Result &result) {
+ exists_ = result.exists_;
+ stale_ = result.stale_;
+ partial_ = result.partial_;
+ row_ = result.row_;
+ if (!result.cells_.empty()) {
+ for (const auto &cell : result.cells_) {
+ cells_.push_back(cell);
+ result_map_[cell->Family()][cell->Qualifier()][cell->Timestamp()] =
+ cell->Value();
+ }
+ }
+}
+const std::vector<std::shared_ptr<Cell> > &Result::Cells() const {
+ return cells_;
+}
+
+std::vector<std::shared_ptr<Cell> > Result::ColumnCells(
+ const std::string &family, const std::string &qualifier) const {
+ std::vector<std::shared_ptr<Cell> > column_cells;
+ // TODO implement a BinarySearch here ?
+ for (const auto &cell : cells_) {
+ if (cell->Family() == family && cell->Qualifier() == qualifier) {
+ column_cells.push_back(cell);
+ }
+ }
+ return column_cells;
+}
+
+const std::shared_ptr<Cell> Result::ColumnLatestCell(
+ const std::string &family, const std::string &qualifier) const {
+ // TODO implement a BinarySearch here ?
+ for (const auto &cell : cells_) {
+ // We find the latest(first) occurrence of the Cell for a given column and
+ // qualifier and break
+ if (cell->Family() == family && cell->Qualifier() == qualifier) {
+ return cell;
+ }
+ }
+ return nullptr;
+}
+
+std::shared_ptr<std::string> Result::Value(const std::string &family,
+ const std::string &qualifier) const {
+ std::shared_ptr<Cell> latest_cell(ColumnLatestCell(family, qualifier));
+ if (latest_cell.get()) {
+ return std::make_shared<std::string>(latest_cell->Value());
+ }
+ return nullptr;
+}
+
+bool Result::IsEmpty() const { return cells_.empty(); }
+
+const std::string &Result::Row() const { return row_; }
+
+const int Result::Size() const { return cells_.size(); }
+
+const ResultMap &Result::Map() const { return result_map_; }
+
+const ResultFamilyMap Result::FamilyMap(const std::string &family) const {
+ ResultFamilyMap family_map;
+ if (!IsEmpty()) {
+ for (auto itr = result_map_.begin(); itr != result_map_.end(); ++itr) {
+ if (family == itr->first) {
+ for (auto qitr = itr->second.begin(); qitr != itr->second.end();
+ ++qitr) {
+ for (auto vitr = qitr->second.begin(); vitr != qitr->second.end();
+ ++vitr) {
+ // We break after inserting the first value. Result.java takes only
+ // the first value
+ family_map[qitr->first] = vitr->second;
+ break;
+ }
+ }
+ }
+ }
+ }
+ return family_map;
+}
+} /* namespace hbase */
diff --git a/core/result.h b/core/result.h
new file mode 100644
index 0000000..ecfdd55
--- /dev/null
+++ b/core/result.h
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <functional>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "core/cell.h"
+
+namespace hbase {
+
+/**
+ * @brief Map of families to all versions of its qualifiers and values
+ * We need to have a reverse ordered map, when storing TS -> value, so that the
+ * most recent value is stored first
+ */
+using ResultMap = std::map<
+ std::string, std::map<std::string, std::map<int64_t, std::string,
+ std::greater<int64_t> > > >;
+
+/**
+ * @brief Map of qualifiers to values.
+ */
+using ResultFamilyMap = std::map<std::string, std::string>;
+
+class Result {
+ public:
+ /**
+ * Constructors
+ */
+ Result(const std::vector<std::shared_ptr<Cell> > &cells, bool exists,
+ bool stale, bool partial);
+ Result(const Result &result);
+ ~Result();
+
+ /**
+ * @brief Return the vector of Cells backing this Result instance. This vector
+ * will be ordered in the same manner
+ * as the one which was passed while creation of the Result instance.
+ */
+ const std::vector<std::shared_ptr<Cell> > &Cells() const;
+
+ /**
+ * @brief Return a vector of Cells for the family and qualifier or empty list
+ * if the column
+ * did not exist in the result.
+ * @param family - column family
+ * @param qualifier - column qualifier
+ */
+ std::vector<std::shared_ptr<Cell> > ColumnCells(
+ const std::string &family, const std::string &qualifier) const;
+
+ /**
+ * @brief Returns the Cell for the most recent timestamp for a given family
+ * and qualifier.
+ * Returns map of qualifiers to values, only includes latest values
+ * @param family - column family.
+ * @param qualifier - column qualifier
+ */
+ const std::shared_ptr<Cell> ColumnLatestCell(
+ const std::string &family, const std::string &qualifier) const;
+
+ /**
+ * @brief Get the latest version of the specified family and qualifier.
+ * @param family - column family
+ * @param qualifier - column qualifier
+ */
+ std::shared_ptr<std::string> Value(const std::string &family,
+ const std::string &qualifier) const;
+
+ /**
+ * @brief Returns if the underlying Cell vector is empty or not
+ */
+ bool IsEmpty() const;
+
+ /**
+ * @brief Retrieves the row key that corresponds to the row from which this
+ * Result was created.
+ */
+ const std::string &Row() const;
+
+ /**
+ * @brief Returns the size of the underlying Cell vector
+ */
+ const int Size() const;
+
+ /**
+ * @brief Map of families to all versions of its qualifiers and values.
+ * Returns a three level Map of the form:
+ * Map<family,Map<qualifier,Map<timestamp,value>>>>
+ * All other map returning methods make use of this map internally
+ * The Map is created when the Result instance is created
+ */
+ const ResultMap &Map() const;
+
+ /**
+ * @brief Map of qualifiers to values.
+ * Returns a Map of the form: Map<qualifier,value>
+ * @param family - column family to get
+ */
+ const ResultFamilyMap FamilyMap(const std::string &family) const;
+
+ private:
+ bool exists_ = false;
+ bool stale_ = false;
+ bool partial_ = false;
+ std::string row_ = "";
+ std::vector<std::shared_ptr<Cell> > cells_;
+ ResultMap result_map_;
+};
+} /* namespace hbase */
[hbase] 69/137: HBASE-17465 [C++] implement request retry mechanism
over RPC (Xiaobing Zhou)
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 670943a5867ad5833e287124e39fc35cb704c961
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Fri Mar 3 19:15:16 2017 -0800
HBASE-17465 [C++] implement request retry mechanism over RPC (Xiaobing Zhou)
---
Makefile | 2 +-
bin/start-docker.sh | 2 +-
connection/connection-factory.cc | 7 +-
connection/connection-factory.h | 3 +-
connection/connection-pool.cc | 4 +-
connection/rpc-client.cc | 42 +----
connection/rpc-client.h | 42 +----
core/BUCK | 14 ++
core/async-rpc-retrying-caller-factory.cc | 22 +++
core/async-rpc-retrying-caller-factory.h | 124 ++++++++++++++
core/async-rpc-retrying-caller.cc | 22 +++
core/async-rpc-retrying-caller.h | 266 ++++++++++++++++++++++++++++++
core/async-rpc-retrying-test.cc | 255 ++++++++++++++++++++++++++++
core/client.cc | 3 +-
core/client.h | 2 +-
core/filter.h | 2 +-
core/hbase-rpc-controller.cc | 22 +++
core/hbase-rpc-controller.h | 56 +++++++
core/location-cache.cc | 1 +
core/region-location.h | 10 +-
core/response_converter.cc | 1 +
core/response_converter.h | 1 +
core/table.cc | 4 +
core/table.h | 5 +
{utils => exceptions}/BUCK | 17 +-
exceptions/exception.h | 104 ++++++++++++
utils/BUCK | 7 +-
utils/connection-util.cc | 26 +++
utils/connection-util.h | 62 +++++++
utils/sys-util.h | 39 +++++
utils/time-util.h | 52 ++++++
31 files changed, 1113 insertions(+), 106 deletions(-)
diff --git a/Makefile b/Makefile
index 84ae556..b926220 100644
--- a/Makefile
+++ b/Makefile
@@ -22,7 +22,7 @@ LD:=g++
DEBUG_PATH = build/debug
RELEASE_PATH = build/release
PROTO_SRC_DIR = build/if
-MODULES = connection core serde test-util utils security
+MODULES = connection core serde test-util utils security exceptions
SRC_DIR = $(MODULES)
DEBUG_BUILD_DIR = $(addprefix $(DEBUG_PATH)/,$(MODULES))
RELEASE_BUILD_DIR = $(addprefix $(RELEASE_PATH)/,$(MODULES))
diff --git a/bin/start-docker.sh b/bin/start-docker.sh
index 1380cdf..8b017a0 100755
--- a/bin/start-docker.sh
+++ b/bin/start-docker.sh
@@ -56,7 +56,7 @@ docker build -t hbase_native .
# After the image is built run the thing
docker run -p 16050:16050/tcp \
- -v ${BASE_DIR}/..:/usr/src/hbase \
+ -v ${BASE_DIR}/..:/usr/src/hbase \
-v ~/.m2:/root/.m2 \
-it hbase_native /bin/bash
popd
diff --git a/connection/connection-factory.cc b/connection/connection-factory.cc
index 2f7e75c..832b00f 100644
--- a/connection/connection-factory.cc
+++ b/connection/connection-factory.cc
@@ -31,11 +31,10 @@ using std::chrono::milliseconds;
using std::chrono::nanoseconds;
ConnectionFactory::ConnectionFactory(std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool,
- std::shared_ptr<Codec> codec,
- nanoseconds connect_timeout)
+ std::shared_ptr<Codec> codec, nanoseconds connect_timeout)
: connect_timeout_(connect_timeout),
- io_pool_(io_pool),
- pipeline_factory_(std::make_shared<RpcPipelineFactory>(codec)) {}
+ io_pool_(io_pool),
+ pipeline_factory_(std::make_shared<RpcPipelineFactory>(codec)) {}
std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> ConnectionFactory::MakeBootstrap() {
auto client = std::make_shared<wangle::ClientBootstrap<SerializePipeline>>();
diff --git a/connection/connection-factory.h b/connection/connection-factory.h
index fbcb6ef..32d0bf7 100644
--- a/connection/connection-factory.h
+++ b/connection/connection-factory.h
@@ -44,8 +44,7 @@ class ConnectionFactory {
* There should only be one ConnectionFactory per client.
*/
ConnectionFactory(std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool,
- std::shared_ptr<Codec> codec,
- nanoseconds connect_timeout = nanoseconds(0));
+ std::shared_ptr<Codec> codec, nanoseconds connect_timeout = nanoseconds(0));
/** Default Destructor */
virtual ~ConnectionFactory() = default;
diff --git a/connection/connection-pool.cc b/connection/connection-pool.cc
index b18ee89..4fe4610 100644
--- a/connection/connection-pool.cc
+++ b/connection/connection-pool.cc
@@ -22,6 +22,7 @@
#include <folly/SocketAddress.h>
#include <wangle/service/Service.h>
+#include <folly/Logging.h>
#include <memory>
#include <utility>
@@ -34,8 +35,7 @@ using folly::SharedMutexWritePriority;
using folly::SocketAddress;
ConnectionPool::ConnectionPool(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
- std::shared_ptr<Codec> codec,
- nanoseconds connect_timeout)
+ std::shared_ptr<Codec> codec, nanoseconds connect_timeout)
: cf_(std::make_shared<ConnectionFactory>(io_executor, codec, connect_timeout)),
clients_(),
connections_(),
diff --git a/connection/rpc-client.cc b/connection/rpc-client.cc
index c61a73e..5fa1138 100644
--- a/connection/rpc-client.cc
+++ b/connection/rpc-client.cc
@@ -18,27 +18,17 @@
*/
#include "connection/rpc-client.h"
+
+#include <folly/Logging.h>
#include <unistd.h>
#include <wangle/concurrent/IOThreadPoolExecutor.h>
+#include <memory>
+#include <string>
using hbase::RpcClient;
-using hbase::AbstractRpcChannel;
namespace hbase {
-class RpcChannelImplementation : public AbstractRpcChannel {
- public:
- RpcChannelImplementation(std::shared_ptr<RpcClient> rpc_client, const std::string& host,
- uint16_t port, std::shared_ptr<User> ticket, int rpc_timeout)
- : AbstractRpcChannel(rpc_client, host, port, ticket, rpc_timeout) {}
-
- void CallMethod(const MethodDescriptor* method, RpcController* controller, const Message* request,
- Message* response, Closure* done) override {
- rpc_client_->CallMethod(method, controller, request, response, done, host_, port_, ticket_);
- }
-};
-} // namespace hbase
-
RpcClient::RpcClient(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
std::shared_ptr<Codec> codec, nanoseconds connect_timeout)
: io_executor_(io_executor) {
@@ -80,26 +70,4 @@ folly::Future<std::unique_ptr<Response>> RpcClient::AsyncCall(const std::string&
std::shared_ptr<RpcConnection> RpcClient::GetConnection(std::shared_ptr<ConnectionId> remote_id) {
return cp_->GetConnection(remote_id);
}
-
-std::shared_ptr<RpcChannel> RpcClient::CreateRpcChannel(const std::string& host, uint16_t port,
- std::shared_ptr<User> ticket,
- int rpc_timeout) {
- std::shared_ptr<RpcChannelImplementation> channel = std::make_shared<RpcChannelImplementation>(
- shared_from_this(), host, port, ticket, rpc_timeout);
-
- /* static_pointer_cast is safe since RpcChannelImplementation derives
- * from RpcChannel, otherwise, dynamic_pointer_cast should be used. */
- return std::static_pointer_cast<RpcChannel>(channel);
-}
-
-void RpcClient::CallMethod(const MethodDescriptor* method, RpcController* controller,
- const Message* req_msg, Message* resp_msg, Closure* done,
- const std::string& host, uint16_t port, std::shared_ptr<User> ticket) {
- std::shared_ptr<Message> shared_req(const_cast<Message*>(req_msg));
- std::shared_ptr<Message> shared_resp(resp_msg);
-
- std::unique_ptr<Request> req = std::make_unique<Request>(shared_req, shared_resp, method->name());
-
- AsyncCall(host, port, std::move(req), ticket, method->service()->name())
- .then([done, this](std::unique_ptr<Response> resp) { done->Run(); });
-}
+} // namespace hbase
diff --git a/connection/rpc-client.h b/connection/rpc-client.h
index 5c11ab5..d416ceb 100644
--- a/connection/rpc-client.h
+++ b/connection/rpc-client.h
@@ -38,24 +38,15 @@ using hbase::ConnectionPool;
using hbase::RpcConnection;
using hbase::security::User;
-using google::protobuf::MethodDescriptor;
-using google::protobuf::RpcChannel;
using google::protobuf::Message;
-using google::protobuf::RpcController;
-using google::protobuf::Closure;
-
using std::chrono::nanoseconds;
-class RpcChannelImplementation;
-
namespace hbase {
-class RpcClient : public std::enable_shared_from_this<RpcClient> {
- friend class RpcChannelImplementation;
-
+class RpcClient {
public:
- RpcClient(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
- std::shared_ptr<Codec> codec, nanoseconds connect_timeout);
+ RpcClient(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor, std::shared_ptr<Codec> codec,
+ nanoseconds connect_timeout = nanoseconds(0));
virtual ~RpcClient() { Close(); }
@@ -79,40 +70,13 @@ class RpcClient : public std::enable_shared_from_this<RpcClient> {
virtual void Close();
- virtual std::shared_ptr<RpcChannel> CreateRpcChannel(const std::string &host, uint16_t port,
- std::shared_ptr<User> ticket,
- int rpc_timeout);
-
std::shared_ptr<ConnectionPool> connection_pool() const { return cp_; }
private:
- void CallMethod(const MethodDescriptor *method, RpcController *controller, const Message *req_msg,
- Message *resp_msg, Closure *done, const std::string &host, uint16_t port,
- std::shared_ptr<User> ticket);
std::shared_ptr<RpcConnection> GetConnection(std::shared_ptr<ConnectionId> remote_id);
private:
std::shared_ptr<ConnectionPool> cp_;
std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_;
};
-
-class AbstractRpcChannel : public RpcChannel {
- public:
- AbstractRpcChannel(std::shared_ptr<RpcClient> rpc_client, const std::string &host, uint16_t port,
- std::shared_ptr<User> ticket, int rpc_timeout)
- : rpc_client_(rpc_client),
- host_(host),
- port_(port),
- ticket_(ticket),
- rpc_timeout_(rpc_timeout) {}
-
- virtual ~AbstractRpcChannel() = default;
-
- protected:
- std::shared_ptr<RpcClient> rpc_client_;
- std::string host_;
- uint16_t port_;
- std::shared_ptr<User> ticket_;
- int rpc_timeout_;
-};
} // namespace hbase
diff --git a/core/BUCK b/core/BUCK
index e541d8f..2f4f6c1 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -40,6 +40,9 @@ cxx_library(
"request_converter.h",
"response_converter.h",
"table.h",
+ "async-rpc-retrying-caller-factory.h",
+ "async-rpc-retrying-caller.h",
+ "hbase-rpc-controller.h",
],
srcs=[
"cell.cc",
@@ -58,6 +61,8 @@ cxx_library(
"table.cc",
],
deps=[
+ "//exceptions:exceptions",
+ "//utils:utils",
"//connection:connection",
"//if:if",
"//serde:serde",
@@ -96,6 +101,15 @@ cxx_test(
deps=[":core",],
run_test_separately=True,)
cxx_test(
+ name="retry-test",
+ srcs=["async-rpc-retrying-test.cc",],
+ deps=[
+ ":core",
+ "//test-util:test-util",
+ "//exceptions:exceptions",
+ ],
+ run_test_separately=True,)
+cxx_test(
name="time_range-test",
srcs=["time_range-test.cc",],
deps=[":core",],
diff --git a/core/async-rpc-retrying-caller-factory.cc b/core/async-rpc-retrying-caller-factory.cc
new file mode 100644
index 0000000..0ac9cac
--- /dev/null
+++ b/core/async-rpc-retrying-caller-factory.cc
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/async-rpc-retrying-caller-factory.h"
+
+namespace hbase {} // namespace hbase
diff --git a/core/async-rpc-retrying-caller-factory.h b/core/async-rpc-retrying-caller-factory.h
new file mode 100644
index 0000000..3342e29
--- /dev/null
+++ b/core/async-rpc-retrying-caller-factory.h
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <folly/Logging.h>
+#include <folly/io/IOBuf.h>
+#include <folly/io/async/EventBase.h>
+#include <chrono>
+#include <memory>
+#include <string>
+
+#include "connection/rpc-client.h"
+#include "core/async-rpc-retrying-caller.h"
+#include "if/Client.pb.h"
+#include "if/HBase.pb.h"
+
+using hbase::pb::TableName;
+using std::chrono::nanoseconds;
+
+namespace hbase {
+
+template <typename CONN, typename RESP, typename RPC_CLIENT>
+class SingleRequestCallerBuilder
+ : public std::enable_shared_from_this<SingleRequestCallerBuilder<CONN, RESP, RPC_CLIENT>> {
+ public:
+ explicit SingleRequestCallerBuilder(std::shared_ptr<CONN> conn)
+ : conn_(conn),
+ table_name_(nullptr),
+ rpc_timeout_nanos_(0),
+ operation_timeout_nanos_(0),
+ locate_type_(RegionLocateType::kCurrent) {}
+
+ virtual ~SingleRequestCallerBuilder() = default;
+
+ typedef SingleRequestCallerBuilder<CONN, RESP, RPC_CLIENT> GenenericThisType;
+ typedef std::shared_ptr<GenenericThisType> SharedThisPtr;
+
+ SharedThisPtr table(std::shared_ptr<TableName> table_name) {
+ table_name_ = table_name;
+ return shared_this();
+ }
+
+ SharedThisPtr rpc_timeout(nanoseconds rpc_timeout_nanos) {
+ rpc_timeout_nanos_ = rpc_timeout_nanos;
+ return shared_this();
+ }
+
+ SharedThisPtr operation_timeout(nanoseconds operation_timeout_nanos) {
+ operation_timeout_nanos_ = operation_timeout_nanos;
+ return shared_this();
+ }
+
+ SharedThisPtr row(const std::string& row) {
+ row_ = row;
+ return shared_this();
+ }
+
+ SharedThisPtr locate_type(RegionLocateType locate_type) {
+ locate_type_ = locate_type;
+ return shared_this();
+ }
+
+ SharedThisPtr action(Callable<RESP, RPC_CLIENT> callable) {
+ callable_ = callable;
+ return shared_this();
+ }
+
+ folly::Future<RESP> Call() { return Build()->Call(); }
+
+ std::shared_ptr<AsyncSingleRequestRpcRetryingCaller<CONN, RESP, RPC_CLIENT>> Build() {
+ return std::make_shared<AsyncSingleRequestRpcRetryingCaller<CONN, RESP, RPC_CLIENT>>(
+ conn_, table_name_, row_, locate_type_, callable_, conn_->get_conn_conf()->GetPauseNs(),
+ conn_->get_conn_conf()->GetMaxRetries(), operation_timeout_nanos_, rpc_timeout_nanos_,
+ conn_->get_conn_conf()->GetStartLogErrorsCount());
+ }
+
+ private:
+ SharedThisPtr shared_this() {
+ return std::enable_shared_from_this<GenenericThisType>::shared_from_this();
+ }
+
+ private:
+ std::shared_ptr<CONN> conn_;
+ std::shared_ptr<TableName> table_name_;
+ nanoseconds rpc_timeout_nanos_;
+ nanoseconds operation_timeout_nanos_;
+ std::string row_;
+ RegionLocateType locate_type_;
+ Callable<RESP, RPC_CLIENT> callable_;
+}; // end of SingleRequestCallerBuilder
+
+template <typename CONN>
+class AsyncRpcRetryingCallerFactory {
+ private:
+ std::shared_ptr<CONN> conn_;
+
+ public:
+ explicit AsyncRpcRetryingCallerFactory(std::shared_ptr<CONN> conn) : conn_(conn) {}
+
+ virtual ~AsyncRpcRetryingCallerFactory() = default;
+
+ template <typename RESP, typename RPC_CLIENT = hbase::RpcClient>
+ std::shared_ptr<SingleRequestCallerBuilder<CONN, RESP, RPC_CLIENT>> Single() {
+ return std::make_shared<SingleRequestCallerBuilder<CONN, RESP, RPC_CLIENT>>(conn_);
+ }
+};
+
+} // namespace hbase
diff --git a/core/async-rpc-retrying-caller.cc b/core/async-rpc-retrying-caller.cc
new file mode 100644
index 0000000..743b6bb
--- /dev/null
+++ b/core/async-rpc-retrying-caller.cc
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/async-rpc-retrying-caller.h"
+
+namespace hbase {} /* namespace hbase */
diff --git a/core/async-rpc-retrying-caller.h b/core/async-rpc-retrying-caller.h
new file mode 100644
index 0000000..f7a1523
--- /dev/null
+++ b/core/async-rpc-retrying-caller.h
@@ -0,0 +1,266 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <folly/Format.h>
+#include <folly/Logging.h>
+#include <folly/futures/Future.h>
+#include <folly/io/async/EventBase.h>
+#include <folly/io/async/HHWheelTimer.h>
+#include <algorithm>
+#include <chrono>
+#include <functional>
+#include <memory>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
+#include "connection/rpc-client.h"
+#include "core/hbase-rpc-controller.h"
+#include "core/region-location.h"
+#include "exceptions/exception.h"
+#include "if/HBase.pb.h"
+#include "utils/connection-util.h"
+#include "utils/sys-util.h"
+#include "utils/time-util.h"
+
+using std::chrono::nanoseconds;
+using std::chrono::milliseconds;
+
+namespace hbase {
+
+template <typename T>
+using Supplier = std::function<T()>;
+
+template <typename T>
+using Consumer = std::function<void(T)>;
+
+template <typename R, typename S, typename... I>
+using ReqConverter = std::function<R(const S&, const I&...)>;
+
+template <typename R, typename S>
+using RespConverter = std::function<R(const S&)>;
+
+template <typename RESP>
+using RpcCallback = std::function<void(const RESP&)>;
+
+template <typename REQ, typename RESP, typename RPC_CLIENT = hbase::RpcClient>
+using RpcCall = std::function<folly::Future<std::unique_ptr<RESP>>(
+ std::shared_ptr<RPC_CLIENT>, std::shared_ptr<RegionLocation>,
+ std::shared_ptr<HBaseRpcController>, std::unique_ptr<REQ>)>;
+
+template <typename RESP, typename RPC_CLIENT = hbase::RpcClient>
+using Callable = std::function<folly::Future<RESP>(std::shared_ptr<HBaseRpcController>,
+ std::shared_ptr<RegionLocation>,
+ std::shared_ptr<RPC_CLIENT>)>;
+
+template <typename CONN, typename RESP, typename RPC_CLIENT = hbase::RpcClient>
+class AsyncSingleRequestRpcRetryingCaller {
+ public:
+ AsyncSingleRequestRpcRetryingCaller(std::shared_ptr<CONN> conn,
+ std::shared_ptr<hbase::pb::TableName> table_name,
+ const std::string& row, RegionLocateType locate_type,
+ Callable<RESP, RPC_CLIENT> callable, nanoseconds pause_ns,
+ uint32_t max_retries, nanoseconds operation_timeout_nanos,
+ nanoseconds rpc_timeout_nanos,
+ uint32_t start_log_errors_count)
+ : conn_(conn),
+ table_name_(table_name),
+ row_(row),
+ locate_type_(locate_type),
+ callable_(callable),
+ pause_ns_(pause_ns),
+ max_retries_(max_retries),
+ operation_timeout_nanos_(operation_timeout_nanos),
+ rpc_timeout_nanos_(rpc_timeout_nanos),
+ start_log_errors_count_(start_log_errors_count),
+ promise_(std::make_shared<folly::Promise<RESP>>()),
+ tries_(1) {
+ controller_ = conn_->get_rpc_controller_factory()->NewController();
+ start_ns_ = TimeUtil::GetNowNanos();
+ max_attempts_ = ConnectionUtils::Retries2Attempts(max_retries);
+ exceptions_ = std::make_shared<std::vector<ThrowableWithExtraContext>>();
+ retry_timer_ = folly::HHWheelTimer::newTimer(&event_base_);
+ }
+
+ virtual ~AsyncSingleRequestRpcRetryingCaller() {}
+
+ folly::Future<RESP> Call() {
+ auto f = promise_->getFuture();
+ LocateThenCall();
+ return f;
+ }
+
+ private:
+ void LocateThenCall() {
+ int64_t locate_timeout_ns;
+ if (operation_timeout_nanos_.count() > 0) {
+ locate_timeout_ns = RemainingTimeNs();
+ if (locate_timeout_ns <= 0) {
+ CompleteExceptionally();
+ return;
+ }
+ } else {
+ locate_timeout_ns = -1L;
+ }
+
+ conn_->get_locator()
+ ->GetRegionLocation(table_name_, row_, locate_type_, locate_timeout_ns)
+ .then([this](RegionLocation& loc) { Call(loc); })
+ .onError([this](const std::exception& e) {
+ OnError(e,
+ [this]() -> std::string {
+ return "Locate '" + row_ + "' in " + table_name_->namespace_() + "::" +
+ table_name_->qualifier() + " failed, tries = " + std::to_string(tries_) +
+ ", maxAttempts = " + std::to_string(max_attempts_) + ", timeout = " +
+ TimeUtil::ToMillisStr(operation_timeout_nanos_) +
+ " ms, time elapsed = " + TimeUtil::ElapsedMillisStr(this->start_ns_) +
+ " ms";
+ },
+ [](const std::exception& error) {});
+ });
+ }
+
+ void OnError(const std::exception& error, Supplier<std::string> err_msg,
+ Consumer<std::exception> update_cached_location) {
+ ThrowableWithExtraContext twec(std::make_shared<std::exception>(error),
+ TimeUtil::GetNowNanos());
+ exceptions_->push_back(twec);
+ if (SysUtil::InstanceOf<DoNotRetryIOException, std::exception>(error) ||
+ tries_ >= max_retries_) {
+ CompleteExceptionally();
+ return;
+ }
+
+ int64_t delay_ns;
+ if (operation_timeout_nanos_.count() > 0) {
+ int64_t max_delay_ns = RemainingTimeNs() - ConnectionUtils::kSleepDeltaNs;
+ if (max_delay_ns <= 0) {
+ CompleteExceptionally();
+ return;
+ }
+ delay_ns =
+ std::min(max_delay_ns, ConnectionUtils::GetPauseTime(pause_ns_.count(), tries_ - 1));
+ } else {
+ delay_ns = ConnectionUtils::GetPauseTime(pause_ns_.count(), tries_ - 1);
+ }
+ update_cached_location(error);
+ tries_++;
+ retry_timer_->scheduleTimeoutFn([this]() { LocateThenCall(); },
+ milliseconds(TimeUtil::ToMillis(delay_ns)));
+ }
+
+ void Call(const RegionLocation& loc) {
+ int64_t call_timeout_ns;
+ if (operation_timeout_nanos_.count() > 0) {
+ call_timeout_ns = this->RemainingTimeNs();
+ if (call_timeout_ns <= 0) {
+ this->CompleteExceptionally();
+ return;
+ }
+ call_timeout_ns = std::min(call_timeout_ns, rpc_timeout_nanos_.count());
+ } else {
+ call_timeout_ns = rpc_timeout_nanos_.count();
+ }
+
+ std::shared_ptr<RPC_CLIENT> rpc_client;
+ try {
+ rpc_client = conn_->GetRpcClient();
+ } catch (const IOException& e) {
+ OnError(e,
+ [&, this]() -> std::string {
+ return "Get async rpc_client to " +
+ folly::sformat("{0}:{1}", loc.server_name().host_name(),
+ loc.server_name().port()) +
+ " for '" + row_ + "' in " + loc.DebugString() + " of " +
+ table_name_->namespace_() + "::" + table_name_->qualifier() +
+ " failed, tries = " + std::to_string(tries_) + ", maxAttempts = " +
+ std::to_string(max_attempts_) + ", timeout = " +
+ TimeUtil::ToMillisStr(this->operation_timeout_nanos_) +
+ " ms, time elapsed = " + TimeUtil::ElapsedMillisStr(this->start_ns_) + " ms";
+ },
+ [&, this](const std::exception& error) {
+ conn_->get_locator()->UpdateCachedLocation(loc, error);
+ });
+ return;
+ }
+
+ ResetController(controller_, call_timeout_ns);
+
+ callable_(controller_, std::make_shared<RegionLocation>(loc), rpc_client)
+ .then([this](const RESP& resp) { this->promise_->setValue(std::move(resp)); })
+ .onError([&, this](const std::exception& e) {
+ OnError(e,
+ [&, this]() -> std::string {
+ return "Call to " + folly::sformat("{0}:{1}", loc.server_name().host_name(),
+ loc.server_name().port()) +
+ " for '" + row_ + "' in " + loc.DebugString() + " of " +
+ table_name_->namespace_() + "::" + table_name_->qualifier() +
+ " failed, tries = " + std::to_string(tries_) + ", maxAttempts = " +
+ std::to_string(max_attempts_) + ", timeout = " +
+ TimeUtil::ToMillisStr(this->operation_timeout_nanos_) +
+ " ms, time elapsed = " + TimeUtil::ElapsedMillisStr(this->start_ns_) +
+ " ms";
+ },
+ [&, this](const std::exception& error) {
+ conn_->get_locator()->UpdateCachedLocation(loc, error);
+ });
+ return;
+ });
+ }
+
+ void CompleteExceptionally() {
+ this->promise_->setException(RetriesExhaustedException(tries_ - 1, exceptions_));
+ }
+
+ int64_t RemainingTimeNs() {
+ return operation_timeout_nanos_.count() - (TimeUtil::GetNowNanos() - start_ns_);
+ }
+
+ static void ResetController(std::shared_ptr<HBaseRpcController> controller,
+ const int64_t& timeout_ns) {
+ controller->Reset();
+ if (timeout_ns >= 0) {
+ controller->set_call_timeout(
+ milliseconds(std::min(static_cast<int64_t>(INT_MAX), TimeUtil::ToMillis(timeout_ns))));
+ }
+ }
+
+ private:
+ folly::HHWheelTimer::UniquePtr retry_timer_;
+ std::shared_ptr<CONN> conn_;
+ std::shared_ptr<hbase::pb::TableName> table_name_;
+ std::string row_;
+ RegionLocateType locate_type_;
+ Callable<RESP, RPC_CLIENT> callable_;
+ nanoseconds pause_ns_;
+ uint32_t max_retries_;
+ nanoseconds operation_timeout_nanos_;
+ nanoseconds rpc_timeout_nanos_;
+ uint32_t start_log_errors_count_;
+ std::shared_ptr<folly::Promise<RESP>> promise_;
+ std::shared_ptr<HBaseRpcController> controller_;
+ uint64_t start_ns_;
+ uint32_t tries_;
+ std::shared_ptr<std::vector<ThrowableWithExtraContext>> exceptions_;
+ uint32_t max_attempts_;
+ folly::EventBase event_base_;
+};
+
+} /* namespace hbase */
diff --git a/core/async-rpc-retrying-test.cc b/core/async-rpc-retrying-test.cc
new file mode 100644
index 0000000..a9b0017
--- /dev/null
+++ b/core/async-rpc-retrying-test.cc
@@ -0,0 +1,255 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <folly/Logging.h>
+#include <folly/Memory.h>
+#include <folly/futures/Future.h>
+#include <gmock/gmock.h>
+#include <google/protobuf/stubs/callback.h>
+#include <wangle/concurrent/IOThreadPoolExecutor.h>
+
+#include <functional>
+#include <string>
+
+#include "connection/request.h"
+#include "connection/response.h"
+#include "connection/rpc-client.h"
+#include "core/async-rpc-retrying-caller-factory.h"
+#include "core/async-rpc-retrying-caller.h"
+#include "core/client.h"
+#include "core/hbase-rpc-controller.h"
+#include "core/keyvalue-codec.h"
+#include "core/region-location.h"
+#include "core/request_converter.h"
+#include "core/response_converter.h"
+#include "core/result.h"
+#include "exceptions/exception.h"
+#include "if/Client.pb.h"
+#include "if/HBase.pb.h"
+#include "test-util/test-util.h"
+
+using namespace google::protobuf;
+using namespace hbase;
+using namespace hbase::pb;
+using namespace std::placeholders;
+using namespace testing;
+using ::testing::Return;
+using ::testing::_;
+using std::chrono::nanoseconds;
+
+class MockRpcControllerFactory {
+ public:
+ MOCK_METHOD0(NewController, std::shared_ptr<HBaseRpcController>());
+};
+
+class MockAsyncConnectionConfiguration {
+ public:
+ MOCK_METHOD0(GetPauseNs, nanoseconds());
+ MOCK_METHOD0(GetMaxRetries, int32_t());
+ MOCK_METHOD0(GetStartLogErrorsCount, int32_t());
+ MOCK_METHOD0(GetReadRpcTimeoutNs, nanoseconds());
+ MOCK_METHOD0(GetOperationTimeoutNs, nanoseconds());
+};
+
+class AsyncRegionLocator {
+ public:
+ explicit AsyncRegionLocator(std::shared_ptr<RegionLocation> region_location)
+ : region_location_(region_location) {}
+ ~AsyncRegionLocator() = default;
+
+ folly::Future<RegionLocation> GetRegionLocation(std::shared_ptr<hbase::pb::TableName>,
+ const std::string&, RegionLocateType, int64_t) {
+ folly::Promise<RegionLocation> promise;
+ promise.setValue(*region_location_);
+ return promise.getFuture();
+ }
+
+ void UpdateCachedLocation(const RegionLocation&, const std::exception&) {}
+
+ private:
+ std::shared_ptr<RegionLocation> region_location_;
+};
+
+class MockAsyncConnection {
+ public:
+ MOCK_METHOD0(get_conn_conf, std::shared_ptr<MockAsyncConnectionConfiguration>());
+ MOCK_METHOD0(get_rpc_controller_factory, std::shared_ptr<MockRpcControllerFactory>());
+ MOCK_METHOD0(get_locator, std::shared_ptr<AsyncRegionLocator>());
+ MOCK_METHOD0(GetRpcClient, std::shared_ptr<hbase::RpcClient>());
+};
+
+template <typename CONN>
+class MockRawAsyncTableImpl {
+ public:
+ explicit MockRawAsyncTableImpl(std::shared_ptr<CONN> conn)
+ : conn_(conn), promise_(std::make_shared<folly::Promise<hbase::Result>>()) {}
+ virtual ~MockRawAsyncTableImpl() = default;
+
+ /* implement this in real RawAsyncTableImpl. */
+
+ /* in real RawAsyncTableImpl, this should be private. */
+ folly::Future<hbase::Result> GetCall(std::shared_ptr<hbase::RpcClient> rpc_client,
+ std::shared_ptr<HBaseRpcController> controller,
+ std::shared_ptr<RegionLocation> loc, const hbase::Get& get) {
+ hbase::RpcCall<hbase::Request, hbase::Response, hbase::RpcClient> rpc_call = [](
+ std::shared_ptr<hbase::RpcClient> rpc_client, std::shared_ptr<RegionLocation> loc,
+ std::shared_ptr<HBaseRpcController> controller,
+ std::unique_ptr<hbase::Request> preq) -> folly::Future<std::unique_ptr<hbase::Response>> {
+ return rpc_client->AsyncCall(loc->server_name().host_name(), loc->server_name().port(),
+ std::move(preq), User::defaultUser(), "ClientService");
+ };
+
+ return Call<hbase::Get, hbase::Request, hbase::Response, hbase::Result>(
+ rpc_client, controller, loc, get, &hbase::RequestConverter::ToGetRequest, rpc_call,
+ &hbase::ResponseConverter::FromGetResponse);
+ }
+
+ /* in real RawAsyncTableImpl, this should be private. */
+ template <typename REQ, typename PREQ, typename PRESP, typename RESP>
+ folly::Future<RESP> Call(
+ std::shared_ptr<hbase::RpcClient> rpc_client, std::shared_ptr<HBaseRpcController> controller,
+ std::shared_ptr<RegionLocation> loc, const REQ& req,
+ const ReqConverter<std::unique_ptr<PREQ>, REQ, std::string>& req_converter,
+ const hbase::RpcCall<PREQ, PRESP, hbase::RpcClient>& rpc_call,
+ const RespConverter<std::unique_ptr<RESP>, PRESP>& resp_converter) {
+ rpc_call(rpc_client, loc, controller, std::move(req_converter(req, loc->region_name())))
+ .then([&, this](std::unique_ptr<PRESP> presp) {
+ std::unique_ptr<hbase::Result> result = hbase::ResponseConverter::FromGetResponse(*presp);
+ promise_->setValue(std::move(*result));
+ })
+ .onError([this](const std::exception& e) { promise_->setException(e); });
+ return promise_->getFuture();
+ }
+
+ private:
+ std::shared_ptr<CONN> conn_;
+ std::shared_ptr<folly::Promise<hbase::Result>> promise_;
+};
+
+TEST(AsyncRpcRetryTest, TestGetBasic) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+
+ // Using TestUtil to populate test data
+ hbase::TestUtil* test_util = new hbase::TestUtil();
+ test_util->RunShellCmd("create 't', 'd'");
+ test_util->RunShellCmd("put 't', 'test2', 'd:2', 'value2'");
+ test_util->RunShellCmd("put 't', 'test2', 'd:extra', 'value for extra'");
+
+ // Create TableName and Row to be fetched from HBase
+ auto tn = folly::to<hbase::pb::TableName>("t");
+ auto row = "test2";
+
+ // Get to be performed on above HBase Table
+ hbase::Get get(row);
+
+ // Create Configuration
+ hbase::Configuration conf;
+
+ // Create a client
+ Client client(conf);
+
+ // Get connection to HBase Table
+ auto table = client.Table(tn);
+ ASSERT_TRUE(table) << "Unable to get connection to Table.";
+
+ /* init region location and rpc channel */
+ auto region_location = table->GetRegionLocation(row);
+
+ auto io_executor_ = std::make_shared<wangle::IOThreadPoolExecutor>(1);
+ auto codec = std::make_shared<hbase::KeyValueCodec>();
+ auto rpc_client = std::make_shared<RpcClient>(io_executor_, codec);
+
+ /* init rpc controller */
+ auto controller = std::make_shared<HBaseRpcController>();
+
+ /* init rpc controller factory */
+ auto controller_factory = std::make_shared<MockRpcControllerFactory>();
+ EXPECT_CALL((*controller_factory), NewController()).Times(1).WillRepeatedly(Return(controller));
+
+ /* init connection configuration */
+ auto connection_conf = std::make_shared<MockAsyncConnectionConfiguration>();
+ EXPECT_CALL((*connection_conf), GetPauseNs())
+ .Times(1)
+ .WillRepeatedly(Return(nanoseconds(100000000)));
+ EXPECT_CALL((*connection_conf), GetMaxRetries()).Times(1).WillRepeatedly(Return(31));
+ EXPECT_CALL((*connection_conf), GetStartLogErrorsCount()).Times(1).WillRepeatedly(Return(9));
+ EXPECT_CALL((*connection_conf), GetReadRpcTimeoutNs())
+ .Times(1)
+ .WillRepeatedly(Return(nanoseconds(60000000000)));
+ EXPECT_CALL((*connection_conf), GetOperationTimeoutNs())
+ .Times(1)
+ .WillRepeatedly(Return(nanoseconds(1200000000000)));
+
+ /* init region locator */
+ auto region_locator = std::make_shared<AsyncRegionLocator>(region_location);
+
+ /* init hbase client connection */
+ auto conn = std::make_shared<MockAsyncConnection>();
+ EXPECT_CALL((*conn), get_conn_conf()).Times(AtLeast(1)).WillRepeatedly(Return(connection_conf));
+ EXPECT_CALL((*conn), get_rpc_controller_factory())
+ .Times(AtLeast(1))
+ .WillRepeatedly(Return(controller_factory));
+ EXPECT_CALL((*conn), get_locator()).Times(AtLeast(1)).WillRepeatedly(Return(region_locator));
+ EXPECT_CALL((*conn), GetRpcClient()).Times(AtLeast(1)).WillRepeatedly(Return(rpc_client));
+
+ /* init retry caller factory */
+ auto tableImpl = std::make_shared<MockRawAsyncTableImpl<MockAsyncConnection>>(conn);
+ AsyncRpcRetryingCallerFactory<MockAsyncConnection> caller_factory(conn);
+
+ /* init request caller builder */
+ auto builder = caller_factory.Single<hbase::Result>();
+
+ /* call with retry to get result */
+ try {
+ auto async_caller =
+ builder->table(std::make_shared<TableName>(tn))
+ ->row(row)
+ ->rpc_timeout(conn->get_conn_conf()->GetReadRpcTimeoutNs())
+ ->operation_timeout(conn->get_conn_conf()->GetOperationTimeoutNs())
+ ->action(
+ [=, &get](
+ std::shared_ptr<hbase::HBaseRpcController> controller,
+ std::shared_ptr<hbase::RegionLocation> loc,
+ std::shared_ptr<hbase::RpcClient> rpc_client) -> folly::Future<hbase::Result> {
+ return tableImpl->GetCall(rpc_client, controller, loc, get);
+ })
+ ->Build();
+
+ hbase::Result result = async_caller->Call().get();
+
+ /*Stopping the connection as we are getting segfault due to some folly issue
+ The connection stays open and we don't want that.
+ So we are stopping the connection.
+ We can remove this once we have fixed the folly part */
+ delete test_util;
+
+ // Test the values, should be same as in put executed on hbase shell
+ ASSERT_TRUE(!result.IsEmpty()) << "Result shouldn't be empty.";
+ EXPECT_EQ("test2", result.Row());
+ EXPECT_EQ("value2", *(result.Value("d", "2")));
+ EXPECT_EQ("value for extra", *(result.Value("d", "extra")));
+ } catch (std::exception& e) {
+ LOG(ERROR) << e.what();
+ throw e;
+ }
+
+ table->Close();
+ client.Close();
+}
diff --git a/core/client.cc b/core/client.cc
index 240da72..f0483ef 100644
--- a/core/client.cc
+++ b/core/client.cc
@@ -57,7 +57,8 @@ void Client::init(const hbase::Configuration &conf) {
} else {
LOG(WARNING) << "Not using RPC Cell Codec";
}
- rpc_client_ = std::make_shared<hbase::RpcClient>(io_executor_, codec, conn_conf_->connect_timeout());
+ rpc_client_ =
+ std::make_shared<hbase::RpcClient>(io_executor_, codec, conn_conf_->connect_timeout());
location_cache_ =
std::make_shared<hbase::LocationCache>(conf_, cpu_executor_, rpc_client_->connection_pool());
}
diff --git a/core/client.h b/core/client.h
index a96d6f3..e73ab70 100644
--- a/core/client.h
+++ b/core/client.h
@@ -89,7 +89,7 @@ class Client {
bool is_closed_ = false;
/** Methods */
- void init(const hbase::Configuration &conf);
+ void init(const hbase::Configuration& conf);
};
} // namespace hbase
diff --git a/core/filter.h b/core/filter.h
index b5b7133..10accaa 100644
--- a/core/filter.h
+++ b/core/filter.h
@@ -20,9 +20,9 @@
#pragma once
#include <memory>
+#include <set>
#include <string>
#include <utility>
-#include <set>
#include <vector>
#include "if/Comparator.pb.h"
diff --git a/core/hbase-rpc-controller.cc b/core/hbase-rpc-controller.cc
new file mode 100644
index 0000000..bc53781
--- /dev/null
+++ b/core/hbase-rpc-controller.cc
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/hbase-rpc-controller.h"
+
+namespace hbase {} /* namespace hbase */
diff --git a/core/hbase-rpc-controller.h b/core/hbase-rpc-controller.h
new file mode 100644
index 0000000..661c810
--- /dev/null
+++ b/core/hbase-rpc-controller.h
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <google/protobuf/service.h>
+#include <chrono>
+#include <string>
+
+using google::protobuf::RpcController;
+using google::protobuf::Closure;
+
+using std::chrono::milliseconds;
+
+namespace hbase {
+
+class HBaseRpcController : public RpcController {
+ public:
+ HBaseRpcController() {}
+ virtual ~HBaseRpcController() = default;
+
+ void set_call_timeout(const milliseconds& call_timeout) {
+ // TODO:
+ }
+
+ void Reset() override {}
+
+ bool Failed() const override { return false; }
+
+ std::string ErrorText() const override { return ""; }
+
+ void StartCancel() override {}
+
+ void SetFailed(const std::string& reason) override {}
+
+ bool IsCanceled() const override { return false; }
+
+ void NotifyOnCancel(Closure* callback) override {}
+};
+
+} /* namespace hbase */
diff --git a/core/location-cache.cc b/core/location-cache.cc
index da9f64a..17032fe 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -25,6 +25,7 @@
#include <utility>
+#include <folly/Logging.h>
#include "connection/response.h"
#include "connection/rpc-connection.h"
#include "if/Client.pb.h"
diff --git a/core/region-location.h b/core/region-location.h
index b0411cb..4087d94 100644
--- a/core/region-location.h
+++ b/core/region-location.h
@@ -26,6 +26,8 @@
namespace hbase {
+enum RegionLocateType { kBefore, kCurrent, kAfter };
+
/**
* @brief class to hold where a region is located.
*
@@ -49,17 +51,17 @@ class RegionLocation {
/**
* Get a reference to the regio info
*/
- const hbase::pb::RegionInfo ®ion_info() { return ri_; }
+ const hbase::pb::RegionInfo ®ion_info() const { return ri_; }
/**
* Get a reference to the server name
*/
- const hbase::pb::ServerName &server_name() { return sn_; }
+ const hbase::pb::ServerName &server_name() const { return sn_; }
/**
* Get a reference to the region name.
*/
- const std::string ®ion_name() { return region_name_; }
+ const std::string ®ion_name() const { return region_name_; }
/**
* Get a service. This could be closed or null. It's the caller's
@@ -79,7 +81,7 @@ class RegionLocation {
*/
void set_server_name(hbase::pb::ServerName sn) { sn_ = sn; }
- const std::string DebugString() {
+ const std::string DebugString() const {
return "region_info:" + ri_.ShortDebugString() + ", server_name:" + sn_.ShortDebugString();
}
diff --git a/core/response_converter.cc b/core/response_converter.cc
index 19a3554..2497306 100644
--- a/core/response_converter.cc
+++ b/core/response_converter.cc
@@ -19,6 +19,7 @@
#include "core/response_converter.h"
+#include <string>
#include <vector>
#include "core/cell.h"
diff --git a/core/response_converter.h b/core/response_converter.h
index 859644b..759b1ce 100644
--- a/core/response_converter.h
+++ b/core/response_converter.h
@@ -20,6 +20,7 @@
#pragma once
#include <memory>
+#include <vector>
#include "connection/response.h"
#include "core/result.h"
#include "if/Client.pb.h"
diff --git a/core/table.cc b/core/table.cc
index 4e30d4b..ba4dc29 100644
--- a/core/table.cc
+++ b/core/table.cc
@@ -71,4 +71,8 @@ void Table::Close() {
is_closed_ = true;
}
+std::shared_ptr<RegionLocation> Table::GetRegionLocation(const std::string &row) {
+ return location_cache_->LocateRegion(*table_name_, row).get();
+}
+
} /* namespace hbase */
diff --git a/core/table.h b/core/table.h
index 0e98cd2..f82382e 100644
--- a/core/table.h
+++ b/core/table.h
@@ -57,6 +57,11 @@ class Table {
*/
void Close();
+ /**
+ * @brief - Get region location for a row in current table.
+ */
+ std::shared_ptr<RegionLocation> GetRegionLocation(const std::string &row);
+
private:
std::shared_ptr<TableName> table_name_;
std::shared_ptr<hbase::LocationCache> location_cache_;
diff --git a/utils/BUCK b/exceptions/BUCK
similarity index 71%
copy from utils/BUCK
copy to exceptions/BUCK
index 796f2f5..a23654c 100644
--- a/utils/BUCK
+++ b/exceptions/BUCK
@@ -16,14 +16,9 @@
# limitations under the License.
cxx_library(
- name="utils",
- exported_headers=["user-util.h", "version.h"],
- srcs=["user-util.cc",],
- deps=['//third-party:folly',],
- tests=[":user-util-test"],
- visibility=['PUBLIC',],
- compiler_flags=['-Weffc++'],)
-cxx_test(
- name="user-util-test",
- srcs=["user-util-test.cc",],
- deps=[":utils",],)
+ name="exceptions",
+ exported_headers=["exception.h",],
+ srcs=[],
+ deps=["//third-party:folly",],
+ compiler_flags=['-Weffc++'],
+ visibility=['//core/...'],)
\ No newline at end of file
diff --git a/exceptions/exception.h b/exceptions/exception.h
new file mode 100644
index 0000000..c0c4142
--- /dev/null
+++ b/exceptions/exception.h
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <exception>
+#include <string>
+#include <vector>
+#include <folly/io/IOBuf.h>
+
+namespace hbase {
+
+class ThrowableWithExtraContext {
+public:
+ ThrowableWithExtraContext(std::shared_ptr<std::exception> cause,
+ const long& when) :
+ cause_(cause), when_(when), extras_("") {
+ }
+
+ ThrowableWithExtraContext(std::shared_ptr<std::exception> cause,
+ const long& when, const std::string& extras) :
+ cause_(cause), when_(when), extras_(extras) {
+ }
+
+ std::string ToString() {
+ // TODO:
+ // return new Date(this.when).toString() + ", " + extras + ", " + t.toString();
+ return extras_ + ", " + cause_->what();
+ }
+
+ std::shared_ptr<std::exception> cause() {
+ return cause_;
+ }
+private:
+ std::shared_ptr<std::exception> cause_;
+ long when_;
+ std::string extras_;
+};
+
+class IOException: public std::logic_error {
+public:
+ IOException(
+ const std::string& what) :
+ logic_error(what), cause_(nullptr) {}
+ IOException(
+ const std::string& what,
+ std::shared_ptr<std::exception> cause) :
+ logic_error(what), cause_(cause) {}
+ virtual ~IOException() = default;
+
+ std::shared_ptr<std::exception> cause() {
+ return cause_;
+ }
+private:
+ const std::shared_ptr<std::exception> cause_;
+};
+
+class RetriesExhaustedException: public IOException {
+public:
+ RetriesExhaustedException(
+ const int& num_retries,
+ std::shared_ptr<std::vector<ThrowableWithExtraContext>> exceptions) :
+ IOException(
+ GetMessage(num_retries, exceptions),
+ exceptions->empty() ? nullptr : (*exceptions)[exceptions->size() - 1].cause()){
+ }
+ virtual ~RetriesExhaustedException() = default;
+
+private:
+ std::string GetMessage(
+ const int& num_retries,
+ std::shared_ptr<std::vector<ThrowableWithExtraContext>> exceptions) {
+ std::string buffer("Failed after attempts=");
+ buffer.append(std::to_string(num_retries + 1));
+ buffer.append(", exceptions:\n");
+ for (auto it = exceptions->begin(); it != exceptions->end(); it++) {
+ buffer.append(it->ToString());
+ buffer.append("\n");
+ }
+ return buffer;
+ }
+};
+
+class HBaseIOException : public IOException {
+};
+
+class DoNotRetryIOException : public HBaseIOException {
+};
+} // namespace hbase
diff --git a/utils/BUCK b/utils/BUCK
index 796f2f5..eae929e 100644
--- a/utils/BUCK
+++ b/utils/BUCK
@@ -17,8 +17,11 @@
cxx_library(
name="utils",
- exported_headers=["user-util.h", "version.h"],
- srcs=["user-util.cc",],
+ exported_headers=[
+ "user-util.h", "version.h", "connection-util.h", "sys-util.h",
+ "time-util.h"
+ ],
+ srcs=["user-util.cc", "connection-util.cc"],
deps=['//third-party:folly',],
tests=[":user-util-test"],
visibility=['PUBLIC',],
diff --git a/utils/connection-util.cc b/utils/connection-util.cc
new file mode 100644
index 0000000..76689bf
--- /dev/null
+++ b/utils/connection-util.cc
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "utils/connection-util.h"
+
+namespace hbase {
+
+const std::vector<uint32_t> ConnectionUtils::kRetryBackoff = {1, 2, 3, 5, 10, 20, 40,
+ 100, 100, 100, 100, 200, 200};
+} /* namespace hbase */
diff --git a/utils/connection-util.h b/utils/connection-util.h
new file mode 100644
index 0000000..f52c2f9
--- /dev/null
+++ b/utils/connection-util.h
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <algorithm>
+#include <climits>
+#include <cstdlib>
+#include <memory>
+#include <vector>
+#include "utils/time-util.h"
+
+namespace hbase {
+class ConnectionUtils {
+ public:
+ static int Retries2Attempts(const int& retries) {
+ return std::max(1, retries == INT_MAX ? INT_MAX : retries + 1);
+ }
+
+ /* Add a delta to avoid timeout immediately after a retry sleeping. */
+ static const uint64_t kSleepDeltaNs = 1000000;
+
+ static const std::vector<uint32_t> kRetryBackoff;
+ /**
+ * Calculate pause time. Built on {@link kRetryBackoff}.
+ * @param pause time to pause
+ * @param tries amount of tries
+ * @return How long to wait after <code>tries</code> retries
+ */
+ static int64_t GetPauseTime(const int64_t& pause, const int32_t& tries) {
+ int32_t ntries = tries;
+ if (static_cast<size_t>(ntries) >= kRetryBackoff.size()) {
+ ntries = kRetryBackoff.size() - 1;
+ }
+ if (ntries < 0) {
+ ntries = 0;
+ }
+
+ int64_t normal_pause = pause * kRetryBackoff[ntries];
+ // 1% possible jitter
+ float r = static_cast<float>(std::rand()) / static_cast<float>(RAND_MAX);
+ int64_t jitter = (int64_t)(normal_pause * r * 0.01f);
+ return normal_pause + jitter;
+ }
+};
+} /* namespace hbase */
diff --git a/utils/sys-util.h b/utils/sys-util.h
new file mode 100644
index 0000000..68f00d7
--- /dev/null
+++ b/utils/sys-util.h
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <type_traits>
+
+namespace hbase {
+
+class SysUtil {
+ public:
+ template <class BASE, typename DERIVED>
+ static constexpr bool InstanceOf(const DERIVED& object) {
+ return !dynamic_cast<const BASE*>(&object);
+ }
+
+ template <typename BASE, typename DERIVED>
+ static constexpr bool InstanceOf() {
+ return std::is_base_of<BASE, DERIVED>();
+ }
+};
+
+} /* namespace hbase */
diff --git a/utils/time-util.h b/utils/time-util.h
new file mode 100644
index 0000000..bbc3b35
--- /dev/null
+++ b/utils/time-util.h
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <chrono>
+#include <string>
+using std::chrono::nanoseconds;
+using std::chrono::milliseconds;
+
+namespace hbase {
+class TimeUtil {
+ public:
+ static int64_t ToMillis(const int64_t& nanos) {
+ return std::chrono::duration_cast<milliseconds>(nanoseconds(nanos)).count();
+ }
+
+ static std::string ToMillisStr(const nanoseconds& nanos) {
+ return std::to_string(std::chrono::duration_cast<milliseconds>(nanos).count());
+ }
+
+ static int64_t GetNowNanos() {
+ auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
+ return std::chrono::duration_cast<nanoseconds>(duration).count();
+ }
+
+ static int64_t ElapsedMillis(const int64_t& start_ns) {
+ return std::chrono::duration_cast<milliseconds>(nanoseconds(GetNowNanos() - start_ns)).count();
+ }
+
+ static std::string ElapsedMillisStr(const int64_t& start_ns) {
+ return std::to_string(
+ std::chrono::duration_cast<milliseconds>(nanoseconds(GetNowNanos() - start_ns)).count());
+ }
+};
+} /* namespace hbase */
[hbase] 32/137: HBASE-15774 Fix Upgrade lock usage in connection
pool.
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit f107bbbb1fac741a619040b6c67a9d8c38f35f91
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Tue May 10 17:44:41 2016 -0700
HBASE-15774 Fix Upgrade lock usage in connection pool.
Summary:
Use upgrade lock better.
Fix a clang warning around initializing the UpgradeHolder incorrectly.
Remove dead code. ( We'l need to add it back when there's a better plan)
Add on more comments.
Test Plan: buck test --all
Differential Revision: https://reviews.facebook.net/D58005
---
connection/connection-pool-test.cc | 12 +++----
connection/connection-pool.cc | 47 +++++++++++++++++++++-----
connection/connection-pool.h | 6 ++--
core/client.h | 2 +-
core/location-cache.cc | 68 ++++++++------------------------------
5 files changed, 62 insertions(+), 73 deletions(-)
diff --git a/connection/connection-pool-test.cc b/connection/connection-pool-test.cc
index c0c346f..bd2d585 100644
--- a/connection/connection-pool-test.cc
+++ b/connection/connection-pool-test.cc
@@ -79,9 +79,9 @@ TEST(TestConnectionPool, TestOnlyCreateOnce) {
sn.set_host_name(hostname);
sn.set_port(port);
- auto result = cp.get(sn);
+ auto result = cp.Get(sn);
ASSERT_TRUE(result != nullptr);
- result = cp.get(sn);
+ result = cp.Get(sn);
}
TEST(TestConnectionPool, TestOnlyCreateMultipleDispose) {
@@ -102,13 +102,13 @@ TEST(TestConnectionPool, TestOnlyCreateMultipleDispose) {
ConnectionPool cp{mock_cf};
{
- auto result_one = cp.get(folly::to<ServerName>(
+ auto result_one = cp.Get(folly::to<ServerName>(
hostname_one + ":" + folly::to<std::string>(port)));
- auto result_two = cp.get(folly::to<ServerName>(
+ auto result_two = cp.Get(folly::to<ServerName>(
hostname_two + ":" + folly::to<std::string>(port)));
}
- auto result_one = cp.get(
+ auto result_one = cp.Get(
folly::to<ServerName>(hostname_one + ":" + folly::to<std::string>(port)));
- auto result_two = cp.get(
+ auto result_two = cp.Get(
folly::to<ServerName>(hostname_two + ":" + folly::to<std::string>(port)));
}
diff --git a/connection/connection-pool.cc b/connection/connection-pool.cc
index 90e2056..aa3d094 100644
--- a/connection/connection-pool.cc
+++ b/connection/connection-pool.cc
@@ -48,25 +48,54 @@ ConnectionPool::~ConnectionPool() {
clients_.clear();
}
-std::shared_ptr<HBaseService> ConnectionPool::get(const ServerName &sn) {
- // Create a read lock.
- SharedMutexWritePriority::UpgradeHolder holder(map_mutex_);
+std::shared_ptr<HBaseService> ConnectionPool::Get(const ServerName &sn) {
+ // Try and get th cached connection.
+ auto found_ptr = GetCached(sn);
+ // If there's no connection then create it.
+ if (found_ptr == nullptr) {
+ found_ptr = GetNew(sn);
+ }
+ return found_ptr;
+}
+
+std::shared_ptr<HBaseService> ConnectionPool::GetCached(const ServerName &sn) {
+ SharedMutexWritePriority::ReadHolder holder(map_mutex_);
auto found = connections_.find(sn);
- if (found == connections_.end() || found->second == nullptr) {
- // Move the upgradable lock into the write lock if the connection
- // hasn't been found.
- SharedMutexWritePriority::WriteHolder holder(std::move(holder));
+ if (found == connections_.end()) {
+ return nullptr;
+ }
+ return found->second;
+}
+
+std::shared_ptr<HBaseService> ConnectionPool::GetNew(const ServerName &sn) {
+ // Grab the upgrade lock. While we are double checking other readers can
+ // continue on
+ SharedMutexWritePriority::UpgradeHolder u_holder{map_mutex_};
+
+ // Now check if someone else created the connection before we got the lock
+ // This is safe since we hold the upgrade lock.
+ // upgrade lock is more power than the reader lock.
+ auto found = connections_.find(sn);
+ if (found != connections_.end() && found->second != nullptr) {
+ return found->second;
+ } else {
+ // Yeah it looks a lot like there's no connection
+ SharedMutexWritePriority::WriteHolder w_holder{std::move(u_holder)};
+
+ // Make double sure there are not stale connections hanging around.
+ connections_.erase(sn);
+
+ // Nope we are the ones who should create the new connection.
auto client = cf_->MakeBootstrap();
auto dispatcher = cf_->Connect(client, sn.host_name(), sn.port());
clients_.insert(std::make_pair(sn, client));
connections_.insert(std::make_pair(sn, dispatcher));
return dispatcher;
}
- return found->second;
}
-void ConnectionPool::close(const ServerName &sn) {
+void ConnectionPool::Close(const ServerName &sn) {
SharedMutexWritePriority::WriteHolder holder{map_mutex_};
auto found = connections_.find(sn);
diff --git a/connection/connection-pool.h b/connection/connection-pool.h
index 60f00de..605a81b 100644
--- a/connection/connection-pool.h
+++ b/connection/connection-pool.h
@@ -80,14 +80,16 @@ public:
* Get a connection to the server name. Start time is ignored.
* This can be a blocking operation for a short time.
*/
- std::shared_ptr<HBaseService> get(const hbase::pb::ServerName &sn);
+ std::shared_ptr<HBaseService> Get(const hbase::pb::ServerName &sn);
/**
* Close/remove a connection.
*/
- void close(const hbase::pb::ServerName &sn);
+ void Close(const hbase::pb::ServerName &sn);
private:
+ std::shared_ptr<HBaseService> GetCached(const hbase::pb::ServerName& sn);
+ std::shared_ptr<HBaseService> GetNew(const hbase::pb::ServerName& sn);
std::unordered_map<hbase::pb::ServerName, std::shared_ptr<HBaseService>,
ServerNameHash, ServerNameEquals>
connections_;
diff --git a/core/client.h b/core/client.h
index 4db82c4..ba24bb9 100644
--- a/core/client.h
+++ b/core/client.h
@@ -48,9 +48,9 @@ public:
~Client();
private:
- LocationCache location_cache_;
std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_;
+ LocationCache location_cache_;
};
} // namespace hbase
diff --git a/core/location-cache.cc b/core/location-cache.cc
index 9f2a0ef..6ba8add 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -115,7 +115,7 @@ Future<std::shared_ptr<RegionLocation>>
LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
return this->LocateMeta()
.via(cpu_executor_.get())
- .then([this](ServerName sn) { return this->cp_.get(sn); })
+ .then([this](ServerName sn) { return this->cp_.Get(sn); })
.then([tn, row, this](std::shared_ptr<HBaseService> service) {
return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
})
@@ -134,67 +134,25 @@ LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
})
.then([this](std::shared_ptr<RegionLocation> rl) {
// Now fill out the connection.
- rl->set_service(cp_.get(rl->server_name()));
+ rl->set_service(cp_.Get(rl->server_name()));
return rl;
});
}
-/**
- * Filter to remove a service from the location cache and the connection cache
- * on errors
- * or on cloase.
- */
-class RemoveServiceFilter
- : public ServiceFilter<std::unique_ptr<Request>, Response> {
-
-public:
- /** Create a new filter. */
- RemoveServiceFilter(std::shared_ptr<HBaseService> service, ServerName sn,
- ConnectionPool &cp)
- : ServiceFilter<unique_ptr<Request>, Response>(service), sn_(sn),
- cp_(cp) {}
-
- /**
- * Close will remove the connection from all caches.
- */
- folly::Future<folly::Unit> close() override {
- if (!released.exchange(true)) {
- return this->service_->close().then([this]() {
- // TODO(eclark): remove the service from the meta cache.
- this->cp_.close(this->sn_);
- });
- } else {
- return folly::makeFuture();
- }
- }
-
- /** Has this been closed */
- virtual bool isAvailable() override {
- return !released && service_->isAvailable();
- }
-
- /** Send the message. */
- folly::Future<Response> operator()(unique_ptr<Request> req) override {
- // TODO(eclark): add in an on error handler that will
- // remove the region location from the cache if needed.
- // Also close the connection if this is likely to be an error
- // that needs to get a new connection.
- return (*this->service_)(std::move(req));
- }
-
-private:
- std::atomic<bool> released{false};
- hbase::pb::ServerName sn_;
- ConnectionPool &cp_;
-};
-
std::shared_ptr<RegionLocation>
LocationCache::CreateLocation(const Response &resp) {
auto resp_msg = static_pointer_cast<ScanResponse>(resp.resp_msg());
auto &results = resp_msg->results().Get(0);
auto &cells = results.cell();
- auto ri = folly::to<RegionInfo>(cells.Get(0).value());
- auto sn = folly::to<ServerName>(cells.Get(1).value());
- return std::make_shared<RegionLocation>(cells.Get(0).row(), std::move(ri), sn,
- nullptr);
+
+ // TODO(eclark): There should probably be some better error
+ // handling around this.
+ auto cell_zero = cells.Get(0).value();
+ auto cell_one = cells.Get(1).value();
+ auto row = cells.Get(0).row();
+
+ auto region_info = folly::to<RegionInfo>(cell_zero);
+ auto server_name = folly::to<ServerName>(cell_one);
+ return std::make_shared<RegionLocation>(row, std::move(region_info),
+ server_name, nullptr);
}
[hbase] 40/137: HBASE-16174 Hook cell test up,
and fix broken cell test.
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit cdf8a65a9555ef43376eef0977f4aec6012ca396
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Tue Jul 5 11:13:01 2016 -0700
HBASE-16174 Hook cell test up, and fix broken cell test.
---
core/BUCK | 8 ++++++++
core/cell-test.cc | 2 +-
2 files changed, 9 insertions(+), 1 deletion(-)
diff --git a/core/BUCK b/core/BUCK
index 1c926e3..c615426 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -20,6 +20,7 @@ cxx_library(
name="core",
exported_headers=[
"client.h",
+ "cell.h",
"hbase_macros.h",
"region-location.h",
"location-cache.h",
@@ -28,6 +29,7 @@ cxx_library(
"meta-utils.h",
],
srcs=[
+ "cell.cc",
"client.cc",
"location-cache.cc",
"meta-utils.cc",
@@ -51,6 +53,12 @@ cxx_test(name="location-cache-test",
deps=[":core",
"//test-util:test-util", ],
run_test_separately=True, )
+cxx_test(name="cell-test",
+ srcs=[
+ "cell-test.cc",
+ ],
+ deps=[":core", ],
+ run_test_separately=True, )
cxx_binary(name="simple-client",
srcs=["simple-client.cc", ],
deps=[":core", "//connection:connection"], )
diff --git a/core/cell-test.cc b/core/cell-test.cc
index cbe50eb..49f567b 100644
--- a/core/cell-test.cc
+++ b/core/cell-test.cc
@@ -100,7 +100,7 @@ TEST(CellTest, MultipleCellsTest) {
TEST(CellTest, CellRowTest) {
std::string row = "only-row";
- std::string family = "";
+ std::string family = "D";
std::string column = "";
std::string value = "";
long timestamp = std::numeric_limits<long>::max();
[hbase] 86/137: HBASE-15903 Delete Object
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 4545eddf12efc390fd110c2b0d23571a08f4f6fa
Author: tedyu <yu...@gmail.com>
AuthorDate: Wed May 31 11:13:14 2017 -0700
HBASE-15903 Delete Object
---
core/BUCK | 7 +++
core/client-test.cc | 71 ++++++++++++++++++++++++-
core/delete-test.cc | 124 ++++++++++++++++++++++++++++++++++++++++++
core/delete.cc | 133 ++++++++++++++++++++++++++++++++++++++++++++++
core/delete.h | 111 ++++++++++++++++++++++++++++++++++++++
core/raw-async-table.cc | 15 ++++++
core/raw-async-table.h | 2 +
core/request-converter.cc | 13 +++++
core/request-converter.h | 3 ++
core/table.cc | 5 ++
core/table.h | 6 +++
11 files changed, 489 insertions(+), 1 deletion(-)
diff --git a/core/BUCK b/core/BUCK
index 9cea1f6..870c63f 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -38,6 +38,7 @@ cxx_library(
"get.h",
"mutation.h",
"put.h",
+ "delete.h",
"scan.h",
"result.h",
"request-converter.h",
@@ -68,6 +69,7 @@ cxx_library(
"get.cc",
"mutation.cc",
"put.cc",
+ "delete.cc",
"scan.cc",
"raw-async-table.cc",
"result.cc",
@@ -151,6 +153,11 @@ cxx_test(
],
run_test_separately=True,)
cxx_test(
+ name="delete-test",
+ srcs=["delete-test.cc",],
+ deps=[":core",],
+ run_test_separately=True,)
+cxx_test(
name="put-test",
srcs=[
"put-test.cc",
diff --git a/core/client-test.cc b/core/client-test.cc
index 1c6ec4a..b58ca0a 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -22,6 +22,7 @@
#include "core/cell.h"
#include "core/client.h"
#include "core/configuration.h"
+#include "core/delete.h"
#include "core/get.h"
#include "core/hbase-configuration-loader.h"
#include "core/put.h"
@@ -115,6 +116,75 @@ TEST_F(ClientTest, DefaultConfiguration) {
client.Close();
}
+TEST_F(ClientTest, PutGetDelete) {
+ // Using TestUtil to populate test data
+ std::string tableName = "t1";
+ ClientTest::test_util->CreateTable(tableName, "d");
+
+ // Create TableName and Row to be fetched from HBase
+ auto tn = folly::to<hbase::pb::TableName>(tableName);
+ auto row = "test1";
+
+ // Create a client
+ hbase::Client client(*ClientTest::test_util->conf());
+
+ // Get connection to HBase Table
+ auto table = client.Table(tn);
+ ASSERT_TRUE(table) << "Unable to get connection to Table.";
+
+ // Perform Puts
+ std::string valExtra = "value for extra";
+ std::string valExt = "value for ext";
+ table->Put(Put{row}.AddColumn("d", "1", "value1"));
+ // Put two values for column "extra"
+ table->Put(Put{row}.AddColumn("d", "extra", "1st val extra"));
+ table->Put(Put{row}.AddColumn("d", "extra", valExtra));
+ table->Put(Put{row}.AddColumn("d", "ext", valExt));
+
+ // Perform the Get
+ hbase::Get get(row);
+ auto result = table->Get(get);
+
+ // Test the values, should be same as in put executed on hbase shell
+ ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty.";
+ EXPECT_EQ("test1", result->Row());
+ EXPECT_EQ("value1", *(result->Value("d", "1")));
+ EXPECT_EQ(valExtra, *(result->Value("d", "extra")));
+ auto cell = *(result->ColumnCells("d", "extra"))[0];
+ auto tsExtra = cell.Timestamp();
+
+ // delete column "1"
+ table->Delete(hbase::Delete{row}.AddColumn("d", "1"));
+ result = table->Get(get);
+ ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty.";
+ ASSERT_TRUE(result->Value("d", "1") == nullptr) << "Column 1 should be gone";
+ EXPECT_EQ(valExtra, *(result->Value("d", "extra")));
+
+ // delete cell from column "extra" with timestamp tsExtra
+ table->Delete(hbase::Delete{row}.AddColumn("d", "extra", tsExtra));
+ result = table->Get(get);
+ ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty.";
+ ASSERT_TRUE(result->Value("d", "1") == nullptr) << "Column 1 should be gone";
+ ASSERT_TRUE(result->Value("d", "extra") != nullptr) << "Column extra should have value";
+ EXPECT_EQ(valExt, *(result->Value("d", "ext"))) << "Column ext should have value";
+
+ // delete all cells from "extra" column
+ table->Delete(hbase::Delete{row}.AddColumns("d", "extra"));
+ result = table->Get(get);
+ ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty.";
+ ASSERT_TRUE(result->Value("d", "1") == nullptr) << "Column 1 should be gone";
+ ASSERT_TRUE(result->Value("d", "extra") == nullptr) << "Column extra should be gone";
+ EXPECT_EQ(valExt, *(result->Value("d", "ext"))) << "Column ext should have value";
+
+ // Delete the row and verify that subsequent Get returns nothing
+ table->Delete(hbase::Delete{row}.AddFamily("d"));
+ result = table->Get(get);
+ ASSERT_TRUE(result->IsEmpty()) << "Result should be empty.";
+
+ table->Close();
+ client.Close();
+}
+
TEST_F(ClientTest, PutGet) {
// Using TestUtil to populate test data
ClientTest::test_util->CreateTable("t", "d");
@@ -147,7 +217,6 @@ TEST_F(ClientTest, PutGet) {
table->Close();
client.Close();
}
-
TEST_F(ClientTest, GetForNonExistentTable) {
// Create TableName and Row to be fetched from HBase
auto tn = folly::to<hbase::pb::TableName>("t_not_exists");
diff --git a/core/delete-test.cc b/core/delete-test.cc
new file mode 100644
index 0000000..19e844b
--- /dev/null
+++ b/core/delete-test.cc
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include "core/mutation.h"
+#include "core/delete.h"
+#include "utils/time-util.h"
+
+using hbase::Delete;
+using hbase::Cell;
+using hbase::CellType;
+using hbase::Mutation;
+using hbase::TimeUtil;
+
+const constexpr int64_t Mutation::kLatestTimestamp;
+
+TEST(Delete, Row) {
+ Delete del{"foo"};
+ EXPECT_EQ("foo", del.row());
+}
+
+TEST(Delete, Timestamp) {
+ Delete del{"row"};
+
+ // test default timestamp
+ EXPECT_EQ(Mutation::kLatestTimestamp, del.TimeStamp());
+
+ // set custom timestamp
+ auto ts = TimeUtil::ToMillis(TimeUtil::GetNowNanos());
+ del.SetTimeStamp(ts);
+ EXPECT_EQ(ts, del.TimeStamp());
+
+ // Add a column with custom timestamp
+ del.AddColumn("f", "q");
+ auto &cell = del.FamilyMap().at("f")[0];
+ EXPECT_EQ(ts, cell->Timestamp());
+}
+
+TEST(Delete, HasFamilies) {
+ Delete del{"row"};
+
+ EXPECT_EQ(false, del.HasFamilies());
+
+ del.AddColumn("f", "q");
+ EXPECT_EQ(true, del.HasFamilies());
+}
+
+TEST(Delete, Add) {
+ CellType cell_type = CellType::DELETE;
+ std::string row = "row";
+ std::string family = "family";
+ std::string column = "column";
+ int64_t timestamp = std::numeric_limits<int64_t>::max();
+ auto cell = std::make_unique<Cell>(row, family, column, timestamp, "", cell_type);
+
+ // add first cell
+ Delete del{"row"};
+ del.Add(std::move(cell));
+ EXPECT_EQ(1, del.FamilyMap().size());
+ EXPECT_EQ(1, del.FamilyMap().at(family).size());
+
+ // add a non-matching row
+ auto cell2 = std::make_unique<Cell>(row, family, column, timestamp, "", cell_type);
+ Delete del2{"foo"};
+ ASSERT_THROW(del2.Add(std::move(cell2)), std::runtime_error); // rows don't match
+
+ // add a second cell with same family
+ auto cell3 = std::make_unique<Cell>(row, family, "column-2", timestamp, "", cell_type);
+ del.Add(std::move(cell3));
+ EXPECT_EQ(1, del.FamilyMap().size());
+ EXPECT_EQ(2, del.FamilyMap().at(family).size());
+
+ // add a cell to a different family
+ auto cell4 = std::make_unique<Cell>(row, "family-2", "column-2", timestamp, "", cell_type);
+ del.Add(std::move(cell4));
+ EXPECT_EQ(2, del.FamilyMap().size());
+ EXPECT_EQ(1, del.FamilyMap().at("family-2").size());
+}
+
+TEST(Delete, AddColumn) {
+ std::string row = "row";
+ std::string family = "family";
+ std::string column = "column";
+
+ Delete del{"row"};
+ del.AddColumn(family, column);
+ EXPECT_EQ(1, del.FamilyMap().size());
+ EXPECT_EQ(1, del.FamilyMap().at(family).size());
+
+ // add a second cell with same family
+ del.AddColumn(family, "column-2");
+ EXPECT_EQ(1, del.FamilyMap().size());
+ EXPECT_EQ(2, del.FamilyMap().at(family).size());
+
+ // add a cell to a different family
+ del.AddColumn("family-2", column);
+ EXPECT_EQ(2, del.FamilyMap().size());
+ EXPECT_EQ(1, del.FamilyMap().at("family-2").size());
+
+ // use the AddColumn overload
+ auto ts = TimeUtil::ToMillis(TimeUtil::GetNowNanos());
+ del.AddColumn(family, column, ts);
+ EXPECT_EQ(2, del.FamilyMap().size());
+ EXPECT_EQ(3, del.FamilyMap().at(family).size());
+ auto &cell = del.FamilyMap().at(family)[2];
+ EXPECT_EQ(ts, cell->Timestamp());
+}
diff --git a/core/delete.cc b/core/delete.cc
new file mode 100644
index 0000000..b44971f
--- /dev/null
+++ b/core/delete.cc
@@ -0,0 +1,133 @@
+
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/delete.h"
+#include <folly/Conv.h>
+#include <algorithm>
+#include <limits>
+#include <stdexcept>
+#include <utility>
+
+namespace hbase {
+
+/**
+ * @brief Add the specified column to this Delete operation.
+ * This is an expensive call in that on the server-side, it first does a
+ * get to find the latest versions timestamp. Then it adds a delete using
+ * the fetched cells timestamp.
+ * @param family family name
+ * @param qualifier column qualifier
+ */
+Delete& Delete::AddColumn(const std::string& family, const std::string& qualifier) {
+ return AddColumn(family, qualifier, timestamp_);
+}
+
+/**
+ * @brief Add the specified column to this Delete operation.
+ * @param family family name
+ * @param qualifier column qualifier
+ * @param timestamp version timestamp
+ */
+Delete& Delete::AddColumn(const std::string& family, const std::string& qualifier,
+ int64_t timestamp) {
+ if (timestamp < 0) {
+ throw std::runtime_error("Timestamp cannot be negative. ts=" +
+ folly::to<std::string>(timestamp));
+ }
+
+ return Add(std::make_unique<Cell>(row_, family, qualifier, timestamp, "",
+ hbase::CellType::DELETE));
+}
+/**
+ * Delete all versions of the specified column.
+ * @param family family name
+ * @param qualifier column qualifier
+ */
+Delete& Delete::AddColumns(const std::string& family, const std::string& qualifier) {
+ return AddColumns(family, qualifier, timestamp_);
+}
+/**
+ * Delete all versions of the specified column with a timestamp less than
+ * or equal to the specified timestamp.
+ * @param family family name
+ * @param qualifier column qualifier
+ * @param timestamp maximum version timestamp
+ */
+Delete& Delete::AddColumns(const std::string& family, const std::string& qualifier,
+ int64_t timestamp) {
+ if (timestamp < 0) {
+ throw std::runtime_error("Timestamp cannot be negative. ts=" +
+ folly::to<std::string>(timestamp));
+ }
+
+ return Add(std::make_unique<Cell>(row_, family, qualifier, timestamp, "",
+ hbase::CellType::DELETE_COLUMN));
+}
+/**
+ * Delete all versions of all columns of the specified family.
+ * <p>
+ * Overrides previous calls to deleteColumn and deleteColumns for the
+ * specified family.
+ * @param family family name
+ */
+Delete& Delete::AddFamily(const std::string& family) {
+ return AddFamily(family, timestamp_);
+}
+
+/**
+ * Delete all columns of the specified family with a timestamp less than
+ * or equal to the specified timestamp.
+ * <p>
+ * Overrides previous calls to deleteColumn and deleteColumns for the
+ * specified family.
+ * @param family family name
+ * @param timestamp maximum version timestamp
+ */
+Delete& Delete::AddFamily(const std::string& family, int64_t timestamp) {
+ const auto &it = family_map_.find(family);
+ if (family_map_.end() != it) {
+ it->second.clear();
+ } else {
+ family_map_[family];
+ }
+ return Add(std::make_unique<Cell>(row_, family, "", timestamp, "",
+ hbase::CellType::DELETE_FAMILY));
+}
+/**
+ * Delete all columns of the specified family with a timestamp equal to
+ * the specified timestamp.
+ * @param family family name
+ * @param timestamp version timestamp
+ */
+Delete& Delete::AddFamilyVersion(const std::string& family, int64_t timestamp) {
+ return Add(std::make_unique<Cell>(row_, family, "", timestamp, "",
+ hbase::CellType::DELETE_FAMILY_VERSION));
+}
+Delete& Delete::Add(std::unique_ptr<Cell> cell) {
+ if (cell->Row() != row_) {
+ throw std::runtime_error("The row in " + cell->DebugString() +
+ " doesn't match the original one " + row_);
+ }
+
+ family_map_[cell->Family()].push_back(std::move(cell));
+ return *this;
+}
+} // namespace hbase
diff --git a/core/delete.h b/core/delete.h
new file mode 100644
index 0000000..9ebb5a6
--- /dev/null
+++ b/core/delete.h
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+#include "core/cell.h"
+#include "core/mutation.h"
+
+namespace hbase {
+
+class Delete : public Mutation {
+ public:
+ /**
+ * Constructors
+ */
+ /*
+ * If no further operations are done, this will delete everything
+ * associated with the specified row (all versions of all columns in all
+ * families), with timestamp from current point in time to the past.
+ * Cells defining timestamp for a future point in time
+ * (timestamp > current time) will not be deleted.
+ */
+ explicit Delete(const std::string& row) : Mutation(row) {}
+
+ Delete(const std::string& row, int64_t timestamp) : Mutation(row, timestamp) {}
+ Delete(const Delete& cdelete) : Mutation(cdelete) {}
+ Delete& operator=(const Delete& cdelete) {
+ Mutation::operator=(cdelete);
+ return *this;
+ }
+
+ ~Delete() = default;
+
+ /**
+ * @brief Add the specified column to this Delete operation.
+ * @param family family name
+ * @param qualifier column qualifier
+ */
+ Delete& AddColumn(const std::string& family, const std::string& qualifier);
+
+ /**
+ * @brief Add the specified column to this Delete operation.
+ * @param family family name
+ * @param qualifier column qualifier
+ * @param timestamp version timestamp
+ */
+ Delete& AddColumn(const std::string& family, const std::string& qualifier, int64_t timestamp);
+
+ /**
+ * @brief Deletes all versions of the specified column
+ * @param family family name
+ * @param qualifier column qualifier
+ */
+ Delete& AddColumns(const std::string& family, const std::string& qualifier);
+ /**
+ * @brief Deletes all versions of the specified column with a timestamp less than
+ * or equal to the specified timestamp
+ * @param family family name
+ * @param qualifier column qualifier
+ * @param timestamp version timestamp
+ */
+ Delete& AddColumns(const std::string& family, const std::string& qualifier, int64_t timestamp);
+ /**
+ * @brief Add the specified family to this Delete operation.
+ * @param family family name
+ */
+ Delete& AddFamily(const std::string& family);
+
+ /**
+ * @brief Deletes all columns of the specified family with a timestamp less than
+ * or equal to the specified timestamp
+ * @param family family name
+ * @param timestamp version timestamp
+ */
+ Delete& AddFamily(const std::string& family, int64_t timestamp);
+ /**
+ * @brief Deletes all columns of the specified family with a timestamp
+ * equal to the specified timestamp
+ * @param family family name
+ * @param timestamp version timestamp
+ */
+ Delete& AddFamilyVersion(const std::string& family, int64_t timestamp);
+ /**
+ * Advanced use only.
+ * Add an existing delete marker to this Delete object.
+ */
+ Delete& Add(std::unique_ptr<Cell> cell);
+};
+
+} // namespace hbase
diff --git a/core/raw-async-table.cc b/core/raw-async-table.cc
index 1a66a04..26967eb 100644
--- a/core/raw-async-table.cc
+++ b/core/raw-async-table.cc
@@ -94,6 +94,21 @@ folly::Future<folly::Unit> RawAsyncTable::Put(const hbase::Put& put) {
return caller->Call().then([caller](const auto r) { return r; });
}
+folly::Future<folly::Unit> RawAsyncTable::Delete(const hbase::Delete& del) {
+ auto caller =
+ CreateCallerBuilder<folly::Unit>(del.row(), connection_conf_->write_rpc_timeout())
+ ->action([=, &del](std::shared_ptr<hbase::HBaseRpcController> controller,
+ std::shared_ptr<hbase::RegionLocation> loc,
+ std::shared_ptr<hbase::RpcClient> rpc_client) -> folly::Future<folly::Unit> {
+ return Call<hbase::Delete, hbase::Request, hbase::Response, folly::Unit>(
+ rpc_client, controller, loc, del, &hbase::RequestConverter::DeleteToMutateRequest,
+ [](const Response& r) -> folly::Unit { return folly::unit; });
+ })
+ ->Build();
+
+ return caller->Call().then([caller](const auto r) { return r; });
+}
+
folly::Future<std::vector<folly::Try<std::shared_ptr<Result>>>> RawAsyncTable::Get(
const std::vector<hbase::Get>& gets) {
return this->Batch(gets);
diff --git a/core/raw-async-table.h b/core/raw-async-table.h
index 9d9ed91..d042d27 100644
--- a/core/raw-async-table.h
+++ b/core/raw-async-table.h
@@ -29,6 +29,7 @@
#include "core/async-rpc-retrying-caller-factory.h"
#include "core/async-rpc-retrying-caller.h"
#include "core/connection-configuration.h"
+#include "core/delete.h"
#include "core/get.h"
#include "core/put.h"
#include "core/result.h"
@@ -51,6 +52,7 @@ class RawAsyncTable {
folly::Future<std::shared_ptr<Result>> Get(const hbase::Get& get);
+ folly::Future<folly::Unit> Delete(const hbase::Delete& del);
folly::Future<folly::Unit> Put(const hbase::Put& put);
void Close() {}
diff --git a/core/request-converter.cc b/core/request-converter.cc
index eb293f5..85b4d6d 100644
--- a/core/request-converter.cc
+++ b/core/request-converter.cc
@@ -219,4 +219,17 @@ std::unique_ptr<Request> RequestConverter::ToMutateRequest(const Put &put,
VLOG(3) << "Req is " << pb_req->req_msg()->ShortDebugString();
return pb_req;
}
+
+std::unique_ptr<Request> RequestConverter::DeleteToMutateRequest(const Delete &del,
+ const std::string ®ion_name) {
+ auto pb_req = Request::mutate();
+ auto pb_msg = std::static_pointer_cast<hbase::pb::MutateRequest>(pb_req->req_msg());
+ RequestConverter::SetRegion(region_name, pb_msg->mutable_region());
+
+ pb_msg->set_allocated_mutation(
+ ToMutation(MutationType::MutationProto_MutationType_DELETE, del, -1).release());
+
+ VLOG(3) << "Req is " << pb_req->req_msg()->ShortDebugString();
+ return pb_req;
+}
} /* namespace hbase */
diff --git a/core/request-converter.h b/core/request-converter.h
index b604d18..6db1962 100644
--- a/core/request-converter.h
+++ b/core/request-converter.h
@@ -25,6 +25,7 @@
#include "connection/request.h"
#include "core/action.h"
#include "core/cell.h"
+#include "core/delete.h"
#include "core/get.h"
#include "core/mutation.h"
#include "core/put.h"
@@ -66,6 +67,8 @@ class RequestConverter {
static std::unique_ptr<Request> ToMultiRequest(const ActionsByRegion ®ion_requests);
+ static std::unique_ptr<Request> DeleteToMutateRequest(const Delete &del,const std::string ®ion_name);
+
static std::unique_ptr<Request> ToMutateRequest(const Put &put, const std::string ®ion_name);
static std::unique_ptr<pb::MutationProto> ToMutation(const MutationType type,
diff --git a/core/table.cc b/core/table.cc
index a2f31d9..dd26adf 100644
--- a/core/table.cc
+++ b/core/table.cc
@@ -57,6 +57,11 @@ void Table::Put(const hbase::Put &put) {
future.get(operation_timeout());
}
+void Table::Delete(const hbase::Delete &del) {
+ auto future = async_table_->Delete(del);
+ future.get(operation_timeout());
+}
+
milliseconds Table::operation_timeout() const {
return TimeUtil::ToMillis(async_connection_->connection_conf()->operation_timeout());
}
diff --git a/core/table.h b/core/table.h
index 0ee8312..c4217d3 100644
--- a/core/table.h
+++ b/core/table.h
@@ -61,6 +61,12 @@ class Table {
*/
void Put(const hbase::Put &put);
+ /**
+ * @brief - Deletes some data in the table.
+ * @param - del Delete object to perform HBase Delete operation.
+ */
+ void Delete(const hbase::Delete &del);
+
// TODO: Batch Puts
/**
[hbase] 113/137: HBASE-18536. [C++] Add fault injection infra
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit cd6e21f03a63e0784b28608e2d5d6428a98259d6
Author: Xiaobing Zhou <xz...@hortonworks.com>
AuthorDate: Mon Aug 7 17:51:34 2017 -0700
HBASE-18536. [C++] Add fault injection infra
Signed-off-by: Enis Soztutar <en...@apache.org>
---
connection/BUCK | 3 +++
connection/rpc-fault-injector-inl.h | 41 ++++++++++++++++++++++++++++++
connection/rpc-fault-injector.cc | 21 ++++++++++++++++
connection/rpc-fault-injector.h | 50 +++++++++++++++++++++++++++++++++++++
4 files changed, 115 insertions(+)
diff --git a/connection/BUCK b/connection/BUCK
index aaf8fdb..a87d27a 100644
--- a/connection/BUCK
+++ b/connection/BUCK
@@ -35,6 +35,8 @@ cxx_library(
"sasl-util.h",
"rpc-test-server.h",
"rpc-test-server-handler.h",
+ "rpc-fault-injector.h",
+ "rpc-fault-injector-inl.h",
],
srcs=[
"client-dispatcher.cc",
@@ -48,6 +50,7 @@ cxx_library(
"sasl-util.cc",
"rpc-test-server.cc",
"rpc-test-server-handler.cc",
+ "rpc-fault-injector.cc",
],
deps=[
"//if:if",
diff --git a/connection/rpc-fault-injector-inl.h b/connection/rpc-fault-injector-inl.h
new file mode 100644
index 0000000..8bbaddf
--- /dev/null
+++ b/connection/rpc-fault-injector-inl.h
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+namespace hbase {
+
+template <typename T>
+std::shared_ptr<T> RpcFaultInjector<T>::instance = std::make_shared<T>();
+
+template <typename T>
+RpcFaultInjector<T>::RpcFaultInjector() {}
+
+template <typename T>
+RpcFaultInjector<T>::~RpcFaultInjector() {}
+
+template <typename T>
+std::shared_ptr<T> RpcFaultInjector<T>::Get() {
+ return instance;
+}
+
+template <typename T>
+void RpcFaultInjector<T>::Set(std::shared_ptr<T> injector) {
+ instance = injector;
+}
+} /* namespace hbase */
diff --git a/connection/rpc-fault-injector.cc b/connection/rpc-fault-injector.cc
new file mode 100644
index 0000000..16e2034
--- /dev/null
+++ b/connection/rpc-fault-injector.cc
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include "rpc-fault-injector.h"
+
+namespace hbase {} /* namespace hbase */
diff --git a/connection/rpc-fault-injector.h b/connection/rpc-fault-injector.h
new file mode 100644
index 0000000..2733b7d
--- /dev/null
+++ b/connection/rpc-fault-injector.h
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <folly/io/async/AsyncTransport.h>
+#include "connection/pipeline.h"
+
+namespace hbase {
+
+template <typename T>
+class RpcFaultInjector {
+ public:
+ RpcFaultInjector();
+ virtual ~RpcFaultInjector();
+
+ static std::shared_ptr<T> Get();
+ static void Set(std::shared_ptr<T> instance);
+
+ private:
+ static std::shared_ptr<T> instance;
+};
+
+class RpcClientFaultInjector : public RpcFaultInjector<RpcClientFaultInjector> {
+ public:
+ RpcClientFaultInjector() {}
+ virtual ~RpcClientFaultInjector() {}
+ /**
+ * Here goes virtual functions for injecting various faults. They should be no-ops by default.
+ * Sub classes of RpcClientFaultInjector will override by providing concrete faults.
+ */
+};
+} /* namespace hbase */
+
+#include "connection/rpc-fault-injector-inl.h"
[hbase] 27/137: HBASE-15777 Fix needs header in client handler
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 14e1b9c152e112f0d1ccb3f958215b9cee66ee1a
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Thu May 5 13:51:18 2016 -0700
HBASE-15777 Fix needs header in client handler
---
connection/client-handler.cc | 31 ++++++++++++++++++-------------
connection/client-handler.h | 19 ++++++++++++++++++-
2 files changed, 36 insertions(+), 14 deletions(-)
diff --git a/connection/client-handler.cc b/connection/client-handler.cc
index b92ad89..4fdb7ae 100644
--- a/connection/client-handler.cc
+++ b/connection/client-handler.cc
@@ -37,7 +37,7 @@ using hbase::pb::GetResponse;
using google::protobuf::Message;
ClientHandler::ClientHandler(std::string user_name)
- : user_name_(user_name), need_send_header_(true), serde_(),
+ : user_name_(user_name), serde_(), header_info_(),
resp_msgs_(
make_unique<folly::AtomicHashMap<
uint32_t, std::shared_ptr<google::protobuf::Message>>>(5000)) {}
@@ -81,22 +81,27 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
}
}
-// TODO(eclark): Figure out how to handle the
-// network errors that are going to come.
Future<Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
// Keep track of if we have sent the header.
- if (UNLIKELY(need_send_header_)) {
- need_send_header_ = false;
+ //
+ // even though the bool is atomic we can load it lazily here.
+ if (UNLIKELY(header_info_->need_.load(std::memory_order_relaxed))) {
- // Should we be sending just one fireWrite?
- // Right now we're sending one for the header
- // and one for the request.
+ // Grab the lock.
+ // We need to make sure that no one gets past here without there being a
+ // hearder sent.
+ std::lock_guard<std::mutex> lock(header_info_->mutex_);
+
+ // Now see if we are the first thread to get through.
//
- // That doesn't seem like too bad, but who knows.
- auto pre = serde_.Preamble();
- auto header = serde_.Header(user_name_);
- pre->appendChain(std::move(header));
- ctx->fireWrite(std::move(pre));
+ // If this is the first thread to get through then the
+ // need_send_header will have been true before this.
+ if (header_info_->need_.exchange(false)) {
+ auto pre = serde_.Preamble();
+ auto header = serde_.Header(user_name_);
+ pre->appendChain(std::move(header));
+ ctx->fireWrite(std::move(pre));
+ }
}
resp_msgs_->insert(r->call_id(), r->resp_msg());
diff --git a/connection/client-handler.h b/connection/client-handler.h
index be5143c..1a4275f 100644
--- a/connection/client-handler.h
+++ b/connection/client-handler.h
@@ -21,6 +21,8 @@
#include <folly/AtomicHashMap.h>
#include <wangle/channel/Handler.h>
+#include <atomic>
+#include <mutex>
#include <string>
#include "serde/rpc.h"
@@ -29,6 +31,7 @@
namespace hbase {
class Request;
class Response;
+class HeaderInfo;
}
namespace google {
namespace protobuf {
@@ -37,6 +40,7 @@ class Message;
}
namespace hbase {
+
class ClientHandler : public wangle::Handler<std::unique_ptr<folly::IOBuf>,
Response, std::unique_ptr<Request>,
std::unique_ptr<folly::IOBuf>> {
@@ -47,7 +51,7 @@ public:
std::unique_ptr<Request> r) override;
private:
- bool need_send_header_;
+ std::unique_ptr<HeaderInfo> header_info_;
std::string user_name_;
RpcSerde serde_;
@@ -56,4 +60,17 @@ private:
uint32_t, std::shared_ptr<google::protobuf::Message>>>
resp_msgs_;
};
+
+/**
+ * Class to contain the info about if the connection header and preamble has
+ * been sent.
+ *
+ * We use a serperate class here so that ClientHandler is relocatable.
+ */
+class HeaderInfo {
+public:
+ HeaderInfo() : need_(true), mutex_() {}
+ std::atomic<bool> need_;
+ std::mutex mutex_;
+};
} // namespace hbase
[hbase] 31/137: HBASE-15792 Add on a test for locating a region
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 191935d67718e1410c9886854104afb7587f403a
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Mon May 9 12:06:54 2016 -0700
HBASE-15792 Add on a test for locating a region
Summary:
Add on a TestUtil class that will start a cluster.
Add on the ability to run a shell command in a cluster.
Throw an exception if the table isn't the correct table when looking up a region.
Add a test to test that getting a region throws when the table doesn't exist.
Add a test to make sure that getting a region location works when the table does exist.
Test Plan:
Added unit tests.
Changed unit tests.
Differential Revision: https://reviews.facebook.net/D57891
---
bin/start-local-hbase.sh | 5 +--
connection/client-handler.cc | 3 +-
core/BUCK | 4 +--
core/client.cc | 1 +
core/location-cache-test.cc | 29 ++++++++++++++--
core/location-cache.cc | 10 +++++-
core/location-cache.h | 10 +++++-
bin/start-local-hbase.sh => test-util/BUCK | 28 +++++++---------
test-util/test-util.cc | 54 ++++++++++++++++++++++++++++++
core/test-env.cc => test-util/test-util.h | 48 ++++++++++++++------------
10 files changed, 146 insertions(+), 46 deletions(-)
diff --git a/bin/start-local-hbase.sh b/bin/start-local-hbase.sh
index cfc71f9..49637d6 100755
--- a/bin/start-local-hbase.sh
+++ b/bin/start-local-hbase.sh
@@ -21,7 +21,8 @@
rm -rf /tmp/hbase-*
# Start the master/regionservers.
-$PWD/../bin/start-hbase.sh
+T_DIR=${1:-"/tmp/hbase-testing"}
+$PWD/../bin/start-hbase.sh -Dhbase.tmp.dir="${T_DIR}"
until [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010/jmx) == "200" ]
do
@@ -30,4 +31,4 @@ do
done
# This sucks, but master can easily be up and meta not be assigned yet.
-sleep 30
+sleep 10
diff --git a/connection/client-handler.cc b/connection/client-handler.cc
index 2e3fcd3..3409b80 100644
--- a/connection/client-handler.cc
+++ b/connection/client-handler.cc
@@ -37,7 +37,8 @@ using hbase::pb::GetResponse;
using google::protobuf::Message;
ClientHandler::ClientHandler(std::string user_name)
- : user_name_(user_name), serde_(), once_flag_(std::make_unique<std::once_flag>()),
+ : user_name_(user_name), serde_(),
+ once_flag_(std::make_unique<std::once_flag>()),
resp_msgs_(
make_unique<folly::AtomicHashMap<
uint32_t, std::shared_ptr<google::protobuf::Message>>>(5000)) {}
diff --git a/core/BUCK b/core/BUCK
index 485f9ba..1c926e3 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -46,10 +46,10 @@ cxx_library(
], )
cxx_test(name="location-cache-test",
srcs=[
- "test-env.cc",
"location-cache-test.cc",
],
- deps=[":core", ],
+ deps=[":core",
+ "//test-util:test-util", ],
run_test_separately=True, )
cxx_binary(name="simple-client",
srcs=["simple-client.cc", ],
diff --git a/core/client.cc b/core/client.cc
index 1e80998..82ecd22 100644
--- a/core/client.cc
+++ b/core/client.cc
@@ -22,6 +22,7 @@
#include <glog/logging.h>
#include <string>
+#include <unistd.h>
using namespace folly;
using namespace std;
diff --git a/core/location-cache-test.cc b/core/location-cache-test.cc
index 8bc6383..53db6fc 100644
--- a/core/location-cache-test.cc
+++ b/core/location-cache-test.cc
@@ -16,14 +16,22 @@
* limitations under the License.
*
*/
+#include "core/location-cache.h"
+
#include <folly/Memory.h>
#include <gtest/gtest.h>
-#include "location-cache.h"
+#include <chrono>
+
+#include "if/HBase.pb.h"
+#include "serde/table-name.h"
+#include "test-util/test-util.h"
using namespace hbase;
+using namespace std::chrono;
TEST(LocationCacheTest, TestGetMetaNodeContents) {
- // TODO(elliott): need to make a test utility for this.
+ TestUtil test_util{};
+
auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
LocationCache cache{"localhost:2181", cpu, io};
@@ -33,3 +41,20 @@ TEST(LocationCacheTest, TestGetMetaNodeContents) {
ASSERT_TRUE(result.has_port());
ASSERT_TRUE(result.has_host_name());
}
+
+TEST(LocationCacheTest, TestGetRegionLocation) {
+ TestUtil test_util{};
+ auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
+ auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
+ LocationCache cache{"localhost:2181", cpu, io};
+
+ // If there is no table this should throw an exception
+ auto tn = folly::to<hbase::pb::TableName>("t");
+ auto row = "test";
+ ASSERT_ANY_THROW(cache.LocateFromMeta(tn, row).get(milliseconds(1000)));
+ test_util.RunShellCmd("create 't', 'd'");
+ auto loc = cache.LocateFromMeta(tn, row).get(milliseconds(1000));
+ ASSERT_TRUE(loc != nullptr);
+ cpu->stop();
+ io->stop();
+}
diff --git a/core/location-cache.cc b/core/location-cache.cc
index 6c018f9..9f2a0ef 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -116,7 +116,7 @@ LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
return this->LocateMeta()
.via(cpu_executor_.get())
.then([this](ServerName sn) { return this->cp_.get(sn); })
- .then([&](std::shared_ptr<HBaseService> service) {
+ .then([tn, row, this](std::shared_ptr<HBaseService> service) {
return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
})
.then([this](Response resp) {
@@ -124,6 +124,14 @@ LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
// a region location.
return this->CreateLocation(std::move(resp));
})
+ .then([tn, this](std::shared_ptr<RegionLocation> rl) {
+ // Make sure that the correct location was found.
+ if (rl->region_info().table_name().namespace_() != tn.namespace_() ||
+ rl->region_info().table_name().qualifier() != tn.qualifier()) {
+ throw std::runtime_error("Doesn't look like table exists.");
+ }
+ return rl;
+ })
.then([this](std::shared_ptr<RegionLocation> rl) {
// Now fill out the connection.
rl->set_service(cp_.get(rl->server_name()));
diff --git a/core/location-cache.h b/core/location-cache.h
index e077750..d435530 100644
--- a/core/location-cache.h
+++ b/core/location-cache.h
@@ -50,7 +50,9 @@ public:
/**
* Constructor.
* @param quorum_spec Where to connect for Zookeeper.
- * @param executor The cpu executor to run on.
+ * @param cpu_executor executor used to run non network IO based
+ * continuations.
+ * @param io_executor executor used to talk to the network
*/
LocationCache(std::string quorum_spec,
std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_exector,
@@ -70,6 +72,12 @@ public:
/**
* Go read meta and find out where a region is located.
+ *
+ * @param tn Table name of the table to look up. This object must live until
+ * after the future is returned
+ *
+ * @param row of the table to look up. This object must live until after the
+ * future is returned
*/
folly::Future<std::shared_ptr<RegionLocation>>
LocateFromMeta(const hbase::pb::TableName &tn, const std::string &row);
diff --git a/bin/start-local-hbase.sh b/test-util/BUCK
old mode 100755
new mode 100644
similarity index 67%
copy from bin/start-local-hbase.sh
copy to test-util/BUCK
index cfc71f9..e5e6ea2
--- a/bin/start-local-hbase.sh
+++ b/test-util/BUCK
@@ -1,5 +1,3 @@
-#!/usr/bin/env bash
-
##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -17,17 +15,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# Clean up from any other tests.
-rm -rf /tmp/hbase-*
-
-# Start the master/regionservers.
-$PWD/../bin/start-hbase.sh
-
-until [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010/jmx) == "200" ]
-do
- printf "Waiting for local HBase cluster to start\n"
- sleep 1
-done
-
-# This sucks, but master can easily be up and meta not be assigned yet.
-sleep 30
+cxx_library(name="test-util",
+ exported_headers=[
+ "test-util.h",
+ ],
+ srcs=["test-util.cc"],
+ deps=[
+ "//third-party:folly",
+ ],
+ visibility=[
+ 'PUBLIC',
+ ],
+ )
\ No newline at end of file
diff --git a/test-util/test-util.cc b/test-util/test-util.cc
new file mode 100644
index 0000000..e5fba48
--- /dev/null
+++ b/test-util/test-util.cc
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "test-util/test-util.h"
+
+#include <folly/Format.h>
+
+using hbase::TestUtil;
+using folly::Random;
+
+const static int STR_LEN = 32;
+
+std::string TestUtil::RandString() {
+ auto s = std::string(STR_LEN, 'z');
+
+ for (int i = 0; i < STR_LEN; i++) {
+ auto r = Random::rand32('a', 'z');
+ s[i] = static_cast<char>(r);
+ }
+ return s;
+}
+
+TestUtil::TestUtil() : temp_dir_(TestUtil::RandString()) {
+ auto p = temp_dir_.path().string();
+ auto cmd = std::string{"bin/start-local-hbase.sh " + p};
+ auto res_code = std::system(cmd.c_str());
+ CHECK(res_code == 0);
+}
+TestUtil::~TestUtil() {
+ auto res_code = std::system("bin/stop-local-hbase.sh");
+ CHECK(res_code == 0);
+}
+
+void TestUtil::RunShellCmd(const std::string& command) {
+ auto cmd_string = folly::sformat("echo \"{}\" | ../bin/hbase shell", command);
+ auto res_code = std::system(cmd_string.c_str());
+ CHECK(res_code == 0);
+}
diff --git a/core/test-env.cc b/test-util/test-util.h
similarity index 57%
rename from core/test-env.cc
rename to test-util/test-util.h
index 277abd9..395b157 100644
--- a/core/test-env.cc
+++ b/test-util/test-util.h
@@ -16,30 +16,36 @@
* limitations under the License.
*
*/
+#pragma once
-#include <gtest/gtest.h>
+#include <folly/experimental/TestUtil.h>
+#include <folly/Random.h>
-namespace {
+#include <cstdlib>
+#include <string>
-class NativeClientTestEnv : public ::testing::Environment {
+namespace hbase {
+/**
+ * @brief Class to deal with a local instance cluster for testing.
+ */
+class TestUtil {
public:
- void SetUp() override {
- // start local HBase cluster to be reused by all tests
- auto result = system("bin/start-local-hbase.sh");
- ASSERT_EQ(0, result);
- }
-
- void TearDown() override {
- // shutdown local HBase cluster
- auto result = system("bin/stop-local-hbase.sh");
- ASSERT_EQ(0, result);
- }
-};
-} // anonymous
+ /**
+ * Creating a TestUtil will spin up a cluster.
+ */
+ TestUtil();
+ /**
+ * Destroying a TestUtil will spin down a cluster.
+ */
+ ~TestUtil();
+ /**
+ * Run a command in the hbase shell.
+ */
+ void RunShellCmd(const std::string& command);
+ static std::string RandString();
-int main(int argc, char **argv) {
- testing::InitGoogleTest(&argc, argv);
- ::testing::AddGlobalTestEnvironment(new NativeClientTestEnv());
- return RUN_ALL_TESTS();
-}
+private:
+ folly::test::TemporaryDirectory temp_dir_;
+};
+} // namespace hbase
[hbase] 16/137: HBASE-15696 Move region location cache
serialization into serde
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit c0fc766fde904b37c4a09641c803d0714ec92c38
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Fri Apr 22 22:21:45 2016 -0700
HBASE-15696 Move region location cache serialization into serde
Summary: Use IOBuf for zk
Test Plan: Added a unit test
Differential Revision: https://reviews.facebook.net/D57147
---
Dockerfile | 7 +-
connection/client-dispatcher.h | 8 ++
connection/client-handler.cc | 7 +-
connection/client-handler.h | 2 +-
connection/pipeline.h | 2 +-
core/client.cc | 4 +-
core/client.h | 4 +-
core/location-cache-test.cc | 2 +-
core/location-cache.cc | 48 +++-----
core/location-cache.h | 4 +-
core/simple-client.cc | 6 +-
serde/BUCK | 10 ++
serde/client-deserializer-test.cc | 4 +-
serde/client-deserializer.cc | 4 +-
serde/client-serializer-test.cc | 2 +-
serde/client-serializer.cc | 2 +-
serde/client-serializer.h | 2 +-
serde/zk-deserializer-test.cc | 123 +++++++++++++++++++++
serde/zk-deserializer.cc | 78 +++++++++++++
utils/user-util-test.cc => serde/zk-deserializer.h | 29 ++---
utils/user-util-test.cc | 1 +
21 files changed, 277 insertions(+), 72 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 9ccc0d3..1524c56 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -19,12 +19,11 @@ FROM pjameson/buck-folly-watchman
ARG CC=/usr/bin/gcc-5
ARG CXX=/usr/bin/g++-5
-ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 -pthread"
-ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 -pthread"
+ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -pthread"
+ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -pthread"
-RUN apt-get install -y clang-format-3.7 vim maven inetutils-ping python-pip && \
+RUN apt-get install -y vim maven inetutils-ping python-pip && \
pip install yapf && \
- ln -sf /usr/bin/clang-format-3.7 /usr/bin/clang-format && \
apt-get -qq clean && \
apt-get -y -qq autoremove && \
rm -rf /var/lib/{apt,dpkg,cache,log}/ && \
diff --git a/connection/client-dispatcher.h b/connection/client-dispatcher.h
index c3987c9..89c7119 100644
--- a/connection/client-dispatcher.h
+++ b/connection/client-dispatcher.h
@@ -37,6 +37,14 @@ public:
private:
std::unordered_map<uint32_t, folly::Promise<Response>> requests_;
+ // Start at some number way above what could
+ // be there for un-initialized call id counters.
+ //
+ // This makes it easier to make sure that the're are
+ // no access to un-initialized variables.
+ //
+ // uint32_t has a max of 4Billion so 10 more or less is
+ // not a big deal.
uint32_t current_call_id_ = 10;
};
} // namespace hbase
diff --git a/connection/client-handler.cc b/connection/client-handler.cc
index 74b23ef..7b7794d 100644
--- a/connection/client-handler.cc
+++ b/connection/client-handler.cc
@@ -19,15 +19,15 @@
#include "connection/client-handler.h"
-#include <glog/logging.h>
#include <folly/Likely.h>
+#include <glog/logging.h>
#include <string>
#include "connection/request.h"
#include "connection/response.h"
-#include "if/RPC.pb.h"
#include "if/Client.pb.h"
+#include "if/RPC.pb.h"
using namespace hbase;
using namespace folly;
@@ -60,6 +60,9 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
}
}
+
+// TODO(eclark): Figure out how to handle the
+// network errors that are going to come.
Future<Unit> ClientHandler::write(Context *ctx, Request r) {
// Keep track of if we have sent the header.
if (UNLIKELY(need_send_header_)) {
diff --git a/connection/client-handler.h b/connection/client-handler.h
index 38c5725..dbaf5a0 100644
--- a/connection/client-handler.h
+++ b/connection/client-handler.h
@@ -22,8 +22,8 @@
#include <string>
-#include "serde/client-serializer.h"
#include "serde/client-deserializer.h"
+#include "serde/client-serializer.h"
// Forward decs.
namespace hbase {
diff --git a/connection/pipeline.h b/connection/pipeline.h
index 68ade48..8114fab 100644
--- a/connection/pipeline.h
+++ b/connection/pipeline.h
@@ -18,8 +18,8 @@
*/
#pragma once
-#include <wangle/service/Service.h>
#include <folly/io/IOBufQueue.h>
+#include <wangle/service/Service.h>
#include "connection/request.h"
#include "connection/response.h"
diff --git a/core/client.cc b/core/client.cc
index 893894f..266c239 100644
--- a/core/client.cc
+++ b/core/client.cc
@@ -21,8 +21,8 @@
#include <folly/Logging.h>
#include <folly/Random.h>
-#include <glog/logging.h>
#include <gflags/gflags.h>
+#include <glog/logging.h>
#include <wangle/concurrent/GlobalExecutor.h>
#include <string>
@@ -36,5 +36,5 @@ using namespace hbase::pb;
namespace hbase {
Client::Client(string quorum_spec)
- : location_cache(quorum_spec, wangle::getCPUExecutor()) {}
+ : location_cache_(quorum_spec, wangle::getCPUExecutor()) {}
}
diff --git a/core/client.h b/core/client.h
index c2dc226..b583285 100644
--- a/core/client.h
+++ b/core/client.h
@@ -19,8 +19,8 @@
#pragma once
-#include <folly/io/IOBuf.h>
#include <folly/futures/Future.h>
+#include <folly/io/IOBuf.h>
#include <string>
@@ -36,7 +36,7 @@ public:
folly::Future<GetResult> get(const GetRequest &get_request);
private:
- LocationCache location_cache;
+ LocationCache location_cache_;
};
} // namespace hbase
diff --git a/core/location-cache-test.cc b/core/location-cache-test.cc
index 70ca6f1..f3166fb 100644
--- a/core/location-cache-test.cc
+++ b/core/location-cache-test.cc
@@ -16,8 +16,8 @@
* limitations under the License.
*
*/
-#include <gtest/gtest.h>
#include <folly/Memory.h>
+#include <gtest/gtest.h>
#include <wangle/concurrent/GlobalExecutor.h>
#include "location-cache.h"
diff --git a/core/location-cache.cc b/core/location-cache.cc
index 52e86e3..5925f4a 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -16,11 +16,13 @@
* limitations under the License.
*
*/
-#include "location-cache.h"
+#include "core/location-cache.h"
#include <folly/Logging.h>
+#include <folly/io/IOBuf.h>
#include "if/ZooKeeper.pb.h"
+#include "serde/zk-deserializer.h"
using namespace std;
using namespace folly;
@@ -28,7 +30,7 @@ using namespace hbase::pb;
using namespace hbase;
// TODO(eclark): make this configurable on client creation
-static const char META_LOCATION[] = "/hbase/meta-region-server";
+static const char META_ZNODE_NAME[] = "/hbase/meta-region-server";
LocationCache::LocationCache(string quorum_spec,
shared_ptr<folly::Executor> executor)
@@ -66,45 +68,25 @@ void LocationCache::RefreshMetaLocation() {
}
ServerName LocationCache::ReadMetaLocation() {
- char contents[4096];
+ auto buf = IOBuf::create(4096);
+ ZkDeserializer derser;
+
// This needs to be int rather than size_t as that's what ZK expects.
- int len = sizeof(contents);
+ int len = buf->capacity();
// TODO(elliott): handle disconnects/reconntion as needed.
- int zk_result = zoo_get(this->zk_, META_LOCATION, 0, contents, &len, nullptr);
+ int zk_result =
+ zoo_get(this->zk_, META_ZNODE_NAME, 0,
+ reinterpret_cast<char *>(buf->writableData()), &len, nullptr);
+ LOG(ERROR) << "len = " << len;
if (zk_result != ZOK || len < 9) {
LOG(ERROR) << "Error getting meta location.";
throw runtime_error("Error getting meta location");
}
- // There should be a magic number for recoverable zk
- if (static_cast<uint8_t>(contents[0]) != 255) {
- LOG(ERROR) << "Magic number not in ZK znode data expected 255 got ="
- << unsigned(static_cast<uint8_t>(contents[0]));
- throw runtime_error("Magic number not in znode data");
- }
- // pos will keep track of skipped bytes.
- int pos = 1;
- // How long is the id?
- int id_len = 0;
- for (int i = 0; i < 4; ++i) {
- id_len = id_len << 8;
- id_len = id_len | static_cast<uint8_t>(contents[pos]);
- ++pos;
- }
- // Skip the id
- pos += id_len;
- // Then all protobuf's for HBase are prefixed with a magic string.
- // PBUF, so we skip that.
- // TODO(eclark): check to make sure that the magic string is correct
- // though I am not sure that will get us much.
- pos += 4;
+ buf->append(len);
MetaRegionServer mrs;
- // Try to decode the protobuf.
- // If there's an error bail out.
- if (mrs.ParseFromArray(contents + pos, len - pos) == false) {
- LOG(ERROR) << "Error parsing Protobuf Message";
- throw runtime_error("Error parsing protobuf");
+ if (derser.parse(buf.get(), &mrs) == false) {
+ LOG(ERROR) << "Unable to decode";
}
-
return mrs.server();
}
diff --git a/core/location-cache.h b/core/location-cache.h
index 28a1ee1..ce97c43 100644
--- a/core/location-cache.h
+++ b/core/location-cache.h
@@ -18,10 +18,10 @@
*/
#pragma once
-#include <zookeeper/zookeeper.h>
+#include <folly/Executor.h>
#include <folly/futures/Future.h>
#include <folly/futures/SharedPromise.h>
-#include <folly/Executor.h>
+#include <zookeeper/zookeeper.h>
#include <memory>
#include <mutex>
diff --git a/core/simple-client.cc b/core/simple-client.cc
index 065f70b..5e6f82e 100644
--- a/core/simple-client.cc
+++ b/core/simple-client.cc
@@ -22,13 +22,13 @@
#include <gflags/gflags.h>
#include <wangle/concurrent/GlobalExecutor.h>
-#include <iostream>
#include <chrono>
+#include <iostream>
-#include "core/client.h"
#include "connection/connection-factory.h"
-#include "if/ZooKeeper.pb.h"
+#include "core/client.h"
#include "if/Client.pb.h"
+#include "if/ZooKeeper.pb.h"
using namespace folly;
using namespace std;
diff --git a/serde/BUCK b/serde/BUCK
index 207607f..0014c0b 100644
--- a/serde/BUCK
+++ b/serde/BUCK
@@ -19,10 +19,12 @@ cxx_library(name="serde",
exported_headers=[
"client-serializer.h",
"client-deserializer.h",
+ "zk-deserializer.h",
],
srcs=[
"client-serializer.cc",
"client-deserializer.cc",
+ "zk-deserializer.cc",
],
deps=[
"//if:if",
@@ -52,3 +54,11 @@ cxx_test(name="client-deserializer-test",
":serde",
"//if:if",
], )
+cxx_test(name="zk-deserializer-test",
+ srcs=[
+ "zk-deserializer-test.cc",
+ ],
+ deps=[
+ ":serde",
+ "//if:if",
+ ], )
diff --git a/serde/client-deserializer-test.cc b/serde/client-deserializer-test.cc
index bb57e50..9fef093 100644
--- a/serde/client-deserializer-test.cc
+++ b/serde/client-deserializer-test.cc
@@ -17,12 +17,12 @@
*
*/
-#include <gtest/gtest.h>
#include <folly/io/IOBuf.h>
+#include <gtest/gtest.h>
+#include "if/Client.pb.h"
#include "serde/client-deserializer.h"
#include "serde/client-serializer.h"
-#include "if/Client.pb.h"
using namespace hbase;
using folly::IOBuf;
diff --git a/serde/client-deserializer.cc b/serde/client-deserializer.cc
index 118b0d1..acca7ea 100644
--- a/serde/client-deserializer.cc
+++ b/serde/client-deserializer.cc
@@ -19,10 +19,10 @@
#include "serde/client-deserializer.h"
-#include <google/protobuf/message.h>
+#include <folly/Logging.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
-#include <folly/Logging.h>
+#include <google/protobuf/message.h>
using namespace hbase;
diff --git a/serde/client-serializer-test.cc b/serde/client-serializer-test.cc
index b32b55d..9bf38af 100644
--- a/serde/client-serializer-test.cc
+++ b/serde/client-serializer-test.cc
@@ -22,9 +22,9 @@
#include <string>
-#include "serde/client-serializer.h"
#include "if/HBase.pb.h"
#include "if/RPC.pb.h"
+#include "serde/client-serializer.h"
using namespace hbase;
using namespace hbase::pb;
diff --git a/serde/client-serializer.cc b/serde/client-serializer.cc
index 881b6e4..09b81c8 100644
--- a/serde/client-serializer.cc
+++ b/serde/client-serializer.cc
@@ -18,8 +18,8 @@
*/
#include "serde/client-serializer.h"
-#include <folly/io/Cursor.h>
#include <folly/Logging.h>
+#include <folly/io/Cursor.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
diff --git a/serde/client-serializer.h b/serde/client-serializer.h
index 685095d..9c819fe 100644
--- a/serde/client-serializer.h
+++ b/serde/client-serializer.h
@@ -18,9 +18,9 @@
*/
#pragma once
+#include <cstdint>
#include <folly/io/IOBuf.h>
#include <string>
-#include <cstdint>
// Forward
namespace google {
diff --git a/serde/zk-deserializer-test.cc b/serde/zk-deserializer-test.cc
new file mode 100644
index 0000000..92d85a0
--- /dev/null
+++ b/serde/zk-deserializer-test.cc
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "serde/zk-deserializer.h"
+
+#include <folly/Logging.h>
+#include <folly/io/Cursor.h>
+#include <folly/io/IOBuf.h>
+#include <gtest/gtest.h>
+
+#include "if/ZooKeeper.pb.h"
+
+using namespace hbase;
+using namespace hbase::pb;
+using namespace folly;
+using namespace std;
+using namespace folly::io;
+
+// Test that would test if there's nothing there.
+TEST(TestZkDesializer, TestThrowNoMagicNum) {
+ ZkDeserializer deser;
+ MetaRegionServer mrs;
+
+ auto buf = IOBuf::create(100);
+ buf->append(100);
+ RWPrivateCursor c{buf.get()};
+ c.write<uint8_t>(99);
+ ASSERT_THROW(deser.parse(buf.get(), &mrs), runtime_error);
+}
+
+// Test if the protobuf is in a format that we can't decode
+TEST(TestZkDesializer, TestBadProtoThrow) {
+ ZkDeserializer deser;
+ MetaRegionServer mrs;
+ string magic{"PBUF"};
+
+ // Set ServerName
+ mrs.mutable_server()->set_host_name("test");
+ mrs.mutable_server()->set_port(567);
+ mrs.mutable_server()->set_start_code(9567);
+
+ // One byte magic number
+ // four bytes for id length
+ // four bytes for id
+ // four bytes for PBUF
+ uint32_t start_len = 1 + 4 + 4 + 4;
+ // How large the protobuf will be
+ uint32_t pbuf_size = mrs.ByteSize();
+
+ auto buf = IOBuf::create(start_len + pbuf_size);
+ buf->append(start_len + pbuf_size);
+ RWPrivateCursor c{buf.get()};
+
+ // Write the magic number
+ c.write<uint8_t>(255);
+ // Write the id len
+ c.writeBE<uint32_t>(4);
+ // Write the id
+ c.write<uint32_t>(13);
+ // Write the PBUF string
+ c.push(reinterpret_cast<const uint8_t *>(magic.c_str()), 4);
+
+ // Create the protobuf
+ MetaRegionServer out;
+ ASSERT_THROW(deser.parse(buf.get(), &out), runtime_error);
+}
+
+// Test to make sure the whole thing works.
+TEST(TestZkDesializer, TestNoThrow) {
+ ZkDeserializer deser;
+ MetaRegionServer mrs;
+ string magic{"PBUF"};
+
+ // Set ServerName
+ mrs.mutable_server()->set_host_name("test");
+ mrs.mutable_server()->set_port(567);
+ mrs.mutable_server()->set_start_code(9567);
+
+ // One byte magic number
+ // four bytes for id length
+ // four bytes for id
+ // four bytes for PBUF
+ uint32_t start_len = 1 + 4 + 4 + 4;
+ // How large the protobuf will be
+ uint32_t pbuf_size = mrs.ByteSize();
+
+ auto buf = IOBuf::create(start_len + pbuf_size);
+ buf->append(start_len + pbuf_size);
+ RWPrivateCursor c{buf.get()};
+
+ // Write the magic number
+ c.write<uint8_t>(255);
+ // Write the id len
+ c.writeBE<uint32_t>(4);
+ // Write the id
+ c.write<uint32_t>(13);
+ // Write the PBUF string
+ c.push(reinterpret_cast<const uint8_t *>(magic.c_str()), 4);
+
+ // Now write the serialized protobuf
+ mrs.SerializeWithCachedSizesToArray(buf->writableData() + start_len);
+
+ // Create the protobuf
+ MetaRegionServer out;
+ ASSERT_TRUE(deser.parse(buf.get(), &out));
+ ASSERT_EQ(mrs.server().host_name(), out.server().host_name());
+}
diff --git a/serde/zk-deserializer.cc b/serde/zk-deserializer.cc
new file mode 100644
index 0000000..33cf809
--- /dev/null
+++ b/serde/zk-deserializer.cc
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "serde/zk-deserializer.h"
+
+#include <folly/io/Cursor.h>
+#include <folly/io/IOBuf.h>
+#include <google/protobuf/message.h>
+
+using hbase::ZkDeserializer;
+using std::runtime_error;
+using folly::IOBuf;
+using folly::io::Cursor;
+using google::protobuf::Message;
+
+static const std::string MAGIC_STRING = "PBUF";
+
+bool ZkDeserializer::parse(IOBuf *buf, Message *out) {
+
+ // The format is like this
+ // 1 byte of magic number. 255
+ // 4 bytes of id length.
+ // id_length number of bytes for the id of who put up the znode
+ // 4 bytes of a magic string PBUF
+ // Then the protobuf serialized without a varint header.
+
+ Cursor c{buf};
+
+ // There should be a magic number for recoverable zk
+ uint8_t magic_num = c.read<uint8_t>();
+ if (magic_num != 255) {
+ LOG(ERROR) << "Magic number not in ZK znode data expected 255 got ="
+ << unsigned(magic_num);
+ throw runtime_error("Magic number not in znode data");
+ }
+ // How long is the id?
+ uint32_t id_len = c.readBE<uint32_t>();
+
+ if (id_len >= c.length()) {
+ LOG(ERROR) << "After skiping the if from zookeeper data there's not enough "
+ "left to read anything else";
+ throw runtime_error("Not enough bytes to decode from zookeeper");
+ }
+
+ // Skip the id
+ c.skip(id_len);
+
+ // Make sure that the magic string is there.
+ if (MAGIC_STRING != c.readFixedString(4)) {
+ LOG(ERROR) << "There was no PBUF magic string.";
+ throw runtime_error("No PBUF magic string in the zookpeeper data.");
+ }
+
+ // Try to decode the protobuf.
+ // If there's an error bail out.
+ if (out->ParseFromArray(c.data(), c.length()) == false) {
+ LOG(ERROR) << "Error parsing Protobuf Message";
+ throw runtime_error("Error parsing protobuf");
+ }
+
+ return true;
+}
diff --git a/utils/user-util-test.cc b/serde/zk-deserializer.h
similarity index 75%
copy from utils/user-util-test.cc
copy to serde/zk-deserializer.h
index 2a7434f..aa91661 100644
--- a/utils/user-util-test.cc
+++ b/serde/zk-deserializer.h
@@ -16,19 +16,20 @@
* limitations under the License.
*
*/
+#pragma once
-#include <gtest/gtest.h>
-#include <string>
-#include <folly/Logging.h>
-
-#include "utils/user-util.h"
-
-using namespace std;
-using namespace hbase;
-
-TEST(TestUserUtil, TestGetSomething) {
- UserUtil u_util;
- string name = u_util.user_name();
-
- ASSERT_GT(name.length(), 0);
+namespace google {
+namespace protobuf {
+class Message;
}
+}
+namespace folly {
+class IOBuf;
+}
+
+namespace hbase {
+class ZkDeserializer {
+public:
+ bool parse(folly::IOBuf *buf, google::protobuf::Message *out);
+};
+} // namespace hbase
diff --git a/utils/user-util-test.cc b/utils/user-util-test.cc
index 2a7434f..1e72bcc 100644
--- a/utils/user-util-test.cc
+++ b/utils/user-util-test.cc
@@ -30,5 +30,6 @@ TEST(TestUserUtil, TestGetSomething) {
UserUtil u_util;
string name = u_util.user_name();
+ // TODO shell out to whoami to check this.
ASSERT_GT(name.length(), 0);
}
[hbase] 96/137: HBASE-18173 Append class - Addendum to fix
formatting
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit fa7c7a996873600ce027cda09dd42b12fd378cf6
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Wed Jun 7 17:49:47 2017 -0700
HBASE-18173 Append class - Addendum to fix formatting
---
core/append-test.cc | 3 +--
core/append.cc | 5 ++---
core/client-test.cc | 4 ++--
core/increment-test.cc | 2 +-
core/increment.cc | 7 +++---
core/raw-async-table.cc | 22 ++++++++++++-------
core/request-converter.cc | 34 ++++++++++++++---------------
core/request-converter.h | 7 +++---
utils/bytes-util.cc | 55 +++++++++++++++++++++++------------------------
9 files changed, 71 insertions(+), 68 deletions(-)
diff --git a/core/append-test.cc b/core/append-test.cc
index 619826c..2216034 100644
--- a/core/append-test.cc
+++ b/core/append-test.cc
@@ -19,8 +19,8 @@
#include <glog/logging.h>
#include <gtest/gtest.h>
-#include "core/mutation.h"
#include "core/append.h"
+#include "core/mutation.h"
#include "utils/time-util.h"
using hbase::Append;
@@ -103,4 +103,3 @@ TEST(Append, Add) {
EXPECT_EQ(2, append.FamilyMap().size());
EXPECT_EQ(1, append.FamilyMap().at("family-2").size());
}
-
diff --git a/core/append.cc b/core/append.cc
index 18ee45a..95349ae 100644
--- a/core/append.cc
+++ b/core/append.cc
@@ -35,10 +35,9 @@ namespace hbase {
* @param value value to append
*/
Append& Append::Add(const std::string& family, const std::string& qualifier,
- const std::string& value) {
+ const std::string& value) {
family_map_[family].push_back(std::move(
- std::make_unique<Cell>(row_, family, qualifier, timestamp_, value,
- hbase::CellType::PUT)));
+ std::make_unique<Cell>(row_, family, qualifier, timestamp_, value, hbase::CellType::PUT)));
return *this;
}
Append& Append::Add(std::unique_ptr<Cell> cell) {
diff --git a/core/client-test.cc b/core/client-test.cc
index 1ee0a83..4972e05 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -25,8 +25,8 @@
#include "core/configuration.h"
#include "core/delete.h"
#include "core/get.h"
-#include "core/increment.h"
#include "core/hbase-configuration-loader.h"
+#include "core/increment.h"
#include "core/put.h"
#include "core/result.h"
#include "core/table.h"
@@ -244,7 +244,7 @@ TEST_F(ClientTest, Increment) {
result = table->Increment(hbase::Increment{row}.AddColumn("d", "1", incr2));
EXPECT_EQ(row, result->Row());
- EXPECT_EQ(incr1+incr2, hbase::BytesUtil::ToInt64(*(result->Value("d", "1"))));
+ EXPECT_EQ(incr1 + incr2, hbase::BytesUtil::ToInt64(*(result->Value("d", "1"))));
}
TEST_F(ClientTest, PutGet) {
diff --git a/core/increment-test.cc b/core/increment-test.cc
index 3d7b1d4..aa9b864 100644
--- a/core/increment-test.cc
+++ b/core/increment-test.cc
@@ -19,8 +19,8 @@
#include <glog/logging.h>
#include <gtest/gtest.h>
-#include "core/mutation.h"
#include "core/increment.h"
+#include "core/mutation.h"
#include "core/put.h"
#include "utils/time-util.h"
diff --git a/core/increment.cc b/core/increment.cc
index 1a84266..a21a764 100644
--- a/core/increment.cc
+++ b/core/increment.cc
@@ -38,10 +38,9 @@ namespace hbase {
* @param amount amount to increment by
*/
Increment& Increment::AddColumn(const std::string& family, const std::string& qualifier,
- int64_t amount) {
- family_map_[family].push_back(std::move(
- std::make_unique<Cell>(row_, family, qualifier, timestamp_, BytesUtil::ToString(amount),
- hbase::CellType::PUT)));
+ int64_t amount) {
+ family_map_[family].push_back(std::move(std::make_unique<Cell>(
+ row_, family, qualifier, timestamp_, BytesUtil::ToString(amount), hbase::CellType::PUT)));
return *this;
}
Increment& Increment::Add(std::unique_ptr<Cell> cell) {
diff --git a/core/raw-async-table.cc b/core/raw-async-table.cc
index 2a98d54..413dc6c 100644
--- a/core/raw-async-table.cc
+++ b/core/raw-async-table.cc
@@ -79,12 +79,15 @@ folly::Future<std::shared_ptr<Result>> RawAsyncTable::Get(const hbase::Get& get)
}
folly::Future<std::shared_ptr<Result>> RawAsyncTable::Increment(const hbase::Increment& incr) {
auto caller =
- CreateCallerBuilder<std::shared_ptr<Result>>(incr.row(), connection_conf_->write_rpc_timeout())
+ CreateCallerBuilder<std::shared_ptr<Result>>(incr.row(),
+ connection_conf_->write_rpc_timeout())
->action([=, &incr](std::shared_ptr<hbase::HBaseRpcController> controller,
- std::shared_ptr<hbase::RegionLocation> loc,
- std::shared_ptr<hbase::RpcClient> rpc_client) -> folly::Future<std::shared_ptr<Result>> {
+ std::shared_ptr<hbase::RegionLocation> loc,
+ std::shared_ptr<hbase::RpcClient>
+ rpc_client) -> folly::Future<std::shared_ptr<Result>> {
return Call<hbase::Increment, hbase::Request, hbase::Response, std::shared_ptr<Result>>(
- rpc_client, controller, loc, incr, &hbase::RequestConverter::IncrementToMutateRequest,
+ rpc_client, controller, loc, incr,
+ &hbase::RequestConverter::IncrementToMutateRequest,
&hbase::ResponseConverter::FromMutateResponse);
})
->Build();
@@ -126,12 +129,15 @@ folly::Future<folly::Unit> RawAsyncTable::Delete(const hbase::Delete& del) {
folly::Future<std::shared_ptr<Result>> RawAsyncTable::Append(const hbase::Append& append) {
auto caller =
- CreateCallerBuilder<std::shared_ptr<Result>>(append.row(), connection_conf_->write_rpc_timeout())
+ CreateCallerBuilder<std::shared_ptr<Result>>(append.row(),
+ connection_conf_->write_rpc_timeout())
->action([=, &append](std::shared_ptr<hbase::HBaseRpcController> controller,
- std::shared_ptr<hbase::RegionLocation> loc,
- std::shared_ptr<hbase::RpcClient> rpc_client) -> folly::Future<std::shared_ptr<Result>> {
+ std::shared_ptr<hbase::RegionLocation> loc,
+ std::shared_ptr<hbase::RpcClient>
+ rpc_client) -> folly::Future<std::shared_ptr<Result>> {
return Call<hbase::Append, hbase::Request, hbase::Response, std::shared_ptr<Result>>(
- rpc_client, controller, loc, append, &hbase::RequestConverter::AppendToMutateRequest,
+ rpc_client, controller, loc, append,
+ &hbase::RequestConverter::AppendToMutateRequest,
&hbase::ResponseConverter::FromMutateResponse);
})
->Build();
diff --git a/core/request-converter.cc b/core/request-converter.cc
index 54fdfc5..042d5b1 100644
--- a/core/request-converter.cc
+++ b/core/request-converter.cc
@@ -288,29 +288,29 @@ std::unique_ptr<Request> RequestConverter::DeleteToMutateRequest(const Delete &d
VLOG(3) << "Req is " << pb_req->req_msg()->ShortDebugString();
return pb_req;
}
-std::unique_ptr<Request> RequestConverter::IncrementToMutateRequest(const Increment &incr,
- const std::string ®ion_name) {
- auto pb_req = Request::mutate();
- auto pb_msg = std::static_pointer_cast<hbase::pb::MutateRequest>(pb_req->req_msg());
- RequestConverter::SetRegion(region_name, pb_msg->mutable_region());
+std::unique_ptr<Request> RequestConverter::IncrementToMutateRequest(
+ const Increment &incr, const std::string ®ion_name) {
+ auto pb_req = Request::mutate();
+ auto pb_msg = std::static_pointer_cast<hbase::pb::MutateRequest>(pb_req->req_msg());
+ RequestConverter::SetRegion(region_name, pb_msg->mutable_region());
- pb_msg->set_allocated_mutation(
- ToMutation(MutationType::MutationProto_MutationType_INCREMENT, incr, -1).release());
+ pb_msg->set_allocated_mutation(
+ ToMutation(MutationType::MutationProto_MutationType_INCREMENT, incr, -1).release());
- VLOG(3) << "Req is " << pb_req->req_msg()->ShortDebugString();
- return pb_req;
+ VLOG(3) << "Req is " << pb_req->req_msg()->ShortDebugString();
+ return pb_req;
}
std::unique_ptr<Request> RequestConverter::AppendToMutateRequest(const Append &append,
- const std::string ®ion_name) {
- auto pb_req = Request::mutate();
- auto pb_msg = std::static_pointer_cast<hbase::pb::MutateRequest>(pb_req->req_msg());
- RequestConverter::SetRegion(region_name, pb_msg->mutable_region());
+ const std::string ®ion_name) {
+ auto pb_req = Request::mutate();
+ auto pb_msg = std::static_pointer_cast<hbase::pb::MutateRequest>(pb_req->req_msg());
+ RequestConverter::SetRegion(region_name, pb_msg->mutable_region());
- pb_msg->set_allocated_mutation(
- ToMutation(MutationType::MutationProto_MutationType_APPEND, append, -1).release());
+ pb_msg->set_allocated_mutation(
+ ToMutation(MutationType::MutationProto_MutationType_APPEND, append, -1).release());
- VLOG(3) << "Req is " << pb_req->req_msg()->ShortDebugString();
- return pb_req;
+ VLOG(3) << "Req is " << pb_req->req_msg()->ShortDebugString();
+ return pb_req;
}
} /* namespace hbase */
diff --git a/core/request-converter.h b/core/request-converter.h
index a9d65d6..51440d8 100644
--- a/core/request-converter.h
+++ b/core/request-converter.h
@@ -24,8 +24,8 @@
#include <vector>
#include "connection/request.h"
#include "core/action.h"
-#include "core/cell.h"
#include "core/append.h"
+#include "core/cell.h"
#include "core/delete.h"
#include "core/get.h"
#include "core/increment.h"
@@ -85,14 +85,15 @@ class RequestConverter {
static std::unique_ptr<Request> ToMutateRequest(const Put &put, const std::string ®ion_name);
static std::unique_ptr<Request> IncrementToMutateRequest(const Increment &incr,
- const std::string ®ion_name);
+ const std::string ®ion_name);
static std::unique_ptr<pb::MutationProto> ToMutation(const MutationType type,
const Mutation &mutation,
const int64_t nonce);
static std::unique_ptr<Request> AppendToMutateRequest(const Append &append,
- const std::string ®ion_name);
+ const std::string ®ion_name);
+
private:
// Constructor not required. We have all static methods to create PB requests.
RequestConverter();
diff --git a/utils/bytes-util.cc b/utils/bytes-util.cc
index e201d55..a937782 100644
--- a/utils/bytes-util.cc
+++ b/utils/bytes-util.cc
@@ -19,10 +19,10 @@
#include "utils/bytes-util.h"
+#include <bits/stdc++.h>
+#include <boost/predef.h>
#include <memory>
#include <string>
-#include<bits/stdc++.h>
-#include <boost/predef.h>
#include <glog/logging.h>
@@ -31,40 +31,39 @@ namespace hbase {
constexpr char BytesUtil::kHexChars[];
std::string BytesUtil::ToString(int64_t val) {
- std::string res;
+ std::string res;
#if BOOST_ENDIAN_BIG_BYTE || BOOST_ENDIAN_BIG_WORD
- for (int i = 7; i > 0; i--) {
- res += (int8_t) (val & 0xffu);
- val = val >> 8;
- }
- res += (int8_t) val;
+ for (int i = 7; i > 0; i--) {
+ res += (int8_t)(val & 0xffu);
+ val = val >> 8;
+ }
+ res += (int8_t)val;
#else
- int64_t mask = 0xff00000000000000u;
- for (int i = 56; i >= 1; i -= 8) {
- auto num = ((val & mask) >> i);
- res += num;
- mask = mask >> 8;
- }
- res += (val & 0xff);
+ int64_t mask = 0xff00000000000000u;
+ for (int i = 56; i >= 1; i -= 8) {
+ auto num = ((val & mask) >> i);
+ res += num;
+ mask = mask >> 8;
+ }
+ res += (val & 0xff);
#endif
- return res;
+ return res;
}
int64_t BytesUtil::ToInt64(std::string str) {
- if (str.length() < 8) {
- throw std::runtime_error("There are not enough bytes. Expected: 8, actual: " + str.length());
- }
- const unsigned char *bytes = reinterpret_cast<unsigned char *>(const_cast<char*>(str.c_str()));
- int64_t l = 0;
- for(int i = 0; i < 8; i++) {
- l <<= 8;
- l ^= bytes[i];
- }
- return l;
-
+ if (str.length() < 8) {
+ throw std::runtime_error("There are not enough bytes. Expected: 8, actual: " + str.length());
+ }
+ const unsigned char *bytes = reinterpret_cast<unsigned char *>(const_cast<char *>(str.c_str()));
+ int64_t l = 0;
+ for (int i = 0; i < 8; i++) {
+ l <<= 8;
+ l ^= bytes[i];
+ }
+ return l;
}
-std::string BytesUtil::ToStringBinary(const std::string& b, size_t off, size_t len) {
+std::string BytesUtil::ToStringBinary(const std::string &b, size_t off, size_t len) {
std::string result;
// Just in case we are passed a 'len' that is > buffer length...
if (off >= b.size()) {
[hbase] 54/137: HBASE-17395 [C++] Use custom line wrapping in
formatting
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 2a72ee4fbc91b297f725c6e7063f330a8892cc2d
Author: Sudeep Sunthankar <su...@hashmapinc.com>
AuthorDate: Fri Dec 30 10:03:56 2016 +1100
HBASE-17395 [C++] Use custom line wrapping in formatting
Signed-off-by: Enis Soztutar <en...@apache.org>
---
bin/format-code.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/bin/format-code.sh b/bin/format-code.sh
index 55fe69e..6796acd 100755
--- a/bin/format-code.sh
+++ b/bin/format-code.sh
@@ -19,5 +19,5 @@ set -euo pipefail
IFS=$'\n\t'
-find core connection serde utils test-util -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i --style=Google
-find core connection serde utils third-party -name "BUCK" | xargs -P8 yapf -i --style=google
+find core connection serde utils test-util security -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i --style=='{BasedOnStyle: Google, ColumnLimit: 100}'
+find core connection serde utils third-party security -name "BUCK" | xargs -P8 yapf -i --style=google
[hbase] 132/137: HBASE-18725 [C++] Install header files as well as
library - ADDENDUM to only install .h files for if/
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit f89c22e07b535b053771d9fbb916a10c7752bb20
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Fri Sep 15 17:55:30 2017 -0700
HBASE-18725 [C++] Install header files as well as library - ADDENDUM to only install .h files for if/
---
Makefile | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/Makefile b/Makefile
index 4d6430c..cd818ca 100644
--- a/Makefile
+++ b/Makefile
@@ -104,7 +104,8 @@ protos: $(PROTO_CXX_DIR) $(PROTOSRC) $(PROTOHDR)
install_headers:
cp -r $(INCLUDE_HBASE)/hbase $(LOCAL_INCLUDE_DIR)
- cp -r $(PROTO_CXX_DIR) $(LOCAL_INCLUDE_DIR)/hbase/
+ mkdir -p $(LOCAL_INCLUDE_DIR)/hbase/if
+ cp -r $(PROTO_CXX_DIR)/*.h $(LOCAL_INCLUDE_DIR)/hbase/if
uninstall_headers:
rm -rf $(LOCAL_INCLUDE_DIR)/hbase
[hbase] 119/137: HBASE-18579 Enable core dump by default for docker
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 9fb0544bcf864d0d810ba9c272511d3596ec5fe8
Author: tedyu <yu...@gmail.com>
AuthorDate: Mon Aug 14 07:06:23 2017 -0700
HBASE-18579 Enable core dump by default for docker
---
bin/start-docker.sh | 2 +-
docker-files/Dockerfile | 4 +++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/bin/start-docker.sh b/bin/start-docker.sh
index 53325c1..391238d 100755
--- a/bin/start-docker.sh
+++ b/bin/start-docker.sh
@@ -55,7 +55,7 @@ fi;
docker build -t hbase_native -f docker-files/Dockerfile .
# After the image is built run the thing
-docker run -h="securecluster" -p 16050:16050/tcp \
+docker run --privileged=true -h="securecluster" -p 16050:16050/tcp \
-v ${BASE_DIR}/..:/usr/src/hbase \
-v ~/.m2:/root/.m2 \
-it hbase_native /bin/bash
diff --git a/docker-files/Dockerfile b/docker-files/Dockerfile
index efd9a9d..ac3cb47 100644
--- a/docker-files/Dockerfile
+++ b/docker-files/Dockerfile
@@ -117,6 +117,8 @@ RUN cd /usr/src/ && \
ctest && \
make install
-ENTRYPOINT /usr/sbin/krb5kdc -P /var/run/krb5kdc.pid && /bin/bash
+RUN echo "enabled=1" >> /etc/default/apport
+
+ENTRYPOINT /usr/sbin/krb5kdc -P /var/run/krb5kdc.pid && echo "/tmp/core.%h.%e.%t" >> /proc/sys/kernel/core_pattern && sysctl -p && ulimit -c unlimited && /bin/bash
WORKDIR /usr/src/hbase/hbase-native-client
[hbase] 135/137: HBASE-20009 Add license header to krb5.conf
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 5785b609b75f71bcc1adb3af7a698e8e2a6aab3f
Author: tedyu <yu...@gmail.com>
AuthorDate: Wed Feb 28 10:19:54 2018 -0800
HBASE-20009 Add license header to krb5.conf
---
docker-files/krb5.conf | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/docker-files/krb5.conf b/docker-files/krb5.conf
index d449928..955c6dc 100644
--- a/docker-files/krb5.conf
+++ b/docker-files/krb5.conf
@@ -1,3 +1,21 @@
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
[logging]
default = FILE:/var/log/krb5libs.log
kdc = FILE:/var/log/krb5kdc.log
[hbase] 116/137: HBASE-18537 [C++] Improvements to load-client
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit efd0a46fc0581e14973e80912972c889a25509db
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Fri Aug 11 11:09:34 2017 -0700
HBASE-18537 [C++] Improvements to load-client
---
core/load-client.cc | 213 +++++++++++++++++++++++++++++++-------------------
core/simple-client.cc | 4 +-
2 files changed, 135 insertions(+), 82 deletions(-)
diff --git a/core/load-client.cc b/core/load-client.cc
index 67f0d57..8cceeef 100644
--- a/core/load-client.cc
+++ b/core/load-client.cc
@@ -63,16 +63,17 @@ static constexpr const char *appendPrefix = "a";
std::string PrefixZero(int total_width, int num) {
std::string str = std::to_string(num);
- auto prefix_len = total_width - str.length();
- if (prefix_len > 0) return std::string(total_width - str.length(), '0') + str;
+ int prefix_len = total_width - str.length();
+ if (prefix_len > 0) {
+ return std::string(prefix_len, '0') + str;
+ }
return str;
}
bool Verify(std::shared_ptr<hbase::Result> result, std::string family, int m) {
auto col = std::to_string(m);
- auto int_val = hbase::BytesUtil::ToInt64(*(result->Value(family, incrPrefix + col)));
- if (int_val != m) {
- LOG(ERROR) << "value is not " << col << " for " << result->Row();
+ if (!result->Value(family, col)) {
+ LOG(ERROR) << "Column:" << col << " is not found for " << result->Row();
return false;
}
auto l = *(result->Value(family, col));
@@ -80,11 +81,52 @@ bool Verify(std::shared_ptr<hbase::Result> result, std::string family, int m) {
LOG(ERROR) << "value " << *(result->Value(family, "1")) << " is not " << col;
return false;
}
- l = *(result->Value(family, appendPrefix + col));
- if (l != col) {
- LOG(ERROR) << "value " << *(result->Value(family, "1")) << " is not " << col;
+ if (FLAGS_appends) {
+ if (!result->Value(family, incrPrefix + col)) {
+ LOG(ERROR) << "Column:" << (incrPrefix + col) << " is not found for " << result->Row();
+ return false;
+ }
+ auto int_val = hbase::BytesUtil::ToInt64(*(result->Value(family, incrPrefix + col)));
+ if (int_val != m) {
+ LOG(ERROR) << "value is not " << col << " for " << result->Row();
+ return false;
+ }
+ if (!result->Value(family, appendPrefix + col)) {
+ LOG(ERROR) << "Column:" << (appendPrefix + col) << " is not found for " << result->Row();
+ return false;
+ }
+ l = *(result->Value(family, appendPrefix + col));
+ if (l != col) {
+ LOG(ERROR) << "value " << *(result->Value(family, "1")) << " is not " << col;
+ return false;
+ }
+ }
+
+ return true;
+}
+
+bool Verify(std::shared_ptr<hbase::Result> result, const std::string &row,
+ const std::vector<std::string> &families) {
+ if (result == nullptr || result->IsEmpty()) {
+ LOG(ERROR) << "didn't get result";
return false;
}
+ if (result->Row().compare(row) != 0) {
+ LOG(ERROR) << "row " << result->Row() << " is not the expected: " << row;
+ return false;
+ }
+ // Test the values
+ for (auto family : families) {
+ if (!result->Value(family, kNumColumn)) {
+ LOG(ERROR) << "Column:" << kNumColumn << " is not found for " << result->Row();
+ return false;
+ }
+ auto cols = std::stoi(*(result->Value(family, kNumColumn)));
+ VLOG(3) << "Result for row:" << row << " contains " << std::to_string(cols) << " columns";
+ for (int m = 1; m <= cols; m++) {
+ if (!Verify(result, family, m)) return false;
+ }
+ }
return true;
}
@@ -95,35 +137,46 @@ bool DoScan(int iteration, uint64_t max_row, uint64_t rows, std::unique_ptr<Tabl
auto end = start + rows;
auto width = std::to_string(max_row).length();
scan.SetStartRow(PrefixZero(width, start));
- scan.SetStopRow(PrefixZero(width, end));
+ if (end != max_row && end != max_row + 1) {
+ scan.SetStopRow(PrefixZero(width, end));
+ }
+
+ auto start_ns = TimeUtil::GetNowNanos();
auto scanner = table->Scan(scan);
- auto cnt = start;
+ auto cnt = 0;
auto r = scanner->Next();
while (r != nullptr) {
- auto row = PrefixZero(width, cnt);
- if (r->Row().compare(row) != 0) {
- LOG(ERROR) << "row " << r->Row() << " is not the expected: " << row;
+ auto row = PrefixZero(width, start + cnt);
+ if (!Verify(r, row, families)) {
return false;
}
- for (auto family : families) {
- auto cols = std::stoi(*(r->Value(family, kNumColumn)));
- VLOG(3) << "scan gets " << std::to_string(cols) << " columns";
- for (int m = 1; m <= cols; m++) {
- if (!Verify(r, family, m)) return false;
- }
- }
cnt++;
r = scanner->Next();
+ if (cnt != 0 && cnt % FLAGS_report_num_rows == 0) {
+ LOG(INFO) << "(Thread " << iteration << ") "
+ << "Scan iterated over " << cnt << " results in "
+ << TimeUtil::ElapsedMillis(start_ns) << " ms.";
+ }
+ }
+ if (cnt != rows) {
+ LOG(ERROR) << "(Thread " << iteration << ") "
+ << "Expected number of results does not match. expected:" << rows
+ << ", actual:" << cnt;
+ return false;
}
- LOG(INFO) << "Iteration " << iteration << " scanned " << std::to_string(cnt - start) << " rows";
+ LOG(INFO) << "(Thread " << iteration << ") "
+ << "scanned " << std::to_string(cnt) << " rows in " << TimeUtil::ElapsedMillis(start_ns)
+ << " ms.";
return true;
}
bool DoGet(int iteration, uint64_t max_row, uint64_t rows, std::unique_ptr<Table> table,
- const std::vector<std::string> &families, int batch_num_rows) {
+ const std::vector<std::string> &families, uint64_t batch_num_rows) {
auto width = std::to_string(max_row).length();
+ auto start_ns = TimeUtil::GetNowNanos();
for (uint64_t k = iteration; k <= max_row;) {
+ uint64_t total_read = 0;
std::vector<hbase::Get> gets;
for (uint64_t i = 0; i < batch_num_rows && k <= max_row; ++i, k += FLAGS_threads) {
std::string row = PrefixZero(width, k);
@@ -132,29 +185,34 @@ bool DoGet(int iteration, uint64_t max_row, uint64_t rows, std::unique_ptr<Table
}
VLOG(3) << "getting for " << batch_num_rows << " rows";
auto results = table->Get(gets);
+ if (results.size() != gets.size()) {
+ LOG(ERROR) << "(Thread " << iteration << ") "
+ << "Expected number of results does not match. expected:" << gets.size()
+ << ", actual:" << results.size();
+ return false;
+ }
for (uint64_t i = 0; i < batch_num_rows && i < results.size(); ++i) {
- auto result = results[i];
- if (result == nullptr) {
- LOG(ERROR) << "didn't get result";
+ if (!Verify(results[i], gets[i].row(), families)) {
return false;
}
- // Test the values
- for (auto family : families) {
- auto cols = std::stoi(*(result->Value(family, kNumColumn)));
- VLOG(3) << "gets " << std::to_string(cols) << " columns";
- for (int m = 1; m <= cols; m++) {
- if (!Verify(result, family, m)) return false;
- }
- }
+ }
+ total_read += gets.size();
+ if (total_read != 0 && total_read % FLAGS_report_num_rows == 0) {
+ LOG(INFO) << "(Thread " << iteration << ") "
+ << "Sent " << total_read << " Multi-Get requests in "
+ << TimeUtil::ElapsedMillis(start_ns) << " ms.";
}
k += batch_num_rows;
}
- LOG(INFO) << "Sent " << rows << " gets in iteration " << iteration;
+ LOG(INFO) << "(Thread " << iteration << ") "
+ << "Sent " << rows << " gets"
+ << " in " << TimeUtil::ElapsedMillis(start_ns) << " ms.";
return true;
}
void DoPut(int iteration, uint64_t max_row, uint64_t rows, int cols, std::unique_ptr<Table> table,
const std::vector<std::string> &families) {
+ auto start_ns = TimeUtil::GetNowNanos();
auto width = std::to_string(max_row).length();
for (uint64_t j = 0; j < rows; j++) {
std::string row = PrefixZero(width, iteration * rows + j);
@@ -167,15 +225,20 @@ void DoPut(int iteration, uint64_t max_row, uint64_t rows, int cols, std::unique
}
}
table->Put(put);
- if ((j + 1) % FLAGS_report_num_rows == 0)
- LOG(INFO) << "Written " << std::to_string(j + 1) << " rows";
+ if ((j + 1) % FLAGS_report_num_rows == 0) {
+ LOG(INFO) << "(Thread " << iteration << ") "
+ << "Written " << std::to_string(j + 1) << " rows in "
+ << TimeUtil::ElapsedMillis(start_ns) << " ms.";
+ }
}
- LOG(INFO) << "written " << std::to_string(rows) << " rows in " << std::to_string(iteration)
- << " iteration";
+ LOG(INFO) << "(Thread " << iteration << ") "
+ << "written " << std::to_string(rows) << " rows"
+ << " in " << TimeUtil::ElapsedMillis(start_ns) << " ms.";
}
bool DoAppendIncrement(int iteration, uint64_t max_row, uint64_t rows, int cols,
std::unique_ptr<Table> table, const std::vector<std::string> &families) {
+ auto start_ns = TimeUtil::GetNowNanos();
auto width = std::to_string(max_row).length();
for (uint64_t j = 0; j < rows; j++) {
std::string row = PrefixZero(width, iteration * rows + j);
@@ -188,20 +251,26 @@ bool DoAppendIncrement(int iteration, uint64_t max_row, uint64_t rows, int cols,
hbase::Increment{row}.AddColumn(family, incrPrefix + std::to_string(k), k));
if (!table->Append(hbase::Append{row}.Add(family, appendPrefix + std::to_string(k),
std::to_string(k)))) {
- LOG(ERROR) << "append for " << row << " family: " << family << " failed";
+ LOG(ERROR) << "(Thread " << iteration << ") "
+ << "append for " << row << " family: " << family << " failed";
return false;
}
}
}
if ((j + 1) % FLAGS_report_num_rows == 0)
- LOG(INFO) << "Written " << std::to_string(j + 1) << " increments";
+ LOG(INFO) << "(Thread " << iteration << ") "
+ << "Written " << std::to_string(j + 1) << " increments"
+ << " in " << TimeUtil::ElapsedMillis(start_ns) << " ms.";
}
+ LOG(INFO) << "(Thread " << iteration << ") "
+ << "written " << std::to_string(rows) << " increments"
+ << " in " << TimeUtil::ElapsedMillis(start_ns) << " ms.";
return true;
}
int main(int argc, char *argv[]) {
- google::SetUsageMessage("Load client to manipulate multiple rows from HBase on the comamnd line");
- google::ParseCommandLineFlags(&argc, &argv, true);
+ gflags::SetUsageMessage("Load client to manipulate multiple rows from HBase on the comamnd line");
+ gflags::ParseCommandLineFlags(&argc, &argv, true);
google::InitGoogleLogging(argv[0]);
google::InstallFailureSignalHandler();
FLAGS_logtostderr = 1;
@@ -244,94 +313,78 @@ int main(int argc, char *argv[]) {
int rows = FLAGS_num_rows / FLAGS_threads;
if (FLAGS_num_rows % FLAGS_threads != 0) rows++;
int cols = FLAGS_num_cols;
- bool succeeded = true;
+ std::atomic<int8_t> succeeded{1}; // not using bool since we want atomic &=
if (FLAGS_puts) {
+ LOG(INFO) << "Sending put requests";
auto start_ns = TimeUtil::GetNowNanos();
std::vector<std::thread> writer_threads;
for (int i = 0; i < FLAGS_threads; i++) {
- writer_threads.push_back(std::thread([=, &client, &tn, &families, &succeeded] {
- // Get connection to HBase Table
+ writer_threads.push_back(std::thread([&, i] {
auto table = client->Table(*tn);
-
DoPut(i, FLAGS_num_rows - 1, rows, cols, std::move(table), families);
- table->Close();
}));
}
- for (std::vector<std::thread>::iterator it = writer_threads.begin(); it != writer_threads.end();
- it++) {
- std::thread thread = std::move(*it);
- thread.join();
+ for (auto &t : writer_threads) {
+ t.join();
}
LOG(INFO) << "Successfully sent " << num_puts << " Put requests in "
<< TimeUtil::ElapsedMillis(start_ns) << " ms.";
}
if (FLAGS_appends) {
+ LOG(INFO) << "Sending append/increment requests";
auto start_ns = TimeUtil::GetNowNanos();
std::vector<std::thread> writer_threads;
for (int i = 0; i < FLAGS_threads; i++) {
- writer_threads.push_back(std::thread([=, &client, &tn, &families, &succeeded] {
- // Get connection to HBase Table
+ writer_threads.push_back(std::thread([&, i] {
auto table = client->Table(*tn);
-
succeeded &=
DoAppendIncrement(i, FLAGS_num_rows - 1, rows, cols, std::move(table), families);
- table->Close();
}));
}
- for (std::vector<std::thread>::iterator it = writer_threads.begin(); it != writer_threads.end();
- it++) {
- std::thread thread = std::move(*it);
- thread.join();
+ for (auto &t : writer_threads) {
+ t.join();
}
LOG(INFO) << "Successfully sent " << num_puts << " append requests in "
<< TimeUtil::ElapsedMillis(start_ns) << " ms.";
}
if (FLAGS_scans) {
+ LOG(INFO) << "Sending scan requests";
auto start_ns = TimeUtil::GetNowNanos();
std::vector<std::thread> reader_threads;
for (int i = 0; i < FLAGS_threads; i++) {
- reader_threads.push_back(std::thread([=, &client, &tn, &families, &succeeded] {
- // Get connection to HBase Table
+ reader_threads.push_back(std::thread([&, i] {
auto table1 = client->Table(*tn);
-
succeeded &= DoScan(i, FLAGS_num_rows - 1, rows, std::move(table1), families);
- table1->Close();
}));
}
- for (std::vector<std::thread>::iterator it = reader_threads.begin(); it != reader_threads.end();
- it++) {
- std::thread thread = std::move(*it);
- thread.join();
+ for (auto &t : reader_threads) {
+ t.join();
}
- LOG(INFO) << (succeeded ? "Successful. " : "Failed. ") << "Spent "
- << TimeUtil::ElapsedMillis(start_ns) << " ms.";
+ LOG(INFO) << (succeeded.load() ? "Successfully " : "Failed. ") << " scannned " << num_puts
+ << " rows in " << TimeUtil::ElapsedMillis(start_ns) << " ms.";
}
if (FLAGS_gets) {
+ LOG(INFO) << "Sending get requests";
auto start_ns = TimeUtil::GetNowNanos();
std::vector<std::thread> reader_threads;
for (int i = 0; i < FLAGS_threads; i++) {
- reader_threads.push_back(std::thread([=, &client, &tn, &families, &succeeded] {
- // Get connection to HBase Table
+ reader_threads.push_back(std::thread([&, i] {
auto table1 = client->Table(*tn);
-
succeeded &=
DoGet(i, FLAGS_num_rows - 1, rows, std::move(table1), families, FLAGS_batch_num_rows);
- table1->Close();
}));
}
- for (std::vector<std::thread>::iterator it = reader_threads.begin(); it != reader_threads.end();
- it++) {
- std::thread thread = std::move(*it);
- thread.join();
+ for (auto &t : reader_threads) {
+ t.join();
}
- LOG(INFO) << (succeeded ? "Successful. " : "Failed. ") << "Spent "
- << TimeUtil::ElapsedMillis(start_ns) << " ms.";
+ LOG(INFO) << (succeeded.load() ? "Successful. " : "Failed. ") << " sent multi-get requests for "
+ << num_puts << " rows in " << TimeUtil::ElapsedMillis(start_ns) << " ms.";
}
client->Close();
- return succeeded ? 0 : -1;
+ return succeeded.load() ? 0 : -1;
}
diff --git a/core/simple-client.cc b/core/simple-client.cc
index d36689e..6730248 100644
--- a/core/simple-client.cc
+++ b/core/simple-client.cc
@@ -82,8 +82,8 @@ void ValidateResult(const Result &result, const std::string &row) {
}
int main(int argc, char *argv[]) {
- google::SetUsageMessage("Simple client to get a single row from HBase on the comamnd line");
- google::ParseCommandLineFlags(&argc, &argv, true);
+ gflags::SetUsageMessage("Simple client to get a single row from HBase on the comamnd line");
+ gflags::ParseCommandLineFlags(&argc, &argv, true);
google::InitGoogleLogging(argv[0]);
google::InstallFailureSignalHandler();
FLAGS_logtostderr = 1;
[hbase] 124/137: HBASE-18728 [C++] The library should be named
libHBaseClient, instead of libHbaseClient
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit a4040d2f7c8ed7efc7af23f80dc7c42ff9a94195
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Thu Aug 31 17:34:37 2017 -0700
HBASE-18728 [C++] The library should be named libHBaseClient, instead of libHbaseClient
---
Makefile | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/Makefile b/Makefile
index 708e907..c879f98 100644
--- a/Makefile
+++ b/Makefile
@@ -51,10 +51,10 @@ INCLUDES := $(addprefix -I,$(INCLUDE_DIR))
LIB_DIR := /usr/local
LIB_LIBDIR := $(LIB_DIR)/lib
LIB_INCDIR := $(LIB_DIR)/include
-LIB_RELEASE := $(RELEASE_PATH)/libHbaseClient.so
-ARC_RELEASE := $(RELEASE_PATH)/libHbaseClient.a
-LIB_DEBUG := $(DEBUG_PATH)/libHbaseClient_d.so
-ARC_DEBUG := $(DEBUG_PATH)/libHbaseClient_d.a
+LIB_RELEASE := $(RELEASE_PATH)/libHBaseClient.so
+ARC_RELEASE := $(RELEASE_PATH)/libHBaseClient.a
+LIB_DEBUG := $(DEBUG_PATH)/libHBaseClient_d.so
+ARC_DEBUG := $(DEBUG_PATH)/libHBaseClient_d.a
build: checkdirs protos $(LIB_DEBUG) $(LIB_RELEASE) $(ARC_DEBUG) $(ARC_RELEASE)
@@ -92,14 +92,14 @@ protos: $(PROTO_CXX_DIR) $(PROTOSRC) $(PROTOHDR)
@make -j8 all -f Makefile.protos
install:
- cp $(LIB_RELEASE) $(LIB_LIBDIR)/libHbaseClient.so
- cp $(ARC_RELEASE) $(LIB_LIBDIR)/libHbaseClient.a
- cp $(LIB_DEBUG) $(LIB_LIBDIR)/libHbaseClient_d.so
- cp $(ARC_DEBUG) $(LIB_LIBDIR)/libHbaseClient_d.a
+ cp $(LIB_RELEASE) $(LIB_LIBDIR)/libHBaseClient.so
+ cp $(ARC_RELEASE) $(LIB_LIBDIR)/libHBaseClient.a
+ cp $(LIB_DEBUG) $(LIB_LIBDIR)/libHBaseClient_d.so
+ cp $(ARC_DEBUG) $(LIB_LIBDIR)/libHBaseClient_d.a
ldconfig
-
+
uninstall:
- rm -f $(LIB_LIBDIR)/libHbaseClient.so $(LIB_LIBDIR)/libHbaseClient.a $(LIB_LIBDIR)/libHbaseClient_d.so $(ARC_DEBUG) $(LIB_LIBDIR)/libHbaseClient_d.a
+ rm -f $(LIB_LIBDIR)/libHBaseClient.so $(LIB_LIBDIR)/libHBaseClient.a $(LIB_LIBDIR)/libHBaseClient_d.so $(ARC_DEBUG) $(LIB_LIBDIR)/libHBaseClient_d.a
ldconfig
$(PROTO_CXX_DIR):
[hbase] 73/137: HBASE-17466 [C++] Test cleanup and minor
improvements
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit e9ab73fbb971d048247322cea34b071f4b473c28
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Mon Mar 13 14:21:24 2017 -0700
HBASE-17466 [C++] Test cleanup and minor improvements
---
connection/BUCK | 12 +--
core/BUCK | 35 ++++----
core/async-rpc-retrying-test.cc | 13 +--
core/client-test.cc | 41 ++-------
core/client.cc | 1 -
core/client.h | 2 -
core/filter-test.cc | 2 +-
core/location-cache-test.cc | 12 ++-
core/location-cache.cc | 15 ++--
core/location-cache.h | 11 +--
core/zk-util-test.cc | 50 +++++++++++
core/zk-util.cc | 58 +++++++++++++
core/zk-util.h | 41 +++++++++
security/BUCK | 4 +-
serde/BUCK | 52 +++---------
test-util/mini-cluster.cc | 180 +++++++++++++++-------------------------
test-util/mini-cluster.h | 23 ++---
test-util/test-util.cc | 41 +++++----
test-util/test-util.h | 23 ++---
third-party/BUCK | 4 +-
20 files changed, 320 insertions(+), 300 deletions(-)
diff --git a/connection/BUCK b/connection/BUCK
index bc05be0..19536d5 100644
--- a/connection/BUCK
+++ b/connection/BUCK
@@ -50,14 +50,8 @@ cxx_library(
"//third-party:wangle",
],
compiler_flags=['-Weffc++'],
- visibility=[
- '//core/...',
- ],)
+ visibility=['//core/...',],)
cxx_test(
name="connection-pool-test",
- srcs=[
- "connection-pool-test.cc",
- ],
- deps=[
- ":connection",
- ],)
+ srcs=["connection-pool-test.cc",],
+ deps=[":connection",],)
diff --git a/core/BUCK b/core/BUCK
index fd4c0dc..3df48d6 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -41,6 +41,7 @@ cxx_library(
"async-rpc-retrying-caller-factory.h",
"async-rpc-retrying-caller.h",
"hbase-rpc-controller.h",
+ "zk-util.h",
],
srcs=[
"cell.cc",
@@ -55,6 +56,7 @@ cxx_library(
"request-converter.cc",
"response-converter.cc",
"table.cc",
+ "zk-util.cc",
],
deps=[
"//exceptions:exceptions",
@@ -83,14 +85,10 @@ cxx_library(
],
deps=["//third-party:folly"],
compiler_flags=['-Weffc++', '-ggdb'],
- visibility=[
- 'PUBLIC',
- ],)
+ visibility=['PUBLIC',],)
cxx_test(
name="location-cache-test",
- srcs=[
- "location-cache-test.cc",
- ],
+ srcs=["location-cache-test.cc",],
deps=[
":core",
"//test-util:test-util",
@@ -98,18 +96,12 @@ cxx_test(
run_test_separately=True,)
cxx_test(
name="cell-test",
- srcs=[
- "cell-test.cc",
- ],
- deps=[
- ":core",
- ],
+ srcs=["cell-test.cc",],
+ deps=[":core",],
run_test_separately=True,)
cxx_test(
name="filter-test",
- srcs=[
- "filter-test.cc",
- ],
+ srcs=["filter-test.cc",],
deps=[
":core",
"//if:if",
@@ -167,9 +159,7 @@ cxx_test(
run_test_separately=True,)
cxx_test(
name="client-test",
- srcs=[
- "client-test.cc",
- ],
+ srcs=["client-test.cc",],
deps=[
":core",
"//if:if",
@@ -177,9 +167,12 @@ cxx_test(
"//test-util:test-util",
],
run_test_separately=True,)
+cxx_test(
+ name="zk-util-test",
+ srcs=["zk-util-test.cc",],
+ deps=[":core",],
+ run_test_separately=True,)
cxx_binary(
name="simple-client",
- srcs=[
- "simple-client.cc",
- ],
+ srcs=["simple-client.cc",],
deps=[":core", "//connection:connection"],)
diff --git a/core/async-rpc-retrying-test.cc b/core/async-rpc-retrying-test.cc
index f2a79a1..81d726f 100644
--- a/core/async-rpc-retrying-test.cc
+++ b/core/async-rpc-retrying-test.cc
@@ -143,11 +143,10 @@ class MockRawAsyncTableImpl {
};
TEST(AsyncRpcRetryTest, TestGetBasic) {
- // Remove already configured env if present.
- unsetenv("HBASE_CONF");
-
// Using TestUtil to populate test data
- hbase::TestUtil* test_util = new hbase::TestUtil();
+ auto test_util = std::make_unique<hbase::TestUtil>();
+ test_util->StartMiniCluster(2);
+
test_util->CreateTable("t", "d");
test_util->TablePut("t", "test2", "d", "2", "value2");
test_util->TablePut("t", "test2", "d", "extra", "value for extra");
@@ -231,12 +230,6 @@ TEST(AsyncRpcRetryTest, TestGetBasic) {
hbase::Result result = async_caller->Call().get();
- /*Stopping the connection as we are getting segfault due to some folly issue
- The connection stays open and we don't want that.
- So we are stopping the connection.
- We can remove this once we have fixed the folly part */
- delete test_util;
-
// Test the values, should be same as in put executed on hbase shell
ASSERT_TRUE(!result.IsEmpty()) << "Result shouldn't be empty.";
EXPECT_EQ("test2", result.Row());
diff --git a/core/client-test.cc b/core/client-test.cc
index 6f74e9c..72680c6 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -59,7 +59,8 @@ class ClientTest : public ::testing::Test {
static std::unique_ptr<hbase::TestUtil> test_util;
static void SetUpTestCase() {
- test_util = std::make_unique<hbase::TestUtil>(2, ClientTest::kDefHBaseConfPath.c_str());
+ test_util = std::make_unique<hbase::TestUtil>();
+ test_util->StartMiniCluster(2);
}
};
std::unique_ptr<hbase::TestUtil> ClientTest::test_util = nullptr;
@@ -86,9 +87,9 @@ const std::string ClientTest::kHBaseXmlData(
"the License.\n "
"*/\n-->\n<configuration>\n\n</configuration>");
-TEST(Client, EmptyConfigurationPassedToClient) { ASSERT_ANY_THROW(hbase::Client client); }
+TEST_F(ClientTest, EmptyConfigurationPassedToClient) { ASSERT_ANY_THROW(hbase::Client client); }
-TEST(Client, ConfigurationPassedToClient) {
+TEST_F(ClientTest, ConfigurationPassedToClient) {
// Remove already configured env if present.
unsetenv("HBASE_CONF");
ClientTest::CreateHBaseConfWithEnv();
@@ -101,7 +102,7 @@ TEST(Client, ConfigurationPassedToClient) {
client.Close();
}
-TEST(Client, DefaultConfiguration) {
+TEST_F(ClientTest, DefaultConfiguration) {
// Remove already configured env if present.
unsetenv("HBASE_CONF");
ClientTest::CreateHBaseConfWithEnv();
@@ -112,10 +113,6 @@ TEST(Client, DefaultConfiguration) {
}
TEST_F(ClientTest, Get) {
- // Remove already configured env if present.
- unsetenv("HBASE_CONF");
- ClientTest::CreateHBaseConfWithEnv();
-
// Using TestUtil to populate test data
ClientTest::test_util->CreateTable("t", "d");
ClientTest::test_util->TablePut("t", "test2", "d", "2", "value2");
@@ -128,12 +125,8 @@ TEST_F(ClientTest, Get) {
// Get to be performed on above HBase Table
hbase::Get get(row);
- // Create Configuration
- hbase::HBaseConfigurationLoader loader;
- auto conf = loader.LoadDefaultResources();
-
// Create a client
- hbase::Client client(conf.value());
+ hbase::Client client(*ClientTest::test_util->conf());
// Get connection to HBase Table
auto table = client.Table(tn);
@@ -152,11 +145,7 @@ TEST_F(ClientTest, Get) {
client.Close();
}
-TEST(Client, GetForNonExistentTable) {
- // Remove already configured env if present.
- unsetenv("HBASE_CONF");
- ClientTest::CreateHBaseConfWithEnv();
-
+TEST_F(ClientTest, GetForNonExistentTable) {
// Create TableName and Row to be fetched from HBase
auto tn = folly::to<hbase::pb::TableName>("t_not_exists");
auto row = "test2";
@@ -164,12 +153,8 @@ TEST(Client, GetForNonExistentTable) {
// Get to be performed on above HBase Table
hbase::Get get(row);
- // Create Configuration
- hbase::HBaseConfigurationLoader loader;
- auto conf = loader.LoadDefaultResources();
-
// Create a client
- hbase::Client client(conf.value());
+ hbase::Client client(*ClientTest::test_util->conf());
// Get connection to HBase Table
auto table = client.Table(tn);
@@ -183,10 +168,6 @@ TEST(Client, GetForNonExistentTable) {
}
TEST_F(ClientTest, GetForNonExistentRow) {
- // Remove already configured env if present.
- unsetenv("HBASE_CONF");
- ClientTest::CreateHBaseConfWithEnv();
-
// Using TestUtil to populate test data
ClientTest::test_util->CreateTable("t_exists", "d");
@@ -197,12 +178,8 @@ TEST_F(ClientTest, GetForNonExistentRow) {
// Get to be performed on above HBase Table
hbase::Get get(row);
- // Create Configuration
- hbase::HBaseConfigurationLoader loader;
- auto conf = loader.LoadDefaultResources();
-
// Create a client
- hbase::Client client(conf.value());
+ hbase::Client client(*ClientTest::test_util->conf());
// Get connection to HBase Table
auto table = client.Table(tn);
diff --git a/core/client.cc b/core/client.cc
index f0483ef..3f889d4 100644
--- a/core/client.cc
+++ b/core/client.cc
@@ -41,7 +41,6 @@ Client::Client(const hbase::Configuration &conf) { init(conf); }
void Client::init(const hbase::Configuration &conf) {
conf_ = std::make_shared<hbase::Configuration>(conf);
- auto zk_quorum = conf_->Get(kHBaseZookeeperQuorum_, kDefHBaseZookeeperQuorum_);
conn_conf_ = std::make_shared<hbase::ConnectionConfiguration>(*conf_);
// start thread pools
diff --git a/core/client.h b/core/client.h
index 6e3c889..040bea0 100644
--- a/core/client.h
+++ b/core/client.h
@@ -70,8 +70,6 @@ class Client {
private:
/** Constants */
- static constexpr const char* kHBaseZookeeperQuorum_ = "hbase.zookeeper.quorum";
- static constexpr const char* kDefHBaseZookeeperQuorum_ = "localhost:2181";
/** Parameter name for HBase client IO thread pool size. Defaults to num cpus */
static constexpr const char* kClientIoThreadPoolSize = "hbase.client.io.thread.pool.size";
/** Parameter name for HBase client CPU thread pool size. Defaults to (2 * num cpus) */
diff --git a/core/filter-test.cc b/core/filter-test.cc
index 6529b5a..40081cc 100644
--- a/core/filter-test.cc
+++ b/core/filter-test.cc
@@ -41,6 +41,7 @@ class FilterTest : public ::testing::Test {
protected:
static void SetUpTestCase() {
test_util_ = std::make_unique<TestUtil>();
+ test_util_->StartMiniCluster(2);
}
static void TearDownTestCase() { test_util_.release(); }
@@ -51,7 +52,6 @@ class FilterTest : public ::testing::Test {
static std::unique_ptr<TestUtil> test_util_;
};
-
std::unique_ptr<TestUtil> FilterTest::test_util_ = nullptr;
TEST_F(FilterTest, GetWithColumnPrefixFilter) {
diff --git a/core/location-cache-test.cc b/core/location-cache-test.cc
index 75a56d1..4159af4 100644
--- a/core/location-cache-test.cc
+++ b/core/location-cache-test.cc
@@ -34,6 +34,7 @@ class LocationCacheTest : public ::testing::Test {
protected:
static void SetUpTestCase() {
test_util_ = std::make_unique<TestUtil>();
+ test_util_->StartMiniCluster(2);
}
static void TearDownTestCase() { test_util_.release(); }
@@ -46,7 +47,6 @@ class LocationCacheTest : public ::testing::Test {
std::unique_ptr<TestUtil> LocationCacheTest::test_util_ = nullptr;
-
TEST_F(LocationCacheTest, TestGetMetaNodeContents) {
auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
@@ -105,8 +105,9 @@ TEST_F(LocationCacheTest, TestCaching) {
ASSERT_EQ(loc, cache.GetCachedLocation(tn_1, row_a));
// test with two regions
- std::string empty;
- LocationCacheTest::test_util_->CreateTable("t2", "d", "b", empty);
+ std::vector<std::string> keys;
+ keys.push_back("b");
+ LocationCacheTest::test_util_->CreateTable("t2", "d", keys);
ASSERT_FALSE(cache.IsLocationCached(tn_2, "a"));
loc = cache.LocateRegion(tn_2, "a").get(milliseconds(1000));
@@ -121,7 +122,10 @@ TEST_F(LocationCacheTest, TestCaching) {
ASSERT_EQ(loc, cache.GetCachedLocation(tn_2, "ba"));
// test with three regions
- LocationCacheTest::test_util_->CreateTable("t3", "d", "b", "c");
+ keys.clear();
+ keys.push_back("b");
+ keys.push_back("c");
+ LocationCacheTest::test_util_->CreateTable("t3", "d", keys);
ASSERT_FALSE(cache.IsLocationCached(tn_3, "c"));
ASSERT_FALSE(cache.IsLocationCached(tn_3, "ca"));
diff --git a/core/location-cache.cc b/core/location-cache.cc
index 9550614..505f48c 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -18,13 +18,12 @@
*/
#include "core/location-cache.h"
+#include <folly/Conv.h>
#include <folly/Logging.h>
#include <folly/io/IOBuf.h>
#include <wangle/concurrent/CPUThreadPoolExecutor.h>
#include <wangle/concurrent/IOThreadPoolExecutor.h>
-#include <utility>
-
#include <folly/Logging.h>
#include "connection/response.h"
#include "connection/rpc-connection.h"
@@ -34,6 +33,8 @@
#include "serde/server-name.h"
#include "serde/zk.h"
+#include <utility>
+
using namespace std;
using namespace folly;
using hbase::RpcConnection;
@@ -63,8 +64,8 @@ LocationCache::LocationCache(std::shared_ptr<hbase::Configuration> conf,
zk_(nullptr),
cached_locations_(),
locations_lock_() {
- quorum_spec_ = conf_->Get(kHBaseZookeeperQuorum_, kDefHBaseZookeeperQuorum_);
- zk_ = zookeeper_init(quorum_spec_.c_str(), nullptr, 1000, 0, 0, 0);
+ zk_quorum_ = ZKUtil::ParseZooKeeperQuorum(*conf_);
+ zk_ = zookeeper_init(zk_quorum_.c_str(), nullptr, 1000, 0, 0, 0);
}
LocationCache::~LocationCache() {
@@ -100,15 +101,13 @@ ServerName LocationCache::ReadMetaLocation() {
// This needs to be int rather than size_t as that's what ZK expects.
int len = buf->capacity();
- std::string zk_node = conf_->Get(kHBaseMetaZnodeName_, kDefHBaseMetaZnodeName_);
- zk_node += "/" + kHBaseMetaRegionServer_;
+ std::string zk_node = ZKUtil::MetaZNode(*conf_);
// TODO(elliott): handle disconnects/reconntion as needed.
int zk_result = zoo_get(this->zk_, zk_node.c_str(), 0,
reinterpret_cast<char *>(buf->writableData()), &len, nullptr);
if (zk_result != ZOK || len < 9) {
LOG(ERROR) << "Error getting meta location.";
- throw runtime_error("Error getting meta location. Quorum " +
- conf_->Get(kHBaseZookeeperQuorum_, ""));
+ throw runtime_error("Error getting meta location. Quorum: " + zk_quorum_);
}
buf->append(len);
diff --git a/core/location-cache.h b/core/location-cache.h
index 4d66e06..c1c5f62 100644
--- a/core/location-cache.h
+++ b/core/location-cache.h
@@ -36,6 +36,7 @@
#include "core/meta-utils.h"
#include "core/region-location.h"
#include "serde/table-name.h"
+#include "zk-util.h"
namespace hbase {
// Forward
@@ -172,6 +173,8 @@ class LocationCache {
*/
void ClearCachedLocation(const hbase::pb::TableName &tn, const std::string &row);
+ const std::string &zk_quorum() { return zk_quorum_; }
+
private:
void RefreshMetaLocation();
hbase::pb::ServerName ReadMetaLocation();
@@ -180,15 +183,9 @@ class LocationCache {
const hbase::pb::TableName &tn);
std::shared_ptr<hbase::PerTableLocationMap> GetNewTableLocations(const hbase::pb::TableName &tn);
- const std::string kHBaseZookeeperQuorum_ = "hbase.zookeeper.quorum";
- const std::string kDefHBaseZookeeperQuorum_ = "localhost:2181";
- const std::string kHBaseMetaZnodeName_ = "zookeeper.znode.parent";
- const std::string kDefHBaseMetaZnodeName_ = "/hbase";
- const std::string kHBaseMetaRegionServer_ = "meta-region-server";
-
/* data */
std::shared_ptr<hbase::Configuration> conf_;
- std::string quorum_spec_;
+ std::string zk_quorum_;
std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> meta_promise_;
std::mutex meta_lock_;
diff --git a/core/zk-util-test.cc b/core/zk-util-test.cc
new file mode 100644
index 0000000..b0cefdc
--- /dev/null
+++ b/core/zk-util-test.cc
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include <gtest/gtest.h>
+
+#include "core/zk-util.h"
+
+using hbase::Configuration;
+using hbase::ZKUtil;
+
+TEST(ZKUtilTest, ParseZooKeeperQuorum) {
+ Configuration conf{};
+ conf.Set(ZKUtil::kHBaseZookeeperQuorum_, "s1");
+ conf.SetInt(ZKUtil::kHBaseZookeeperClientPort_, 100);
+
+ ASSERT_EQ("s1:100", ZKUtil::ParseZooKeeperQuorum(conf));
+
+ conf.Set(ZKUtil::kHBaseZookeeperQuorum_, "s1:42");
+
+ ASSERT_EQ("s1:42", ZKUtil::ParseZooKeeperQuorum(conf));
+
+ conf.Set(ZKUtil::kHBaseZookeeperQuorum_, "s1,s2,s3");
+ ASSERT_EQ("s1:100,s2:100,s3:100", ZKUtil::ParseZooKeeperQuorum(conf));
+
+ conf.Set(ZKUtil::kHBaseZookeeperQuorum_, "s1:42,s2:42,s3:42");
+ ASSERT_EQ("s1:42,s2:42,s3:42", ZKUtil::ParseZooKeeperQuorum(conf));
+}
+
+TEST(ZKUtilTest, MetaZNode) {
+ Configuration conf{};
+ ASSERT_EQ("/hbase/meta-region-server", ZKUtil::MetaZNode(conf));
+
+ conf.Set(ZKUtil::kHBaseZnodeParent_, "/hbase-secure");
+ ASSERT_EQ("/hbase-secure/meta-region-server", ZKUtil::MetaZNode(conf));
+}
diff --git a/core/zk-util.cc b/core/zk-util.cc
new file mode 100644
index 0000000..50ea92a
--- /dev/null
+++ b/core/zk-util.cc
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/zk-util.h"
+
+#include <folly/Conv.h>
+#include <boost/algorithm/string.hpp>
+
+#include <vector>
+
+namespace hbase {
+
+/**
+ * Returns a "proper" zookeeper quorum string, from hbase's broken quorum string formats. In
+ * hbase.zookeeper.quorum, the ports are not listed explicitly per server (eg. s1,s2,s3),
+ * however ZooKeeper expects the string of the format s1:2181,s2:2181,s3:2181. This code
+ * appends the "clientPort" to each node in the quorum string if not there.
+ */
+std::string ZKUtil::ParseZooKeeperQuorum(const hbase::Configuration& conf) {
+ auto zk_quorum = conf.Get(kHBaseZookeeperQuorum_, kDefHBaseZookeeperQuorum_);
+ auto zk_port = conf.GetInt(kHBaseZookeeperClientPort_, kDefHBaseZookeeperClientPort_);
+
+ std::vector<std::string> zk_quorum_parts;
+ boost::split(zk_quorum_parts, zk_quorum, boost::is_any_of(","), boost::token_compress_on);
+ std::vector<std::string> servers;
+ for (auto server : zk_quorum_parts) {
+ if (boost::contains(server, ":")) {
+ servers.push_back(server);
+ } else {
+ servers.push_back(server + ":" + folly::to<std::string>(zk_port));
+ }
+ }
+ return boost::join(servers, ",");
+}
+
+std::string ZKUtil::MetaZNode(const hbase::Configuration& conf) {
+ std::string zk_node = conf.Get(kHBaseZnodeParent_, kDefHBaseZnodeParent_) + "/";
+ zk_node += kHBaseMetaRegionServer_;
+ return zk_node;
+}
+
+} // namespace hbase
diff --git a/core/zk-util.h b/core/zk-util.h
new file mode 100644
index 0000000..8f2d627
--- /dev/null
+++ b/core/zk-util.h
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <cstdlib>
+#include <string>
+#include "core/configuration.h"
+
+namespace hbase {
+
+class ZKUtil {
+ public:
+ static constexpr const char* kHBaseZookeeperQuorum_ = "hbase.zookeeper.quorum";
+ static constexpr const char* kDefHBaseZookeeperQuorum_ = "localhost:2181";
+ static constexpr const char* kHBaseZookeeperClientPort_ = "hbase.zookeeper.property.clientPort";
+ static constexpr const int32_t kDefHBaseZookeeperClientPort_ = 2181;
+ static constexpr const char* kHBaseZnodeParent_ = "zookeeper.znode.parent";
+ static constexpr const char* kDefHBaseZnodeParent_ = "/hbase";
+ static constexpr const char* kHBaseMetaRegionServer_ = "meta-region-server";
+
+ static std::string ParseZooKeeperQuorum(const hbase::Configuration& conf);
+
+ static std::string MetaZNode(const hbase::Configuration& conf);
+};
+} // namespace hbase
diff --git a/security/BUCK b/security/BUCK
index 1199797..7383028 100644
--- a/security/BUCK
+++ b/security/BUCK
@@ -19,9 +19,7 @@
# to a single server.
cxx_library(
name="security",
- exported_headers=[
- "user.h",
- ],
+ exported_headers=["user.h",],
srcs=[],
deps=["//core:conf"],
compiler_flags=['-Weffc++'],
diff --git a/serde/BUCK b/serde/BUCK
index 76d1c72..38e7b4d 100644
--- a/serde/BUCK
+++ b/serde/BUCK
@@ -41,54 +41,28 @@ cxx_library(
":region-info-deserializer-test",
],
compiler_flags=['-Weffc++'],
- visibility=[
- 'PUBLIC',
- ],)
+ visibility=['PUBLIC',],)
cxx_test(
name="table-name-test",
- srcs=[
- "table-name-test.cc",
- ],
- deps=[
- ":serde",
- ],)
+ srcs=["table-name-test.cc",],
+ deps=[":serde",],)
cxx_test(
name="server-name-test",
- srcs=[
- "server-name-test.cc",
- ],
- deps=[
- ":serde",
- ],)
+ srcs=["server-name-test.cc",],
+ deps=[":serde",],)
cxx_test(
name="client-serializer-test",
- srcs=[
- "client-serializer-test.cc",
- ],
- deps=[
- ":serde",
- ],)
+ srcs=["client-serializer-test.cc",],
+ deps=[":serde",],)
cxx_test(
name="client-deserializer-test",
- srcs=[
- "client-deserializer-test.cc",
- ],
- deps=[
- ":serde",
- ],)
+ srcs=["client-deserializer-test.cc",],
+ deps=[":serde",],)
cxx_test(
name="zk-deserializer-test",
- srcs=[
- "zk-deserializer-test.cc",
- ],
- deps=[
- ":serde",
- ],)
+ srcs=["zk-deserializer-test.cc",],
+ deps=[":serde",],)
cxx_test(
name="region-info-deserializer-test",
- srcs=[
- "region-info-deserializer-test.cc",
- ],
- deps=[
- ":serde",
- ],)
+ srcs=["region-info-deserializer-test.cc",],
+ deps=[":serde",],)
diff --git a/test-util/mini-cluster.cc b/test-util/mini-cluster.cc
index 88a091c..fb58fc1 100644
--- a/test-util/mini-cluster.cc
+++ b/test-util/mini-cluster.cc
@@ -18,13 +18,12 @@
*/
#include "test-util/mini-cluster.h"
-#include <glog/logging.h>
#include <fcntl.h>
-#include <boost/filesystem/operations.hpp>
+#include <glog/logging.h>
#include <boost/filesystem/fstream.hpp>
+#include <boost/filesystem/operations.hpp>
#include <fstream>
-
using hbase::MiniCluster;
JNIEnv *MiniCluster::CreateVM(JavaVM **jvm) {
@@ -60,7 +59,7 @@ JNIEnv *MiniCluster::CreateVM(JavaVM **jvm) {
}
fd.close();
}
- auto options = std::string {"-Djava.class.path="} + clspath;
+ auto options = std::string{"-Djava.class.path="} + clspath;
jvm_options.optionString = const_cast<char *>(options.c_str());
args.options = &jvm_options;
args.ignoreUnrecognized = 0;
@@ -74,40 +73,11 @@ JNIEnv *MiniCluster::CreateVM(JavaVM **jvm) {
return env_;
}
-void MiniCluster::WriteConf(jobject conf, const std::string& filepath) {
- jclass class_fdesc = env_->FindClass("java/io/FileDescriptor");
- // construct a new FileDescriptor
- jmethodID const_fdesc = env_->GetMethodID(class_fdesc, "<init>", "()V");
-
- jobject file = env_->NewObject(class_fdesc, const_fdesc);
- jfieldID field_fd = env_->GetFieldID(class_fdesc, "fd", "I");
-
- int fd = open(filepath.c_str(), O_RDWR | O_NONBLOCK | O_CREAT, S_IRWXU);
- if (fd < 0) {
- LOG(INFO) << "Couldn't open file " << filepath.c_str();
- exit(-1);
- }
- env_->SetIntField(file, field_fd, fd);
-
- jclass cls_outstream = env_->FindClass("java/io/FileOutputStream");
- jmethodID ctor_stream = env_->GetMethodID(cls_outstream, "<init>", "(Ljava/io/FileDescriptor;)V");
- if (ctor_stream == NULL) {
- LOG(INFO) << "Couldn't get ctor for FileOutputStream";
- exit(-1);
- }
- jobject file_outstream = env_->NewObject(cls_outstream, ctor_stream, file);
- if (file_outstream == NULL) {
- LOG(INFO) << "Couldn't create FileOutputStream";
- exit(-1);
- }
- jmethodID writeXmlMid = env_->GetMethodID(conf_class_, "writeXml", "(Ljava/io/OutputStream;)V");
- env_->CallObjectMethod(conf, writeXmlMid, file_outstream);
-}
void MiniCluster::Setup() {
jmethodID constructor;
pthread_mutex_lock(&count_mutex_);
if (env_ == NULL) {
- env_ = CreateVM(&jvm);
+ env_ = CreateVM(&jvm_);
if (env_ == NULL) {
exit(-1);
}
@@ -119,7 +89,7 @@ void MiniCluster::Setup() {
exit(-1);
}
jmethodID mid = env_->GetStaticMethodID(testing_util_class_, "createLocalHTU",
- "()Lorg/apache/hadoop/hbase/HBaseTestingUtility;");
+ "()Lorg/apache/hadoop/hbase/HBaseTestingUtility;");
htu_ = env_->CallStaticObjectMethod(testing_util_class_, mid);
// this should be converted to a globalref I think to avoid the underlying java obj getting
// GC'ed
@@ -128,12 +98,12 @@ void MiniCluster::Setup() {
exit(-1);
}
get_conn_mid_ = env_->GetMethodID(testing_util_class_, "getConnection",
- "()Lorg/apache/hadoop/hbase/client/Connection;");
- jclass connClass = env_->FindClass("org/apache/hadoop/hbase/client/Connection");
+ "()Lorg/apache/hadoop/hbase/client/Connection;");
+ jclass conn_class = env_->FindClass("org/apache/hadoop/hbase/client/Connection");
get_admin_mid_ =
- env_->GetMethodID(connClass, "getAdmin", "()Lorg/apache/hadoop/hbase/client/Admin;");
+ env_->GetMethodID(conn_class, "getAdmin", "()Lorg/apache/hadoop/hbase/client/Admin;");
get_table_mid_ = env_->GetMethodID(
- connClass, "getTable",
+ conn_class, "getTable",
"(Lorg/apache/hadoop/hbase/TableName;)Lorg/apache/hadoop/hbase/client/Table;");
if (get_table_mid_ == NULL) {
LOG(INFO) << "Couldn't find getConnection";
@@ -145,9 +115,10 @@ void MiniCluster::Setup() {
LOG(INFO) << "Couldn't find move";
exit(-1);
}
- create_table_mid_ = env_->GetMethodID(testing_util_class_, "createTable",
- "(Lorg/apache/hadoop/hbase/TableName;Ljava/lang/String;)Lorg/"
- "apache/hadoop/hbase/client/Table;");
+ create_table_mid_ =
+ env_->GetMethodID(testing_util_class_, "createTable",
+ "(Lorg/apache/hadoop/hbase/TableName;Ljava/lang/String;)Lorg/"
+ "apache/hadoop/hbase/client/Table;");
create_table_with_split_mid_ = env_->GetMethodID(
testing_util_class_, "createTable",
"(Lorg/apache/hadoop/hbase/TableName;[[B[[B)Lorg/apache/hadoop/hbase/client/Table;");
@@ -166,12 +137,13 @@ void MiniCluster::Setup() {
jclass hbaseMiniClusterClass = env_->FindClass("org/apache/hadoop/hbase/MiniHBaseCluster");
stop_rs_mid_ =
env_->GetMethodID(hbaseMiniClusterClass, "stopRegionServer",
- "(I)Lorg/apache/hadoop/hbase/util/JVMClusterUtil$RegionServerThread;");
+ "(I)Lorg/apache/hadoop/hbase/util/JVMClusterUtil$RegionServerThread;");
get_conf_mid_ = env_->GetMethodID(hbaseMiniClusterClass, "getConfiguration",
- "()Lorg/apache/hadoop/conf/Configuration;");
+ "()Lorg/apache/hadoop/conf/Configuration;");
conf_class_ = env_->FindClass("org/apache/hadoop/conf/Configuration");
- set_conf_mid_ = env_->GetMethodID(conf_class_, "set", "(Ljava/lang/String;Ljava/lang/String;)V");
+ set_conf_mid_ =
+ env_->GetMethodID(conf_class_, "set", "(Ljava/lang/String;Ljava/lang/String;)V");
if (set_conf_mid_ == NULL) {
LOG(INFO) << "Couldn't find method getConf in MiniHBaseCluster";
exit(-1);
@@ -182,7 +154,7 @@ void MiniCluster::Setup() {
put_mid_ = env_->GetMethodID(tableClass, "put", "(Lorg/apache/hadoop/hbase/client/Put;)V");
jclass connFactoryClass = env_->FindClass("org/apache/hadoop/hbase/client/ConnectionFactory");
create_conn_mid_ = env_->GetStaticMethodID(connFactoryClass, "createConnection",
- "()Lorg/apache/hadoop/hbase/client/Connection;");
+ "()Lorg/apache/hadoop/hbase/client/Connection;");
if (create_conn_mid_ == NULL) {
LOG(INFO) << "Couldn't find createConnection";
exit(-1);
@@ -209,49 +181,49 @@ JNIEnv *MiniCluster::env() {
return env_;
}
// converts C char* to Java byte[]
-jbyteArray MiniCluster::StrToByteChar(const std::string& str) {
- char *p = const_cast<char*>(str.c_str());
- int n = 0;
- while (*p++) {
- n++;
- }
- if (n == NULL) return NULL;
+jbyteArray MiniCluster::StrToByteChar(const std::string &str) {
+ char *p = const_cast<char *>(str.c_str());
+ int n = str.length();
jbyteArray arr = env_->NewByteArray(n);
env_->SetByteArrayRegion(arr, 0, n, reinterpret_cast<const jbyte *>(str.c_str()));
return arr;
}
-jobject MiniCluster::CreateTable(std::string tblNam, std::string familyName) {
- jstring tblNameStr = env_->NewStringUTF(tblNam.c_str());
- jobject tblName = env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_, tblNameStr);
- jstring famStr = env_->NewStringUTF(familyName.c_str());
- jobject tbl = env_->CallObjectMethod(htu_, create_table_mid_, tblName, famStr);
- return tbl;
+jobject MiniCluster::CreateTable(const std::string &table, const std::string &family) {
+ jstring table_name_str = env_->NewStringUTF(table.c_str());
+ jobject table_name =
+ env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_, table_name_str);
+ jstring family_str = env_->NewStringUTF(family.c_str());
+ jobject table_obj = env_->CallObjectMethod(htu_, create_table_mid_, table_name, family_str);
+ return table_obj;
}
-jobject MiniCluster::CreateTable(std::string tblNam, std::string familyName, std::string key1,
- std::string key2) {
- jstring tblNameStr = env_->NewStringUTF(tblNam.c_str());
- jobject tblName = env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_, tblNameStr);
- jclass arrayElemType = env_->FindClass("[B");
- jobjectArray famArray = env_->NewObjectArray(1, arrayElemType, env_->NewByteArray(1));
- env_->SetObjectArrayElement(famArray, 0, StrToByteChar(familyName));
+jobject MiniCluster::CreateTable(const std::string &table, const std::string &family,
+ const std::vector<std::string> &keys) {
+ jstring table_name_str = env_->NewStringUTF(table.c_str());
+ jobject table_name =
+ env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_, table_name_str);
+ jclass array_element_type = env_->FindClass("[B");
- int len = 2;
- if (key2.empty()) len = 1;
- jobjectArray keyArray = env_->NewObjectArray(len, arrayElemType, env_->NewByteArray(1));
+ jobjectArray family_array = env_->NewObjectArray(1, array_element_type, env_->NewByteArray(1));
+ env_->SetObjectArrayElement(family_array, 0, StrToByteChar(family));
- env_->SetObjectArrayElement(keyArray, 0, StrToByteChar(key1));
- if (!key2.empty()) {
- env_->SetObjectArrayElement(keyArray, 1, StrToByteChar(key2));
+ jobjectArray key_array =
+ env_->NewObjectArray(keys.size(), array_element_type, env_->NewByteArray(1));
+
+ int i = 0;
+ for (auto key : keys) {
+ env_->SetObjectArrayElement(key_array, i++, StrToByteChar(key));
}
- jobject tbl = env_->CallObjectMethod(htu_, create_table_with_split_mid_, tblName, famArray, keyArray);
+
+ jobject tbl = env_->CallObjectMethod(htu_, create_table_with_split_mid_, table_name, family_array,
+ key_array);
return tbl;
}
-jobject MiniCluster::StopRegionServer(jobject cluster, int idx) {
+jobject MiniCluster::StopRegionServer(int idx) {
env();
- return env_->CallObjectMethod(cluster, stop_rs_mid_, (jint)idx);
+ return env_->CallObjectMethod(cluster_, stop_rs_mid_, (jint)idx);
}
// returns the Configuration for the cluster
@@ -267,8 +239,9 @@ jobject MiniCluster::admin() {
return admin;
}
-jobject MiniCluster::TablePut(const std::string table, const std::string row, const std::string fam,
- const std::string col, const std::string value) {
+jobject MiniCluster::TablePut(const std::string &table, const std::string &row,
+ const std::string &family, const std::string &column,
+ const std::string &value) {
env();
jobject conn = env_->CallObjectMethod(htu(), get_conn_mid_);
jobject put = env_->NewObject(put_class_, put_ctor_, StrToByteChar(row));
@@ -276,47 +249,30 @@ jobject MiniCluster::TablePut(const std::string table, const std::string row, co
LOG(INFO) << "Couldn't create Put";
exit(-1);
}
- env_->CallObjectMethod(put, add_col_mid_, StrToByteChar(fam), StrToByteChar(col),
- StrToByteChar(value));
- jobject tblName =
- env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_,
- env_->NewStringUTF(table.c_str()));
- jobject tableObj = env_->CallObjectMethod(conn, get_table_mid_, tblName);
- env_->CallObjectMethod(tableObj, put_mid_, put);
- return tableObj;
+ env_->CallObjectMethod(put, add_col_mid_, StrToByteChar(family), StrToByteChar(column),
+ StrToByteChar(value));
+ jobject table_name_obj = env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_,
+ env_->NewStringUTF(table.c_str()));
+ jobject table_obj = env_->CallObjectMethod(conn, get_table_mid_, table_name_obj);
+ env_->CallObjectMethod(table_obj, put_mid_, put);
+ return table_obj;
}
// moves region to server
-void MiniCluster::MoveRegion(std::string region, std::string server) {
+void MiniCluster::MoveRegion(const std::string ®ion, const std::string &server) {
jobject admin_ = admin();
- env_->CallObjectMethod(admin_, move_mid_, StrToByteChar(region),
- StrToByteChar(server));
+ env_->CallObjectMethod(admin_, move_mid_, StrToByteChar(region), StrToByteChar(server));
}
-jobject MiniCluster::StartCluster(int numRegionServers, std::string conf_path) {
+jobject MiniCluster::StartCluster(int num_region_servers) {
env();
jmethodID mid = env_->GetMethodID(testing_util_class_, "startMiniCluster",
- "(I)Lorg/apache/hadoop/hbase/MiniHBaseCluster;");
+ "(I)Lorg/apache/hadoop/hbase/MiniHBaseCluster;");
if (mid == NULL) {
LOG(INFO) << "Couldn't find method startMiniCluster in the class HBaseTestingUtility";
exit(-1);
}
- cluster_ = env_->CallObjectMethod(htu(), mid, (jint)numRegionServers);
- jobject conf = GetConf();
- jstring jport = (jstring)env_->CallObjectMethod(
- conf, conf_get_mid_, env_->NewStringUTF("hbase.zookeeper.property.clientPort"));
- const char *port = env_->GetStringUTFChars(jport, 0);
- LOG(INFO) << "retrieved port " << port;
- std::string quorum("localhost:");
- env_->CallObjectMethod(conf, set_conf_mid_, env_->NewStringUTF("hbase.zookeeper.quorum"),
- env_->NewStringUTF((quorum + port).c_str()));
- if (!conf_path.empty()) {
- // Directory will be created if not present
- if (!boost::filesystem::exists(conf_path)) {
- boost::filesystem::create_directories(conf_path);
- }
- WriteConf(conf, conf_path + "/hbase-site.xml");
- }
+ cluster_ = env_->CallObjectMethod(htu(), mid, static_cast<jint>(num_region_servers));
return cluster_;
}
@@ -324,16 +280,16 @@ void MiniCluster::StopCluster() {
env();
jmethodID mid = env_->GetMethodID(testing_util_class_, "shutdownMiniCluster", "()V");
env_->CallVoidMethod(htu(), mid);
- if (jvm != NULL) {
- jvm->DestroyJavaVM();
- jvm = NULL;
+ if (jvm_ != NULL) {
+ jvm_->DestroyJavaVM();
+ jvm_ = NULL;
}
}
-const std::string MiniCluster::GetConfValue(std::string key) {
+const std::string MiniCluster::GetConfValue(const std::string &key) {
jobject conf = GetConf();
- jstring jval = (jstring)env_->CallObjectMethod(conf, conf_get_mid_,
- env_->NewStringUTF(key.c_str()));
+ jstring jval =
+ (jstring)env_->CallObjectMethod(conf, conf_get_mid_, env_->NewStringUTF(key.c_str()));
const char *val = env_->GetStringUTFChars(jval, 0);
return val;
}
diff --git a/test-util/mini-cluster.h b/test-util/mini-cluster.h
index 54842e1..a9502b5 100644
--- a/test-util/mini-cluster.h
+++ b/test-util/mini-cluster.h
@@ -19,27 +19,29 @@
#pragma once
#include <string>
+#include <vector>
#include "jni.h"
namespace hbase {
class MiniCluster {
public:
- jobject StartCluster(int numRegionServers, std::string conf_path);
+ jobject StartCluster(int32_t num_region_servers);
void StopCluster();
- jobject CreateTable(std::string tblNam, std::string familyName);
- jobject CreateTable(std::string tblNam, std::string familyName, std::string key1, std::string k2);
- jobject StopRegionServer(jobject cluster, int idx);
+ jobject CreateTable(const std::string &table, const std::string &family);
+ jobject CreateTable(const std::string &table, const std::string &family,
+ const std::vector<std::string> &keys);
+ jobject StopRegionServer(int idx);
// moves region to server
- void MoveRegion(std::string region, std::string server);
+ void MoveRegion(const std::string ®ion, const std::string &server);
// returns the Configuration instance for the cluster
jobject GetConf();
// returns the value for config key retrieved from cluster
- const std::string GetConfValue(std::string key);
+ const std::string GetConfValue(const std::string &key);
// Does Put into table for family fam, qualifier col with value
- jobject TablePut(const std::string table, const std::string row, const std::string fam,
- const std::string col, const std::string value);
+ jobject TablePut(const std::string &table, const std::string &row, const std::string &family,
+ const std::string &column, const std::string &value);
private:
JNIEnv *env_;
@@ -66,13 +68,12 @@ class MiniCluster {
jobject htu_;
jobject cluster_;
pthread_mutex_t count_mutex_;
- JavaVM *jvm;
+ JavaVM *jvm_;
JNIEnv *CreateVM(JavaVM **jvm);
- void WriteConf(jobject conf, const std::string& filepath);
void Setup();
jobject htu();
JNIEnv *env();
- jbyteArray StrToByteChar(const std::string& str);
+ jbyteArray StrToByteChar(const std::string &str);
jobject admin();
};
} /*namespace hbase*/
diff --git a/test-util/test-util.cc b/test-util/test-util.cc
index 13c4ec4..dfe4260 100644
--- a/test-util/test-util.cc
+++ b/test-util/test-util.cc
@@ -22,6 +22,8 @@
#include <folly/Format.h>
+#include "core/zk-util.h"
+
using hbase::TestUtil;
using folly::Random;
@@ -42,33 +44,28 @@ std::string TestUtil::RandString(int len) {
return s;
}
-TestUtil::TestUtil() : TestUtil::TestUtil(2, "") {}
-
-TestUtil::TestUtil(int servers, const std::string& confPath)
- : temp_dir_(TestUtil::RandString()), numRegionServers(servers), conf_path(confPath) {
- auto p = temp_dir_.path().string();
- StartMiniCluster(2);
- std::string quorum("localhost:");
- const std::string port = mini->GetConfValue("hbase.zookeeper.property.clientPort");
- conf()->Set("hbase.zookeeper.quorum", quorum + port);
-}
+TestUtil::TestUtil() : temp_dir_(TestUtil::RandString()) {}
TestUtil::~TestUtil() { StopMiniCluster(); }
-void TestUtil::StartMiniCluster(int num_region_servers) {
- mini = std::make_unique<MiniCluster>();
- mini->StartCluster(num_region_servers, conf_path);
+void TestUtil::StartMiniCluster(int32_t num_region_servers) {
+ mini_ = std::make_unique<MiniCluster>();
+ mini_->StartCluster(num_region_servers);
+
+ conf()->Set(ZKUtil::kHBaseZookeeperQuorum_, mini_->GetConfValue(ZKUtil::kHBaseZookeeperQuorum_));
+ conf()->Set(ZKUtil::kHBaseZookeeperClientPort_,
+ mini_->GetConfValue(ZKUtil::kHBaseZookeeperClientPort_));
}
-void TestUtil::StopMiniCluster() { mini->StopCluster(); }
+void TestUtil::StopMiniCluster() { mini_->StopCluster(); }
-void TestUtil::CreateTable(std::string tblNam, std::string familyName) {
- mini->CreateTable(tblNam, familyName);
+void TestUtil::CreateTable(const std::string &table, const std::string &family) {
+ mini_->CreateTable(table, family);
}
-void TestUtil::CreateTable(std::string tblNam, std::string familyName, std::string key1,
- std::string k2) {
- mini->CreateTable(tblNam, familyName, key1, k2);
+void TestUtil::CreateTable(const std::string &table, const std::string &family,
+ const std::vector<std::string> &keys) {
+ mini_->CreateTable(table, family, keys);
}
-void TestUtil::TablePut(std::string table, std::string row, std::string fam, std::string col,
- std::string value) {
- mini->TablePut(table, row, fam, col, value);
+void TestUtil::TablePut(const std::string &table, const std::string &row, const std::string &family,
+ const std::string &column, const std::string &value) {
+ mini_->TablePut(table, row, family, column, value);
}
diff --git a/test-util/test-util.h b/test-util/test-util.h
index 1734870..611404d 100644
--- a/test-util/test-util.h
+++ b/test-util/test-util.h
@@ -24,6 +24,7 @@
#include <cstdlib>
#include <memory>
#include <string>
+#include <vector>
#include "core/configuration.h"
#include "test-util/mini-cluster.h"
@@ -33,14 +34,7 @@ namespace hbase {
*/
class TestUtil {
public:
- /**
- * Creating a TestUtil will spin up a cluster.
- */
TestUtil();
- /**
- * Creating a TestUtil will spin up a cluster with numRegionServers region servers.
- */
- TestUtil(int numRegionServers, const std::string& confPath);
/**
* Destroying a TestUtil will spin down a cluster and remove the test dir.
@@ -61,19 +55,18 @@ class TestUtil {
/**
* Starts mini hbase cluster with specified number of region servers
*/
- void StartMiniCluster(int num_region_servers);
+ void StartMiniCluster(int32_t num_region_servers);
void StopMiniCluster();
- void CreateTable(std::string tblNam, std::string familyName);
- void CreateTable(std::string tblNam, std::string familyName, std::string key1, std::string k2);
- void TablePut(std::string table, std::string row, std::string fam, std::string col,
- std::string value);
+ void CreateTable(const std::string &table, const std::string &family);
+ void CreateTable(const std::string &table, const std::string &family,
+ const std::vector<std::string> &keys);
+ void TablePut(const std::string &table, const std::string &row, const std::string &family,
+ const std::string &column, const std::string &value);
private:
- std::unique_ptr<MiniCluster> mini;
+ std::unique_ptr<MiniCluster> mini_;
folly::test::TemporaryDirectory temp_dir_;
- int numRegionServers = 2;
- std::string conf_path;
std::shared_ptr<Configuration> conf_ = std::make_shared<Configuration>();
};
} // namespace hbase
diff --git a/third-party/BUCK b/third-party/BUCK
index a55a6fb..f37eb4e 100644
--- a/third-party/BUCK
+++ b/third-party/BUCK
@@ -112,6 +112,4 @@ cxx_library(
('googletest/googlemock', 'src/*.cc'),
]),
exported_deps=dynamic_rules,
- visibility=[
- 'PUBLIC',
- ],)
+ visibility=['PUBLIC',],)
[hbase] 87/137: HBASE-17860 Implement secure native client
connection - ADDENDUM to remove Dockerfile
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 2710ab9345e7b9ce0ffb421cf16f05d866884b99
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Wed May 31 11:34:51 2017 -0700
HBASE-17860 Implement secure native client connection - ADDENDUM to remove Dockerfile
---
Dockerfile | 58 ----------------------------------------------------------
1 file changed, 58 deletions(-)
diff --git a/Dockerfile b/Dockerfile
deleted file mode 100644
index 8b56590..0000000
--- a/Dockerfile
+++ /dev/null
@@ -1,58 +0,0 @@
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM pjameson/buck-folly-watchman:20160511
-
-ARG CC=/usr/bin/gcc-5
-ARG CXX=/usr/bin/g++-5
-ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -pthread"
-ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -pthread"
-
-ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
-
-RUN apt-get install -y vim maven inetutils-ping python-pip doxygen graphviz clang-format && \
- pip install yapf && \
- apt-get -qq clean && \
- apt-get -y -qq autoremove && \
- rm -rf /var/lib/{apt,dpkg,cache,log}/ && \
- rm -rf /tmp/*
-
-RUN git clone https://github.com/google/protobuf.git /usr/src/protobuf && \
- cd /usr/src/protobuf/ && \
- git checkout 2.7.0 && \
- mkdir gmock && \
- ldconfig && \
- ./autogen.sh && \
- ./configure && \
- make && \
- make install && \
- make clean && \
- rm -rf .git && \
- cd /usr/src && \
- wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && \
- tar zxf zookeeper-3.4.8.tar.gz && \
- rm -rf zookeeper-3.4.8.tar.gz && \
- cd zookeeper-3.4.8 && \
- cd src/c && \
- ldconfig && \
- ./configure && \
- make && \
- make install && \
- make clean && \
- ldconfig
-
-WORKDIR /usr/src/hbase/hbase-native-client
[hbase] 136/137: HBASE-22038 fix buiding failures
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 5eaad7c74844d43377608636212b893efed2d5b6
Author: xujunhong <xu...@xiaomi.com>
AuthorDate: Tue Mar 12 05:11:25 2019 -0700
HBASE-22038 fix buiding failures
Signed-off-by: Guanghao Zhang <zg...@apache.org>
---
.gitignore | 2 +
bin/copy-protobuf.sh | 2 +-
docker-files/Dockerfile | 25 +-
src/hbase/if/AccessControl.proto | 123 -----
src/hbase/if/Admin.proto | 310 -------------
src/hbase/if/Aggregate.proto | 63 ---
src/hbase/if/Authentication.proto | 82 ----
src/hbase/if/Cell.proto | 67 ---
src/hbase/if/Client.proto | 478 --------------------
src/hbase/if/ClusterId.proto | 34 --
src/hbase/if/ClusterStatus.proto | 227 ----------
src/hbase/if/Comparator.proto | 74 ---
src/hbase/if/Encryption.proto | 33 --
src/hbase/if/ErrorHandling.proto | 58 ---
src/hbase/if/FS.proto | 45 --
src/hbase/if/Filter.proto | 171 -------
src/hbase/if/HBase.proto | 253 -----------
src/hbase/if/HFile.proto | 49 --
src/hbase/if/LoadBalancer.proto | 29 --
src/hbase/if/MapReduce.proto | 37 --
src/hbase/if/Master.proto | 828 ----------------------------------
src/hbase/if/MasterProcedure.proto | 285 ------------
src/hbase/if/MultiRowMutation.proto | 45 --
src/hbase/if/Procedure.proto | 119 -----
src/hbase/if/Quota.proto | 76 ----
src/hbase/if/RPC.proto | 137 ------
src/hbase/if/RSGroup.proto | 34 --
src/hbase/if/RSGroupAdmin.proto | 136 ------
src/hbase/if/RegionNormalizer.proto | 28 --
src/hbase/if/RegionServerStatus.proto | 158 -------
src/hbase/if/RowProcessor.proto | 45 --
src/hbase/if/SecureBulkLoad.proto | 72 ---
src/hbase/if/Snapshot.proto | 66 ---
src/hbase/if/Tracing.proto | 33 --
src/hbase/if/VisibilityLabels.proto | 83 ----
src/hbase/if/WAL.proto | 173 -------
src/hbase/if/ZooKeeper.proto | 176 --------
src/hbase/if/test.proto | 43 --
src/hbase/if/test_rpc_service.proto | 36 --
39 files changed, 15 insertions(+), 4720 deletions(-)
diff --git a/.gitignore b/.gitignore
index 5f3e387..b83bb81 100644
--- a/.gitignore
+++ b/.gitignore
@@ -43,3 +43,5 @@ Makefile
# Copied from hbase-common at build-time
include/hbase/utils/version.h
+
+.idea/*
diff --git a/bin/copy-protobuf.sh b/bin/copy-protobuf.sh
index 47544c8..c928461 100755
--- a/bin/copy-protobuf.sh
+++ b/bin/copy-protobuf.sh
@@ -19,6 +19,6 @@ set -euo pipefail
IFS=$'\n\t'
BIN_DIR=$(dirname "$0")
-PB_SOURCE_DIR="${BIN_DIR}/../../hbase-protocol/src/main/protobuf/"
+PB_SOURCE_DIR="${BIN_DIR}/../../hbase-protocol-shaded/src/main/protobuf/"
PB_DEST_DIR="${BIN_DIR}/../src/hbase/if/"
rsync -r --exclude BUCK ${PB_SOURCE_DIR} ${PB_DEST_DIR}
diff --git a/docker-files/Dockerfile b/docker-files/Dockerfile
index 045f4f5..84868a9 100644
--- a/docker-files/Dockerfile
+++ b/docker-files/Dockerfile
@@ -27,14 +27,14 @@ ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
RUN apt-get update && \
apt-get install -y vim maven inetutils-ping python-pip doxygen graphviz clang-format valgrind \
wget libgflags-dev libgoogle-glog-dev dh-autoreconf pkg-config libssl-dev build-essential \
- libboost-all-dev libevent-dev libdouble-conversion-dev cmake libkrb5-dev git openjdk-8-jdk && \
+ libboost-all-dev libevent-dev libdouble-conversion-dev cmake libkrb5-dev git openjdk-8-jdk curl unzip && \
pip install yapf && \
apt-get -qq clean && \
apt-get -y -qq autoremove && \
rm -rf /var/lib/{apt,dpkg,cache,log}/ && \
rm -rf /tmp/*
-RUN wget ftp://ftp.cyrusimap.org/cyrus-sasl/cyrus-sasl-2.1.26.tar.gz ; \
+RUN wget https://www.cyrusimap.org/releases/cyrus-sasl-2.1.26.tar.gz ; \
tar zxf cyrus-sasl-2.1.26.tar.gz ; \
cd cyrus-sasl-2.1.26 ; \
./configure ; \
@@ -43,7 +43,7 @@ RUN wget ftp://ftp.cyrusimap.org/cyrus-sasl/cyrus-sasl-2.1.26.tar.gz ; \
cp /usr/local/lib/sasl2/* /usr/lib/sasl2/
RUN apt-get update && \
- apt-get install -y debconf-utils debconf-set-selections && \
+ apt-get install -y debconf-utils && \
echo "krb5-config krb5-config/kerberos_servers string localhost" | debconf-set-selections ; \
echo "krb5-config krb5-config/admin_server string localhost" | debconf-set-selections ; \
echo "krb5-config krb5-config/add_servers_realm string EXAMPLE.COM" | debconf-set-selections ; \
@@ -64,7 +64,7 @@ COPY docker-files/krb5.conf /etc
RUN git clone https://github.com/google/protobuf.git /usr/src/protobuf && \
cd /usr/src/protobuf/ && \
- git checkout 2.7.0 && \
+ git checkout 3.5.1.1 && \
mkdir gmock && \
ldconfig && \
./autogen.sh && \
@@ -72,9 +72,9 @@ RUN git clone https://github.com/google/protobuf.git /usr/src/protobuf && \
make -j4 && \
make install && \
make clean && \
- rm -rf .git && \
+ rm -rf .git* && \
cd /usr/src && \
- wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && \
+ wget https://archive.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && \
tar zxf zookeeper-3.4.8.tar.gz && \
rm -rf zookeeper-3.4.8.tar.gz && \
cd zookeeper-3.4.8 && \
@@ -89,11 +89,10 @@ RUN git clone https://github.com/google/protobuf.git /usr/src/protobuf && \
# Update folly. `make check` removed due to a failing test, explicitly installing gtest ourselves
RUN cd /usr/src/ && \
ver=2017.09.04.00 && \
- wget https://github.com/facebook/folly/archive/v$ver.tar.gz && \
- tar zxf v$ver.tar.gz && \
- rm -rf v$ver.tar.gz && \
+ wget https://github.com/facebook/folly/archive/v$ver.tar.gz -O folly-$ver.tar.gz && \
+ tar zxf folly-$ver.tar.gz && \
+ rm -rf folly-$ver.tar.gz && \
cd folly-$ver/folly/test && \
- rm -rf gtest && \
wget https://github.com/google/googletest/archive/release-1.8.0.tar.gz && \
tar zxf release-1.8.0.tar.gz && \
rm -f release-1.8.0.tar.gz && \
@@ -113,9 +112,9 @@ RUN cd /usr/src/ && \
# Update wangle
RUN cd /usr/src/ && \
ver=2017.09.04.00 && \
- wget https://github.com/facebook/wangle/archive/v$ver.tar.gz && \
- tar zxf v$ver.tar.gz && \
- rm -rf v$ver.tar.gz && \
+ wget https://github.com/facebook/wangle/archive/v$ver.tar.gz -O wangle-$ver.tar.gz && \
+ tar zxf wangle-$ver.tar.gz && \
+ rm -rf wangle-$ver.tar.gz && \
cd wangle-$ver/wangle && \
ldconfig && \
cmake . -DBUILD_TESTS=OFF && \
diff --git a/src/hbase/if/AccessControl.proto b/src/hbase/if/AccessControl.proto
deleted file mode 100644
index e67540b..0000000
--- a/src/hbase/if/AccessControl.proto
+++ /dev/null
@@ -1,123 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "AccessControlProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-import "HBase.proto";
-
-message Permission {
- enum Action {
- READ = 0;
- WRITE = 1;
- EXEC = 2;
- CREATE = 3;
- ADMIN = 4;
- }
- enum Type {
- Global = 1;
- Namespace = 2;
- Table = 3;
- }
- required Type type = 1;
- optional GlobalPermission global_permission = 2;
- optional NamespacePermission namespace_permission = 3;
- optional TablePermission table_permission = 4;
-}
-
-message TablePermission {
- optional TableName table_name = 1;
- optional bytes family = 2;
- optional bytes qualifier = 3;
- repeated Permission.Action action = 4;
-}
-
-message NamespacePermission {
- optional bytes namespace_name = 1;
- repeated Permission.Action action = 2;
-}
-
-message GlobalPermission {
- repeated Permission.Action action = 1;
-}
-
-message UserPermission {
- required bytes user = 1;
- required Permission permission = 3;
-}
-
-/**
- * Content of the /hbase/acl/<table or namespace> znode.
- */
-message UsersAndPermissions {
- message UserPermissions {
- required bytes user = 1;
- repeated Permission permissions = 2;
- }
-
- repeated UserPermissions user_permissions = 1;
-}
-
-message GrantRequest {
- required UserPermission user_permission = 1;
-}
-
-message GrantResponse {
-}
-
-message RevokeRequest {
- required UserPermission user_permission = 1;
-}
-
-message RevokeResponse {
-}
-
-message GetUserPermissionsRequest {
- optional Permission.Type type = 1;
- optional TableName table_name = 2;
- optional bytes namespace_name = 3;
-}
-
-message GetUserPermissionsResponse {
- repeated UserPermission user_permission = 1;
-}
-
-message CheckPermissionsRequest {
- repeated Permission permission = 1;
-}
-
-message CheckPermissionsResponse {
-}
-
-service AccessControlService {
- rpc Grant(GrantRequest)
- returns (GrantResponse);
-
- rpc Revoke(RevokeRequest)
- returns (RevokeResponse);
-
- rpc GetUserPermissions(GetUserPermissionsRequest)
- returns (GetUserPermissionsResponse);
-
- rpc CheckPermissions(CheckPermissionsRequest)
- returns (CheckPermissionsResponse);
-}
diff --git a/src/hbase/if/Admin.proto b/src/hbase/if/Admin.proto
deleted file mode 100644
index a1905a4..0000000
--- a/src/hbase/if/Admin.proto
+++ /dev/null
@@ -1,310 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file contains protocol buffers that are used for Admin service.
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "AdminProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-import "Client.proto";
-import "HBase.proto";
-import "WAL.proto";
-
-message GetRegionInfoRequest {
- required RegionSpecifier region = 1;
- optional bool compaction_state = 2;
-}
-
-message GetRegionInfoResponse {
- required RegionInfo region_info = 1;
- optional CompactionState compaction_state = 2;
- optional bool isRecovering = 3;
-
- enum CompactionState {
- NONE = 0;
- MINOR = 1;
- MAJOR = 2;
- MAJOR_AND_MINOR = 3;
- }
-}
-
-/**
- * Get a list of store files for a set of column families in a particular region.
- * If no column family is specified, get the store files for all column families.
- */
-message GetStoreFileRequest {
- required RegionSpecifier region = 1;
- repeated bytes family = 2;
-}
-
-message GetStoreFileResponse {
- repeated string store_file = 1;
-}
-
-message GetOnlineRegionRequest {
-}
-
-message GetOnlineRegionResponse {
- repeated RegionInfo region_info = 1;
-}
-
-message OpenRegionRequest {
- repeated RegionOpenInfo open_info = 1;
- // the intended server for this RPC.
- optional uint64 serverStartCode = 2;
- // wall clock time from master
- optional uint64 master_system_time = 5;
-
- message RegionOpenInfo {
- required RegionInfo region = 1;
- optional uint32 version_of_offline_node = 2;
- repeated ServerName favored_nodes = 3;
- // open region for distributedLogReplay
- optional bool openForDistributedLogReplay = 4;
- }
-}
-
-message OpenRegionResponse {
- repeated RegionOpeningState opening_state = 1;
-
- enum RegionOpeningState {
- OPENED = 0;
- ALREADY_OPENED = 1;
- FAILED_OPENING = 2;
- }
-}
-
-message WarmupRegionRequest {
-
- required RegionInfo regionInfo = 1;
-}
-
-message WarmupRegionResponse {
-}
-
-/**
- * Closes the specified region and will use or not use ZK during the close
- * according to the specified flag.
- */
-message CloseRegionRequest {
- required RegionSpecifier region = 1;
- optional uint32 version_of_closing_node = 2;
- optional bool transition_in_ZK = 3 [default = true];
- optional ServerName destination_server = 4;
- // the intended server for this RPC.
- optional uint64 serverStartCode = 5;
-}
-
-message CloseRegionResponse {
- required bool closed = 1;
-}
-
-/**
- * Flushes the MemStore of the specified region.
- * <p>
- * This method is synchronous.
- */
-message FlushRegionRequest {
- required RegionSpecifier region = 1;
- optional uint64 if_older_than_ts = 2;
- optional bool write_flush_wal_marker = 3; // whether to write a marker to WAL even if not flushed
-}
-
-message FlushRegionResponse {
- required uint64 last_flush_time = 1;
- optional bool flushed = 2;
- optional bool wrote_flush_wal_marker = 3;
-}
-
-/**
- * Splits the specified region.
- * <p>
- * This method currently flushes the region and then forces a compaction which
- * will then trigger a split. The flush is done synchronously but the
- * compaction is asynchronous.
- */
-message SplitRegionRequest {
- required RegionSpecifier region = 1;
- optional bytes split_point = 2;
-}
-
-message SplitRegionResponse {
-}
-
-/**
- * Compacts the specified region. Performs a major compaction if specified.
- * <p>
- * This method is asynchronous.
- */
-message CompactRegionRequest {
- required RegionSpecifier region = 1;
- optional bool major = 2;
- optional bytes family = 3;
-}
-
-message CompactRegionResponse {
-}
-
-message UpdateFavoredNodesRequest {
- repeated RegionUpdateInfo update_info = 1;
-
- message RegionUpdateInfo {
- required RegionInfo region = 1;
- repeated ServerName favored_nodes = 2;
- }
-}
-
-message UpdateFavoredNodesResponse {
- optional uint32 response = 1;
-}
-
-/**
- * Merges the specified regions.
- * <p>
- * This method currently closes the regions and then merges them
- */
-message MergeRegionsRequest {
- required RegionSpecifier region_a = 1;
- required RegionSpecifier region_b = 2;
- optional bool forcible = 3 [default = false];
- // wall clock time from master
- optional uint64 master_system_time = 4;
-}
-
-message MergeRegionsResponse {
-}
-
-// Protocol buffer version of WAL for replication
-message WALEntry {
- required WALKey key = 1;
- // Following may be null if the KVs/Cells are carried along the side in a cellblock (See
- // RPC for more on cellblocks). If Cells/KVs are in a cellblock, this next field is null
- // and associated_cell_count has count of Cells associated w/ this WALEntry
- repeated bytes key_value_bytes = 2;
- // If Cell data is carried alongside in a cellblock, this is count of Cells in the cellblock.
- optional int32 associated_cell_count = 3;
-}
-
-/**
- * Replicates the given entries. The guarantee is that the given entries
- * will be durable on the slave cluster if this method returns without
- * any exception. hbase.replication has to be set to true for this to work.
- */
-message ReplicateWALEntryRequest {
- repeated WALEntry entry = 1;
- optional string replicationClusterId = 2;
- optional string sourceBaseNamespaceDirPath = 3;
- optional string sourceHFileArchiveDirPath = 4;
-}
-
-message ReplicateWALEntryResponse {
-}
-
-message RollWALWriterRequest {
-}
-
-/*
- * Roll request responses no longer include regions to flush
- * this list will always be empty when talking to a 1.0 server
- */
-message RollWALWriterResponse {
- // A list of encoded name of regions to flush
- repeated bytes region_to_flush = 1;
-}
-
-message StopServerRequest {
- required string reason = 1;
-}
-
-message StopServerResponse {
-}
-
-message GetServerInfoRequest {
-}
-
-message ServerInfo {
- required ServerName server_name = 1;
- optional uint32 webui_port = 2;
-}
-
-message GetServerInfoResponse {
- required ServerInfo server_info = 1;
-}
-
-message UpdateConfigurationRequest {
-}
-
-message UpdateConfigurationResponse {
-}
-
-service AdminService {
- rpc GetRegionInfo(GetRegionInfoRequest)
- returns(GetRegionInfoResponse);
-
- rpc GetStoreFile(GetStoreFileRequest)
- returns(GetStoreFileResponse);
-
- rpc GetOnlineRegion(GetOnlineRegionRequest)
- returns(GetOnlineRegionResponse);
-
- rpc OpenRegion(OpenRegionRequest)
- returns(OpenRegionResponse);
-
- rpc WarmupRegion(WarmupRegionRequest)
- returns(WarmupRegionResponse);
-
- rpc CloseRegion(CloseRegionRequest)
- returns(CloseRegionResponse);
-
- rpc FlushRegion(FlushRegionRequest)
- returns(FlushRegionResponse);
-
- rpc SplitRegion(SplitRegionRequest)
- returns(SplitRegionResponse);
-
- rpc CompactRegion(CompactRegionRequest)
- returns(CompactRegionResponse);
-
- rpc MergeRegions(MergeRegionsRequest)
- returns(MergeRegionsResponse);
-
- rpc ReplicateWALEntry(ReplicateWALEntryRequest)
- returns(ReplicateWALEntryResponse);
-
- rpc Replay(ReplicateWALEntryRequest)
- returns(ReplicateWALEntryResponse);
-
- rpc RollWALWriter(RollWALWriterRequest)
- returns(RollWALWriterResponse);
-
- rpc GetServerInfo(GetServerInfoRequest)
- returns(GetServerInfoResponse);
-
- rpc StopServer(StopServerRequest)
- returns(StopServerResponse);
-
- rpc UpdateFavoredNodes(UpdateFavoredNodesRequest)
- returns(UpdateFavoredNodesResponse);
-
- rpc UpdateConfiguration(UpdateConfigurationRequest)
- returns(UpdateConfigurationResponse);
-}
diff --git a/src/hbase/if/Aggregate.proto b/src/hbase/if/Aggregate.proto
deleted file mode 100644
index 4d32e70..0000000
--- a/src/hbase/if/Aggregate.proto
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "AggregateProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-import "Client.proto";
-
-message AggregateRequest {
- /** The request passed to the AggregateService consists of three parts
- * (1) the (canonical) classname of the ColumnInterpreter implementation
- * (2) the Scan query
- * (3) any bytes required to construct the ColumnInterpreter object
- * properly
- */
- required string interpreter_class_name = 1;
- required Scan scan = 2;
- optional bytes interpreter_specific_bytes = 3;
-}
-
-message AggregateResponse {
- /**
- * The AggregateService methods all have a response that either is a Pair
- * or a simple object. When it is a Pair both first_part and second_part
- * have defined values (and the second_part is not present in the response
- * when the response is not a pair). Refer to the AggregateImplementation
- * class for an overview of the AggregateResponse object constructions.
- */
- repeated bytes first_part = 1;
- optional bytes second_part = 2;
-}
-
-/** Refer to the AggregateImplementation class for an overview of the
- * AggregateService method implementations and their functionality.
- */
-service AggregateService {
- rpc GetMax (AggregateRequest) returns (AggregateResponse);
- rpc GetMin (AggregateRequest) returns (AggregateResponse);
- rpc GetSum (AggregateRequest) returns (AggregateResponse);
- rpc GetRowNum (AggregateRequest) returns (AggregateResponse);
- rpc GetAvg (AggregateRequest) returns (AggregateResponse);
- rpc GetStd (AggregateRequest) returns (AggregateResponse);
- rpc GetMedian (AggregateRequest) returns (AggregateResponse);
-}
diff --git a/src/hbase/if/Authentication.proto b/src/hbase/if/Authentication.proto
deleted file mode 100644
index 2f64799..0000000
--- a/src/hbase/if/Authentication.proto
+++ /dev/null
@@ -1,82 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "AuthenticationProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-message AuthenticationKey {
- required int32 id = 1;
- required int64 expiration_date = 2;
- required bytes key = 3;
-}
-
-
-message TokenIdentifier {
- enum Kind {
- HBASE_AUTH_TOKEN = 0;
- }
- required Kind kind = 1;
- required bytes username = 2;
- required int32 key_id = 3;
- optional int64 issue_date = 4;
- optional int64 expiration_date = 5;
- optional int64 sequence_number = 6;
-}
-
-
-// Serialization of the org.apache.hadoop.security.token.Token class
-// Note that this is a Hadoop class, so fields may change!
-message Token {
- // the TokenIdentifier in serialized form
- // Note: we can't use the protobuf directly because the Hadoop Token class
- // only stores the serialized bytes
- optional bytes identifier = 1;
- optional bytes password = 2;
- optional bytes service = 3;
-}
-
-
-// RPC request & response messages
-message GetAuthenticationTokenRequest {
-}
-
-message GetAuthenticationTokenResponse {
- optional Token token = 1;
-}
-
-message WhoAmIRequest {
-}
-
-message WhoAmIResponse {
- optional string username = 1;
- optional string auth_method = 2;
-}
-
-
-// RPC service
-service AuthenticationService {
- rpc GetAuthenticationToken(GetAuthenticationTokenRequest)
- returns (GetAuthenticationTokenResponse);
-
- rpc WhoAmI(WhoAmIRequest)
- returns (WhoAmIResponse);
-}
diff --git a/src/hbase/if/Cell.proto b/src/hbase/if/Cell.proto
deleted file mode 100644
index 2c61035..0000000
--- a/src/hbase/if/Cell.proto
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// Cell and KeyValue protos
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "CellProtos";
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-/**
- * The type of the key in a Cell
- */
-enum CellType {
- MINIMUM = 0;
- PUT = 4;
-
- DELETE = 8;
- DELETE_COLUMN = 12;
- DELETE_FAMILY = 14;
-
- // MAXIMUM is used when searching; you look from maximum on down.
- MAXIMUM = 255;
-}
-
-/**
- * Protocol buffer version of Cell.
- */
-message Cell {
- optional bytes row = 1;
- optional bytes family = 2;
- optional bytes qualifier = 3;
- optional uint64 timestamp = 4;
- optional CellType cell_type = 5;
- optional bytes value = 6;
- optional bytes tags = 7;
-}
-
-/**
- * Protocol buffer version of KeyValue.
- * It doesn't have those transient parameters
- */
-message KeyValue {
- required bytes row = 1;
- required bytes family = 2;
- required bytes qualifier = 3;
- optional uint64 timestamp = 4;
- optional CellType key_type = 5;
- optional bytes value = 6;
- optional bytes tags = 7;
-}
diff --git a/src/hbase/if/Client.proto b/src/hbase/if/Client.proto
deleted file mode 100644
index 8a4d459..0000000
--- a/src/hbase/if/Client.proto
+++ /dev/null
@@ -1,478 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file contains protocol buffers that are used for Client service.
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "ClientProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-import "HBase.proto";
-import "Filter.proto";
-import "Cell.proto";
-import "Comparator.proto";
-import "MapReduce.proto";
-
-/**
- * The protocol buffer version of Authorizations.
- */
-message Authorizations {
- repeated string label = 1;
-}
-
-/**
- * The protocol buffer version of CellVisibility.
- */
-message CellVisibility {
- required string expression = 1;
-}
-
-/**
- * Container for a list of column qualifier names of a family.
- */
-message Column {
- required bytes family = 1;
- repeated bytes qualifier = 2;
-}
-
-/**
- * Consistency defines the expected consistency level for an operation.
- */
-enum Consistency {
- STRONG = 0;
- TIMELINE = 1;
-}
-
-/**
- * The protocol buffer version of Get.
- * Unless existence_only is specified, return all the requested data
- * for the row that matches exactly.
- */
-message Get {
- required bytes row = 1;
- repeated Column column = 2;
- repeated NameBytesPair attribute = 3;
- optional Filter filter = 4;
- optional TimeRange time_range = 5;
- optional uint32 max_versions = 6 [default = 1];
- optional bool cache_blocks = 7 [default = true];
- optional uint32 store_limit = 8;
- optional uint32 store_offset = 9;
-
- // The result isn't asked for, just check for
- // the existence.
- optional bool existence_only = 10 [default = false];
-
- optional Consistency consistency = 12 [default = STRONG];
- repeated ColumnFamilyTimeRange cf_time_range = 13;
-}
-
-message Result {
- // Result includes the Cells or else it just has a count of Cells
- // that are carried otherwise.
- repeated Cell cell = 1;
- // The below count is set when the associated cells are
- // not part of this protobuf message; they are passed alongside
- // and then this Message is just a placeholder with metadata.
- // The count is needed to know how many to peel off the block of Cells as
- // ours. NOTE: This is different from the pb managed cell_count of the
- // 'cell' field above which is non-null when the cells are pb'd.
- optional int32 associated_cell_count = 2;
-
- // used for Get to check existence only. Not set if existence_only was not set to true
- // in the query.
- optional bool exists = 3;
-
- // Whether or not the results are coming from possibly stale data
- optional bool stale = 4 [default = false];
-
- // Whether or not the entire result could be returned. Results will be split when
- // the RPC chunk size limit is reached. Partial results contain only a subset of the
- // cells for a row and must be combined with a result containing the remaining cells
- // to form a complete result
- optional bool partial = 5 [default = false];
-}
-
-/**
- * The get request. Perform a single Get operation.
- */
-message GetRequest {
- required RegionSpecifier region = 1;
- required Get get = 2;
-}
-
-message GetResponse {
- optional Result result = 1;
-}
-
-/**
- * Condition to check if the value of a given cell (row,
- * family, qualifier) matches a value via a given comparator.
- *
- * Condition is used in check and mutate operations.
- */
-message Condition {
- required bytes row = 1;
- required bytes family = 2;
- required bytes qualifier = 3;
- required CompareType compare_type = 4;
- required Comparator comparator = 5;
-}
-
-
-/**
- * A specific mutation inside a mutate request.
- * It can be an append, increment, put or delete based
- * on the mutation type. It can be fully filled in or
- * only metadata present because data is being carried
- * elsewhere outside of pb.
- */
-message MutationProto {
- optional bytes row = 1;
- optional MutationType mutate_type = 2;
- repeated ColumnValue column_value = 3;
- optional uint64 timestamp = 4;
- repeated NameBytesPair attribute = 5;
- optional Durability durability = 6 [default = USE_DEFAULT];
-
- // For some mutations, a result may be returned, in which case,
- // time range can be specified for potential performance gain
- optional TimeRange time_range = 7;
- // The below count is set when the associated cells are NOT
- // part of this protobuf message; they are passed alongside
- // and then this Message is a placeholder with metadata. The
- // count is needed to know how many to peel off the block of Cells as
- // ours. NOTE: This is different from the pb managed cell_count of the
- // 'cell' field above which is non-null when the cells are pb'd.
- optional int32 associated_cell_count = 8;
-
- optional uint64 nonce = 9;
-
- enum Durability {
- USE_DEFAULT = 0;
- SKIP_WAL = 1;
- ASYNC_WAL = 2;
- SYNC_WAL = 3;
- FSYNC_WAL = 4;
- }
-
- enum MutationType {
- APPEND = 0;
- INCREMENT = 1;
- PUT = 2;
- DELETE = 3;
- }
-
- enum DeleteType {
- DELETE_ONE_VERSION = 0;
- DELETE_MULTIPLE_VERSIONS = 1;
- DELETE_FAMILY = 2;
- DELETE_FAMILY_VERSION = 3;
- }
-
- message ColumnValue {
- required bytes family = 1;
- repeated QualifierValue qualifier_value = 2;
-
- message QualifierValue {
- optional bytes qualifier = 1;
- optional bytes value = 2;
- optional uint64 timestamp = 3;
- optional DeleteType delete_type = 4;
- optional bytes tags = 5;
- }
- }
-}
-
-/**
- * The mutate request. Perform a single Mutate operation.
- *
- * Optionally, you can specify a condition. The mutate
- * will take place only if the condition is met. Otherwise,
- * the mutate will be ignored. In the response result,
- * parameter processed is used to indicate if the mutate
- * actually happened.
- */
-message MutateRequest {
- required RegionSpecifier region = 1;
- required MutationProto mutation = 2;
- optional Condition condition = 3;
- optional uint64 nonce_group = 4;
-}
-
-message MutateResponse {
- optional Result result = 1;
-
- // used for mutate to indicate processed only
- optional bool processed = 2;
-}
-
-/**
- * Instead of get from a table, you can scan it with optional filters.
- * You can specify the row key range, time range, the columns/families
- * to scan and so on.
- *
- * This scan is used the first time in a scan request. The response of
- * the initial scan will return a scanner id, which should be used to
- * fetch result batches later on before it is closed.
- */
-message Scan {
- repeated Column column = 1;
- repeated NameBytesPair attribute = 2;
- optional bytes start_row = 3;
- optional bytes stop_row = 4;
- optional Filter filter = 5;
- optional TimeRange time_range = 6;
- optional uint32 max_versions = 7 [default = 1];
- optional bool cache_blocks = 8 [default = true];
- optional uint32 batch_size = 9;
- optional uint64 max_result_size = 10;
- optional uint32 store_limit = 11;
- optional uint32 store_offset = 12;
- optional bool load_column_families_on_demand = 13; /* DO NOT add defaults to load_column_families_on_demand. */
- optional bool small = 14;
- optional bool reversed = 15 [default = false];
- optional Consistency consistency = 16 [default = STRONG];
- optional uint32 caching = 17;
- optional bool allow_partial_results = 18;
- repeated ColumnFamilyTimeRange cf_time_range = 19;
-}
-
-/**
- * A scan request. Initially, it should specify a scan. Later on, you
- * can use the scanner id returned to fetch result batches with a different
- * scan request.
- *
- * The scanner will remain open if there are more results, and it's not
- * asked to be closed explicitly.
- *
- * You can fetch the results and ask the scanner to be closed to save
- * a trip if you are not interested in remaining results.
- */
-message ScanRequest {
- optional RegionSpecifier region = 1;
- optional Scan scan = 2;
- optional uint64 scanner_id = 3;
- optional uint32 number_of_rows = 4;
- optional bool close_scanner = 5;
- optional uint64 next_call_seq = 6;
- optional bool client_handles_partials = 7;
- optional bool client_handles_heartbeats = 8;
- optional bool track_scan_metrics = 9;
- optional bool renew = 10 [default = false];
-}
-
-/**
- * The scan response. If there are no more results, more_results will
- * be false. If it is not specified, it means there are more.
- */
-message ScanResponse {
- // This field is filled in if we are doing cellblocks. A cellblock is made up
- // of all Cells serialized out as one cellblock BUT responses from a server
- // have their Cells grouped by Result. So we can reconstitute the
- // Results on the client-side, this field is a list of counts of Cells
- // in each Result that makes up the response. For example, if this field
- // has 3, 3, 3 in it, then we know that on the client, we are to make
- // three Results each of three Cells each.
- repeated uint32 cells_per_result = 1;
-
- optional uint64 scanner_id = 2;
- optional bool more_results = 3;
- optional uint32 ttl = 4;
- // If cells are not carried in an accompanying cellblock, then they are pb'd here.
- // This field is mutually exclusive with cells_per_result (since the Cells will
- // be inside the pb'd Result)
- repeated Result results = 5;
- optional bool stale = 6;
-
- // This field is filled in if we are doing cellblocks. In the event that a row
- // could not fit all of its cells into a single RPC chunk, the results will be
- // returned as partials, and reconstructed into a complete result on the client
- // side. This field is a list of flags indicating whether or not the result
- // that the cells belong to is a partial result. For example, if this field
- // has false, false, true in it, then we know that on the client side, we need to
- // make another RPC request since the last result was only a partial.
- repeated bool partial_flag_per_result = 7;
-
- // A server may choose to limit the number of results returned to the client for
- // reasons such as the size in bytes or quantity of results accumulated. This field
- // will true when more results exist in the current region.
- optional bool more_results_in_region = 8;
-
- // This field is filled in if the server is sending back a heartbeat message.
- // Heartbeat messages are sent back to the client to prevent the scanner from
- // timing out. Seeing a heartbeat message communicates to the Client that the
- // server would have continued to scan had the time limit not been reached.
- optional bool heartbeat_message = 9;
-
- // This field is filled in if the client has requested that scan metrics be tracked.
- // The metrics tracked here are sent back to the client to be tracked together with
- // the existing client side metrics.
- optional ScanMetrics scan_metrics = 10;
-}
-
-/**
- * Atomically bulk load multiple HFiles (say from different column families)
- * into an open region.
- */
-message BulkLoadHFileRequest {
- required RegionSpecifier region = 1;
- repeated FamilyPath family_path = 2;
- optional bool assign_seq_num = 3;
-
- message FamilyPath {
- required bytes family = 1;
- required string path = 2;
- }
-}
-
-message BulkLoadHFileResponse {
- required bool loaded = 1;
-}
-
-message CoprocessorServiceCall {
- required bytes row = 1;
- required string service_name = 2;
- required string method_name = 3;
- required bytes request = 4;
-}
-
-message CoprocessorServiceResult {
- optional NameBytesPair value = 1;
-}
-
-message CoprocessorServiceRequest {
- required RegionSpecifier region = 1;
- required CoprocessorServiceCall call = 2;
-}
-
-message CoprocessorServiceResponse {
- required RegionSpecifier region = 1;
- required NameBytesPair value = 2;
-}
-
-// Either a Get or a Mutation
-message Action {
- // If part of a multi action, useful aligning
- // result with what was originally submitted.
- optional uint32 index = 1;
- optional MutationProto mutation = 2;
- optional Get get = 3;
- optional CoprocessorServiceCall service_call = 4;
-}
-
-/**
- * Actions to run against a Region.
- */
-message RegionAction {
- required RegionSpecifier region = 1;
- // When set, run mutations as atomic unit.
- optional bool atomic = 2;
- repeated Action action = 3;
-}
-
-/*
-* Statistics about the current load on the region
-*/
-message RegionLoadStats {
- // Percent load on the memstore. Guaranteed to be positive, between 0 and 100.
- optional int32 memstoreLoad = 1 [default = 0];
- // Percent JVM heap occupancy. Guaranteed to be positive, between 0 and 100.
- // We can move this to "ServerLoadStats" should we develop them.
- optional int32 heapOccupancy = 2 [default = 0];
- // Compaction pressure. Guaranteed to be positive, between 0 and 100.
- optional int32 compactionPressure = 3 [default = 0];
-}
-
-message MultiRegionLoadStats{
- repeated RegionSpecifier region = 1;
- repeated RegionLoadStats stat = 2;
-}
-
-/**
- * Either a Result or an Exception NameBytesPair (keyed by
- * exception name whose value is the exception stringified)
- * or maybe empty if no result and no exception.
- */
-message ResultOrException {
- // If part of a multi call, save original index of the list of all
- // passed so can align this response w/ original request.
- optional uint32 index = 1;
- optional Result result = 2;
- optional NameBytesPair exception = 3;
- // result if this was a coprocessor service call
- optional CoprocessorServiceResult service_result = 4;
- // current load on the region
- optional RegionLoadStats loadStats = 5 [deprecated=true];
-}
-
-/**
- * The result of a RegionAction.
- */
-message RegionActionResult {
- repeated ResultOrException resultOrException = 1;
- // If the operation failed globally for this region, this exception is set
- optional NameBytesPair exception = 2;
-}
-
-/**
- * Execute a list of actions on a given region in order.
- * Nothing prevents a request to contains a set of RegionAction on the same region.
- * For this reason, the matching between the MultiRequest and the MultiResponse is not
- * done by the region specifier but by keeping the order of the RegionActionResult vs.
- * the order of the RegionAction.
- */
-message MultiRequest {
- repeated RegionAction regionAction = 1;
- optional uint64 nonceGroup = 2;
- optional Condition condition = 3;
-}
-
-message MultiResponse {
- repeated RegionActionResult regionActionResult = 1;
- // used for mutate to indicate processed only
- optional bool processed = 2;
- optional MultiRegionLoadStats regionStatistics = 3;
-}
-
-
-service ClientService {
- rpc Get(GetRequest)
- returns(GetResponse);
-
- rpc Mutate(MutateRequest)
- returns(MutateResponse);
-
- rpc Scan(ScanRequest)
- returns(ScanResponse);
-
- rpc BulkLoadHFile(BulkLoadHFileRequest)
- returns(BulkLoadHFileResponse);
-
- rpc ExecService(CoprocessorServiceRequest)
- returns(CoprocessorServiceResponse);
-
- rpc ExecRegionServerService(CoprocessorServiceRequest)
- returns(CoprocessorServiceResponse);
-
- rpc Multi(MultiRequest)
- returns(MultiResponse);
-}
diff --git a/src/hbase/if/ClusterId.proto b/src/hbase/if/ClusterId.proto
deleted file mode 100644
index aed8cfc..0000000
--- a/src/hbase/if/ClusterId.proto
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file contains protocol buffers that are shared throughout HBase
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "ClusterIdProtos";
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-/**
- * Content of the '/hbase/hbaseid', cluster id, znode.
- * Also cluster of the ${HBASE_ROOTDIR}/hbase.id file.
- */
-message ClusterId {
- // This is the cluster id, a uuid as a String
- required string cluster_id = 1;
-}
diff --git a/src/hbase/if/ClusterStatus.proto b/src/hbase/if/ClusterStatus.proto
deleted file mode 100644
index 54bc0c3..0000000
--- a/src/hbase/if/ClusterStatus.proto
+++ /dev/null
@@ -1,227 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file contains protocol buffers that are used for ClustStatus
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "ClusterStatusProtos";
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-import "HBase.proto";
-import "ClusterId.proto";
-import "FS.proto";
-
-message RegionState {
- required RegionInfo region_info = 1;
- required State state = 2;
- optional uint64 stamp = 3;
- enum State {
- OFFLINE = 0; // region is in an offline state
- PENDING_OPEN = 1; // sent rpc to server to open but has not begun
- OPENING = 2; // server has begun to open but not yet done
- OPEN = 3; // server opened region and updated meta
- PENDING_CLOSE = 4; // sent rpc to server to close but has not begun
- CLOSING = 5; // server has begun to close but not yet done
- CLOSED = 6; // server closed region and updated meta
- SPLITTING = 7; // server started split of a region
- SPLIT = 8; // server completed split of a region
- FAILED_OPEN = 9; // failed to open, and won't retry any more
- FAILED_CLOSE = 10; // failed to close, and won't retry any more
- MERGING = 11; // server started merge a region
- MERGED = 12; // server completed merge of a region
- SPLITTING_NEW = 13; // new region to be created when RS splits a parent
- // region but hasn't be created yet, or master doesn't
- // know it's already created
- MERGING_NEW = 14; // new region to be created when RS merges two
- // daughter regions but hasn't be created yet, or
- // master doesn't know it's already created
- }
-}
-
-message RegionInTransition {
- required RegionSpecifier spec = 1;
- required RegionState region_state = 2;
-}
-
-/**
- * sequence Id of a store
- */
-message StoreSequenceId {
- required bytes family_name = 1;
- required uint64 sequence_id = 2;
-}
-
-/**
- * contains a sequence id of a region which should be the minimum of its store sequence ids and
- * list of sequence ids of the region's stores
- */
-message RegionStoreSequenceIds {
- required uint64 last_flushed_sequence_id = 1;
- repeated StoreSequenceId store_sequence_id = 2;
-}
-
-message RegionLoad {
- /** the region specifier */
- required RegionSpecifier region_specifier = 1;
-
- /** the number of stores for the region */
- optional uint32 stores = 2;
-
- /** the number of storefiles for the region */
- optional uint32 storefiles = 3;
-
- /** the total size of the store files for the region, uncompressed, in MB */
- optional uint32 store_uncompressed_size_MB = 4;
-
- /** the current total size of the store files for the region, in MB */
- optional uint32 storefile_size_MB = 5;
-
- /** the current size of the memstore for the region, in MB */
- optional uint32 memstore_size_MB = 6;
-
- /**
- * The current total size of root-level store file indexes for the region,
- * in MB. The same as {@link #rootIndexSizeKB} but in MB.
- */
- optional uint32 storefile_index_size_MB = 7;
-
- /** the current total read requests made to region */
- optional uint64 read_requests_count = 8;
-
- /** the current total write requests made to region */
- optional uint64 write_requests_count = 9;
-
- /** the total compacting key values in currently running compaction */
- optional uint64 total_compacting_KVs = 10;
-
- /** the completed count of key values in currently running compaction */
- optional uint64 current_compacted_KVs = 11;
-
- /** The current total size of root-level indexes for the region, in KB. */
- optional uint32 root_index_size_KB = 12;
-
- /** The total size of all index blocks, not just the root level, in KB. */
- optional uint32 total_static_index_size_KB = 13;
-
- /**
- * The total size of all Bloom filter blocks, not just loaded into the
- * block cache, in KB.
- */
- optional uint32 total_static_bloom_size_KB = 14;
-
- /** the most recent sequence Id from cache flush */
- optional uint64 complete_sequence_id = 15;
-
- /** The current data locality for region in the regionserver */
- optional float data_locality = 16;
-
- optional uint64 last_major_compaction_ts = 17 [default = 0];
-
- /** the most recent sequence Id of store from cache flush */
- repeated StoreSequenceId store_complete_sequence_id = 18;
-
- /** the current total filtered read requests made to region */
- optional uint64 filtered_read_requests_count = 19;
-}
-
-/* Server-level protobufs */
-
-message ReplicationLoadSink {
- required uint64 ageOfLastAppliedOp = 1;
- required uint64 timeStampsOfLastAppliedOp = 2;
-}
-
-message ReplicationLoadSource {
- required string peerID = 1;
- required uint64 ageOfLastShippedOp = 2;
- required uint32 sizeOfLogQueue = 3;
- required uint64 timeStampOfLastShippedOp = 4;
- required uint64 replicationLag = 5;
-}
-
-message ServerLoad {
- /** Number of requests since last report. */
- optional uint64 number_of_requests = 1;
-
- /** Total Number of requests from the start of the region server. */
- optional uint64 total_number_of_requests = 2;
-
- /** the amount of used heap, in MB. */
- optional uint32 used_heap_MB = 3;
-
- /** the maximum allowable size of the heap, in MB. */
- optional uint32 max_heap_MB = 4;
-
- /** Information on the load of individual regions. */
- repeated RegionLoad region_loads = 5;
-
- /**
- * Regionserver-level coprocessors, e.g., WALObserver implementations.
- * Region-level coprocessors, on the other hand, are stored inside RegionLoad
- * objects.
- */
- repeated Coprocessor coprocessors = 6;
-
- /**
- * Time when incremental (non-total) counts began being calculated (e.g. number_of_requests)
- * time is measured as the difference, measured in milliseconds, between the current time
- * and midnight, January 1, 1970 UTC.
- */
- optional uint64 report_start_time = 7;
-
- /**
- * Time when report was generated.
- * time is measured as the difference, measured in milliseconds, between the current time
- * and midnight, January 1, 1970 UTC.
- */
- optional uint64 report_end_time = 8;
-
- /**
- * The port number that this region server is hosing an info server on.
- */
- optional uint32 info_server_port = 9;
-
- /**
- * The replicationLoadSource for the replication Source status of this region server.
- */
- repeated ReplicationLoadSource replLoadSource = 10;
-
- /**
- * The replicationLoadSink for the replication Sink status of this region server.
- */
- optional ReplicationLoadSink replLoadSink = 11;
-}
-
-message LiveServerInfo {
- required ServerName server = 1;
- required ServerLoad server_load = 2;
-}
-
-message ClusterStatus {
- optional HBaseVersionFileContent hbase_version = 1;
- repeated LiveServerInfo live_servers = 2;
- repeated ServerName dead_servers = 3;
- repeated RegionInTransition regions_in_transition = 4;
- optional ClusterId cluster_id = 5;
- repeated Coprocessor master_coprocessors = 6;
- optional ServerName master = 7;
- repeated ServerName backup_masters = 8;
- optional bool balancer_on = 9;
-}
diff --git a/src/hbase/if/Comparator.proto b/src/hbase/if/Comparator.proto
deleted file mode 100644
index 496b68d..0000000
--- a/src/hbase/if/Comparator.proto
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file contains protocol buffers that are used for filters
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "ComparatorProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-// This file contains protocol buffers that are used for comparators (e.g. in filters)
-
-message Comparator {
- required string name = 1;
- optional bytes serialized_comparator = 2;
-}
-
-message ByteArrayComparable {
- optional bytes value = 1;
-}
-
-message BinaryComparator {
- required ByteArrayComparable comparable = 1;
-}
-
-message LongComparator {
- required ByteArrayComparable comparable = 1;
-}
-
-message BinaryPrefixComparator {
- required ByteArrayComparable comparable = 1;
-}
-
-message BitComparator {
- required ByteArrayComparable comparable = 1;
- required BitwiseOp bitwise_op = 2;
-
- enum BitwiseOp {
- AND = 1;
- OR = 2;
- XOR = 3;
- }
-}
-
-message NullComparator {
-}
-
-message RegexStringComparator {
- required string pattern = 1;
- required int32 pattern_flags = 2;
- required string charset = 3;
- optional string engine = 4;
-}
-
-message SubstringComparator {
- required string substr = 1;
-}
diff --git a/src/hbase/if/Encryption.proto b/src/hbase/if/Encryption.proto
deleted file mode 100644
index 97ab5b2..0000000
--- a/src/hbase/if/Encryption.proto
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file contains protocol buffers used for encryption
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "EncryptionProtos";
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-message WrappedKey {
- required string algorithm = 1;
- required uint32 length = 2;
- required bytes data = 3;
- optional bytes iv = 4;
- optional bytes hash = 5;
-}
diff --git a/src/hbase/if/ErrorHandling.proto b/src/hbase/if/ErrorHandling.proto
deleted file mode 100644
index be9a743..0000000
--- a/src/hbase/if/ErrorHandling.proto
+++ /dev/null
@@ -1,58 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file contains protocol buffers that are used for error handling
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "ErrorHandlingProtos";
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-/**
- * Protobuf version of a java.lang.StackTraceElement
- * so we can serialize exceptions.
- */
-message StackTraceElementMessage {
- optional string declaring_class = 1;
- optional string method_name = 2;
- optional string file_name = 3;
- optional int32 line_number = 4;
-}
-
-/**
- * Cause of a remote failure for a generic exception. Contains
- * all the information for a generic exception as well as
- * optional info about the error for generic info passing
- * (which should be another protobuffed class).
- */
-message GenericExceptionMessage {
- optional string class_name = 1;
- optional string message = 2;
- optional bytes error_info = 3;
- repeated StackTraceElementMessage trace = 4;
-}
-
-/**
- * Exception sent across the wire when a remote task needs
- * to notify other tasks that it failed and why
- */
-message ForeignExceptionMessage {
- optional string source = 1;
- optional GenericExceptionMessage generic_exception = 2;
-}
diff --git a/src/hbase/if/FS.proto b/src/hbase/if/FS.proto
deleted file mode 100644
index 9e93120..0000000
--- a/src/hbase/if/FS.proto
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file contains protocol buffers that are written into the filesystem
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "FSProtos";
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-/**
- * The ${HBASE_ROOTDIR}/hbase.version file content
- */
-message HBaseVersionFileContent {
- required string version = 1;
-}
-
-/**
- * Reference file content used when we split an hfile under a region.
- */
-message Reference {
- required bytes splitkey = 1;
- enum Range {
- TOP = 0;
- BOTTOM = 1;
- }
- required Range range = 2;
-}
-
diff --git a/src/hbase/if/Filter.proto b/src/hbase/if/Filter.proto
deleted file mode 100644
index 1fa6697..0000000
--- a/src/hbase/if/Filter.proto
+++ /dev/null
@@ -1,171 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file contains protocol buffers that are used for filters
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "FilterProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-import "HBase.proto";
-import "Comparator.proto";
-
-message Filter {
- required string name = 1;
- optional bytes serialized_filter = 2;
-}
-
-message ColumnCountGetFilter {
- required int32 limit = 1;
-}
-
-message ColumnPaginationFilter {
- required int32 limit = 1;
- optional int32 offset = 2;
- optional bytes column_offset = 3;
-}
-
-message ColumnPrefixFilter {
- required bytes prefix = 1;
-}
-
-message ColumnRangeFilter {
- optional bytes min_column = 1;
- optional bool min_column_inclusive = 2;
- optional bytes max_column = 3;
- optional bool max_column_inclusive = 4;
-}
-
-message CompareFilter {
- required CompareType compare_op = 1;
- optional Comparator comparator = 2;
-}
-
-message DependentColumnFilter {
- required CompareFilter compare_filter = 1;
- optional bytes column_family = 2;
- optional bytes column_qualifier = 3;
- optional bool drop_dependent_column = 4;
-}
-
-message FamilyFilter {
- required CompareFilter compare_filter = 1;
-}
-
-message FilterList {
- required Operator operator = 1;
- repeated Filter filters = 2;
-
- enum Operator {
- MUST_PASS_ALL = 1;
- MUST_PASS_ONE = 2;
- }
-}
-
-message FilterWrapper {
- required Filter filter = 1;
-}
-
-message FirstKeyOnlyFilter {
-}
-
-message FirstKeyValueMatchingQualifiersFilter {
- repeated bytes qualifiers = 1;
-}
-
-message FuzzyRowFilter {
- repeated BytesBytesPair fuzzy_keys_data = 1;
-}
-
-message InclusiveStopFilter {
- optional bytes stop_row_key = 1;
-}
-
-message KeyOnlyFilter {
- required bool len_as_val = 1;
-}
-
-message MultipleColumnPrefixFilter {
- repeated bytes sorted_prefixes = 1;
-}
-
-message PageFilter {
- required int64 page_size = 1;
-}
-
-message PrefixFilter {
- optional bytes prefix = 1;
-}
-
-message QualifierFilter {
- required CompareFilter compare_filter = 1;
-}
-
-message RandomRowFilter {
- required float chance = 1;
-}
-
-message RowFilter {
- required CompareFilter compare_filter = 1;
-}
-
-message SingleColumnValueExcludeFilter {
- required SingleColumnValueFilter single_column_value_filter = 1;
-}
-
-message SingleColumnValueFilter {
- optional bytes column_family = 1;
- optional bytes column_qualifier = 2;
- required CompareType compare_op = 3;
- required Comparator comparator = 4;
- optional bool filter_if_missing = 5;
- optional bool latest_version_only = 6;
-}
-
-message SkipFilter {
- required Filter filter = 1;
-}
-
-message TimestampsFilter {
- repeated int64 timestamps = 1 [packed=true];
- optional bool can_hint = 2;
-}
-
-message ValueFilter {
- required CompareFilter compare_filter = 1;
-}
-
-message WhileMatchFilter {
- required Filter filter = 1;
-}
-message FilterAllFilter {
-}
-
-message RowRange {
- optional bytes start_row = 1;
- optional bool start_row_inclusive = 2;
- optional bytes stop_row = 3;
- optional bool stop_row_inclusive =4;
-}
-
-message MultiRowRangeFilter {
- repeated RowRange row_range_list = 1;
-}
\ No newline at end of file
diff --git a/src/hbase/if/HBase.proto b/src/hbase/if/HBase.proto
deleted file mode 100644
index c36b214..0000000
--- a/src/hbase/if/HBase.proto
+++ /dev/null
@@ -1,253 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file contains protocol buffers that are shared throughout HBase
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "HBaseProtos";
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-import "Cell.proto";
-
-/**
- * Table Name
- */
-message TableName {
- required bytes namespace = 1;
- required bytes qualifier = 2;
-}
-
-/**
- * Table Schema
- * Inspired by the rest TableSchema
- */
-message TableSchema {
- optional TableName table_name = 1;
- repeated BytesBytesPair attributes = 2;
- repeated ColumnFamilySchema column_families = 3;
- repeated NameStringPair configuration = 4;
-}
-
-/** Denotes state of the table */
-message TableState {
- // Table's current state
- enum State {
- ENABLED = 0;
- DISABLED = 1;
- DISABLING = 2;
- ENABLING = 3;
- }
- // This is the table's state.
- required State state = 1;
-}
-
-/**
- * Column Family Schema
- * Inspired by the rest ColumSchemaMessage
- */
-message ColumnFamilySchema {
- required bytes name = 1;
- repeated BytesBytesPair attributes = 2;
- repeated NameStringPair configuration = 3;
-}
-
-/**
- * Protocol buffer version of HRegionInfo.
- */
-message RegionInfo {
- required uint64 region_id = 1;
- required TableName table_name = 2;
- optional bytes start_key = 3;
- optional bytes end_key = 4;
- optional bool offline = 5;
- optional bool split = 6;
- optional int32 replica_id = 7 [default = 0];
-}
-
-/**
- * Protocol buffer for favored nodes
- */
-message FavoredNodes {
- repeated ServerName favored_node = 1;
-}
-
-/**
- * Container protocol buffer to specify a region.
- * You can specify region by region name, or the hash
- * of the region name, which is known as encoded
- * region name.
- */
-message RegionSpecifier {
- required RegionSpecifierType type = 1;
- required bytes value = 2;
-
- enum RegionSpecifierType {
- // <tablename>,<startkey>,<regionId>.<encodedName>
- REGION_NAME = 1;
-
- // hash of <tablename>,<startkey>,<regionId>
- ENCODED_REGION_NAME = 2;
- }
-}
-
-/**
- * A range of time. Both from and to are Java time
- * stamp in milliseconds. If you don't specify a time
- * range, it means all time. By default, if not
- * specified, from = 0, and to = Long.MAX_VALUE
- */
-message TimeRange {
- optional uint64 from = 1;
- optional uint64 to = 2;
-}
-
-/* ColumnFamily Specific TimeRange */
-message ColumnFamilyTimeRange {
- required bytes column_family = 1;
- required TimeRange time_range = 2;
-}
-
-/* Comparison operators */
-enum CompareType {
- LESS = 0;
- LESS_OR_EQUAL = 1;
- EQUAL = 2;
- NOT_EQUAL = 3;
- GREATER_OR_EQUAL = 4;
- GREATER = 5;
- NO_OP = 6;
-}
-
-/**
- * Protocol buffer version of ServerName
- */
-message ServerName {
- required string host_name = 1;
- optional uint32 port = 2;
- optional uint64 start_code = 3;
-}
-
-// Comment data structures
-
-message Coprocessor {
- required string name = 1;
-}
-
-message NameStringPair {
- required string name = 1;
- required string value = 2;
-}
-
-message NameBytesPair {
- required string name = 1;
- optional bytes value = 2;
-}
-
-message BytesBytesPair {
- required bytes first = 1;
- required bytes second = 2;
-}
-
-message NameInt64Pair {
- optional string name = 1;
- optional int64 value = 2;
-}
-
-/**
- * Description of the snapshot to take
- */
-message SnapshotDescription {
- required string name = 1;
- optional string table = 2; // not needed for delete, but checked for in taking snapshot
- optional int64 creation_time = 3 [default = 0];
- enum Type {
- DISABLED = 0;
- FLUSH = 1;
- SKIPFLUSH = 2;
- }
- optional Type type = 4 [default = FLUSH];
- optional int32 version = 5;
- optional string owner = 6;
-}
-
-/**
- * Description of the distributed procedure to take
- */
-message ProcedureDescription {
- required string signature = 1; // the unique signature of the procedure
- optional string instance = 2; // the procedure instance name
- optional int64 creation_time = 3 [default = 0];
- repeated NameStringPair configuration = 4;
-}
-
-message EmptyMsg {
-}
-
-enum TimeUnit {
- NANOSECONDS = 1;
- MICROSECONDS = 2;
- MILLISECONDS = 3;
- SECONDS = 4;
- MINUTES = 5;
- HOURS = 6;
- DAYS = 7;
-}
-
-message LongMsg {
- required int64 long_msg = 1;
-}
-
-message DoubleMsg {
- required double double_msg = 1;
-}
-
-message BigDecimalMsg {
- required bytes bigdecimal_msg = 1;
-}
-
-message UUID {
- required uint64 least_sig_bits = 1;
- required uint64 most_sig_bits = 2;
-}
-
-message NamespaceDescriptor {
- required bytes name = 1;
- repeated NameStringPair configuration = 2;
-}
-
-// Rpc client version info proto. Included in ConnectionHeader on connection setup
-message VersionInfo {
- required string version = 1;
- required string url = 2;
- required string revision = 3;
- required string user = 4;
- required string date = 5;
- required string src_checksum = 6;
- optional uint32 version_major = 7;
- optional uint32 version_minor = 8;
-}
-
-/**
- * Description of the region server info
- */
-message RegionServerInfo {
- optional int32 infoPort = 1;
- optional VersionInfo version_info = 2;
-}
diff --git a/src/hbase/if/HFile.proto b/src/hbase/if/HFile.proto
deleted file mode 100644
index 5c5e4f3..0000000
--- a/src/hbase/if/HFile.proto
+++ /dev/null
@@ -1,49 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "HFileProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-import "HBase.proto";
-
-// Map of name/values
-message FileInfoProto {
- repeated BytesBytesPair map_entry = 1;
-}
-
-// HFile file trailer
-message FileTrailerProto {
- optional uint64 file_info_offset = 1;
- optional uint64 load_on_open_data_offset = 2;
- optional uint64 uncompressed_data_index_size = 3;
- optional uint64 total_uncompressed_bytes = 4;
- optional uint32 data_index_count = 5;
- optional uint32 meta_index_count = 6;
- optional uint64 entry_count = 7;
- optional uint32 num_data_index_levels = 8;
- optional uint64 first_data_block_offset = 9;
- optional uint64 last_data_block_offset = 10;
- optional string comparator_class_name = 11;
- optional uint32 compression_codec = 12;
- optional bytes encryption_key = 13;
-}
diff --git a/src/hbase/if/LoadBalancer.proto b/src/hbase/if/LoadBalancer.proto
deleted file mode 100644
index f9c5d0d..0000000
--- a/src/hbase/if/LoadBalancer.proto
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file contains protocol buffers to represent the state of the load balancer.
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "LoadBalancerProtos";
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-message LoadBalancerState {
- optional bool balancer_on = 1;
-}
diff --git a/src/hbase/if/MapReduce.proto b/src/hbase/if/MapReduce.proto
deleted file mode 100644
index f96ffdf..0000000
--- a/src/hbase/if/MapReduce.proto
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
- //This file includes protocol buffers used in MapReduce only.
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "MapReduceProtos";
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-import "HBase.proto";
-
-message ScanMetrics {
- repeated NameInt64Pair metrics = 1;
-}
-
-message TableSnapshotRegionSplit {
- repeated string locations = 2;
- optional TableSchema table = 3;
- optional RegionInfo region = 4;
-}
diff --git a/src/hbase/if/Master.proto b/src/hbase/if/Master.proto
deleted file mode 100644
index ad8111e..0000000
--- a/src/hbase/if/Master.proto
+++ /dev/null
@@ -1,828 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// All to do with the Master. Includes schema management since these
-// changes are run by the Master process.
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "MasterProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-import "HBase.proto";
-import "Client.proto";
-import "ClusterStatus.proto";
-import "ErrorHandling.proto";
-import "Procedure.proto";
-import "Quota.proto";
-
-/* Column-level protobufs */
-
-message AddColumnRequest {
- required TableName table_name = 1;
- required ColumnFamilySchema column_families = 2;
- optional uint64 nonce_group = 3 [default = 0];
- optional uint64 nonce = 4 [default = 0];
-}
-
-message AddColumnResponse {
- optional uint64 proc_id = 1;
-}
-
-message DeleteColumnRequest {
- required TableName table_name = 1;
- required bytes column_name = 2;
- optional uint64 nonce_group = 3 [default = 0];
- optional uint64 nonce = 4 [default = 0];
-}
-
-message DeleteColumnResponse {
- optional uint64 proc_id = 1;
-}
-
-message ModifyColumnRequest {
- required TableName table_name = 1;
- required ColumnFamilySchema column_families = 2;
- optional uint64 nonce_group = 3 [default = 0];
- optional uint64 nonce = 4 [default = 0];
-}
-
-message ModifyColumnResponse {
- optional uint64 proc_id = 1;
-}
-
-/* Region-level Protos */
-
-message MoveRegionRequest {
- required RegionSpecifier region = 1;
- optional ServerName dest_server_name = 2;
-}
-
-message MoveRegionResponse {
-}
-
-/**
- * Dispatch merging the specified regions.
- */
-message DispatchMergingRegionsRequest {
- required RegionSpecifier region_a = 1;
- required RegionSpecifier region_b = 2;
- optional bool forcible = 3 [default = false];
-}
-
-message DispatchMergingRegionsResponse {
-}
-
-message AssignRegionRequest {
- required RegionSpecifier region = 1;
-}
-
-message AssignRegionResponse {
-}
-
-message UnassignRegionRequest {
- required RegionSpecifier region = 1;
- optional bool force = 2 [default = false];
-}
-
-message UnassignRegionResponse {
-}
-
-message OfflineRegionRequest {
- required RegionSpecifier region = 1;
-}
-
-message OfflineRegionResponse {
-}
-
-/* Table-level protobufs */
-
-message CreateTableRequest {
- required TableSchema table_schema = 1;
- repeated bytes split_keys = 2;
- optional uint64 nonce_group = 3 [default = 0];
- optional uint64 nonce = 4 [default = 0];
-}
-
-message CreateTableResponse {
- optional uint64 proc_id = 1;
-}
-
-message DeleteTableRequest {
- required TableName table_name = 1;
- optional uint64 nonce_group = 2 [default = 0];
- optional uint64 nonce = 3 [default = 0];
-}
-
-message DeleteTableResponse {
- optional uint64 proc_id = 1;
-}
-
-message TruncateTableRequest {
- required TableName tableName = 1;
- optional bool preserveSplits = 2 [default = false];
- optional uint64 nonce_group = 3 [default = 0];
- optional uint64 nonce = 4 [default = 0];
-}
-
-message TruncateTableResponse {
- optional uint64 proc_id = 1;
-}
-
-message EnableTableRequest {
- required TableName table_name = 1;
- optional uint64 nonce_group = 2 [default = 0];
- optional uint64 nonce = 3 [default = 0];
-}
-
-message EnableTableResponse {
- optional uint64 proc_id = 1;
-}
-
-message DisableTableRequest {
- required TableName table_name = 1;
- optional uint64 nonce_group = 2 [default = 0];
- optional uint64 nonce = 3 [default = 0];
-}
-
-message DisableTableResponse {
- optional uint64 proc_id = 1;
-}
-
-message ModifyTableRequest {
- required TableName table_name = 1;
- required TableSchema table_schema = 2;
- optional uint64 nonce_group = 3 [default = 0];
- optional uint64 nonce = 4 [default = 0];
-}
-
-message ModifyTableResponse {
- optional uint64 proc_id = 1;
-}
-
-/* Namespace-level protobufs */
-
-message CreateNamespaceRequest {
- required NamespaceDescriptor namespaceDescriptor = 1;
- optional uint64 nonce_group = 2 [default = 0];
- optional uint64 nonce = 3 [default = 0];
-}
-
-message CreateNamespaceResponse {
- optional uint64 proc_id = 1;
-}
-
-message DeleteNamespaceRequest {
- required string namespaceName = 1;
- optional uint64 nonce_group = 2 [default = 0];
- optional uint64 nonce = 3 [default = 0];
-}
-
-message DeleteNamespaceResponse {
- optional uint64 proc_id = 1;
-}
-
-message ModifyNamespaceRequest {
- required NamespaceDescriptor namespaceDescriptor = 1;
- optional uint64 nonce_group = 2 [default = 0];
- optional uint64 nonce = 3 [default = 0];
-}
-
-message ModifyNamespaceResponse {
- optional uint64 proc_id = 1;
-}
-
-message GetNamespaceDescriptorRequest {
- required string namespaceName = 1;
-}
-
-message GetNamespaceDescriptorResponse {
- required NamespaceDescriptor namespaceDescriptor = 1;
-}
-
-message ListNamespaceDescriptorsRequest {
-}
-
-message ListNamespaceDescriptorsResponse {
- repeated NamespaceDescriptor namespaceDescriptor = 1;
-}
-
-message ListTableDescriptorsByNamespaceRequest {
- required string namespaceName = 1;
-}
-
-message ListTableDescriptorsByNamespaceResponse {
- repeated TableSchema tableSchema = 1;
-}
-
-message ListTableNamesByNamespaceRequest {
- required string namespaceName = 1;
-}
-
-message ListTableNamesByNamespaceResponse {
- repeated TableName tableName = 1;
-}
-
-/* Cluster-level protobufs */
-
-
-message ShutdownRequest {
-}
-
-message ShutdownResponse {
-}
-
-message StopMasterRequest {
-}
-
-message StopMasterResponse {
-}
-
-message BalanceRequest {
- optional bool force = 1;
-}
-
-message BalanceResponse {
- required bool balancer_ran = 1;
-}
-
-message SetBalancerRunningRequest {
- required bool on = 1;
- optional bool synchronous = 2;
-}
-
-message SetBalancerRunningResponse {
- optional bool prev_balance_value = 1;
-}
-
-message IsBalancerEnabledRequest {
-}
-
-message IsBalancerEnabledResponse {
- required bool enabled = 1;
-}
-
-enum MasterSwitchType {
- SPLIT = 0;
- MERGE = 1;
-}
-
-message SetSplitOrMergeEnabledRequest {
- required bool enabled = 1;
- optional bool synchronous = 2;
- repeated MasterSwitchType switch_types = 3;
- optional bool skip_lock = 4;
-}
-
-message SetSplitOrMergeEnabledResponse {
- repeated bool prev_value = 1;
-}
-
-message IsSplitOrMergeEnabledRequest {
- required MasterSwitchType switch_type = 1;
-}
-
-message IsSplitOrMergeEnabledResponse {
- required bool enabled = 1;
-}
-
-message ReleaseSplitOrMergeLockAndRollbackRequest {
-}
-
-message ReleaseSplitOrMergeLockAndRollbackResponse {
-}
-
-message NormalizeRequest {
-}
-
-message NormalizeResponse {
- required bool normalizer_ran = 1;
-}
-
-message SetNormalizerRunningRequest {
- required bool on = 1;
-}
-
-message SetNormalizerRunningResponse {
- optional bool prev_normalizer_value = 1;
-}
-
-message IsNormalizerEnabledRequest {
-}
-
-message IsNormalizerEnabledResponse {
- required bool enabled = 1;
-}
-
-message RunCatalogScanRequest {
-}
-
-message RunCatalogScanResponse {
- optional int32 scan_result = 1;
-}
-
-message EnableCatalogJanitorRequest {
- required bool enable = 1;
-}
-
-message EnableCatalogJanitorResponse {
- optional bool prev_value = 1;
-}
-
-message IsCatalogJanitorEnabledRequest {
-}
-
-message IsCatalogJanitorEnabledResponse {
- required bool value = 1;
-}
-
-message SnapshotRequest {
- required SnapshotDescription snapshot = 1;
-}
-
-message SnapshotResponse {
- required int64 expected_timeout = 1;
-}
-
-message GetCompletedSnapshotsRequest {
-}
-
-message GetCompletedSnapshotsResponse {
- repeated SnapshotDescription snapshots = 1;
-}
-
-message DeleteSnapshotRequest {
- required SnapshotDescription snapshot = 1;
-}
-
-message DeleteSnapshotResponse {
-}
-
-message RestoreSnapshotRequest {
- required SnapshotDescription snapshot = 1;
- optional uint64 nonce_group = 2 [default = 0];
- optional uint64 nonce = 3 [default = 0];
-}
-
-message RestoreSnapshotResponse {
- required uint64 proc_id = 1;
-}
-
-/* if you don't send the snapshot, then you will get it back
- * in the response (if the snapshot is done) so you can check the snapshot
- */
-message IsSnapshotDoneRequest {
- optional SnapshotDescription snapshot = 1;
-}
-
-message IsSnapshotDoneResponse {
- optional bool done = 1 [default = false];
- optional SnapshotDescription snapshot = 2;
-}
-
-message IsRestoreSnapshotDoneRequest {
- optional SnapshotDescription snapshot = 1;
-}
-
-message IsRestoreSnapshotDoneResponse {
- optional bool done = 1 [default = false];
-}
-
-message GetSchemaAlterStatusRequest {
- required TableName table_name = 1;
-}
-
-message GetSchemaAlterStatusResponse {
- optional uint32 yet_to_update_regions = 1;
- optional uint32 total_regions = 2;
-}
-
-message GetTableDescriptorsRequest {
- repeated TableName table_names = 1;
- optional string regex = 2;
- optional bool include_sys_tables = 3 [default=false];
- optional string namespace = 4;
-}
-
-message GetTableDescriptorsResponse {
- repeated TableSchema table_schema = 1;
-}
-
-message GetTableNamesRequest {
- optional string regex = 1;
- optional bool include_sys_tables = 2 [default=false];
- optional string namespace = 3;
-}
-
-message GetTableNamesResponse {
- repeated TableName table_names = 1;
-}
-
-message GetTableStateRequest {
- required TableName table_name = 1;
-}
-
-message GetTableStateResponse {
- required TableState table_state = 1;
-}
-
-message GetClusterStatusRequest {
-}
-
-message GetClusterStatusResponse {
- required ClusterStatus cluster_status = 1;
-}
-
-message IsMasterRunningRequest {
-}
-
-message IsMasterRunningResponse {
- required bool is_master_running = 1;
-}
-
-message ExecProcedureRequest {
- required ProcedureDescription procedure = 1;
-}
-
-message ExecProcedureResponse {
- optional int64 expected_timeout = 1;
- optional bytes return_data = 2;
-}
-
-message IsProcedureDoneRequest {
- optional ProcedureDescription procedure = 1;
-}
-
-message IsProcedureDoneResponse {
- optional bool done = 1 [default = false];
- optional ProcedureDescription snapshot = 2;
-}
-
-message GetProcedureResultRequest {
- required uint64 proc_id = 1;
-}
-
-message GetProcedureResultResponse {
- enum State {
- NOT_FOUND = 0;
- RUNNING = 1;
- FINISHED = 2;
- }
-
- required State state = 1;
- optional uint64 start_time = 2;
- optional uint64 last_update = 3;
- optional bytes result = 4;
- optional ForeignExceptionMessage exception = 5;
-}
-
-message AbortProcedureRequest {
- required uint64 proc_id = 1;
- optional bool mayInterruptIfRunning = 2 [default = true];
-}
-
-message AbortProcedureResponse {
- required bool is_procedure_aborted = 1;
-}
-
-message ListProceduresRequest {
-}
-
-message ListProceduresResponse {
- repeated Procedure procedure = 1;
-}
-
-message SetQuotaRequest {
- optional string user_name = 1;
- optional string user_group = 2;
- optional string namespace = 3;
- optional TableName table_name = 4;
-
- optional bool remove_all = 5;
- optional bool bypass_globals = 6;
- optional ThrottleRequest throttle = 7;
-}
-
-message SetQuotaResponse {
-}
-
-message MajorCompactionTimestampRequest {
- required TableName table_name = 1;
-}
-
-message MajorCompactionTimestampForRegionRequest {
- required RegionSpecifier region = 1;
-}
-
-message MajorCompactionTimestampResponse {
- required int64 compaction_timestamp = 1;
-}
-
-message SecurityCapabilitiesRequest {
-}
-
-message SecurityCapabilitiesResponse {
- enum Capability {
- SIMPLE_AUTHENTICATION = 0;
- SECURE_AUTHENTICATION = 1;
- AUTHORIZATION = 2;
- CELL_AUTHORIZATION = 3;
- CELL_VISIBILITY = 4;
- }
-
- repeated Capability capabilities = 1;
-}
-
-service MasterService {
- /** Used by the client to get the number of regions that have received the updated schema */
- rpc GetSchemaAlterStatus(GetSchemaAlterStatusRequest)
- returns(GetSchemaAlterStatusResponse);
-
- /** Get list of TableDescriptors for requested tables. */
- rpc GetTableDescriptors(GetTableDescriptorsRequest)
- returns(GetTableDescriptorsResponse);
-
- /** Get the list of table names. */
- rpc GetTableNames(GetTableNamesRequest)
- returns(GetTableNamesResponse);
-
- /** Return cluster status. */
- rpc GetClusterStatus(GetClusterStatusRequest)
- returns(GetClusterStatusResponse);
-
- /** return true if master is available */
- rpc IsMasterRunning(IsMasterRunningRequest) returns(IsMasterRunningResponse);
-
- /** Adds a column to the specified table. */
- rpc AddColumn(AddColumnRequest)
- returns(AddColumnResponse);
-
- /** Deletes a column from the specified table. Table must be disabled. */
- rpc DeleteColumn(DeleteColumnRequest)
- returns(DeleteColumnResponse);
-
- /** Modifies an existing column on the specified table. */
- rpc ModifyColumn(ModifyColumnRequest)
- returns(ModifyColumnResponse);
-
- /** Move the region region to the destination server. */
- rpc MoveRegion(MoveRegionRequest)
- returns(MoveRegionResponse);
-
- /** Master dispatch merging the regions */
- rpc DispatchMergingRegions(DispatchMergingRegionsRequest)
- returns(DispatchMergingRegionsResponse);
-
- /** Assign a region to a server chosen at random. */
- rpc AssignRegion(AssignRegionRequest)
- returns(AssignRegionResponse);
-
- /**
- * Unassign a region from current hosting regionserver. Region will then be
- * assigned to a regionserver chosen at random. Region could be reassigned
- * back to the same server. Use MoveRegion if you want
- * to control the region movement.
- */
- rpc UnassignRegion(UnassignRegionRequest)
- returns(UnassignRegionResponse);
-
- /**
- * Offline a region from the assignment manager's in-memory state. The
- * region should be in a closed state and there will be no attempt to
- * automatically reassign the region as in unassign. This is a special
- * method, and should only be used by experts or hbck.
- */
- rpc OfflineRegion(OfflineRegionRequest)
- returns(OfflineRegionResponse);
-
- /** Deletes a table */
- rpc DeleteTable(DeleteTableRequest)
- returns(DeleteTableResponse);
-
- /** Truncate a table */
- rpc truncateTable(TruncateTableRequest)
- returns(TruncateTableResponse);
-
- /** Puts the table on-line (only needed if table has been previously taken offline) */
- rpc EnableTable(EnableTableRequest)
- returns(EnableTableResponse);
-
- /** Take table offline */
- rpc DisableTable(DisableTableRequest)
- returns(DisableTableResponse);
-
- /** Modify a table's metadata */
- rpc ModifyTable(ModifyTableRequest)
- returns(ModifyTableResponse);
-
- /** Creates a new table asynchronously */
- rpc CreateTable(CreateTableRequest)
- returns(CreateTableResponse);
-
- /** Shutdown an HBase cluster. */
- rpc Shutdown(ShutdownRequest)
- returns(ShutdownResponse);
-
- /** Stop HBase Master only. Does not shutdown the cluster. */
- rpc StopMaster(StopMasterRequest)
- returns(StopMasterResponse);
-
- /**
- * Run the balancer. Will run the balancer and if regions to move, it will
- * go ahead and do the reassignments. Can NOT run for various reasons.
- * Check logs.
- */
- rpc Balance(BalanceRequest)
- returns(BalanceResponse);
-
- /**
- * Turn the load balancer on or off.
- * If synchronous is true, it waits until current balance() call, if outstanding, to return.
- */
- rpc SetBalancerRunning(SetBalancerRunningRequest)
- returns(SetBalancerRunningResponse);
-
- /**
- * Query whether the Region Balancer is running.
- */
- rpc IsBalancerEnabled(IsBalancerEnabledRequest)
- returns(IsBalancerEnabledResponse);
-
- /**
- * Turn the split or merge switch on or off.
- * If synchronous is true, it waits until current operation call, if outstanding, to return.
- */
- rpc SetSplitOrMergeEnabled(SetSplitOrMergeEnabledRequest)
- returns(SetSplitOrMergeEnabledResponse);
-
- /**
- * Query whether the split or merge switch is on/off.
- */
- rpc IsSplitOrMergeEnabled(IsSplitOrMergeEnabledRequest)
- returns(IsSplitOrMergeEnabledResponse);
-
- /**
- * Release lock and rollback state.
- */
- rpc ReleaseSplitOrMergeLockAndRollback(ReleaseSplitOrMergeLockAndRollbackRequest)
- returns(ReleaseSplitOrMergeLockAndRollbackResponse);
-
- /**
- * Run region normalizer. Can NOT run for various reasons. Check logs.
- */
- rpc Normalize(NormalizeRequest)
- returns(NormalizeResponse);
-
- /**
- * Turn region normalizer on or off.
- */
- rpc SetNormalizerRunning(SetNormalizerRunningRequest)
- returns(SetNormalizerRunningResponse);
-
- /**
- * Query whether region normalizer is enabled.
- */
- rpc IsNormalizerEnabled(IsNormalizerEnabledRequest)
- returns(IsNormalizerEnabledResponse);
-
- /** Get a run of the catalog janitor */
- rpc RunCatalogScan(RunCatalogScanRequest)
- returns(RunCatalogScanResponse);
-
- /**
- * Enable the catalog janitor on or off.
- */
- rpc EnableCatalogJanitor(EnableCatalogJanitorRequest)
- returns(EnableCatalogJanitorResponse);
-
- /**
- * Query whether the catalog janitor is enabled.
- */
- rpc IsCatalogJanitorEnabled(IsCatalogJanitorEnabledRequest)
- returns(IsCatalogJanitorEnabledResponse);
-
- /**
- * Call a master coprocessor endpoint
- */
- rpc ExecMasterService(CoprocessorServiceRequest)
- returns(CoprocessorServiceResponse);
-
- /**
- * Create a snapshot for the given table.
- */
- rpc Snapshot(SnapshotRequest) returns(SnapshotResponse);
-
- /**
- * Get completed snapshots.
- * Returns a list of snapshot descriptors for completed snapshots
- */
- rpc GetCompletedSnapshots(GetCompletedSnapshotsRequest) returns(GetCompletedSnapshotsResponse);
-
- /**
- * Delete an existing snapshot. This method can also be used to clean up an aborted snapshot.
- */
- rpc DeleteSnapshot(DeleteSnapshotRequest) returns(DeleteSnapshotResponse);
-
- /**
- * Determine if the snapshot is done yet.
- */
- rpc IsSnapshotDone(IsSnapshotDoneRequest) returns(IsSnapshotDoneResponse);
-
- /**
- * Restore a snapshot
- */
- rpc RestoreSnapshot(RestoreSnapshotRequest) returns(RestoreSnapshotResponse);
-
- /**
- * Execute a distributed procedure.
- */
- rpc ExecProcedure(ExecProcedureRequest) returns(ExecProcedureResponse);
-
- /**
- * Execute a distributed procedure with return data.
- */
- rpc ExecProcedureWithRet(ExecProcedureRequest) returns(ExecProcedureResponse);
-
- /**
- * Determine if the procedure is done yet.
- */
- rpc IsProcedureDone(IsProcedureDoneRequest) returns(IsProcedureDoneResponse);
-
- /** return true if master is available */
- /** rpc IsMasterRunning(IsMasterRunningRequest) returns(IsMasterRunningResponse); */
-
- /** Modify a namespace's metadata */
- rpc ModifyNamespace(ModifyNamespaceRequest)
- returns(ModifyNamespaceResponse);
-
- /** Creates a new namespace synchronously */
- rpc CreateNamespace(CreateNamespaceRequest)
- returns(CreateNamespaceResponse);
-
- /** Deletes namespace synchronously */
- rpc DeleteNamespace(DeleteNamespaceRequest)
- returns(DeleteNamespaceResponse);
-
- /** Get a namespace descriptor by name */
- rpc GetNamespaceDescriptor(GetNamespaceDescriptorRequest)
- returns(GetNamespaceDescriptorResponse);
-
- /** returns a list of namespaces */
- rpc ListNamespaceDescriptors(ListNamespaceDescriptorsRequest)
- returns(ListNamespaceDescriptorsResponse);
-
- /** returns a list of tables for a given namespace*/
- rpc ListTableDescriptorsByNamespace(ListTableDescriptorsByNamespaceRequest)
- returns(ListTableDescriptorsByNamespaceResponse);
-
- /** returns a list of tables for a given namespace*/
- rpc ListTableNamesByNamespace(ListTableNamesByNamespaceRequest)
- returns(ListTableNamesByNamespaceResponse);
-
- /** returns table state */
- rpc GetTableState(GetTableStateRequest)
- returns(GetTableStateResponse);
-
- /** Apply the new quota settings */
- rpc SetQuota(SetQuotaRequest) returns(SetQuotaResponse);
-
- /** Returns the timestamp of the last major compaction */
- rpc getLastMajorCompactionTimestamp(MajorCompactionTimestampRequest)
- returns(MajorCompactionTimestampResponse);
-
- /** Returns the timestamp of the last major compaction */
- rpc getLastMajorCompactionTimestampForRegion(MajorCompactionTimestampForRegionRequest)
- returns(MajorCompactionTimestampResponse);
-
- rpc getProcedureResult(GetProcedureResultRequest)
- returns(GetProcedureResultResponse);
-
- /** Returns the security capabilities in effect on the cluster */
- rpc getSecurityCapabilities(SecurityCapabilitiesRequest)
- returns(SecurityCapabilitiesResponse);
-
- /** Abort a procedure */
- rpc AbortProcedure(AbortProcedureRequest)
- returns(AbortProcedureResponse);
-
- /** returns a list of procedures */
- rpc ListProcedures(ListProceduresRequest)
- returns(ListProceduresResponse);
-}
diff --git a/src/hbase/if/MasterProcedure.proto b/src/hbase/if/MasterProcedure.proto
deleted file mode 100644
index 87aae6a..0000000
--- a/src/hbase/if/MasterProcedure.proto
+++ /dev/null
@@ -1,285 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "MasterProcedureProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-import "HBase.proto";
-import "RPC.proto";
-
-// ============================================================================
-// WARNING - Compatibility rules
-// ============================================================================
-// This .proto contains the data serialized by the master procedures.
-// Each procedure has some state stored to know, which step were executed
-// and what were the parameters or data created by the previous steps.
-// new code should be able to handle the old format or at least fail cleanly
-// triggering a rollback/cleanup.
-//
-// Procedures that are inheriting from a StateMachineProcedure have an enum:
-// - Do not change the number of the 'State' enums.
-// doing so, will cause executing the wrong 'step' on the pending
-// procedures when they will be replayed.
-// - Do not remove items from the enum, new code must be able to handle
-// all the previous 'steps'. There may be pending procedure ready to be
-// recovered replayed. alternative you can make sure that not-known state
-// will result in a failure that will rollback the already executed steps.
-// ============================================================================
-
-enum CreateTableState {
- CREATE_TABLE_PRE_OPERATION = 1;
- CREATE_TABLE_WRITE_FS_LAYOUT = 2;
- CREATE_TABLE_ADD_TO_META = 3;
- CREATE_TABLE_ASSIGN_REGIONS = 4;
- CREATE_TABLE_UPDATE_DESC_CACHE = 5;
- CREATE_TABLE_POST_OPERATION = 6;
-}
-
-message CreateTableStateData {
- required UserInformation user_info = 1;
- required TableSchema table_schema = 2;
- repeated RegionInfo region_info = 3;
-}
-
-enum ModifyTableState {
- MODIFY_TABLE_PREPARE = 1;
- MODIFY_TABLE_PRE_OPERATION = 2;
- MODIFY_TABLE_UPDATE_TABLE_DESCRIPTOR = 3;
- MODIFY_TABLE_REMOVE_REPLICA_COLUMN = 4;
- MODIFY_TABLE_DELETE_FS_LAYOUT = 5;
- MODIFY_TABLE_POST_OPERATION = 6;
- MODIFY_TABLE_REOPEN_ALL_REGIONS = 7;
-}
-
-message ModifyTableStateData {
- required UserInformation user_info = 1;
- optional TableSchema unmodified_table_schema = 2;
- required TableSchema modified_table_schema = 3;
- required bool delete_column_family_in_modify = 4;
-}
-
-enum TruncateTableState {
- TRUNCATE_TABLE_PRE_OPERATION = 1;
- TRUNCATE_TABLE_REMOVE_FROM_META = 2;
- TRUNCATE_TABLE_CLEAR_FS_LAYOUT = 3;
- TRUNCATE_TABLE_CREATE_FS_LAYOUT = 4;
- TRUNCATE_TABLE_ADD_TO_META = 5;
- TRUNCATE_TABLE_ASSIGN_REGIONS = 6;
- TRUNCATE_TABLE_POST_OPERATION = 7;
-}
-
-message TruncateTableStateData {
- required UserInformation user_info = 1;
- required bool preserve_splits = 2;
- optional TableName table_name = 3;
- optional TableSchema table_schema = 4;
- repeated RegionInfo region_info = 5;
-}
-
-enum DeleteTableState {
- DELETE_TABLE_PRE_OPERATION = 1;
- DELETE_TABLE_REMOVE_FROM_META = 2;
- DELETE_TABLE_CLEAR_FS_LAYOUT = 3;
- DELETE_TABLE_UPDATE_DESC_CACHE = 4;
- DELETE_TABLE_UNASSIGN_REGIONS = 5;
- DELETE_TABLE_POST_OPERATION = 6;
-}
-
-message DeleteTableStateData {
- required UserInformation user_info = 1;
- required TableName table_name = 2;
- repeated RegionInfo region_info = 3;
-}
-
-enum CreateNamespaceState {
- CREATE_NAMESPACE_PREPARE = 1;
- CREATE_NAMESPACE_CREATE_DIRECTORY = 2;
- CREATE_NAMESPACE_INSERT_INTO_NS_TABLE = 3;
- CREATE_NAMESPACE_UPDATE_ZK = 4;
- CREATE_NAMESPACE_SET_NAMESPACE_QUOTA = 5;
-}
-
-message CreateNamespaceStateData {
- required NamespaceDescriptor namespace_descriptor = 1;
-}
-
-enum ModifyNamespaceState {
- MODIFY_NAMESPACE_PREPARE = 1;
- MODIFY_NAMESPACE_UPDATE_NS_TABLE = 2;
- MODIFY_NAMESPACE_UPDATE_ZK = 3;
-}
-
-message ModifyNamespaceStateData {
- required NamespaceDescriptor namespace_descriptor = 1;
- optional NamespaceDescriptor unmodified_namespace_descriptor = 2;
-}
-
-enum DeleteNamespaceState {
- DELETE_NAMESPACE_PREPARE = 1;
- DELETE_NAMESPACE_DELETE_FROM_NS_TABLE = 2;
- DELETE_NAMESPACE_REMOVE_FROM_ZK = 3;
- DELETE_NAMESPACE_DELETE_DIRECTORIES = 4;
- DELETE_NAMESPACE_REMOVE_NAMESPACE_QUOTA = 5;
-}
-
-message DeleteNamespaceStateData {
- required string namespace_name = 1;
- optional NamespaceDescriptor namespace_descriptor = 2;
-}
-
-enum AddColumnFamilyState {
- ADD_COLUMN_FAMILY_PREPARE = 1;
- ADD_COLUMN_FAMILY_PRE_OPERATION = 2;
- ADD_COLUMN_FAMILY_UPDATE_TABLE_DESCRIPTOR = 3;
- ADD_COLUMN_FAMILY_POST_OPERATION = 4;
- ADD_COLUMN_FAMILY_REOPEN_ALL_REGIONS = 5;
-}
-
-message AddColumnFamilyStateData {
- required UserInformation user_info = 1;
- required TableName table_name = 2;
- required ColumnFamilySchema columnfamily_schema = 3;
- optional TableSchema unmodified_table_schema = 4;
-}
-
-enum ModifyColumnFamilyState {
- MODIFY_COLUMN_FAMILY_PREPARE = 1;
- MODIFY_COLUMN_FAMILY_PRE_OPERATION = 2;
- MODIFY_COLUMN_FAMILY_UPDATE_TABLE_DESCRIPTOR = 3;
- MODIFY_COLUMN_FAMILY_POST_OPERATION = 4;
- MODIFY_COLUMN_FAMILY_REOPEN_ALL_REGIONS = 5;
-}
-
-message ModifyColumnFamilyStateData {
- required UserInformation user_info = 1;
- required TableName table_name = 2;
- required ColumnFamilySchema columnfamily_schema = 3;
- optional TableSchema unmodified_table_schema = 4;
-}
-
-enum DeleteColumnFamilyState {
- DELETE_COLUMN_FAMILY_PREPARE = 1;
- DELETE_COLUMN_FAMILY_PRE_OPERATION = 2;
- DELETE_COLUMN_FAMILY_UPDATE_TABLE_DESCRIPTOR = 3;
- DELETE_COLUMN_FAMILY_DELETE_FS_LAYOUT = 4;
- DELETE_COLUMN_FAMILY_POST_OPERATION = 5;
- DELETE_COLUMN_FAMILY_REOPEN_ALL_REGIONS = 6;
-}
-
-message DeleteColumnFamilyStateData {
- required UserInformation user_info = 1;
- required TableName table_name = 2;
- required bytes columnfamily_name = 3;
- optional TableSchema unmodified_table_schema = 4;
-}
-
-enum EnableTableState {
- ENABLE_TABLE_PREPARE = 1;
- ENABLE_TABLE_PRE_OPERATION = 2;
- ENABLE_TABLE_SET_ENABLING_TABLE_STATE = 3;
- ENABLE_TABLE_MARK_REGIONS_ONLINE = 4;
- ENABLE_TABLE_SET_ENABLED_TABLE_STATE = 5;
- ENABLE_TABLE_POST_OPERATION = 6;
-}
-
-message EnableTableStateData {
- required UserInformation user_info = 1;
- required TableName table_name = 2;
- required bool skip_table_state_check = 3;
-}
-
-enum DisableTableState {
- DISABLE_TABLE_PREPARE = 1;
- DISABLE_TABLE_PRE_OPERATION = 2;
- DISABLE_TABLE_SET_DISABLING_TABLE_STATE = 3;
- DISABLE_TABLE_MARK_REGIONS_OFFLINE = 4;
- DISABLE_TABLE_SET_DISABLED_TABLE_STATE = 5;
- DISABLE_TABLE_POST_OPERATION = 6;
-}
-
-message DisableTableStateData {
- required UserInformation user_info = 1;
- required TableName table_name = 2;
- required bool skip_table_state_check = 3;
-}
-
-message RestoreParentToChildRegionsPair {
- required string parent_region_name = 1;
- required string child1_region_name = 2;
- required string child2_region_name = 3;
-}
-
-enum CloneSnapshotState {
- CLONE_SNAPSHOT_PRE_OPERATION = 1;
- CLONE_SNAPSHOT_WRITE_FS_LAYOUT = 2;
- CLONE_SNAPSHOT_ADD_TO_META = 3;
- CLONE_SNAPSHOT_ASSIGN_REGIONS = 4;
- CLONE_SNAPSHOT_UPDATE_DESC_CACHE = 5;
- CLONE_SNAPSHOT_POST_OPERATION = 6;
-}
-
-message CloneSnapshotStateData {
- required UserInformation user_info = 1;
- required SnapshotDescription snapshot = 2;
- required TableSchema table_schema = 3;
- repeated RegionInfo region_info = 4;
- repeated RestoreParentToChildRegionsPair parent_to_child_regions_pair_list = 5;
-}
-
-enum RestoreSnapshotState {
- RESTORE_SNAPSHOT_PRE_OPERATION = 1;
- RESTORE_SNAPSHOT_UPDATE_TABLE_DESCRIPTOR = 2;
- RESTORE_SNAPSHOT_WRITE_FS_LAYOUT = 3;
- RESTORE_SNAPSHOT_UPDATE_META = 4;
-}
-
-message RestoreSnapshotStateData {
- required UserInformation user_info = 1;
- required SnapshotDescription snapshot = 2;
- required TableSchema modified_table_schema = 3;
- repeated RegionInfo region_info_for_restore = 4;
- repeated RegionInfo region_info_for_remove = 5;
- repeated RegionInfo region_info_for_add = 6;
- repeated RestoreParentToChildRegionsPair parent_to_child_regions_pair_list = 7;
-}
-
-message ServerCrashStateData {
- required ServerName server_name = 1;
- optional bool distributed_log_replay = 2;
- repeated RegionInfo regions_on_crashed_server = 3;
- repeated RegionInfo regions_assigned = 4;
- optional bool carrying_meta = 5;
- optional bool should_split_wal = 6 [default = true];
-}
-
-enum ServerCrashState {
- SERVER_CRASH_START = 1;
- SERVER_CRASH_PROCESS_META = 2;
- SERVER_CRASH_GET_REGIONS = 3;
- SERVER_CRASH_NO_SPLIT_LOGS = 4;
- SERVER_CRASH_SPLIT_LOGS = 5;
- SERVER_CRASH_PREPARE_LOG_REPLAY = 6;
- // Removed SERVER_CRASH_CALC_REGIONS_TO_ASSIGN = 7;
- SERVER_CRASH_ASSIGN = 8;
- SERVER_CRASH_WAIT_ON_ASSIGN = 9;
- SERVER_CRASH_FINISH = 100;
-}
diff --git a/src/hbase/if/MultiRowMutation.proto b/src/hbase/if/MultiRowMutation.proto
deleted file mode 100644
index 747afac..0000000
--- a/src/hbase/if/MultiRowMutation.proto
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package hbase.pb;
-
-import "Client.proto";
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "MultiRowMutationProtos";
-option java_generate_equals_and_hash = true;
-option java_generic_services = true;
-option optimize_for = SPEED;
-
-message MultiRowMutationProcessorRequest{
-}
-
-message MultiRowMutationProcessorResponse{
-}
-
-message MutateRowsRequest {
- repeated MutationProto mutation_request = 1;
- optional uint64 nonce_group = 2;
- optional uint64 nonce = 3;
-}
-
-message MutateRowsResponse {
-}
-
-service MultiRowMutationService {
- rpc MutateRows(MutateRowsRequest)
- returns(MutateRowsResponse);
-}
\ No newline at end of file
diff --git a/src/hbase/if/Procedure.proto b/src/hbase/if/Procedure.proto
deleted file mode 100644
index 55e44a4..0000000
--- a/src/hbase/if/Procedure.proto
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "ProcedureProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-import "ErrorHandling.proto";
-
-enum ProcedureState {
- INITIALIZING = 1; // Procedure in construction, not yet added to the executor
- RUNNABLE = 2; // Procedure added to the executor, and ready to be executed
- WAITING = 3; // The procedure is waiting on children to be completed
- WAITING_TIMEOUT = 4; // The procedure is waiting a timout or an external event
- ROLLEDBACK = 5; // The procedure failed and was rolledback
- FINISHED = 6; // The procedure execution is completed. may need a rollback if failed.
-}
-
-/**
- * Procedure metadata, serialized by the ProcedureStore to be able to recover the old state.
- */
-message Procedure {
- // internal "static" state
- required string class_name = 1; // full classname to be able to instantiate the procedure
- optional uint64 parent_id = 2; // parent if not a root-procedure otherwise not set
- required uint64 proc_id = 3;
- required uint64 start_time = 4;
- optional string owner = 5;
-
- // internal "runtime" state
- required ProcedureState state = 6;
- repeated uint32 stack_id = 7; // stack indices in case the procedure was running
- required uint64 last_update = 8;
- optional uint32 timeout = 9;
-
- // user state/results
- optional ForeignExceptionMessage exception = 10;
- optional bytes result = 11; // opaque (user) result structure
- optional bytes state_data = 12; // opaque (user) procedure internal-state
-
- // Nonce to prevent same procedure submit by multiple times
- optional uint64 nonce_group = 13 [default = 0];
- optional uint64 nonce = 14 [default = 0];
-}
-
-/**
- * SequentialProcedure data
- */
-message SequentialProcedureData {
- required bool executed = 1;
-}
-
-/**
- * StateMachineProcedure data
- */
-message StateMachineProcedureData {
- repeated uint32 state = 1;
-}
-
-/**
- * Procedure WAL header
- */
-message ProcedureWALHeader {
- required uint32 version = 1;
- required uint32 type = 2;
- required uint64 log_id = 3;
- required uint64 min_proc_id = 4;
-}
-
-/**
- * Procedure WAL trailer
- */
-message ProcedureWALTrailer {
- required uint32 version = 1;
- required uint64 tracker_pos = 2;
-}
-
-message ProcedureStoreTracker {
- message TrackerNode {
- required uint64 start_id = 1;
- repeated uint64 updated = 2;
- repeated uint64 deleted = 3;
- }
-
- repeated TrackerNode node = 1;
-}
-
-message ProcedureWALEntry {
- enum Type {
- PROCEDURE_WAL_EOF = 1;
- PROCEDURE_WAL_INIT = 2;
- PROCEDURE_WAL_INSERT = 3;
- PROCEDURE_WAL_UPDATE = 4;
- PROCEDURE_WAL_DELETE = 5;
- PROCEDURE_WAL_COMPACT = 6;
- }
-
- required Type type = 1;
- repeated Procedure procedure = 2;
- optional uint64 proc_id = 3;
-}
diff --git a/src/hbase/if/Quota.proto b/src/hbase/if/Quota.proto
deleted file mode 100644
index a8303b1..0000000
--- a/src/hbase/if/Quota.proto
+++ /dev/null
@@ -1,76 +0,0 @@
- /**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "QuotaProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-import "HBase.proto";
-
-enum QuotaScope {
- CLUSTER = 1;
- MACHINE = 2;
-}
-
-message TimedQuota {
- required TimeUnit time_unit = 1;
- optional uint64 soft_limit = 2;
- optional float share = 3;
- optional QuotaScope scope = 4 [default = MACHINE];
-}
-
-enum ThrottleType {
- REQUEST_NUMBER = 1;
- REQUEST_SIZE = 2;
- WRITE_NUMBER = 3;
- WRITE_SIZE = 4;
- READ_NUMBER = 5;
- READ_SIZE = 6;
-}
-
-message Throttle {
- optional TimedQuota req_num = 1;
- optional TimedQuota req_size = 2;
-
- optional TimedQuota write_num = 3;
- optional TimedQuota write_size = 4;
-
- optional TimedQuota read_num = 5;
- optional TimedQuota read_size = 6;
-}
-
-message ThrottleRequest {
- optional ThrottleType type = 1;
- optional TimedQuota timed_quota = 2;
-}
-
-enum QuotaType {
- THROTTLE = 1;
-}
-
-message Quotas {
- optional bool bypass_globals = 1 [default = false];
- optional Throttle throttle = 2;
-}
-
-message QuotaUsage {
-}
diff --git a/src/hbase/if/RPC.proto b/src/hbase/if/RPC.proto
deleted file mode 100644
index 8413d25..0000000
--- a/src/hbase/if/RPC.proto
+++ /dev/null
@@ -1,137 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package hbase.pb;
-
-import "Tracing.proto";
-import "HBase.proto";
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "RPCProtos";
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-// See https://issues.apache.org/jira/browse/HBASE-7898 for high-level
-// description of RPC specification.
-//
-// On connection setup, the client sends six bytes of preamble -- a four
-// byte magic, a byte of version, and a byte of authentication type.
-//
-// We then send a "ConnectionHeader" protobuf of user information and the
-// 'protocol' or 'service' that is to be run over this connection as well as
-// info such as codecs and compression to use when we send cell blocks(see below).
-// This connection header protobuf is prefaced by an int that holds the length
-// of this connection header (this is NOT a varint). The pb connection header
-// is sent with Message#writeTo. The server throws an exception if it doesn't
-// like what it was sent noting what it is objecting too. Otherwise, the server
-// says nothing and is open for business.
-//
-// Hereafter the client makes requests and the server returns responses.
-//
-// Requests look like this:
-//
-// <An int with the total length of the request>
-// <RequestHeader Message written out using Message#writeDelimitedTo>
-// <Optionally a Request Parameter Message written out using Message#writeDelimitedTo>
-// <Optionally a Cell block>
-//
-// ...where the Request Parameter Message is whatever the method name stipulated
-// in the RequestHeader expects; e.g. if the method is a scan, then the pb
-// Request Message is a GetRequest, or a ScanRequest. A block of Cells
-// optionally follows. The presence of a Request param Message and/or a
-// block of Cells will be noted in the RequestHeader.
-//
-// Response is the mirror of the request:
-//
-// <An int with the total length of the response>
-// <ResponseHeader Message written out using Message#writeDelimitedTo>
-// <Optionally a Response Result Message written out using Message#writeDelimitedTo>
-// <Optionally a Cell block>
-//
-// ...where the Response Message is the response type that goes with the
-// method specified when making the request and the follow on Cell blocks may
-// or may not be there -- read the response header to find out if one following.
-// If an exception, it will be included inside the Response Header.
-//
-// Any time we write a pb, we do it with Message#writeDelimitedTo EXCEPT when
-// the connection header is sent; this is prefaced by an int with its length
-// and the pb connection header is then written with Message#writeTo.
-//
-
-// User Information proto. Included in ConnectionHeader on connection setup
-message UserInformation {
- required string effective_user = 1;
- optional string real_user = 2;
-}
-
-// This is sent on connection setup after the connection preamble is sent.
-message ConnectionHeader {
- optional UserInformation user_info = 1;
- optional string service_name = 2;
- // Cell block codec we will use sending over optional cell blocks. Server throws exception
- // if cannot deal. Null means no codec'ing going on so we are pb all the time (SLOW!!!)
- optional string cell_block_codec_class = 3;
- // Compressor we will use if cell block is compressed. Server will throw exception if not supported.
- // Class must implement hadoop's CompressionCodec Interface. Can't compress if no codec.
- optional string cell_block_compressor_class = 4;
- optional VersionInfo version_info = 5;
-}
-
-// Optional Cell block Message. Included in client RequestHeader
-message CellBlockMeta {
- // Length of the following cell block. Could calculate it but convenient having it too hand.
- optional uint32 length = 1;
-}
-
-// At the RPC layer, this message is used to carry
-// the server side exception to the RPC client.
-message ExceptionResponse {
- // Class name of the exception thrown from the server
- optional string exception_class_name = 1;
- // Exception stack trace from the server side
- optional string stack_trace = 2;
- // Optional hostname. Filled in for some exceptions such as region moved
- // where exception gives clue on where the region may have moved.
- optional string hostname = 3;
- optional int32 port = 4;
- // Set if we are NOT to retry on receipt of this exception
- optional bool do_not_retry = 5;
-}
-
-// Header sent making a request.
-message RequestHeader {
- // Monotonically increasing call_id to keep track of RPC requests and their response
- optional uint32 call_id = 1;
- optional RPCTInfo trace_info = 2;
- optional string method_name = 3;
- // If true, then a pb Message param follows.
- optional bool request_param = 4;
- // If present, then an encoded data block follows.
- optional CellBlockMeta cell_block_meta = 5;
- // 0 is NORMAL priority. 200 is HIGH. If no priority, treat it as NORMAL.
- // See HConstants.
- optional uint32 priority = 6;
- optional uint32 timeout = 7;
-}
-
-message ResponseHeader {
- optional uint32 call_id = 1;
- // If present, then request threw an exception and no response message (else we presume one)
- optional ExceptionResponse exception = 2;
- // If present, then an encoded data block follows.
- optional CellBlockMeta cell_block_meta = 3;
-}
diff --git a/src/hbase/if/RSGroup.proto b/src/hbase/if/RSGroup.proto
deleted file mode 100644
index 7358941..0000000
--- a/src/hbase/if/RSGroup.proto
+++ /dev/null
@@ -1,34 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "RSGroupProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-import "HBase.proto";
-
-message RSGroupInfo {
- required string name = 1;
- repeated ServerName servers = 4;
- repeated TableName tables = 3;
-}
-
diff --git a/src/hbase/if/RSGroupAdmin.proto b/src/hbase/if/RSGroupAdmin.proto
deleted file mode 100644
index fda9b09..0000000
--- a/src/hbase/if/RSGroupAdmin.proto
+++ /dev/null
@@ -1,136 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "RSGroupAdminProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-import "HBase.proto";
-import "RSGroup.proto";
-
-/** Group level protobufs */
-
-message ListTablesOfRSGroupRequest {
- required string r_s_group_name = 1;
-}
-
-message ListTablesOfRSGroupResponse {
- repeated TableName table_name = 1;
-}
-
-message GetRSGroupInfoRequest {
- required string r_s_group_name = 1;
-}
-
-message GetRSGroupInfoResponse {
- optional RSGroupInfo r_s_group_info = 1;
-}
-
-message GetRSGroupInfoOfTableRequest {
- required TableName table_name = 1;
-}
-
-message GetRSGroupInfoOfTableResponse {
- optional RSGroupInfo r_s_group_info = 1;
-}
-
-message MoveServersRequest {
- required string target_group = 1;
- repeated ServerName servers = 3;
-}
-
-message MoveServersResponse {
-}
-
-message MoveTablesRequest {
- required string target_group = 1;
- repeated TableName table_name = 2;
-}
-
-message MoveTablesResponse {
-}
-
-message AddRSGroupRequest {
- required string r_s_group_name = 1;
-}
-
-message AddRSGroupResponse {
-}
-
-message RemoveRSGroupRequest {
- required string r_s_group_name = 1;
-}
-
-message RemoveRSGroupResponse {
-}
-
-message BalanceRSGroupRequest {
- required string r_s_group_name = 1;
-}
-
-message BalanceRSGroupResponse {
- required bool balanceRan = 1;
-}
-
-message ListRSGroupInfosRequest {
-}
-
-message ListRSGroupInfosResponse {
- repeated RSGroupInfo r_s_group_info = 1;
-}
-
-message GetRSGroupInfoOfServerRequest {
- required ServerName server = 2;
-}
-
-message GetRSGroupInfoOfServerResponse {
- optional RSGroupInfo r_s_group_info = 1;
-}
-
-service RSGroupAdminService {
- rpc GetRSGroupInfo(GetRSGroupInfoRequest)
- returns (GetRSGroupInfoResponse);
-
- rpc GetRSGroupInfoOfTable(GetRSGroupInfoOfTableRequest)
- returns (GetRSGroupInfoOfTableResponse);
-
- rpc GetRSGroupInfoOfServer(GetRSGroupInfoOfServerRequest)
- returns (GetRSGroupInfoOfServerResponse);
-
- rpc MoveServers(MoveServersRequest)
- returns (MoveServersResponse);
-
- rpc MoveTables(MoveTablesRequest)
- returns (MoveTablesResponse);
-
- rpc AddRSGroup(AddRSGroupRequest)
- returns (AddRSGroupResponse);
-
- rpc RemoveRSGroup(RemoveRSGroupRequest)
- returns (RemoveRSGroupResponse);
-
- rpc BalanceRSGroup(BalanceRSGroupRequest)
- returns (BalanceRSGroupResponse);
-
- rpc ListRSGroupInfos(ListRSGroupInfosRequest)
- returns (ListRSGroupInfosResponse);
-}
diff --git a/src/hbase/if/RegionNormalizer.proto b/src/hbase/if/RegionNormalizer.proto
deleted file mode 100644
index e5305d6..0000000
--- a/src/hbase/if/RegionNormalizer.proto
+++ /dev/null
@@ -1,28 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file contains protocol buffers to represent the state of the load balancer.
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "RegionNormalizerProtos";
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-message RegionNormalizerState {
- optional bool normalizer_on = 1;
-}
diff --git a/src/hbase/if/RegionServerStatus.proto b/src/hbase/if/RegionServerStatus.proto
deleted file mode 100644
index fda9de2..0000000
--- a/src/hbase/if/RegionServerStatus.proto
+++ /dev/null
@@ -1,158 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// This file contains protocol buffers that are used for RegionServerStatusProtocol.
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "RegionServerStatusProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-import "HBase.proto";
-import "ClusterStatus.proto";
-
-message RegionServerStartupRequest {
- /** Port number this regionserver is up on */
- required uint32 port = 1;
-
- /** This servers' startcode */
- required uint64 server_start_code = 2;
-
- /** Current time of the region server in ms */
- required uint64 server_current_time = 3;
-
- /** hostname for region server, optional */
- optional string use_this_hostname_instead = 4;
-}
-
-message RegionServerStartupResponse {
- /**
- * Configuration for the regionserver to use: e.g. filesystem,
- * hbase rootdir, the hostname to use creating the RegionServer ServerName,
- * etc
- */
- repeated NameStringPair map_entries = 1;
-}
-
-message RegionServerReportRequest {
- required ServerName server = 1;
-
- /** load the server is under */
- optional ServerLoad load = 2;
-}
-
-message RegionServerReportResponse {
-}
-
-message ReportRSFatalErrorRequest {
- /** name of the server experiencing the error */
- required ServerName server = 1;
-
- /** informative text to expose in the master logs and UI */
- required string error_message = 2;
-}
-
-message ReportRSFatalErrorResponse {
-}
-
-message GetLastFlushedSequenceIdRequest {
- /** region name */
- required bytes region_name = 1;
-}
-
-message GetLastFlushedSequenceIdResponse {
- /** the last WAL sequence id flushed from MemStore to HFile for the region */
- required uint64 last_flushed_sequence_id = 1;
-
- /** the last WAL sequence id flushed from MemStore to HFile for stores of the region */
- repeated StoreSequenceId store_last_flushed_sequence_id = 2;
-}
-
-message RegionStateTransition {
- required TransitionCode transition_code = 1;
-
- /** Mutliple regions are involved during merging/splitting */
- repeated RegionInfo region_info = 2;
-
- /** For newly opened region, the open seq num is needed */
- optional uint64 open_seq_num = 3;
-
- enum TransitionCode {
- OPENED = 0;
- FAILED_OPEN = 1;
- /** No failed_close, in which case region server will abort */
- CLOSED = 2;
-
- /** Ask master for ok to split/merge region(s) */
- READY_TO_SPLIT = 3;
- READY_TO_MERGE = 4;
-
- SPLIT_PONR = 5;
- MERGE_PONR = 6;
-
- SPLIT = 7;
- MERGED = 8;
- SPLIT_REVERTED = 9;
- MERGE_REVERTED = 10;
- }
-}
-
-message ReportRegionStateTransitionRequest {
- /** This region server's server name */
- required ServerName server = 1;
-
- repeated RegionStateTransition transition = 2;
-}
-
-message ReportRegionStateTransitionResponse {
- /** Error message if failed to update the region state */
- optional string error_message = 1;
-}
-
-service RegionServerStatusService {
- /** Called when a region server first starts. */
- rpc RegionServerStartup(RegionServerStartupRequest)
- returns(RegionServerStartupResponse);
-
- /** Called to report the load the RegionServer is under. */
- rpc RegionServerReport(RegionServerReportRequest)
- returns(RegionServerReportResponse);
-
- /**
- * Called by a region server to report a fatal error that is causing it to
- * abort.
- */
- rpc ReportRSFatalError(ReportRSFatalErrorRequest)
- returns(ReportRSFatalErrorResponse);
-
- /** Called to get the sequence id of the last MemStore entry flushed to an
- * HFile for a specified region. Used by the region server to speed up
- * log splitting. */
- rpc GetLastFlushedSequenceId(GetLastFlushedSequenceIdRequest)
- returns(GetLastFlushedSequenceIdResponse);
-
- /**
- * Called by a region server to report the progress of a region
- * transition. If the request fails, the transition should
- * be aborted.
- */
- rpc ReportRegionStateTransition(ReportRegionStateTransitionRequest)
- returns(ReportRegionStateTransitionResponse);
-}
diff --git a/src/hbase/if/RowProcessor.proto b/src/hbase/if/RowProcessor.proto
deleted file mode 100644
index cf2f30f..0000000
--- a/src/hbase/if/RowProcessor.proto
+++ /dev/null
@@ -1,45 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-/**
- * Defines a protocol to perform multi row transactions.
- * See BaseRowProcessorEndpoint for the implementation.
- * See HRegion#processRowsWithLocks() for details.
- */
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "RowProcessorProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-message ProcessRequest {
- required string row_processor_class_name = 1;
- optional string row_processor_initializer_message_name = 2;
- optional bytes row_processor_initializer_message = 3;
- optional uint64 nonce_group = 4;
- optional uint64 nonce = 5;
-}
-
-message ProcessResponse {
- required bytes row_processor_result = 1;
-}
-
-service RowProcessorService {
- rpc Process(ProcessRequest) returns (ProcessResponse);
-}
diff --git a/src/hbase/if/SecureBulkLoad.proto b/src/hbase/if/SecureBulkLoad.proto
deleted file mode 100644
index 814735b..0000000
--- a/src/hbase/if/SecureBulkLoad.proto
+++ /dev/null
@@ -1,72 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "SecureBulkLoadProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-import 'HBase.proto';
-import 'Client.proto';
-
-message SecureBulkLoadHFilesRequest {
- repeated BulkLoadHFileRequest.FamilyPath family_path = 1;
- optional bool assign_seq_num = 2;
- required DelegationToken fs_token = 3;
- required string bulk_token = 4;
-}
-
-message SecureBulkLoadHFilesResponse {
- required bool loaded = 1;
-}
-
-message DelegationToken {
- optional bytes identifier = 1;
- optional bytes password = 2;
- optional string kind = 3;
- optional string service = 4;
-}
-
-message PrepareBulkLoadRequest {
- required TableName table_name = 1;
-}
-
-message PrepareBulkLoadResponse {
- required string bulk_token = 1;
-}
-
-message CleanupBulkLoadRequest {
- required string bulk_token = 1;
-
-}
-
-message CleanupBulkLoadResponse {
-}
-
-service SecureBulkLoadService {
- rpc PrepareBulkLoad(PrepareBulkLoadRequest)
- returns (PrepareBulkLoadResponse);
-
- rpc SecureBulkLoadHFiles(SecureBulkLoadHFilesRequest)
- returns (SecureBulkLoadHFilesResponse);
-
- rpc CleanupBulkLoad(CleanupBulkLoadRequest)
- returns (CleanupBulkLoadResponse);
-}
diff --git a/src/hbase/if/Snapshot.proto b/src/hbase/if/Snapshot.proto
deleted file mode 100644
index ae1a1e6..0000000
--- a/src/hbase/if/Snapshot.proto
+++ /dev/null
@@ -1,66 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "SnapshotProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-import "FS.proto";
-import "HBase.proto";
-
-message SnapshotFileInfo {
- enum Type {
- HFILE = 1;
- WAL = 2;
- }
-
- required Type type = 1;
-
- optional string hfile = 3;
-
- optional string wal_server = 4;
- optional string wal_name = 5;
-}
-
-message SnapshotRegionManifest {
- optional int32 version = 1;
-
- required RegionInfo region_info = 2;
- repeated FamilyFiles family_files = 3;
-
- message StoreFile {
- required string name = 1;
- optional Reference reference = 2;
-
- // TODO: Add checksums or other fields to verify the file
- optional uint64 file_size = 3;
- }
-
- message FamilyFiles {
- required bytes family_name = 1;
- repeated StoreFile store_files = 2;
- }
-}
-
-message SnapshotDataManifest {
- required TableSchema table_schema = 1;
- repeated SnapshotRegionManifest region_manifests = 2;
-}
diff --git a/src/hbase/if/Tracing.proto b/src/hbase/if/Tracing.proto
deleted file mode 100644
index 5a64cfc..0000000
--- a/src/hbase/if/Tracing.proto
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "TracingProtos";
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-//Used to pass through the information necessary to continue
-//a trace after an RPC is made. All we need is the traceid
-//(so we know the overarching trace this message is a part of), and
-//the id of the current span when this message was sent, so we know
-//what span caused the new span we will create when this message is received.
-message RPCTInfo {
- optional int64 trace_id = 1;
- optional int64 parent_id = 2;
-}
diff --git a/src/hbase/if/VisibilityLabels.proto b/src/hbase/if/VisibilityLabels.proto
deleted file mode 100644
index d2dc44d..0000000
--- a/src/hbase/if/VisibilityLabels.proto
+++ /dev/null
@@ -1,83 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "VisibilityLabelsProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-import "Client.proto";
-
-message VisibilityLabelsRequest {
- repeated VisibilityLabel visLabel = 1;
-}
-
-message VisibilityLabel {
- required bytes label = 1;
- optional uint32 ordinal = 2;
-}
-
-message VisibilityLabelsResponse {
- repeated RegionActionResult result = 1;
-}
-
-message SetAuthsRequest {
- required bytes user = 1;
- repeated bytes auth = 2;
-}
-
-message UserAuthorizations {
- required bytes user = 1;
- repeated uint32 auth = 2;
-}
-
-message MultiUserAuthorizations {
- repeated UserAuthorizations userAuths = 1;
-}
-
-message GetAuthsRequest {
- required bytes user = 1;
-}
-
-message GetAuthsResponse {
- required bytes user = 1;
- repeated bytes auth = 2;
-}
-
-message ListLabelsRequest {
- optional string regex = 1;
-}
-
-message ListLabelsResponse {
- repeated bytes label = 1;
-}
-
-service VisibilityLabelsService {
- rpc addLabels(VisibilityLabelsRequest)
- returns (VisibilityLabelsResponse);
- rpc setAuths(SetAuthsRequest)
- returns (VisibilityLabelsResponse);
- rpc clearAuths(SetAuthsRequest)
- returns (VisibilityLabelsResponse);
- rpc getAuths(GetAuthsRequest)
- returns (GetAuthsResponse);
- rpc listLabels(ListLabelsRequest)
- returns (ListLabelsResponse);
-}
\ No newline at end of file
diff --git a/src/hbase/if/WAL.proto b/src/hbase/if/WAL.proto
deleted file mode 100644
index 2061b22..0000000
--- a/src/hbase/if/WAL.proto
+++ /dev/null
@@ -1,173 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "WALProtos";
-option java_generic_services = false;
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-import "HBase.proto";
-import "Client.proto";
-
-message WALHeader {
- optional bool has_compression = 1;
- optional bytes encryption_key = 2;
- optional bool has_tag_compression = 3;
- optional string writer_cls_name = 4;
- optional string cell_codec_cls_name = 5;
-}
-
-/*
- * Protocol buffer version of WALKey; see WALKey comment, not really a key but WALEdit header
- * for some KVs
- */
-message WALKey {
- required bytes encoded_region_name = 1;
- required bytes table_name = 2;
- required uint64 log_sequence_number = 3;
- required uint64 write_time = 4;
- /*
- This parameter is deprecated in favor of clusters which
- contains the list of clusters that have consumed the change.
- It is retained so that the log created by earlier releases (0.94)
- can be read by the newer releases.
- */
- optional UUID cluster_id = 5 [deprecated=true];
-
- repeated FamilyScope scopes = 6;
- optional uint32 following_kv_count = 7;
-
- /*
- This field contains the list of clusters that have
- consumed the change
- */
- repeated UUID cluster_ids = 8;
-
- optional uint64 nonceGroup = 9;
- optional uint64 nonce = 10;
- optional uint64 orig_sequence_number = 11;
-
-/*
- optional CustomEntryType custom_entry_type = 9;
-
- enum CustomEntryType {
- COMPACTION = 0;
- }
-*/
-}
-
-enum ScopeType {
- REPLICATION_SCOPE_LOCAL = 0;
- REPLICATION_SCOPE_GLOBAL = 1;
-}
-
-message FamilyScope {
- required bytes family = 1;
- required ScopeType scope_type = 2;
-}
-
-/**
- * Custom WAL entries
- */
-
-/**
- * Special WAL entry to hold all related to a compaction.
- * Written to WAL before completing compaction. There is
- * sufficient info in the below message to complete later
- * the * compaction should we fail the WAL write.
- */
-message CompactionDescriptor {
- required bytes table_name = 1; // TODO: WALKey already stores these, might remove
- required bytes encoded_region_name = 2;
- required bytes family_name = 3;
- repeated string compaction_input = 4; // relative to store dir
- repeated string compaction_output = 5;
- required string store_home_dir = 6; // relative to region dir
- optional bytes region_name = 7; // full region name
-}
-
-/**
- * Special WAL entry to hold all related to a flush.
- */
-message FlushDescriptor {
- enum FlushAction {
- START_FLUSH = 0;
- COMMIT_FLUSH = 1;
- ABORT_FLUSH = 2;
- CANNOT_FLUSH = 3; // marker for indicating that a flush has been requested but cannot complete
- }
-
- message StoreFlushDescriptor {
- required bytes family_name = 1;
- required string store_home_dir = 2; //relative to region dir
- repeated string flush_output = 3; // relative to store dir (if this is a COMMIT_FLUSH)
- }
-
- required FlushAction action = 1;
- required bytes table_name = 2;
- required bytes encoded_region_name = 3;
- optional uint64 flush_sequence_number = 4;
- repeated StoreFlushDescriptor store_flushes = 5;
- optional bytes region_name = 6; // full region name
-}
-
-message StoreDescriptor {
- required bytes family_name = 1;
- required string store_home_dir = 2; //relative to region dir
- repeated string store_file = 3; // relative to store dir
- optional uint64 store_file_size_bytes = 4; // size of store file
-}
-
-/**
- * Special WAL entry used for writing bulk load events to WAL
- */
-message BulkLoadDescriptor {
- required TableName table_name = 1;
- required bytes encoded_region_name = 2;
- repeated StoreDescriptor stores = 3;
- required int64 bulkload_seq_num = 4;
-}
-
-/**
- * Special WAL entry to hold all related to a region event (open/close).
- */
-message RegionEventDescriptor {
- enum EventType {
- REGION_OPEN = 0;
- REGION_CLOSE = 1;
- }
-
- required EventType event_type = 1;
- required bytes table_name = 2;
- required bytes encoded_region_name = 3;
- optional uint64 log_sequence_number = 4;
- repeated StoreDescriptor stores = 5;
- optional ServerName server = 6; // Server who opened the region
- optional bytes region_name = 7; // full region name
-}
-
-/**
- * A trailer that is appended to the end of a properly closed WAL file.
- * If missing, this is either a legacy or a corrupted WAL file.
- * N.B. This trailer currently doesn't contain any information and we
- * purposefully don't expose it in the WAL APIs. It's for future growth.
- */
-message WALTrailer {
-}
diff --git a/src/hbase/if/ZooKeeper.proto b/src/hbase/if/ZooKeeper.proto
deleted file mode 100644
index 41c0e0e..0000000
--- a/src/hbase/if/ZooKeeper.proto
+++ /dev/null
@@ -1,176 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// ZNode data in hbase are serialized protobufs with a four byte
-// 'magic' 'PBUF' prefix.
-package hbase.pb;
-
-option java_package = "org.apache.hadoop.hbase.protobuf.generated";
-option java_outer_classname = "ZooKeeperProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-option optimize_for = SPEED;
-
-import "HBase.proto";
-import "ClusterStatus.proto";
-
-/**
- * Content of the meta-region-server znode.
- */
-message MetaRegionServer {
- // The ServerName hosting the meta region currently, or destination server,
- // if meta region is in transition.
- required ServerName server = 1;
- // The major version of the rpc the server speaks. This is used so that
- // clients connecting to the cluster can have prior knowledge of what version
- // to send to a RegionServer. AsyncHBase will use this to detect versions.
- optional uint32 rpc_version = 2;
-
- // State of the region transition. OPEN means fully operational 'hbase:meta'
- optional RegionState.State state = 3;
-}
-
-/**
- * Content of the master znode.
- */
-message Master {
- // The ServerName of the current Master
- required ServerName master = 1;
- // Major RPC version so that clients can know what version the master can accept.
- optional uint32 rpc_version = 2;
- optional uint32 info_port = 3;
-}
-
-/**
- * Content of the '/hbase/running', cluster state, znode.
- */
-message ClusterUp {
- // If this znode is present, cluster is up. Currently
- // the data is cluster start_date.
- required string start_date = 1;
-}
-
-/**
- * WAL SplitLog directory znodes have this for content. Used doing distributed
- * WAL splitting. Holds current state and name of server that originated split.
- */
-message SplitLogTask {
- enum State {
- UNASSIGNED = 0;
- OWNED = 1;
- RESIGNED = 2;
- DONE = 3;
- ERR = 4;
- }
- enum RecoveryMode {
- UNKNOWN = 0;
- LOG_SPLITTING = 1;
- LOG_REPLAY = 2;
- }
- required State state = 1;
- required ServerName server_name = 2;
- optional RecoveryMode mode = 3 [default = UNKNOWN];
-}
-
-/**
- * The znode that holds state of table.
- * Deprected, table state is stored in table descriptor on HDFS.
- */
-message DeprecatedTableState {
- // Table's current state
- enum State {
- ENABLED = 0;
- DISABLED = 1;
- DISABLING = 2;
- ENABLING = 3;
- }
- // This is the table's state. If no znode for a table,
- // its state is presumed enabled. See o.a.h.h.zookeeper.ZKTable class
- // for more.
- required State state = 1 [default = ENABLED];
-}
-
-message TableCF {
- optional TableName table_name = 1;
- repeated bytes families = 2;
-}
-
-/**
- * Used by replication. Holds a replication peer key.
- */
-message ReplicationPeer {
- // clusterkey is the concatenation of the slave cluster's
- // hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent
- required string clusterkey = 1;
- optional string replicationEndpointImpl = 2;
- repeated BytesBytesPair data = 3;
- repeated NameStringPair configuration = 4;
- repeated TableCF table_cfs = 5;
-}
-
-/**
- * Used by replication. Holds whether enabled or disabled
- */
-message ReplicationState {
- enum State {
- ENABLED = 0;
- DISABLED = 1;
- }
- required State state = 1;
-}
-
-/**
- * Used by replication. Holds the current position in an WAL file.
- */
-message ReplicationHLogPosition {
- required int64 position = 1;
-}
-
-/**
- * Used by replication. Used to lock a region server during failover.
- */
-message ReplicationLock {
- required string lock_owner = 1;
-}
-
-/**
- * Metadata associated with a table lock in zookeeper
- */
-message TableLock {
- optional TableName table_name = 1;
- optional ServerName lock_owner = 2;
- optional int64 thread_id = 3;
- optional bool is_shared = 4;
- optional string purpose = 5;
- optional int64 create_time = 6;
-}
-
-/**
- * State of the switch.
- */
-message SwitchState {
- optional bool enabled = 1;
-}
-
-/**
- * State for split and merge, used in hbck
- */
-message SplitAndMergeState {
- optional bool split_enabled = 1;
- optional bool merge_enabled = 2;
-}
diff --git a/src/hbase/if/test.proto b/src/hbase/if/test.proto
deleted file mode 100644
index 72b68e9..0000000
--- a/src/hbase/if/test.proto
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-option java_package = "org.apache.hadoop.hbase.ipc.protobuf.generated";
-option java_outer_classname = "TestProtos";
-option java_generate_equals_and_hash = true;
-
-message EmptyRequestProto {
-}
-
-message EmptyResponseProto {
-}
-
-message EchoRequestProto {
- required string message = 1;
-}
-
-message EchoResponseProto {
- required string message = 1;
-}
-
-message PauseRequestProto {
- required uint32 ms = 1;
-}
-
-message AddrResponseProto {
- required string addr = 1;
-}
diff --git a/src/hbase/if/test_rpc_service.proto b/src/hbase/if/test_rpc_service.proto
deleted file mode 100644
index 2730403..0000000
--- a/src/hbase/if/test_rpc_service.proto
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-option java_package = "org.apache.hadoop.hbase.ipc.protobuf.generated";
-option java_outer_classname = "TestRpcServiceProtos";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-
-import "test.proto";
-
-
-/**
- * A protobuf service for use in tests
- */
-service TestProtobufRpcProto {
- rpc ping(EmptyRequestProto) returns (EmptyResponseProto);
- rpc echo(EchoRequestProto) returns (EchoResponseProto);
- rpc error(EmptyRequestProto) returns (EmptyResponseProto);
- rpc pause(PauseRequestProto) returns (EmptyResponseProto);
- rpc addr(EmptyRequestProto) returns (AddrResponseProto);
- rpc socketNotOpen(EmptyRequestProto) returns (EmptyResponseProto);
-}
[hbase] 04/137: HBASE-14852 Update build env
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit d760ee998952cab1f57f2936b56b70b0f0486f26
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Thu Nov 19 16:28:11 2015 -0800
HBASE-14852 Update build env
Also includes HBASE-14858 Clean up so core is ready for development on a recent version of c++
---
.buckconfig | 2 +
.gitignore | 23 +---
CMakeLists.txt | 157 ---------------------------
Dockerfile | 22 ++++
README.md | 15 ++-
bin/build-all.sh | 41 -------
bin/build-thirdparty.sh | 64 -----------
bin/download-thirdparty.sh | 70 ------------
bin/hbase-client-env.sh | 47 --------
bin/start-docker.sh | 32 ++++++
cmake_modules/FindGTest.cmake | 53 ---------
cmake_modules/FindLibEv.cmake | 47 --------
core/BUCK | 50 +++++++++
{src/core => core}/admin.cc | 0
src/core/client.cc => core/admin.h | 4 +-
src/async/hbase_result.cc => core/client.cc | 25 +++--
src/core/get.cc => core/client.h | 6 +-
{src/core => core}/connection.cc | 4 +-
src/core/admin.h => core/connection.h | 11 +-
src/core/client.cc => core/connection_attr.h | 6 +-
{src/core => core}/delete.cc | 3 +-
src/core/client.h => core/delete.h | 10 +-
{src/core => core}/get.cc | 0
src/core/client.cc => core/get.h | 4 +-
{src/core => core}/hbase_macros.h | 38 +++----
{src/core => core}/mutation.cc | 11 +-
{src/core => core}/mutation.h | 40 ++++---
{src/core => core}/put.cc | 3 +-
src/core/get.h => core/put.h | 11 +-
{src/core => core}/scanner.cc | 0
src/core/client.cc => core/scanner.h | 4 +-
{src/rpc => rpc}/CMakeLists.txt | 0
src/async/CMakeLists.txt | 32 ------
src/async/get-test.cc | 59 ----------
src/async/hbase_admin.cc | 57 ----------
src/async/hbase_admin.h | 69 ------------
src/async/hbase_client.cc | 47 --------
src/async/hbase_client.h | 60 ----------
src/async/hbase_connection.cc | 37 -------
src/async/hbase_connection.h | 52 ---------
src/async/hbase_errno.h | 23 ----
src/async/hbase_get.cc | 61 -----------
src/async/hbase_get.h | 73 -------------
src/async/hbase_mutations.cc | 111 -------------------
src/async/hbase_mutations.h | 119 --------------------
src/async/hbase_result.h | 44 --------
src/async/hbase_scanner.cc | 59 ----------
src/async/hbase_scanner.h | 80 --------------
src/async/mutations-test.cc | 102 -----------------
src/core/CMakeLists.txt | 31 ------
src/core/connection.h | 26 -----
src/core/connection_attr.h | 30 -----
src/core/delete.h | 29 -----
src/core/hbase_connection_attr.cc | 41 -------
src/core/hbase_connection_attr.h | 51 ---------
src/core/hbase_types.h | 83 --------------
src/core/put.h | 29 -----
src/core/scanner.h | 25 -----
src/sync/CMakeLists.txt | 24 ----
src/sync/hbase_admin.cc | 51 ---------
src/sync/hbase_admin.h | 61 -----------
src/sync/hbase_connection.cc | 37 -------
src/sync/hbase_connection.h | 52 ---------
third-party/BUCK | 105 ++++++++++++++++++
64 files changed, 327 insertions(+), 2236 deletions(-)
diff --git a/.buckconfig b/.buckconfig
new file mode 100644
index 0000000..3227a2a
--- /dev/null
+++ b/.buckconfig
@@ -0,0 +1,2 @@
+[cxx]
+ gtest_dep = //third-party/googletest/googletest:google-test
diff --git a/.gitignore b/.gitignore
index f14b3be..b172fe9 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,25 +14,12 @@
#python
*.pyc
-# CMake Generated Files
-CMakeCache.txt
-CMakeFiles
-Makefile
-cmake_install.CMakeCache
-cmake_install.cmake
-install_manifest.txt
-CTestTestfile.cmake
-build
-Testing
+# Buck
+buck-out
+.buckd
+*.swp
# Thirdparty dirs
-
-thirdparty/glog-*
-thirdparty/gtest-*
-thirdparty/gtest
-thirdparty/protobuf-*
-thirdparty/libevent-*
-thirdparty/libev-*
-thirdparty/installed
+third-party/googletest*
diff --git a/CMakeLists.txt b/CMakeLists.txt
deleted file mode 100644
index 819f178..0000000
--- a/CMakeLists.txt
+++ /dev/null
@@ -1,157 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-cmake_minimum_required(VERSION 2.6)
-
-# generate CTest input files
-enable_testing()
-
-# where to find cmake modules
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
-
-# if no build build type is specified, default to debug builds
-if (NOT CMAKE_BUILD_TYPE)
- set(CMAKE_BUILD_TYPE Debug)
-endif(NOT CMAKE_BUILD_TYPE)
-
-STRING(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE)
-
-set(CXX_COMMON_FLAGS "-Wall -Wextra -std=c++11")
-set(C_COMMON_FLAGS "-Wall -Wextra -std=c11")
-
-set(CXX_FLAGS_DEBUG "-ggdb -g")
-set(C_FLAGS_DEBUG "-ggdb")
-
-set(CXX_FLAGS_FASTDEBUG "-ggdb -O1")
-set(C_FLAGS_FASTDEBUG "-ggdb -O1")
-
-set(CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG -Wno-strict-aliasing")
-set(C_FLAGS_RELEASE "-O3 -g -DNDEBUG -Wno-strict-aliasing")
-
-# if no build build type is specified, default to debug builds
-if (NOT CMAKE_BUILD_TYPE)
- set(CMAKE_BUILD_TYPE Debug)
-endif(NOT CMAKE_BUILD_TYPE)
-
-string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE)
-
-# Set compile flags based on the build type.
-message("Configured for ${CMAKE_BUILD_TYPE} build (set with cmake -DCMAKE_BUILD_TYPE={release,debug,...})")
-if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
- set(CMAKE_CXX_FLAGS ${CXX_FLAGS_DEBUG})
- set(CMAKE_C_FLAGS ${C_FLAGS_DEBUG})
-elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "FASTDEBUG")
- set(CMAKE_CXX_FLAGS ${CXX_FLAGS_FASTDEBUG})
- set(CMAKE_C_FLAGS ${C_FLAGS_FASTDEBUG})
-elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")
- set(CMAKE_CXX_FLAGS ${CXX_FLAGS_RELEASE})
- set(CMAKE_C_FLAGS ${C_FLAGS_RELEASE})
-else()
- message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}")
-endif ()
-
-set(CMAKE_CXX_FLAGS "${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}")
-set(CMAKE_C_FLAGS "${C_COMMON_FLAGS} ${CMAKE_C_FLAGS}")
-
-
-# set compile output directory
-string(TOLOWER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_LOWERCASE)
-set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build/${CMAKE_BUILD_TYPE_LOWERCASE}/")
-
-# Link build/latest to the current build directory, to avoid developers
-# accidentally running the latest debug build when in fact they're building
-# release builds.
-file(MAKE_DIRECTORY ${BUILD_OUTPUT_ROOT_DIRECTORY})
-execute_process(COMMAND rm -f ${CMAKE_CURRENT_SOURCE_DIR}/build/latest)
-execute_process(COMMAND ln -sf ${BUILD_OUTPUT_ROOT_DIRECTORY}
- ${CMAKE_CURRENT_SOURCE_DIR}/build/latest)
-
-# where to put generated libraries
-set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
-set(ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
-set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
-
-# where to put generated binaries
-set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}")
-
-include_directories(src)
-
-#####################
-# Third Party
-#####################
-
-# find boost headers and libs
-set(Boost_DEBUG TRUE)
-set(Boost_USE_MULTITHREADED ON)
-find_package(Boost REQUIRED COMPONENTS thread system-mt)
-include_directories(${Boost_INCLUDE_DIRS})
-
-find_package(Protobuf REQUIRED)
-include_directories(${PROTOBUF_INCLUDE_DIRS})
-add_library(protobuf STATIC IMPORTED)
-set_target_properties(protobuf PROPERTIES IMPORTED_LOCATION "${PROTOBUF_STATIC_LIBRARY}")
-
-find_package(LibEv REQUIRED)
-include_directories(${LIBEV_INCLUDE_DIR})
-add_library(libev STATIC IMPORTED)
-
-
-set(HBASE_LIBS
- ${PROTOBUF}
- ${LIBEV}
- ${Boost_LIBRARIES}
-)
-#########
-# Testing Config
-#########
-
-# find GTest headers and libs
-find_package(GTest REQUIRED)
-include_directories(${GTEST_INCLUDE_DIR})
-add_library(gtest STATIC IMPORTED)
-set_target_properties(gtest PROPERTIES IMPORTED_LOCATION "${GTEST_LIBS}")
-
-set(HBASE_TEST_LIBS ${HBASE_LIBS} ${GTEST_LIBS})
-set(HBASE_ASYNC_TEST_LIBS ${HBASE_LIBS} ${GTEST_LIBS} hbase-async)
-set(HBASE_SYNC_TEST_LIBS ${HBASE_LIBS} ${GTEST_LIBS} hbase-sync)
-
-function(ADD_HBASE_ASYNC_TEST TEST_NAME)
- add_executable(${TEST_NAME} ${TEST_NAME}.cc)
- target_link_libraries(${TEST_NAME} ${HBASE_ASYNC_TEST_LIBS})
- add_test(${TEST_NAME} "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
-endfunction()
-
-
-function(ADD_HBASE_SYNC_TEST TEST_NAME)
- add_executable(${TEST_NAME} ${TEST_NAME}.cc)
- target_link_libraries(${TEST_NAME} ${HBASE_ASYNC_TEST_LIBS})
- add_test(${TEST_NAME} "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
-endfunction()
-
-
-#####
-# Actual project definition
-#####
-add_subdirectory(src/async)
-add_subdirectory(src/sync)
-
-add_subdirectory(src/core)
-add_subdirectory(src/rpc)
-
-
-add_library(hbase-async SHARED $<TARGET_OBJECTS:hcore> $<TARGET_OBJECTS:hasync>)
-add_library(hbase-sync SHARED $<TARGET_OBJECTS:hcore> $<TARGET_OBJECTS:hsync>)
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 0000000..70e823b
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,22 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM pjameson/buck-folly-watchman
+
+RUN apt-get install -y libprotobuf-dev protobuf-compiler clang-format-3.7 vim
+
+WORKDIR /usr/local/src/hbase/hbase-native-client
diff --git a/README.md b/README.md
index 0bb4d48..76df596 100644
--- a/README.md
+++ b/README.md
@@ -19,11 +19,10 @@ under the License.
# hbase-native-client
-Native client for HBase 0.96
+Native client for HBase
-This is a C library that implements a
-HBase client. It's thread safe and libEv
-based.
+This is a C/C++ library that implements a
+HBase client.
## Design Philosphy
@@ -51,3 +50,11 @@ the hbase_ prefix is assumed to be implementation private.
All C apis and typedefs will be prefixed with hb_.
All typedefs end with _t.
+
+
+## Docker
+
+The build environment is docker. This should keep a consistent
+build environment for everyone. Buck the build system works
+best with mmap'd files. On OSX this means that vmwarefusion
+works the best. However it should work with just the defaults.
diff --git a/bin/build-all.sh b/bin/build-all.sh
deleted file mode 100755
index a3e45de..0000000
--- a/bin/build-all.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-#! /usr/bin/env bash
-#
-#/**
-# * Copyright The Apache Software Foundation
-# *
-# * Licensed to the Apache Software Foundation (ASF) under one
-# * or more contributor license agreements. See the NOTICE file
-# * distributed with this work for additional information
-# * regarding copyright ownership. The ASF licenses this file
-# * to you under the Apache License, Version 2.0 (the
-# * "License"); you may not use this file except in compliance
-# * with the License. You may obtain a copy of the License at
-# *
-# * http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# */
-
-set -e
-set -x
-
-SOURCE="${BASH_SOURCE[0]}"
-while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
- DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
- SOURCE="$(readlink "$SOURCE")"
- [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
-done
-DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-
-source ${DIR}/hbase-client-env.sh
-
-${DIR}/download-thirdparty.sh
-${DIR}/build-thirdparty.sh
-cd ${HBASE_DIR}
-cmake .
-make clean all
-make test
diff --git a/bin/build-thirdparty.sh b/bin/build-thirdparty.sh
deleted file mode 100755
index 097c1a4..0000000
--- a/bin/build-thirdparty.sh
+++ /dev/null
@@ -1,64 +0,0 @@
-#! /usr/bin/env bash
-#
-#/**
-# * Copyright The Apache Software Foundation
-# *
-# * Licensed to the Apache Software Foundation (ASF) under one
-# * or more contributor license agreements. See the NOTICE file
-# * distributed with this work for additional information
-# * regarding copyright ownership. The ASF licenses this file
-# * to you under the Apache License, Version 2.0 (the
-# * "License"); you may not use this file except in compliance
-# * with the License. You may obtain a copy of the License at
-# *
-# * http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# */
-
-set -x
-set -e
-
-SOURCE="${BASH_SOURCE[0]}"
-while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
- DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
- SOURCE="$(readlink "$SOURCE")"
- [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
-done
-DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-
-source ${DIR}/../bin/hbase-client-env.sh
-
-# On some systems, autotools installs libraries to lib64 rather than lib. Fix
-# this by setting up lib64 as a symlink to lib. We have to do this step first
-# to handle cases where one third-party library depends on another.
-mkdir -p "${HBASE_PREFIX}/lib"
-cd ${HBASE_PREFIX}
-ln -sf lib "${HBASE_PREFIX}/lib64"
-
-if [ ! -f gtest ]; then
- cd ${HBASE_GTEST_DIR}
- cmake .
- make -j4
- cd ..
- ln -sf ${HBASE_GTEST_DIR} gtest
-fi
-
-if [ ! -f ${HBASE_PREFIX}/lib/libprotobuf.a ]; then
- cd ${HBASE_PROTOBUF_DIR}
- ./configure --with-pic --disable-shared --prefix=${HBASE_PREFIX}
- make -j4 install
-fi
-
-if [ ! -f ${HBASE_PREFIX}/lib/libev.a ]; then
- cd ${HBASE_LIBEV_DIR}
- ./configure --with-pic --disable-shared --prefix=${HBASE_PREFIX}
- make -j4 install
-fi
-
-echo "---------------------"
-echo "Thirdparty dependencies built and installed into $HBASE_PREFIX successfully"
diff --git a/bin/download-thirdparty.sh b/bin/download-thirdparty.sh
deleted file mode 100755
index db98aae..0000000
--- a/bin/download-thirdparty.sh
+++ /dev/null
@@ -1,70 +0,0 @@
-#! /usr/bin/env bash
-#
-#/**
-# * Copyright The Apache Software Foundation
-# *
-# * Licensed to the Apache Software Foundation (ASF) under one
-# * or more contributor license agreements. See the NOTICE file
-# * distributed with this work for additional information
-# * regarding copyright ownership. The ASF licenses this file
-# * to you under the Apache License, Version 2.0 (the
-# * "License"); you may not use this file except in compliance
-# * with the License. You may obtain a copy of the License at
-# *
-# * http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# */
-
-set -x
-set -e
-
-SOURCE="${BASH_SOURCE[0]}"
-while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
- DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
- SOURCE="$(readlink "$SOURCE")"
- [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
-done
-DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-
-source ${DIR}/../bin/hbase-client-env.sh
-
-mkdir -p ${HBASE_TP_DIR}
-cd ${HBASE_TP_DIR}
-
-
-if [ ! -d ${HBASE_GTEST_DIR} ]; then
- echo "Fetching gtest"
- wget -c http://googletest.googlecode.com/files/gtest-${HBASE_GTEST_VERSION}.zip
- unzip gtest-${HBASE_GTEST_VERSION}.zip
- rm gtest-${HBASE_GTEST_VERSION}.zip
-fi
-
-if [ ! -d ${HBASE_PROTOBUF_DIR} ]; then
- echo "Fetching protobuf"
- wget -c http://protobuf.googlecode.com/files/protobuf-${HBASE_PROTOBUF_VERSION}.tar.gz
- tar xzf protobuf-${HBASE_PROTOBUF_VERSION}.tar.gz
- rm protobuf-${HBASE_PROTOBUF_VERSION}.tar.gz
-fi
-
-if [ ! -d $HBASE_LIBEV_DIR ]; then
- echo "Fetching libev"
- wget -c http://dist.schmorp.de/libev/libev-${HBASE_LIBEV_VERSION}.tar.gz
- tar zxf libev-${HBASE_LIBEV_VERSION}.tar.gz
- rm -rf libev-${HBASE_LIBEV_VERSION}.tar.gz
-fi
-
-if [ ! -d $HBASE_PREFIX/bin/cpplint.py ]; then
- echo "Fetching cpplint"
- mkdir -p $HBASE_PREFIX/bin/
- wget -O $HBASE_PREFIX/bin/cpplint.py http://google-styleguide.googlecode.com/svn/trunk/cpplint/cpplint.py
- chmod +x $HBASE_PREFIX/bin/cpplint.py
-fi
-
-
-echo "---------------"
-echo "Thirdparty dependencies downloaded successfully"
diff --git a/bin/hbase-client-env.sh b/bin/hbase-client-env.sh
deleted file mode 100644
index 96763da..0000000
--- a/bin/hbase-client-env.sh
+++ /dev/null
@@ -1,47 +0,0 @@
-#! /usr/bin/env bash
-#
-#/**
-# * Copyright The Apache Software Foundation
-# *
-# * Licensed to the Apache Software Foundation (ASF) under one
-# * or more contributor license agreements. See the NOTICE file
-# * distributed with this work for additional information
-# * regarding copyright ownership. The ASF licenses this file
-# * to you under the Apache License, Version 2.0 (the
-# * "License"); you may not use this file except in compliance
-# * with the License. You may obtain a copy of the License at
-# *
-# * http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# */
-
-SOURCE="${BASH_SOURCE[0]}"
-while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
- DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
- SOURCE="$(readlink "$SOURCE")"
- [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to
- # resolve it relative to the path where
- # the symlink file was located
-done
-DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-HBASE_DIR="${DIR}/../"
-HBASE_TP_DIR="${HBASE_DIR}/thirdparty/"
-
-HBASE_PREFIX=$HBASE_TP_DIR/installed
-
-HBASE_GTEST_VERSION=1.7.0
-HBASE_GTEST_DIR=$HBASE_TP_DIR/gtest-$HBASE_GTEST_VERSION
-
-HBASE_PROTOBUF_VERSION=2.5.0
-HBASE_PROTOBUF_DIR=$HBASE_TP_DIR/protobuf-$HBASE_PROTOBUF_VERSION
-
-HBASE_LIBEV_VERSION=4.15
-HBASE_LIBEV_DIR=$HBASE_TP_DIR/libev-$HBASE_LIBEV_VERSION
-
-# use the compiled tools
-export PATH=$HBASE_PREFIX/bin:$PATH
diff --git a/bin/start-docker.sh b/bin/start-docker.sh
new file mode 100755
index 0000000..1c9b02e
--- /dev/null
+++ b/bin/start-docker.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set -x
+
+eval "$(docker-machine env docker-vm)"
+docker build -t hbase_native .
+
+
+mkdir third-party || true
+if [[ ! -d third-party/googletest ]]; then
+ git clone https://github.com/google/googletest.git third-party/googletest
+fi
+
+
+docker run -v ${PWD}/..:/usr/local/src/hbase -it hbase_native /bin/bash
diff --git a/cmake_modules/FindGTest.cmake b/cmake_modules/FindGTest.cmake
deleted file mode 100644
index 8d27876..0000000
--- a/cmake_modules/FindGTest.cmake
+++ /dev/null
@@ -1,53 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set( GTEST_INCLUDE_SEARCH
- ${CMAKE_SOURCE_DIR}/thirdparty/gtest/include
-)
-
-set( GTEST_LIB_SEARCH
- ${CMAKE_SOURCE_DIR}/thirdparty/gtest
-)
-
-find_path(GTEST_INCLUDE
- NAMES gtest/gtest.h
- PATHS ${GTEST_INCLUDE_SEARCH}
- NO_DEFAULT_PATH
-)
-
-find_library(GTEST_LIBRARY
- NAMES gtest
- PATHS ${GTEST_LIB_SEARCH}
- NO_DEFAULT_PATH
-)
-
-find_library(GTEST_LIBRARY_MAIN
- NAMES gtest_main
- PATHS ${GTEST_LIB_SEARCH}
- NO_DEFAULT_PATH
-)
-
-if (GTEST_INCLUDE AND GTEST_LIBRARY AND GTEST_LIBRARY_MAIN)
- set(GTEST_LIBS ${GTEST_LIBRARY} ${GTEST_LIBRARY_MAIN})
- set(GTEST_INCLUDE_DIR ${GTEST_INCLUDE})
- set(GTEST_FOUND TRUE)
-endif()
-
-mark_as_advanced(
- GTEST_INCLUDE_DIR
- GTEST_LIBS
-)
diff --git a/cmake_modules/FindLibEv.cmake b/cmake_modules/FindLibEv.cmake
deleted file mode 100644
index 5fd3e86..0000000
--- a/cmake_modules/FindLibEv.cmake
+++ /dev/null
@@ -1,47 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set( LIBEV_INCLUDE_SEARCH
- ${CMAKE_SOURCE_DIR}/thirdparty/installed/include
-)
-
-set( LIBEV_LIB_SEARCH
- ${CMAKE_SOURCE_DIR}/thirdparty/installed/include
-)
-
-find_path(LIBEV_INCLUDE
- NAMES ev++.h
- PATHS ${LIBEV_INCLUDE_SEARCH}
- NO_DEFAULT_PATH
-)
-
-find_library(LIBEV_LIBRARY
- NAMES ev
- PATHS ${LIBEV_LIB_SEARCH}
- NO_DEFAULT_PATH
-)
-
-if (LIBEV_INCLUDE_PATH AND LIBEV_LIBRARY)
- set(LIBEV_LIBS ${LIBEV_LIBRARY})
- set(LIBEV_INCLUDE_DIR ${LIBEV_INCLUDE})
- set(LIBEV_FOUND TRUE)
-endif()
-
-mark_as_advanced(
- LIBEV_INCLUDE_DIR
- LIBEV_LIBS
-)
diff --git a/core/BUCK b/core/BUCK
new file mode 100644
index 0000000..a7eaa9e
--- /dev/null
+++ b/core/BUCK
@@ -0,0 +1,50 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+cxx_binary(
+ name = "core",
+ headers = [
+ "admin.h",
+ "client.h",
+ "connection.h",
+ "connection_attr.h",
+ "delete.h",
+ "get.h",
+ "hbase_macros.h",
+ "mutation.h",
+ "put.h",
+ "scanner.h",
+ ],
+ srcs = [
+ "admin.cc",
+ "client.cc",
+ "connection.cc",
+ "get.cc",
+ "mutation.cc",
+ "put.cc",
+ "delete.cc",
+ "scanner.cc",
+ ],
+ deps = [
+ "//third-party:folly",
+ "//third-party:wangle",
+ ],
+ visibility = [
+ 'PUBLIC',
+ ],
+)
diff --git a/src/core/admin.cc b/core/admin.cc
similarity index 100%
rename from src/core/admin.cc
rename to core/admin.cc
diff --git a/src/core/client.cc b/core/admin.h
similarity index 96%
copy from src/core/client.cc
copy to core/admin.h
index 91c235c..775181c 100644
--- a/src/core/client.cc
+++ b/core/admin.h
@@ -17,4 +17,6 @@
*
*/
-#include "core/client.h"
+#pragma once
+
+class Admin {};
diff --git a/src/async/hbase_result.cc b/core/client.cc
similarity index 66%
rename from src/async/hbase_result.cc
rename to core/client.cc
index 9351270..98cf38a 100644
--- a/src/async/hbase_result.cc
+++ b/core/client.cc
@@ -17,21 +17,22 @@
*
*/
-#include "async/hbase_result.h"
+#include "core/client.h"
-#include "core/hbase_types.h"
+#include <folly/Logging.h>
+#include <folly/Random.h>
+#include <glog/logging.h>
+#include <gflags/gflags.h>
-int32_t hb_result_get_cells(hb_result_t result,
- hb_cell_t ** cell_ptr, size_t * num_cells) {
- return 0;
-}
+using namespace folly;
-int32_t hb_result_get_table(hb_result_t result,
- char ** table, size_t * table_length) {
- return 0;
-}
+int main(int argc, char *argv[]) {
+ google::ParseCommandLineFlags(&argc, &argv, true);
+ google::InitGoogleLogging(argv[0]);
-int32_t hb_result_get_namespace(hb_result_t result,
- char ** name_space, size_t * name_space_length) {
+ FB_LOG_EVERY_MS(INFO, 10000) << "Hello";
+ for (long i = 0; i < 10000000; i++) {
+ FB_LOG_EVERY_MS(INFO, 1) << Random::rand32();
+ }
return 0;
}
diff --git a/src/core/get.cc b/core/client.h
similarity index 93%
copy from src/core/get.cc
copy to core/client.h
index 9e11332..921cecc 100644
--- a/src/core/get.cc
+++ b/core/client.h
@@ -17,4 +17,8 @@
*
*/
-#include "core/get.h"
+#pragma once
+
+#include <folly/io/IOBuf.h>
+
+class Client {};
diff --git a/src/core/connection.cc b/core/connection.cc
similarity index 91%
rename from src/core/connection.cc
rename to core/connection.cc
index 099f31d..e9a28eb 100644
--- a/src/core/connection.cc
+++ b/core/connection.cc
@@ -17,6 +17,4 @@
*/
#include "core/connection.h"
-void Connection::set_zk_quorum(char * zk_q) {
- this->zk_quorum = zk_q;
-}
+void Connection::set_zk_quorum(char *zk_q) { this->zk_quorum = zk_q; }
diff --git a/src/core/admin.h b/core/connection.h
similarity index 89%
rename from src/core/admin.h
rename to core/connection.h
index 624c4ac..5c9d6b8 100644
--- a/src/core/admin.h
+++ b/core/connection.h
@@ -14,12 +14,13 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
- *
*/
-#ifndef CORE_ADMIN_H_
-#define CORE_ADMIN_H_
+#pragma once
+
+class Connection {
+ char *zk_quorum;
-class Admin {
+public:
+ void set_zk_quorum(char *zk_q);
};
-#endif // CORE_ADMIN_H_
diff --git a/src/core/client.cc b/core/connection_attr.h
similarity index 91%
copy from src/core/client.cc
copy to core/connection_attr.h
index 91c235c..a312005 100644
--- a/src/core/client.cc
+++ b/core/connection_attr.h
@@ -17,4 +17,8 @@
*
*/
-#include "core/client.h"
+#pragma once
+
+#include "core/hbase_macros.h"
+
+class ConnectionAttr {};
diff --git a/src/core/delete.cc b/core/delete.cc
similarity index 97%
rename from src/core/delete.cc
rename to core/delete.cc
index 695f5bf..57030be 100644
--- a/src/core/delete.cc
+++ b/core/delete.cc
@@ -18,5 +18,4 @@
#include "core/delete.h"
-Delete::~Delete() {
-}
+Delete::~Delete() {}
diff --git a/src/core/client.h b/core/delete.h
similarity index 89%
rename from src/core/client.h
rename to core/delete.h
index 68348a8..34f6a6c 100644
--- a/src/core/client.h
+++ b/core/delete.h
@@ -17,9 +17,11 @@
*
*/
-#ifndef CORE_CLIENT_H_
-#define CORE_CLIENT_H_
+#pragma once
-class Client {
+#include "core/mutation.h"
+
+class Delete : public Mutation {
+public:
+ ~Delete();
};
-#endif // CORE_CLIENT_H_
diff --git a/src/core/get.cc b/core/get.cc
similarity index 100%
rename from src/core/get.cc
rename to core/get.cc
diff --git a/src/core/client.cc b/core/get.h
similarity index 96%
copy from src/core/client.cc
copy to core/get.h
index 91c235c..b4b5912 100644
--- a/src/core/client.cc
+++ b/core/get.h
@@ -17,4 +17,6 @@
*
*/
-#include "core/client.h"
+#pragma once
+
+class Get {};
diff --git a/src/core/hbase_macros.h b/core/hbase_macros.h
similarity index 74%
rename from src/core/hbase_macros.h
rename to core/hbase_macros.h
index 71765c8..48304f0 100644
--- a/src/core/hbase_macros.h
+++ b/core/hbase_macros.h
@@ -17,8 +17,7 @@
*
*/
-#ifndef CORE_HBASE_MACROS_H_
-#define CORE_HBASE_MACROS_H_
+#pragma once
#ifdef __cplusplus
extern "C" {
@@ -35,26 +34,23 @@ extern "C" {
* this library, this compiler flag should not be used.
*/
#if defined _WIN32 || defined __CYGWIN__
- #ifdef _LIBHBASE_IMPLEMENTATION_
- #define API __declspec(dllexport)
- #else
- #ifdef _LIBHBASE_TEST_
- #define HBASE_API
- #else
- #define HBASE_API __declspec(dllimport)
- #endif
- #endif
+#ifdef _LIBHBASE_IMPLEMENTATION_
+#define API __declspec(dllexport)
#else
- #if __GNUC__ >= 4
- #define HBASE_API __attribute__ ((visibility ("default")))
- #else
- #define HBASE_API
- #endif
+#ifdef _LIBHBASE_TEST_
+#define HBASE_API
+#else
+#define HBASE_API __declspec(dllimport)
+#endif
+#endif
+#else
+#if __GNUC__ >= 4
+#define HBASE_API __attribute__((visibility("default")))
+#else
+#define HBASE_API
+#endif
#endif
#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // CORE_HBASE_MACROS_H_
-
+} // extern "C"
+#endif // __cplusplus
diff --git a/src/core/mutation.cc b/core/mutation.cc
similarity index 78%
rename from src/core/mutation.cc
rename to core/mutation.cc
index 4855076..52910d5 100644
--- a/src/core/mutation.cc
+++ b/core/mutation.cc
@@ -19,24 +19,23 @@
#include "core/mutation.h"
-void Mutation::set_namespace(char * name_space, size_t name_space_length) {
+void Mutation::set_namespace(char *name_space, size_t name_space_length) {
this->name_space = name_space;
this->name_space_length = name_space_length;
}
-void Mutation::set_table(char * table, size_t table_length) {
+void Mutation::set_table(char *table, size_t table_length) {
this->table = table;
this->table_length = table_length;
}
-void Mutation::set_row(unsigned char * row, size_t row_length) {
+void Mutation::set_row(unsigned char *row, size_t row_length) {
this->row = row;
this->row_length = row_length;
}
-void Mutation::set_durability(hb_durability_type durability) {
+void Mutation::set_durability(durability_type durability) {
this->durability = durability;
}
-Mutation::~Mutation() {
-}
+Mutation::~Mutation() {}
diff --git a/src/core/mutation.h b/core/mutation.h
similarity index 63%
rename from src/core/mutation.h
rename to core/mutation.h
index e35d595..1880571 100644
--- a/src/core/mutation.h
+++ b/core/mutation.h
@@ -17,32 +17,42 @@
*
*/
-#ifndef CORE_MUTATION_H_
-#define CORE_MUTATION_H_
+#pragma once
#include <stdlib.h>
-#include "core/hbase_types.h"
+typedef enum {
+ DELETE_ONE_VERSION,
+ DELETE_MULTIPLE_VERSIONS,
+ DELETE_FAMILY,
+ DELETE_FAMILY_VERSION
+} delete_type;
+
+typedef enum {
+ USE_DEFAULT,
+ SKIP_WAL,
+ ASYNC_WAL,
+ SYNC_WAL,
+ HSYNC_WAL
+} durability_type;
class Mutation {
- char * name_space;
+ char *name_space;
size_t name_space_length;
- char * table;
+ char *table;
size_t table_length;
- unsigned char * row;
+ unsigned char *row;
size_t row_length;
- hb_durability_type durability;
- public:
- void set_namespace(char * name_space, size_t name_space_length);
- void set_table(char * table, size_t table_length);
- void set_row(unsigned char * row, size_t row_length);
- void set_durability(hb_durability_type durability);
+ durability_type durability;
+
+public:
+ void set_namespace(char *name_space, size_t name_space_length);
+ void set_table(char *table, size_t table_length);
+ void set_row(unsigned char *row, size_t row_length);
+ void set_durability(durability_type durability);
virtual ~Mutation();
};
-#endif // CORE_MUTATION_H_
-
-
diff --git a/src/core/put.cc b/core/put.cc
similarity index 98%
rename from src/core/put.cc
rename to core/put.cc
index 8bedc8b..806a478 100644
--- a/src/core/put.cc
+++ b/core/put.cc
@@ -18,5 +18,4 @@
#include "core/put.h"
-Put::~Put() {
-}
+Put::~Put() {}
diff --git a/src/core/get.h b/core/put.h
similarity index 90%
rename from src/core/get.h
rename to core/put.h
index 3a9fd28..d34aa0d 100644
--- a/src/core/get.h
+++ b/core/put.h
@@ -17,10 +17,11 @@
*
*/
-#ifndef CORE_GET_H_
-#define CORE_GET_H_
+#pragma once
-class Get {
-};
+#include "core/mutation.h"
-#endif // CORE_GET_H_
+class Put : public Mutation {
+public:
+ ~Put();
+};
diff --git a/src/core/scanner.cc b/core/scanner.cc
similarity index 100%
rename from src/core/scanner.cc
rename to core/scanner.cc
diff --git a/src/core/client.cc b/core/scanner.h
similarity index 96%
rename from src/core/client.cc
rename to core/scanner.h
index 91c235c..180865a 100644
--- a/src/core/client.cc
+++ b/core/scanner.h
@@ -17,4 +17,6 @@
*
*/
-#include "core/client.h"
+#pragma once
+
+class Scanner {};
diff --git a/src/rpc/CMakeLists.txt b/rpc/CMakeLists.txt
similarity index 100%
rename from src/rpc/CMakeLists.txt
rename to rpc/CMakeLists.txt
diff --git a/src/async/CMakeLists.txt b/src/async/CMakeLists.txt
deleted file mode 100644
index b2ed461..0000000
--- a/src/async/CMakeLists.txt
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set( ASYNC_SRC
- hbase_admin.cc
- hbase_client.cc
- hbase_connection.cc
- hbase_get.cc
- hbase_mutations.cc
- hbase_result.cc
- hbase_scanner.cc
-)
-
-
-add_library(hasync OBJECT ${ASYNC_SRC})
-
-ADD_HBASE_ASYNC_TEST(mutations-test)
-ADD_HBASE_ASYNC_TEST(get-test)
diff --git a/src/async/get-test.cc b/src/async/get-test.cc
deleted file mode 100644
index df94ce2..0000000
--- a/src/async/get-test.cc
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include <pthread.h>
-
-#include "gtest/gtest.h"
-#include "async/hbase_get.h"
-#include "async/hbase_client.h"
-
-
-pthread_cond_t cv;
-pthread_mutex_t mutex;
-
-TEST(GetTest, TestPut) {
- char tn[] = "T1";
- hb_byte_t row[] = "ROW";
-
- hb_client_t client = NULL;
- hb_get_t get = NULL;
- int32_t s1 = -1;
-
- pthread_cond_init(&cv, NULL);
- pthread_mutex_init(&mutex, NULL);
-
- s1 = hb_client_create(&client, NULL);
- EXPECT_EQ(0, s1);
-
- s1 = hb_get_create(&get);
- EXPECT_EQ(0, s1);
-
- hb_get_set_table(get, tn, 2);
- hb_get_set_row(get, row, 3);
-
- /*
- * TODO:
- * This is currently a NO-OP as there is no CB.
- */
- hb_get_send(client, get, NULL, NULL);
-
- hb_client_destroy(client, NULL, NULL);
-
- EXPECT_EQ(0, s1);
-}
diff --git a/src/async/hbase_admin.cc b/src/async/hbase_admin.cc
deleted file mode 100644
index 17ff2a7..0000000
--- a/src/async/hbase_admin.cc
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "async/hbase_admin.h"
-
-#include <stdlib.h>
-#include <stdbool.h>
-
-#include "core/admin.h"
-#include "async/hbase_connection.h"
-
-int32_t hb_admin_create(hb_admin_t* admin_ptr,
- hb_connection_t connection) {
- (*admin_ptr) = reinterpret_cast<hb_admin_t>(new Admin());
- return 0;
-}
-
-/*
- * Disconnect the admin releasing any internal objects
- * or connections created in the background.
- */
-int32_t hb_admin_destroy(hb_admin_t admin,
- hb_admin_disconnection_cb cb, void * extra) {
- if (cb)
- cb(0, admin, extra);
- free(admin);
- return 0;
-}
-
-/*
- * See if a table exists.
- */
-int32_t hb_admin_table_exists(hb_admin_t admin,
- char * name_space, size_t name_space_length,
- char * table, size_t table_length,
- hb_admin_table_exists_cb cb, void * extra) {
- if (cb)
- cb(0, admin, name_space, name_space_length,
- table, table_length, true, extra);
- return 0;
-}
diff --git a/src/async/hbase_admin.h b/src/async/hbase_admin.h
deleted file mode 100644
index f03a83f..0000000
--- a/src/async/hbase_admin.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef ASYNC_HBASE_ADMIN_H_
-#define ASYNC_HBASE_ADMIN_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdlib.h>
-#include <stdbool.h>
-
-#include "core/hbase_macros.h"
-#include "async/hbase_connection.h"
-
-
-typedef void (* hb_admin_disconnection_cb)( int32_t status,
- hb_admin_t admin, void * extra);
-typedef void (* hb_admin_table_exists_cb)(int32_t status, hb_admin_t admin,
- char * name_space, size_t name_space_length,
- char * table, size_t table_length, bool exsists, void * extra);
-
-/**
- * Create a new hb_admin.
- * All fields are initialized to the defaults. If you want to set
- * connection or other properties, set those before calling any
- * RPC functions.
- */
-HBASE_API int32_t hb_admin_create(hb_admin_t* admin_ptr,
- hb_connection_t connection);
-
-/*
- * Disconnect the admin releasing any internal objects
- * or connections created in the background.
- */
-HBASE_API int32_t hb_admin_destroy(hb_admin_t admin,
- hb_admin_disconnection_cb cb, void * extra);
-
-/*
- * See if a table exists.
- */
-HBASE_API int32_t hb_admin_table_exists(hb_admin_t admin,
- char * name_space, size_t name_space_length,
- char * table, size_t table_length,
- hb_admin_table_exists_cb cb, void * extra);
-
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // ASYNC_HBASE_ADMIN_H_
diff --git a/src/async/hbase_client.cc b/src/async/hbase_client.cc
deleted file mode 100644
index 98e3dbf..0000000
--- a/src/async/hbase_client.cc
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "async/hbase_client.h"
-
-#include <pthread.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "core/client.h"
-#include "async/hbase_connection.h"
-
-int32_t hb_client_create(hb_client_t* client_ptr,
- hb_connection_t connection) {
- (*client_ptr) = reinterpret_cast<hb_client_t>(new Client());
- if (client_ptr == NULL)
- return -1; // TODO(eclark): setup the errno file.
- return 0;
-}
-
-int32_t hb_client_destroy(hb_client_t client,
- hb_client_disconnection_cb cb, void * extra) {
- if (client == NULL)
- return -2;
- if (cb) {
- cb(0, client, extra);
- }
- free(client);
- return 0;
-}
-
diff --git a/src/async/hbase_client.h b/src/async/hbase_client.h
deleted file mode 100644
index 34c3d98..0000000
--- a/src/async/hbase_client.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef ASYNC_HBASE_CLIENT_H_
-#define ASYNC_HBASE_CLIENT_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "core/hbase_macros.h"
-#include "core/hbase_types.h"
-
-/*
- * Client disconnection callback typedef
- *
- * This is called after the connections are closed, but just
- * before the client is freed.
- */
-typedef void (* hb_client_disconnection_cb)( int32_t status,
- hb_client_t client, void * extra);
-
-/**
- * Create an hb_client_t.
- *
- * If connection is null then all defaults will be used.
- */
-HBASE_API int32_t hb_client_create(hb_client_t * client_ptr,
- hb_connection_t connection);
-
-/*
- * Disconnect the client releasing any internal objects
- * or connections created in the background.
- */
-HBASE_API int32_t hb_client_destroy(hb_client_t client,
- hb_client_disconnection_cb cb, void * extra);
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // ASYNC_HBASE_CLIENT_H_
-
-
diff --git a/src/async/hbase_connection.cc b/src/async/hbase_connection.cc
deleted file mode 100644
index 57fa4f5..0000000
--- a/src/async/hbase_connection.cc
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "async/hbase_connection.h"
-
-#include "core/connection.h"
-#include "core/hbase_types.h"
-
-extern "C" {
-int32_t hb_connection_create(hb_connection_t * connection_ptr,
- hb_connection_attr_t connection_attr) {
- (*connection_ptr) = reinterpret_cast<hb_connection_t>(new Connection());
- if ((*connection_ptr) == NULL)
- return -1;
- return 0;
-}
-int32_t hb_connection_destroy(hb_connection_t connection) {
- free(connection);
- return 0;
-}
-} // extern "C"
diff --git a/src/async/hbase_connection.h b/src/async/hbase_connection.h
deleted file mode 100644
index 343f36c..0000000
--- a/src/async/hbase_connection.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef ASYNC_HBASE_CONNECTION_H_
-#define ASYNC_HBASE_CONNECTION_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "core/hbase_macros.h"
-#include "core/hbase_types.h"
-#include "core/hbase_connection_attr.h"
-
-#include <stdlib.h>
-
-/**
- * Create an hb_connection.
- *
- * if connection_attr is null everything will be left as default
- */
-HBASE_API int32_t hb_connection_create(hb_connection_t * connection_ptr,
- hb_connection_attr_t connection_attr);
-
-/**
- * Destroy the connection and free all resources allocated at creation
- * time.
- */
-HBASE_API int32_t hb_connection_destroy(hb_connection_t connection);
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // ASYNC_HBASE_CONNECTION_H_
-
diff --git a/src/async/hbase_errno.h b/src/async/hbase_errno.h
deleted file mode 100644
index 1698bbd..0000000
--- a/src/async/hbase_errno.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef ASYNC_HBASE_ERRNO_H_
-#define ASYNC_HBASE_ERRNO_H_
-
-#endif // ASYNC_HBASE_ERRNO_H_
diff --git a/src/async/hbase_get.cc b/src/async/hbase_get.cc
deleted file mode 100644
index 32048d8..0000000
--- a/src/async/hbase_get.cc
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "async/hbase_get.h"
-
-#include <stdlib.h>
-#include <string.h>
-
-#include "core/get.h"
-
-int32_t hb_get_create(hb_get_t * get_ptr) {
- (*get_ptr) = reinterpret_cast<hb_get_t>(new Get());
- if ((*get_ptr) == NULL) {
- return -1;
- }
- return 0;
-}
-
-int32_t hb_get_destroy(hb_get_t get) {
- free(get);
- return 0;
-}
-
-int32_t hb_get_set_row(hb_get_t get, unsigned char * row,
- size_t row_length) {
- return 0;
-}
-
-int32_t hb_get_set_table(hb_get_t get,
- char * table, size_t table_length) {
- return 0;
-}
-
-int32_t hb_get_set_namespace(hb_get_t get,
- char * name_space, size_t name_space_length) {
- return 0;
-}
-
-int32_t hb_get_send(hb_client_t client,
- hb_get_t get, hb_get_cb cb, void * extra) {
- if (cb) {
- cb(0, client, get, NULL, extra);
- }
- return 0;
-}
diff --git a/src/async/hbase_get.h b/src/async/hbase_get.h
deleted file mode 100644
index 7d91c08..0000000
--- a/src/async/hbase_get.h
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef ASYNC_HBASE_GET_H_
-#define ASYNC_HBASE_GET_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "core/hbase_macros.h"
-#include "core/hbase_types.h"
-
-/**
- * Allocate a new get structure.
- * Ownership passes to the caller.
- */
-HBASE_API int32_t hb_get_create(hb_get_t * get_ptr);
-
-/**
- * Destroy and free a get structure.
- */
-HBASE_API int32_t hb_get_destroy(hb_get_t get);
-
-/**
- * set the row of this get.
- */
-HBASE_API int32_t hb_get_set_row(hb_get_t get, hb_byte_t * row,
- size_t row_length);
-
-/**
- * Set the table.
- */
-HBASE_API int32_t hb_get_set_table(hb_get_t get,
- char * table, size_t table_length);
-
-/**
- * Set the namespace this get is targeting.
- */
-HBASE_API int32_t hb_get_set_namespace(hb_get_t get,
- char * name_space, size_t name_space_length);
-
-/*
- * get call back typedef.
- */
-typedef void (* hb_get_cb)(int32_t status, hb_client_t client,
- hb_get_t get, hb_result_t results, void * extra);
-
-HBASE_API int32_t hb_get_send(hb_client_t client,
- hb_get_t get, hb_get_cb cb, void * extra);
-
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // ASYNC_HBASE_GET_H_
diff --git a/src/async/hbase_mutations.cc b/src/async/hbase_mutations.cc
deleted file mode 100644
index 2456dc0..0000000
--- a/src/async/hbase_mutations.cc
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "async/hbase_mutations.h"
-
-#include <stdlib.h>
-
-#include "core/hbase_types.h"
-#include "core/mutation.h"
-#include "core/put.h"
-#include "core/delete.h"
-
-#include "async/hbase_result.h"
-
-extern "C" {
-int32_t hb_put_create(hb_put_t* put_ptr) {
- (*put_ptr) = reinterpret_cast<hb_put_t>(new Put());
- return 0;
-}
-
-int32_t hb_delete_create(hb_delete_t * delete_ptr) {
- (*delete_ptr) = reinterpret_cast<hb_delete_t>(new Delete());
- return 0;
-}
-
-int32_t hb_increment_create(hb_increment_t * increment_ptr) {
- return 0;
-}
-
-int32_t hb_append_create(hb_append_t * append_ptr) {
- return 0;
-}
-
-int32_t hb_mutation_destroy(hb_mutation_t mutation) {
- return 0;
-}
-
-HBASE_API int32_t hb_mutation_set_namespace(hb_mutation_t mutation,
- char * name_space, size_t name_space_length) {
- Mutation * m = reinterpret_cast<Mutation *>(mutation);
- m->set_namespace(name_space, name_space_length);
- return 0;
-}
-
-HBASE_API int32_t hb_mutation_set_table(hb_mutation_t mutation,
- char * table, size_t table_length) {
- Mutation * m = reinterpret_cast<Mutation *>(mutation);
- m->set_namespace(table, table_length);
- return 0;
-}
-
-int32_t hb_mutation_set_row(hb_mutation_t mutation,
- unsigned char * rk, size_t row_length) {
- Mutation * m = reinterpret_cast<Mutation *>(mutation);
- m->set_row(rk, row_length);
- return 0;
-}
-
-int32_t hb_mutation_set_durability(hb_mutation_t mutation,
- hb_durability_type durability) {
- Mutation * m = reinterpret_cast<Mutation *>(mutation);
- m->set_durability(durability);
- return 0;
-}
-
-int32_t hb_put_add_cell(hb_put_t put, hb_cell_t * cell) {
- return 0;
-}
-
-int32_t hb_delete_add_col(hb_increment_t incr,
- unsigned char * family, size_t family_length,
- unsigned char * qual, size_t qual_length) {
- return 0;
-}
-
-int32_t hb_increment_add_value(hb_increment_t incr,
- unsigned char * family, size_t family_length,
- unsigned char * qual, size_t qual_length,
- int64_t ammount) {
- return 0;
-}
-
-int32_t hb_append_add_cell(hb_append_t put, hb_cell_t * cell) {
- return 0;
-}
-
-int32_t hb_mutation_send(hb_client_t client,
- hb_mutation_t mutation, hb_mutation_cb cb,
- void * extra) {
- if (cb) {
- cb(0, client, mutation, NULL, extra);
- }
- return 0;
-}
-} // extern "C"
diff --git a/src/async/hbase_mutations.h b/src/async/hbase_mutations.h
deleted file mode 100644
index 4000390..0000000
--- a/src/async/hbase_mutations.h
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef ASYNC_HBASE_MUTATIONS_H_
-#define ASYNC_HBASE_MUTATIONS_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "core/hbase_types.h"
-#include "async/hbase_result.h"
-
-// Creation methods
-
-/**
- * Create a put.
- * Ownership passes to the caller.
- */
-HBASE_API int32_t hb_put_create(hb_put_t * put_ptr);
-
-/**
- * Create a delete
- * Ownership passes to the caller.
- */
-HBASE_API int32_t hb_delete_create(hb_delete_t * delete_ptr);
-
-/**
- * Create an increment
- * Ownership passes to the caller.
- */
-HBASE_API int32_t hb_increment_create(hb_increment_t * increment_ptr);
-
-/**
- * Create an append
- * Ownership passes to the caller.
- */
-HBASE_API int32_t hb_append_create(hb_append_t * append_ptr);
-
-/**
- * Destroy the mutation.
- * All internal structures are cleaned up. However any backing
- * data structures passed in by the user are not cleaned up.
- */
-HBASE_API int32_t hb_mutation_destroy(hb_mutation_t mutation);
-
-// Shared setters.
-HBASE_API int32_t hb_mutation_set_namespace(hb_mutation_t mutation,
- char * name_space, size_t name_space_length);
-HBASE_API int32_t hb_mutation_set_table(hb_mutation_t mutation,
- char * table, size_t table_length);
-HBASE_API int32_t hb_mutation_set_row(hb_mutation_t mutation,
- unsigned char * rk, size_t row_length);
-HBASE_API int32_t hb_mutation_set_durability(hb_mutation_t mutation,
- hb_durability_type durability);
-
-// Put Setters etc.
-HBASE_API int32_t hb_put_add_cell(hb_put_t put, hb_cell_t * cell);
-
-// Delete
-HBASE_API int32_t hb_delete_add_col(hb_increment_t incr,
- unsigned char * family, size_t family_length,
- unsigned char * qual, size_t qual_length);
-
-// Increment
-HBASE_API int32_t hb_increment_add_value(hb_increment_t incr,
- unsigned char * family, size_t family_length,
- unsigned char * qual, size_t qual_length,
- int64_t ammount);
-
-// Append
-HBASE_API int32_t hb_append_add_cell(hb_append_t put, hb_cell_t * cell);
-
-// Now that the mutations are created and populated
-// The real meat of the client is below.
-
-/*
- * mutation call back typedef
- */
-typedef void (* hb_mutation_cb)(int32_t status,
- hb_client_t client, hb_mutation_t mutation,
- hb_result_t result, void * extra);
-
-/*
- * Queue a single mutation. This mutation will be
- * sent out in the background and can be batched with
- * other requests destined for the same server.
- *
- * The call back will be executed after the response
- * is received from the RegionServer. Even if the
- * mutation was batched with other requests,
- * the call back will be invoked for every mutation
- * individually.
- */
-HBASE_API int32_t hb_mutation_send(hb_client_t client,
- hb_mutation_t mutation, hb_mutation_cb cb,
- void * extra);
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // ASYNC_HBASE_MUTATIONS_H_
diff --git a/src/async/hbase_result.h b/src/async/hbase_result.h
deleted file mode 100644
index eecbbb3..0000000
--- a/src/async/hbase_result.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef ASYNC_HBASE_RESULT_H_
-#define ASYNC_HBASE_RESULT_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "core/hbase_macros.h"
-#include "core/hbase_types.h"
-
-HBASE_API int32_t hb_result_destroy(hb_result_t result);
-
-HBASE_API int32_t hb_result_get_cells(hb_result_t result,
- hb_cell_t ** cell_ptr, size_t * num_cells);
-
-HBASE_API int32_t hb_result_get_table(hb_result_t result,
- char ** table, size_t * table_length);
-HBASE_API int32_t hb_result_get_namespace(hb_result_t result,
- char ** name_space, size_t * name_space_length);
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // ASYNC_HBASE_RESULT_H_
diff --git a/src/async/hbase_scanner.cc b/src/async/hbase_scanner.cc
deleted file mode 100644
index 5a8e555..0000000
--- a/src/async/hbase_scanner.cc
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#include "async/hbase_scanner.h"
-
-#include <stdlib.h>
-
-#include "core/hbase_types.h"
-#include "core/scanner.h"
-
-int32_t hb_scanner_create(hb_scanner_t * scanner_ptr) {
- (*scanner_ptr) = reinterpret_cast<hb_scanner_t>(new Scanner());
- return (*scanner_ptr != NULL)?0:1;
-}
-
-HBASE_API int32_t hb_scanner_set_table(hb_scanner_t scanner,
- char * table, size_t table_length) {
- return 0;
-}
-
-HBASE_API int32_t hb_scanner_set_namespace(hb_scanner_t scanner,
- char * name_space, size_t name_space_length) {
- return 0;
-}
-
-int32_t hb_scanner_set_start_row(hb_scanner_t scanner,
- unsigned char * start_row, size_t start_row_length) {
- return 0;
-}
-
-int32_t hb_scanner_set_end_row(hb_scanner_t scanner,
- unsigned char * end_row, size_t end_row_length) {
- return 0;
-}
-
-int32_t hb_scanner_set_cache_size(hb_scanner_t scanner,
- size_t cache_size) {
- return 0;
-}
-
-int32_t hb_scanner_set_num_versions(hb_scanner_t scanner,
- int8_t num_versions) {
- return 0;
-}
diff --git a/src/async/hbase_scanner.h b/src/async/hbase_scanner.h
deleted file mode 100644
index cd3f544..0000000
--- a/src/async/hbase_scanner.h
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef ASYNC_HBASE_SCANNER_H_
-#define ASYNC_HBASE_SCANNER_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "async/hbase_result.h"
-#include "core/hbase_types.h"
-
-HBASE_API int32_t hb_scanner_create(hb_scanner_t * scanner_ptr);
-
-HBASE_API int32_t hb_scanner_set_table(hb_scanner_t scanner,
- char * table, size_t table_length);
-HBASE_API int32_t hb_scanner_set_namespace(hb_scanner_t scanner,
- char * name_space, size_t name_space_length);
-
-HBASE_API int32_t hb_scanner_set_start_row(hb_scanner_t scanner,
- unsigned char * start_row, size_t start_row_length);
-HBASE_API int32_t hb_scanner_set_end_row(hb_scanner_t scanner,
- unsigned char * end_row, size_t end_row_length);
-
-HBASE_API int32_t hb_scanner_set_cache_size(hb_scanner_t scanner,
- size_t cache_size);
-HBASE_API int32_t hb_scanner_set_batch_size(hb_scanner_t scanner,
- size_t batch_size);
-HBASE_API int32_t hb_scanner_set_num_versions(hb_scanner_t scanner,
- int8_t num_versions);
-
-/*
- * Scanner call back typedef.
- *
- * This will be called when initinalization of the scanner
- * is complete. It will also be called when scanner next
- * returns results.
- */
-typedef void (* hb_scanner_cb)(int32_t status,
- hb_client_t client,
- hb_scanner_t scanner,
- hb_result_t results,
- size_t num_results,
- void * extra);
-/*
- * Get the next results from the scanner
- */
-HBASE_API int32_t hb_scanner_next(hb_client_t client,
- hb_scanner_t scanner, hb_scanner_cb cb, void * extra);
-
-/*
- * Close the scanner releasing any local and server side
- * resources held. The call back is fired just before the
- * scanner's memory is freed.
- */
-HBASE_API int32_t hb_scanner_destroy(hb_client_t client,
- hb_scanner_t scanner, hb_scanner_cb cb, void * extra);
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // ASYNC_HBASE_SCANNER_H_
diff --git a/src/async/mutations-test.cc b/src/async/mutations-test.cc
deleted file mode 100644
index be5898e..0000000
--- a/src/async/mutations-test.cc
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include <pthread.h>
-
-#include "gtest/gtest.h"
-#include "async/hbase_mutations.h"
-#include "async/hbase_client.h"
-
-pthread_cond_t cv;
-pthread_mutex_t mutex;
-
-bool sent = false;
-
-TEST(ClientTest, EasyTest) {
- EXPECT_EQ(1, 1);
-}
-
-void mutate_cb(int32_t status,
- hb_client_t client, hb_mutation_t mutation,
- hb_result_t result, void * extra) {
-
- // Test Stuff.
- EXPECT_EQ(status, 0);
- EXPECT_TRUE(client != NULL);
- EXPECT_TRUE(mutation != NULL);
-
- pthread_mutex_lock(&mutex);
- sent = true;
- pthread_cond_signal(&cv);
- pthread_mutex_unlock(&mutex);
-}
-
-void wait_send() {
- pthread_mutex_lock(&mutex);
- while (!sent) {
- pthread_cond_wait(&cv, &mutex);
- }
- pthread_mutex_unlock(&mutex);
-}
-
-TEST(MutationTest, TestPut) {
- char tn[] = "T1";
- hb_byte_t row[] = "ROW";
- char fam[] = "D";
- hb_byte_t qual[] = "QUAL";
- hb_byte_t data[] = "Z";
-
- hb_client_t client = NULL;
- hb_put_t put = NULL;
- hb_cell_t cell;
-
- cell.family = fam;
- cell.family_length = 1;
-
- cell.qual = qual;
- cell.qual_length = 4;
-
- cell.value = data;
- cell.value_length = 1;
-
- int32_t status = -1;
-
- status = hb_client_create(&client, NULL);
- EXPECT_EQ(0, status);
-
- hb_put_create(&put);
- hb_mutation_set_table((hb_mutation_t) put, tn, 2);
- hb_mutation_set_row((hb_mutation_t) put, row, 3);
- hb_put_add_cell(put, &cell);
-
- pthread_cond_init(&cv, NULL);
- pthread_mutex_init(&mutex, NULL);
-
- status = hb_mutation_send(client, (hb_mutation_t) put, &mutate_cb, NULL);
- EXPECT_EQ(0, status);
-
- // Now wait a while for things to send.
- wait_send();
- EXPECT_EQ(true, sent);
-
- hb_mutation_destroy((hb_mutation_t *) put);
- hb_client_destroy(client, NULL, NULL);
-
- EXPECT_EQ(0, status);
-}
diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt
deleted file mode 100644
index 49f93b9..0000000
--- a/src/core/CMakeLists.txt
+++ /dev/null
@@ -1,31 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set( CORE_SRC
- admin.cc
- client.cc
- connection.cc
- get.cc
- mutation.cc
- put.cc
- delete.cc
- scanner.cc
- hbase_connection_attr.cc
-)
-
-
-add_library(hcore OBJECT ${CORE_SRC})
diff --git a/src/core/connection.h b/src/core/connection.h
deleted file mode 100644
index 463b7b5..0000000
--- a/src/core/connection.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef CORE_CONNECTION_H_
-#define CORE_CONNECTION_H_
-
-class Connection {
- char * zk_quorum;
- public:
- void set_zk_quorum(char * zk_q);
-};
-#endif // CORE_CONNECTION_H_
diff --git a/src/core/connection_attr.h b/src/core/connection_attr.h
deleted file mode 100644
index a7c229e..0000000
--- a/src/core/connection_attr.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef CORE_CONNECTION_ATTR_H_
-#define CORE_CONNECTION_ATTR_H_
-
-#include "core/hbase_macros.h"
-#include "core/hbase_types.h"
-
-class ConnectionAttr {
-};
-
-#endif // CORE_CONNECTION_ATTR_H_
-
diff --git a/src/core/delete.h b/src/core/delete.h
deleted file mode 100644
index 5740fd9..0000000
--- a/src/core/delete.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef CORE_DELETE_H_
-#define CORE_DELETE_H_
-
-#include "core/mutation.h"
-
-class Delete: public Mutation {
- public:
- ~Delete();
-};
-#endif // CORE_DELETE_H_
diff --git a/src/core/hbase_connection_attr.cc b/src/core/hbase_connection_attr.cc
deleted file mode 100644
index 61940e8..0000000
--- a/src/core/hbase_connection_attr.cc
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "core/hbase_connection_attr.h"
-
-#include "core/hbase_macros.h"
-#include "core/hbase_types.h"
-#include "core/connection_attr.h"
-
-extern "C" {
-int32_t hb_connection_attr_create(hb_connection_attr_t * attr_ptr) {
- (*attr_ptr) = new ConnectionAttr();
- return (attr_ptr == NULL)?-1:0;
-}
-
-int32_t hb_connection_attr_set_zk_quorum(hb_connection_t connection,
- char * zk_quorum) {
- return 0;
-}
-
-int32_t hb_connection_attr_set_zk_root(hb_connection_t connection,
- char * zk_root) {
- return 0;
-}
-}
diff --git a/src/core/hbase_connection_attr.h b/src/core/hbase_connection_attr.h
deleted file mode 100644
index c73c818..0000000
--- a/src/core/hbase_connection_attr.h
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef CORE_HBASE_CONNECTION_ATTR_H_
-#define CORE_HBASE_CONNECTION_ATTR_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "core/hbase_macros.h"
-#include "core/hbase_types.h"
-
-#include <stdlib.h>
-
-HBASE_API int32_t hb_connection_attr_create(hb_connection_attr_t * attr_ptr);
-
-/**
- * Set the zk quorum of a connection that will be created.
- */
-HBASE_API int32_t hb_connection_attr_set_zk_quorum(hb_connection_t connection,
- char * zk_quorum);
-
-/**
- * Set the zk root of a connection that will be created.
- */
-HBASE_API int32_t hb_connection_attr_set_zk_root(hb_connection_t connection,
- char * zk_root);
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // CORE_HBASE_CONNECTION_ATTR_H_
-
diff --git a/src/core/hbase_types.h b/src/core/hbase_types.h
deleted file mode 100644
index 8889b92..0000000
--- a/src/core/hbase_types.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef CORE_HBASE_TYPES_H_
-#define CORE_HBASE_TYPES_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdint.h>
-#include <stddef.h>
-
-typedef unsigned char hb_byte_t;
-
-/*
- * Base kv type.
- */
-typedef struct {
- hb_byte_t* row;
- size_t row_length;
-
- char * family;
- size_t family_length;
-
- hb_byte_t* qual;
- size_t qual_length;
-
- hb_byte_t* value;
- size_t value_length;
-
- uint64_t timestamp;
-} hb_cell_t;
-
-typedef enum {
- DELETE_ONE_VERSION,
- DELETE_MULTIPLE_VERSIONS,
- DELETE_FAMILY,
- DELETE_FAMILY_VERSION
-} hb_delete_type;
-
-typedef enum {
- USE_DEFAULT,
- SKIP_WAL,
- ASYNC_WAL,
- SYNC_WAL,
- HSYNC_WAL
-} hb_durability_type;
-
-typedef void* hb_admin_t;
-typedef void* hb_client_t;
-typedef void* hb_connection_attr_t;
-typedef void* hb_connection_t;
-typedef void* hb_get_t;
-typedef void* hb_mutation_t;
-typedef void* hb_put_t;
-typedef void* hb_delete_t;
-typedef void* hb_increment_t;
-typedef void* hb_append_t;
-typedef void* hb_result_t;
-typedef void* hb_scanner_t;
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // CORE_HBASE_TYPES_H_
diff --git a/src/core/put.h b/src/core/put.h
deleted file mode 100644
index 7bca3e3..0000000
--- a/src/core/put.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef CORE_PUT_H_
-#define CORE_PUT_H_
-
-#include "core/mutation.h"
-
-class Put: public Mutation {
- public:
- ~Put();
-};
-#endif // CORE_PUT_H_
diff --git a/src/core/scanner.h b/src/core/scanner.h
deleted file mode 100644
index 257583b..0000000
--- a/src/core/scanner.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef CORE_SCANNER_H_
-#define CORE_SCANNER_H_
-
-class Scanner {
-};
-#endif // CORE_SCANNER_H_
diff --git a/src/sync/CMakeLists.txt b/src/sync/CMakeLists.txt
deleted file mode 100644
index bfb7e6c..0000000
--- a/src/sync/CMakeLists.txt
+++ /dev/null
@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set( SYNC_SRC
- hbase_connection.cc
- hbase_admin.cc
-)
-
-
-add_library(hsync OBJECT ${SYNC_SRC})
diff --git a/src/sync/hbase_admin.cc b/src/sync/hbase_admin.cc
deleted file mode 100644
index d43c8ec..0000000
--- a/src/sync/hbase_admin.cc
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "sync/hbase_admin.h"
-
-#include <stdlib.h>
-#include <stdbool.h>
-
-#include "core/admin.h"
-
-int32_t hb_admin_create(hb_admin_t** admin_ptr) {
- (*admin_ptr) = reinterpret_cast<hb_admin_t *>(new Admin());
- return 0;
-}
-
-/*
- * Disconnect the admin releasing any internal objects
- * or connections created in the background.
- */
-int32_t hb_admin_destroy(hb_admin_t * admin) {
- Admin * adm = reinterpret_cast<Admin *>(admin);
- delete adm;
- return 0;
-}
-
-/*
- * See if a table exists.
- */
-int32_t hb_admin_table_exists(hb_admin_t * admin,
- char * name_space, size_t name_space_length,
- char * table, size_t table_length,
- bool * exists) {
- *exists = true;
- return 0;
-}
diff --git a/src/sync/hbase_admin.h b/src/sync/hbase_admin.h
deleted file mode 100644
index 860bfba..0000000
--- a/src/sync/hbase_admin.h
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef SYNC_HBASE_ADMIN_H_
-#define SYNC_HBASE_ADMIN_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdlib.h>
-#include <stdbool.h>
-
-#include "core/hbase_macros.h"
-#include "core/hbase_types.h"
-#include "sync/hbase_connection.h"
-
-
-/**
- * Create a new hb_admin.
- * All fields are initialized to the defaults. If you want to set
- * connection or other properties, set those before calling any
- * RPC functions.
- */
-HBASE_API int32_t hb_admin_create(hb_admin_t* admin_ptr,
- hb_connection_t connection);
-
-/*
- * Disconnect the admin releasing any internal objects
- * or connections created in the background.
- */
-HBASE_API int32_t hb_admin_destroy(hb_admin_t admin);
-
-/*
- * See if a table exists.
- */
-HBASE_API int32_t hb_admin_table_exists(hb_admin_t admin,
- char * name_space, size_t name_space_length,
- char * table, size_t table_length, bool * exists);
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // SYNC_HBASE_ADMIN_H_
diff --git a/src/sync/hbase_connection.cc b/src/sync/hbase_connection.cc
deleted file mode 100644
index df05376..0000000
--- a/src/sync/hbase_connection.cc
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "sync/hbase_connection.h"
-
-#include "core/connection.h"
-#include "core/hbase_types.h"
-
-extern "C" {
-int32_t hb_connection_create(hb_connection_t * connection_ptr,
- hb_connection_attr_t connection_attr) {
- (*connection_ptr) = reinterpret_cast<hb_connection_t>(new Connection());
- if ((*connection_ptr) == NULL)
- return -1;
- return 0;
-}
-int32_t hb_connection_destroy(hb_connection_t connection) {
- free(connection);
- return 0;
-}
-} // extern "C"
diff --git a/src/sync/hbase_connection.h b/src/sync/hbase_connection.h
deleted file mode 100644
index af9284d..0000000
--- a/src/sync/hbase_connection.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef SYNC_HBASE_CONNECTION_H_
-#define SYNC_HBASE_CONNECTION_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "core/hbase_macros.h"
-#include "core/hbase_types.h"
-#include "core/hbase_connection_attr.h"
-
-#include <stdlib.h>
-
-/**
- * Create an hb_connection.
- *
- * if connection_attr is null everything will be left as default
- */
-HBASE_API int32_t hb_connection_create(hb_connection_t * connection_ptr,
- hb_connection_attr_t connection_attr);
-
-/**
- * Destroy the connection and free all resources allocated at creation
- * time.
- */
-HBASE_API int32_t hb_connection_destroy(hb_connection_t connection);
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // SYNC_HBASE_CONNECTION_H_
-
diff --git a/third-party/BUCK b/third-party/BUCK
new file mode 100644
index 0000000..89d2c93
--- /dev/null
+++ b/third-party/BUCK
@@ -0,0 +1,105 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+def add_system_libs(names = []):
+ rules = []
+ for name in names:
+ prebuilt_cxx_library(
+ name = name,
+ lib_name = name,
+ lib_dir = "/usr/lib/x86_64-linux-gnu"
+ )
+ rules.append(":" + name)
+
+ return rules
+
+system_libs = [
+ "double-conversion",
+ "glog",
+ "gflags",
+ "unwind",
+ "lzma",
+ "boost_regex",
+]
+tp_dep_rules = add_system_libs(system_libs)
+prebuilt_cxx_library(
+ name = "folly",
+ lib_name = "folly",
+ lib_dir = "/usr/local/lib",
+ deps = tp_dep_rules,
+ exported_linker_flags = [
+ "-pthread",
+ "-lstdc++",
+ ],
+ visibility = [
+ 'PUBLIC',
+ ]
+)
+prebuilt_cxx_library(
+ name = "follybenchmark",
+ lib_name = "follybenchmark",
+ lib_dir = "/usr/local/lib",
+ deps = tp_dep_rules + [":folly"],
+ exported_linker_flags = [
+ "-pthread",
+ "-lstdc++",
+ ],
+ visibility = [
+ 'PUBLIC',
+ ]
+)
+prebuilt_cxx_library(
+ name = "wangle",
+ lib_name = "wangle",
+ lib_dir = "/usr/local/lib",
+ deps = tp_dep_rules + [":folly"],
+ exported_linker_flags = [
+ "-pthread",
+ "-lstdc++",
+ ],
+ visibility = [
+ 'PUBLIC',
+ ]
+)
+cxx_library(
+ name = 'google-test',
+ srcs = [
+ 'googletest/googletest/src/gtest-all.cc',
+ 'googletest/googlemock/src/gmock-all.cc',
+ 'googletest/googlemock/src/gmock_main.cc',
+ ],
+ header_namespace = '',
+ exported_headers = subdir_glob([
+ ('googletest/googletest/include', '**/*.h'),
+ ('googletest/googlemock/include', '**/*.h'),
+ ]),
+ headers = subdir_glob([
+ ('googletest/googletest', 'src/*.h'),
+ ('googletest/googletest', 'src/*.cc'),
+ ('googletest/googlemock', 'src/*.h'),
+ ('googletest/googlemock', 'src/*.cc'),
+ ]),
+ exported_linker_flags = [
+ "-pthread",
+ "-lstdc++",
+ ],
+ visibility = [
+ 'PUBLIC',
+ ],
+ deps = [
+ ]
+)
[hbase] 77/137: HBASE-17727 [C++] Make RespConverter work with
RawAsyncTableImpl
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 215a4a16fc11163df696034d70b7bc14e14b2f36
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Tue Mar 28 13:52:31 2017 -0700
HBASE-17727 [C++] Make RespConverter work with RawAsyncTableImpl
---
core/async-rpc-retrying-test.cc | 62 +++++++++++++++++++----------------------
core/raw-async-table.cc | 11 ++++----
core/raw-async-table.h | 8 +++---
core/response-converter.cc | 4 ---
4 files changed, 38 insertions(+), 47 deletions(-)
diff --git a/core/async-rpc-retrying-test.cc b/core/async-rpc-retrying-test.cc
index d0c7921..5086286 100644
--- a/core/async-rpc-retrying-test.cc
+++ b/core/async-rpc-retrying-test.cc
@@ -137,15 +137,15 @@ class MockRawAsyncTableImpl {
/* in real RawAsyncTableImpl, this should be private. */
template <typename REQ, typename PREQ, typename PRESP, typename RESP>
- folly::Future<RESP> Call(
- std::shared_ptr<hbase::RpcClient> rpc_client, std::shared_ptr<HBaseRpcController> controller,
- std::shared_ptr<RegionLocation> loc, const REQ& req,
- const ReqConverter<std::unique_ptr<PREQ>, REQ, std::string>& req_converter,
- const hbase::RpcCall<PREQ, PRESP>& rpc_call,
- const RespConverter<RESP, PRESP>& resp_converter) {
+ folly::Future<RESP> Call(std::shared_ptr<hbase::RpcClient> rpc_client,
+ std::shared_ptr<HBaseRpcController> controller,
+ std::shared_ptr<RegionLocation> loc, const REQ& req,
+ ReqConverter<std::unique_ptr<PREQ>, REQ, std::string> req_converter,
+ const hbase::RpcCall<PREQ, PRESP>& rpc_call,
+ RespConverter<RESP, PRESP> resp_converter) {
rpc_call(rpc_client, loc, controller, std::move(req_converter(req, loc->region_name())))
- .then([&, this](std::unique_ptr<PRESP> presp) {
- std::shared_ptr<hbase::Result> result = hbase::ResponseConverter::FromGetResponse(*presp);
+ .then([&, this, resp_converter](std::unique_ptr<PRESP> presp) {
+ RESP result = resp_converter(*presp);
promise_->setValue(result);
})
.onError([this](const std::exception& e) { promise_->setException(e); });
@@ -210,31 +210,27 @@ TEST(AsyncRpcRetryTest, TestGetBasic) {
auto builder = conn->caller_factory()->Single<std::shared_ptr<hbase::Result>>();
/* call with retry to get result */
- try {
- auto async_caller =
- builder->table(std::make_shared<TableName>(tn))
- ->row(row)
- ->rpc_timeout(conn->connection_conf()->read_rpc_timeout())
- ->operation_timeout(conn->connection_conf()->operation_timeout())
- ->action([=, &get](std::shared_ptr<hbase::HBaseRpcController> controller,
- std::shared_ptr<hbase::RegionLocation> loc,
- std::shared_ptr<hbase::RpcClient> rpc_client)
- -> folly::Future<std::shared_ptr<hbase::Result>> {
- return tableImpl->GetCall(rpc_client, controller, loc, get);
- })
- ->Build();
-
- auto result = async_caller->Call().get();
-
- // Test the values, should be same as in put executed on hbase shell
- ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty.";
- EXPECT_EQ("test2", result->Row());
- EXPECT_EQ("value2", *(result->Value("d", "2")));
- EXPECT_EQ("value for extra", *(result->Value("d", "extra")));
- } catch (std::exception& e) {
- LOG(ERROR) << e.what();
- throw e;
- }
+
+ auto async_caller =
+ builder->table(std::make_shared<TableName>(tn))
+ ->row(row)
+ ->rpc_timeout(conn->connection_conf()->read_rpc_timeout())
+ ->operation_timeout(conn->connection_conf()->operation_timeout())
+ ->action([=, &get](std::shared_ptr<hbase::HBaseRpcController> controller,
+ std::shared_ptr<hbase::RegionLocation> loc,
+ std::shared_ptr<hbase::RpcClient> rpc_client)
+ -> folly::Future<std::shared_ptr<hbase::Result>> {
+ return tableImpl->GetCall(rpc_client, controller, loc, get);
+ })
+ ->Build();
+
+ auto result = async_caller->Call().get();
+
+ // Test the values, should be same as in put executed on hbase shell
+ ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty.";
+ EXPECT_EQ("test2", result->Row());
+ EXPECT_EQ("value2", *(result->Value("d", "2")));
+ EXPECT_EQ("value for extra", *(result->Value("d", "extra")));
table->Close();
client.Close();
diff --git a/core/raw-async-table.cc b/core/raw-async-table.cc
index 641f3c8..88a3382 100644
--- a/core/raw-async-table.cc
+++ b/core/raw-async-table.cc
@@ -16,6 +16,7 @@
* limitations under the License.
*
*/
+#include <utility>
#include "core/raw-async-table.h"
#include "core/request-converter.h"
@@ -41,18 +42,16 @@ template <typename REQ, typename PREQ, typename PRESP, typename RESP>
folly::Future<RESP> RawAsyncTable::Call(
std::shared_ptr<RpcClient> rpc_client, std::shared_ptr<HBaseRpcController> controller,
std::shared_ptr<RegionLocation> loc, const REQ& req,
- const ReqConverter<std::unique_ptr<PREQ>, REQ, std::string>& req_converter,
- const RespConverter<RESP, PRESP>& resp_converter) {
+ const ReqConverter<std::unique_ptr<PREQ>, REQ, std::string> req_converter,
+ const RespConverter<RESP, PRESP> resp_converter) {
std::unique_ptr<PREQ> preq = req_converter(req, loc->region_name());
// No need to make take a callable argument, it is always the same
return rpc_client
->AsyncCall(loc->server_name().host_name(), loc->server_name().port(), std::move(preq),
User::defaultUser(), "ClientService")
- .then([&](const std::unique_ptr<Response>& presp) {
- return ResponseConverter::FromGetResponse(*presp);
- // return resp_converter(*presp); // TODO this is causing SEGFAULT, figure out why
- });
+ .then(
+ [resp_converter](const std::unique_ptr<PRESP>& presp) { return resp_converter(*presp); });
}
Future<std::shared_ptr<Result>> RawAsyncTable::Get(const hbase::Get& get) {
diff --git a/core/raw-async-table.h b/core/raw-async-table.h
index 527c7be..bbdc6bd 100644
--- a/core/raw-async-table.h
+++ b/core/raw-async-table.h
@@ -18,12 +18,12 @@
*/
#pragma once
+#include <folly/futures/Future.h>
+
#include <chrono>
#include <memory>
#include <string>
-#include <folly/futures/Future.h>
-
#include "core/async-connection.h"
#include "core/async-rpc-retrying-caller-factory.h"
#include "core/async-rpc-retrying-caller.h"
@@ -66,8 +66,8 @@ class RawAsyncTable {
folly::Future<RESP> Call(
std::shared_ptr<RpcClient> rpc_client, std::shared_ptr<HBaseRpcController> controller,
std::shared_ptr<RegionLocation> loc, const REQ& req,
- const ReqConverter<std::unique_ptr<PREQ>, REQ, std::string>& req_converter,
- const RespConverter<RESP, PRESP>& resp_converter);
+ const ReqConverter<std::unique_ptr<PREQ>, REQ, std::string> req_converter,
+ const RespConverter<RESP, PRESP> resp_converter);
template <typename RESP>
std::shared_ptr<SingleRequestCallerBuilder<RESP>> CreateCallerBuilder(std::string row,
diff --git a/core/response-converter.cc b/core/response-converter.cc
index b11856c..b2fff34 100644
--- a/core/response-converter.cc
+++ b/core/response-converter.cc
@@ -36,14 +36,12 @@ ResponseConverter::~ResponseConverter() {}
// impl note: we are returning shared_ptr's instead of unique_ptr's because these
// go inside folly::Future's, making the move semantics extremely tricky.
std::shared_ptr<Result> ResponseConverter::FromGetResponse(const Response& resp) {
- LOG(INFO) << "FromGetResponse";
auto get_resp = std::static_pointer_cast<GetResponse>(resp.resp_msg());
return ToResult(get_resp->result(), resp.cell_scanner());
}
std::shared_ptr<Result> ResponseConverter::ToResult(
const hbase::pb::Result& result, const std::unique_ptr<CellScanner>& cell_scanner) {
- LOG(INFO) << "ToResult";
std::vector<std::shared_ptr<Cell>> vcells;
for (auto cell : result.cell()) {
std::shared_ptr<Cell> pcell =
@@ -59,13 +57,11 @@ std::shared_ptr<Result> ResponseConverter::ToResult(
}
// TODO: check associated cell count?
}
- LOG(INFO) << "Returning Result";
return std::make_shared<Result>(vcells, result.exists(), result.stale(), result.partial());
}
std::vector<std::shared_ptr<Result>> ResponseConverter::FromScanResponse(const Response& resp) {
auto scan_resp = std::static_pointer_cast<ScanResponse>(resp.resp_msg());
- LOG(INFO) << "FromScanResponse:" << scan_resp->ShortDebugString();
int num_results = resp.cell_scanner() != nullptr ? scan_resp->cells_per_result_size()
: scan_resp->results_size();
[hbase] 117/137: HBASE-18565 [C++] Fix deadlock in
AsyncScanRetryingCaller and other RPCs
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit f4041f3f7e0a0d3a2115eab0181165e98b3bb59b
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Fri Aug 11 12:23:43 2017 -0700
HBASE-18565 [C++] Fix deadlock in AsyncScanRetryingCaller and other RPCs
---
core/async-rpc-retrying-caller.cc | 4 ++--
core/async-scan-rpc-retrying-caller.cc | 1 +
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/core/async-rpc-retrying-caller.cc b/core/async-rpc-retrying-caller.cc
index cb058b1..8e60991 100644
--- a/core/async-rpc-retrying-caller.cc
+++ b/core/async-rpc-retrying-caller.cc
@@ -148,9 +148,9 @@ void AsyncSingleRequestRpcRetryingCaller<RESP>::OnError(
* establishment time (see ConnectionFactory::Connect()), otherwise, the IOThreadPool thread
* just hangs because it deadlocks itself.
*/
- conn_->retry_executor()->add([&]() {
+ conn_->retry_executor()->add([=]() {
retry_timer_->scheduleTimeoutFn(
- [this]() { conn_->cpu_executor()->add([&]() { LocateThenCall(); }); },
+ [=]() { conn_->cpu_executor()->add([&]() { LocateThenCall(); }); },
std::chrono::milliseconds(TimeUtil::ToMillis(delay_ns)));
});
}
diff --git a/core/async-scan-rpc-retrying-caller.cc b/core/async-scan-rpc-retrying-caller.cc
index fbdf17a..a1e8362 100644
--- a/core/async-scan-rpc-retrying-caller.cc
+++ b/core/async-scan-rpc-retrying-caller.cc
@@ -406,6 +406,7 @@ void AsyncScanRpcRetryingCaller::Call() {
->AsyncCall(region_location_->server_name().host_name(),
region_location_->server_name().port(), std::move(req),
security::User::defaultUser(), "ClientService")
+ .via(conn_->cpu_executor().get())
.then([self, this](const std::unique_ptr<Response>& resp) {
auto scan_resp = std::static_pointer_cast<pb::ScanResponse>(resp->resp_msg());
return OnComplete(controller_, scan_resp, resp->cell_scanner());
[hbase] 63/137: HBASE-17485 [C++] Zookeeper quorum and znode lookup
made configurable (Sudeep Sunthankar & enis)
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 851a40871034f69bd7a4365e3dc823d8daa183b9
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Tue Jan 24 15:41:34 2017 -0800
HBASE-17485 [C++] Zookeeper quorum and znode lookup made configurable (Sudeep Sunthankar & enis)
---
core/client.cc | 4 ++--
core/location-cache-test.cc | 7 +++----
core/location-cache.cc | 14 +++++++-------
core/location-cache.h | 12 ++++++++++--
core/simple-client.cc | 8 +++++++-
test-util/BUCK | 3 ++-
test-util/test-util.h | 8 ++++++++
7 files changed, 39 insertions(+), 17 deletions(-)
diff --git a/core/client.cc b/core/client.cc
index dd568ce..c1efd8b 100644
--- a/core/client.cc
+++ b/core/client.cc
@@ -47,8 +47,8 @@ void Client::init(const hbase::Configuration &conf) {
io_executor_ = std::make_shared<wangle::IOThreadPoolExecutor>(sysconf(_SC_NPROCESSORS_ONLN));
rpc_client_ = std::make_shared<hbase::RpcClient>(io_executor_);
- location_cache_ = std::make_shared<hbase::LocationCache>(zk_quorum, cpu_executor_,
- rpc_client_->connection_pool());
+ location_cache_ =
+ std::make_shared<hbase::LocationCache>(conf_, cpu_executor_, rpc_client_->connection_pool());
}
// We can't have the threads continue running after everything is done
diff --git a/core/location-cache-test.cc b/core/location-cache-test.cc
index 42e7bb3..1ad6c65 100644
--- a/core/location-cache-test.cc
+++ b/core/location-cache-test.cc
@@ -31,11 +31,10 @@ using namespace std::chrono;
TEST(LocationCacheTest, TestGetMetaNodeContents) {
TestUtil test_util{};
-
auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
auto cp = std::make_shared<ConnectionPool>(io);
- LocationCache cache{"localhost:2181", cpu, cp};
+ LocationCache cache{test_util.conf(), cpu, cp};
auto f = cache.LocateMeta();
auto result = f.get();
ASSERT_FALSE(f.hasException());
@@ -51,7 +50,7 @@ TEST(LocationCacheTest, TestGetRegionLocation) {
auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
auto cp = std::make_shared<ConnectionPool>(io);
- LocationCache cache{"localhost:2181", cpu, cp};
+ LocationCache cache{test_util.conf(), cpu, cp};
// If there is no table this should throw an exception
auto tn = folly::to<hbase::pb::TableName>("t");
@@ -70,7 +69,7 @@ TEST(LocationCacheTest, TestCaching) {
auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
auto cp = std::make_shared<ConnectionPool>(io);
- LocationCache cache{"localhost:2181", cpu, cp};
+ LocationCache cache{test_util.conf(), cpu, cp};
auto tn_1 = folly::to<hbase::pb::TableName>("t1");
auto tn_2 = folly::to<hbase::pb::TableName>("t2");
diff --git a/core/location-cache.cc b/core/location-cache.cc
index 66f3eb7..dab5deb 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -50,13 +50,10 @@ using hbase::pb::ServerName;
using hbase::pb::MetaRegionServer;
using hbase::pb::RegionInfo;
-// TODO(eclark): make this configurable on client creation
-static const char META_ZNODE_NAME[] = "/hbase/meta-region-server";
-
-LocationCache::LocationCache(std::string quorum_spec,
+LocationCache::LocationCache(std::shared_ptr<hbase::Configuration> conf,
std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
std::shared_ptr<ConnectionPool> cp)
- : quorum_spec_(quorum_spec),
+ : conf_(conf),
cpu_executor_(cpu_executor),
meta_promise_(nullptr),
meta_lock_(),
@@ -65,7 +62,8 @@ LocationCache::LocationCache(std::string quorum_spec,
zk_(nullptr),
cached_locations_(),
locations_lock_() {
- zk_ = zookeeper_init(quorum_spec.c_str(), nullptr, 1000, 0, 0, 0);
+ quorum_spec_ = conf_->Get(kHBaseZookeeperQuorum_, kDefHBaseZookeeperQuorum_);
+ zk_ = zookeeper_init(quorum_spec_.c_str(), nullptr, 1000, 0, 0, 0);
}
LocationCache::~LocationCache() {
@@ -101,8 +99,10 @@ ServerName LocationCache::ReadMetaLocation() {
// This needs to be int rather than size_t as that's what ZK expects.
int len = buf->capacity();
+ std::string zk_node = conf_->Get(kHBaseMetaZnodeName_, kDefHBaseMetaZnodeName_);
+ zk_node += "/" + kHBaseMetaRegionServer_;
// TODO(elliott): handle disconnects/reconntion as needed.
- int zk_result = zoo_get(this->zk_, META_ZNODE_NAME, 0,
+ int zk_result = zoo_get(this->zk_, zk_node.c_str(), 0,
reinterpret_cast<char *>(buf->writableData()), &len, nullptr);
if (zk_result != ZOK || len < 9) {
LOG(ERROR) << "Error getting meta location.";
diff --git a/core/location-cache.h b/core/location-cache.h
index 22a8ad5..4d66e06 100644
--- a/core/location-cache.h
+++ b/core/location-cache.h
@@ -32,6 +32,7 @@
#include <string>
#include "connection/connection-pool.h"
+#include "core/configuration.h"
#include "core/meta-utils.h"
#include "core/region-location.h"
#include "serde/table-name.h"
@@ -77,12 +78,12 @@ class LocationCache {
public:
/**
* Constructor.
- * @param quorum_spec Where to connect for Zookeeper.
+ * @param conf Configuration instance to fetch Zookeeper Quorum and Zookeeper Znode.
* @param cpu_executor executor used to run non network IO based
* continuations.
* @param io_executor executor used to talk to the network
*/
- LocationCache(std::string quorum_spec,
+ LocationCache(std::shared_ptr<hbase::Configuration> conf,
std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
std::shared_ptr<ConnectionPool> cp);
/**
@@ -179,7 +180,14 @@ class LocationCache {
const hbase::pb::TableName &tn);
std::shared_ptr<hbase::PerTableLocationMap> GetNewTableLocations(const hbase::pb::TableName &tn);
+ const std::string kHBaseZookeeperQuorum_ = "hbase.zookeeper.quorum";
+ const std::string kDefHBaseZookeeperQuorum_ = "localhost:2181";
+ const std::string kHBaseMetaZnodeName_ = "zookeeper.znode.parent";
+ const std::string kDefHBaseMetaZnodeName_ = "/hbase";
+ const std::string kHBaseMetaRegionServer_ = "meta-region-server";
+
/* data */
+ std::shared_ptr<hbase::Configuration> conf_;
std::string quorum_spec_;
std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> meta_promise_;
diff --git a/core/simple-client.cc b/core/simple-client.cc
index dac0d10..3cd0a93 100644
--- a/core/simple-client.cc
+++ b/core/simple-client.cc
@@ -39,6 +39,7 @@
using namespace folly;
using namespace std;
using namespace std::chrono;
+using hbase::Configuration;
using hbase::Response;
using hbase::Request;
using hbase::HBaseService;
@@ -87,9 +88,14 @@ int main(int argc, char *argv[]) {
// Set up thread pools.
auto cpu_pool = std::make_shared<wangle::CPUThreadPoolExecutor>(FLAGS_threads);
auto io_pool = std::make_shared<wangle::IOThreadPoolExecutor>(5);
+ auto cp = std::make_shared<ConnectionPool>(io_pool);
+
+ // Configuration
+ auto conf = std::make_shared<Configuration>();
+ conf->Set("hbase.zookeeper.quorum", FLAGS_zookeeper);
// Create the cache.
- LocationCache cache{FLAGS_zookeeper, cpu_pool, io_pool};
+ LocationCache cache{conf, cpu_pool, cp};
auto row = FLAGS_row;
auto tn = folly::to<TableName>(FLAGS_table);
diff --git a/test-util/BUCK b/test-util/BUCK
index e5e6ea2..c6e41f1 100644
--- a/test-util/BUCK
+++ b/test-util/BUCK
@@ -22,8 +22,9 @@ cxx_library(name="test-util",
srcs=["test-util.cc"],
deps=[
"//third-party:folly",
+ "//core:core"
],
visibility=[
'PUBLIC',
],
- )
\ No newline at end of file
+ )
diff --git a/test-util/test-util.h b/test-util/test-util.h
index 2fc9d69..cc35511 100644
--- a/test-util/test-util.h
+++ b/test-util/test-util.h
@@ -24,6 +24,8 @@
#include <cstdlib>
#include <string>
+#include "core/configuration.h"
+
namespace hbase {
/**
* @brief Class to deal with a local instance cluster for testing.
@@ -55,7 +57,13 @@ class TestUtil {
*/
static std::string RandString(int len = 32);
+ /**
+ * Returns the configuration to talk to the local cluster
+ */
+ std::shared_ptr<Configuration> conf() const { return conf_; }
+
private:
folly::test::TemporaryDirectory temp_dir_;
+ std::shared_ptr<Configuration> conf_ = std::make_shared<Configuration>();
};
} // namespace hbase
[hbase] 129/137: HBASE-18727 [C++] Don't link JNI/JVM stuff into
libHbaseClient
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 012b3188ddba80caba182da2ff2e4e0a9f67a5a0
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Thu Sep 14 12:27:45 2017 -0700
HBASE-18727 [C++] Don't link JNI/JVM stuff into libHbaseClient
---
Makefile | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
diff --git a/Makefile b/Makefile
index c879f98..a291dfa 100644
--- a/Makefile
+++ b/Makefile
@@ -24,21 +24,25 @@ DEBUG_PATH := $(BUILD_PATH)/debug
RELEASE_PATH := $(BUILD_PATH)/release
PROTO_SRC_DIR := if
PROTO_CXX_DIR := $(BUILD_PATH)/if
-MODULES := connection core exceptions security serde test-util utils
+MODULES := connection core exceptions security serde utils
+TEST_MODULES := test-util # These modules contain test code, not included in the build for the lib
SRC_DIR := $(MODULES)
DEBUG_BUILD_DIR := $(addprefix $(DEBUG_PATH)/,$(MODULES))
RELEASE_BUILD_DIR := $(addprefix $(RELEASE_PATH)/,$(MODULES))
-INCLUDE_DIR := . $(BUILD_PATH) $(JAVA_HOME)/include/ $(JAVA_HOME)/include/linux
+INCLUDE_DIR := . $(BUILD_PATH)
+TEST_BUILD_INCLUDE_DIR := $(INLCUDE_DIR) $(JAVA_HOME)/include/ $(JAVA_HOME)/include/linux
#flags to pass to the CPP compiler & linker
CPPFLAGS_DEBUG := -D_GLIBCXX_USE_CXX11_ABI=0 -g -Wall -std=c++14 -pedantic -fPIC -MMD -MP
CPPFLAGS_RELEASE := -D_GLIBCXX_USE_CXX11_ABI=0 -DNDEBUG -O2 -Wall -std=c++14 -pedantic -fPIC -MMD -MP
-LDFLAGS := -lprotobuf -lzookeeper_mt -lsasl2 -lfolly -lwangle -L $(JAVA_HOME)/jre/lib/amd64/server -ljvm
+LDFLAGS := -lprotobuf -lzookeeper_mt -lsasl2 -lfolly -lwangle
+TEST_BUILD_LDFLAGS := $(LDFLAGS) -L $(JAVA_HOME)/jre/lib/amd64/server -ljvm
LINKFLAG := -shared
#define list of source files and object files
ALLSRC := $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc))
-EXCLUDE_SRC := $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*-test.cc)) core/simple-client.cc core/load-client.cc
+EXCLUDE_SRC := $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*-test.cc)) \
+ core/simple-client.cc core/load-client.cc
SRC := $(filter-out $(EXCLUDE_SRC), $(ALLSRC))
PROTOSRC := $(patsubst %.proto, $(addprefix $(BUILD_PATH)/,%.pb.cc),$(wildcard if/*.proto))
PROTOHDR := $(patsubst %.proto, $(addprefix $(BUILD_PATH)/,%.pb.h),$(wildcard if/*.proto))
[hbase] 118/137: HBASE-18576. [C++] Add ping for RPC test
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 725cc4ce888985ac03ee856b20ca0cafa17c0ee4
Author: Xiaobing Zhou <xz...@hortonworks.com>
AuthorDate: Fri Aug 11 15:02:58 2017 -0700
HBASE-18576. [C++] Add ping for RPC test
Signed-off-by: Enis Soztutar <en...@apache.org>
---
connection/rpc-test-server.cc | 3 +++
connection/rpc-test.cc | 30 ++++++++++++++++++++++++++++++
2 files changed, 33 insertions(+)
diff --git a/connection/rpc-test-server.cc b/connection/rpc-test-server.cc
index f350d6a..6132fbb 100644
--- a/connection/rpc-test-server.cc
+++ b/connection/rpc-test-server.cc
@@ -68,6 +68,9 @@ Future<std::unique_ptr<Response>> RpcTestService::operator()(std::unique_ptr<Req
if (method_name == "ping") {
auto pb_resp_msg = std::make_shared<EmptyResponseProto>();
response->set_resp_msg(pb_resp_msg);
+ VLOG(1) << "RPC server:"
+ << " ping called.";
+
} else if (method_name == "echo") {
auto pb_resp_msg = std::make_shared<EchoResponseProto>();
/* get msg from client */
diff --git a/connection/rpc-test.cc b/connection/rpc-test.cc
index e7f678d..4688950 100644
--- a/connection/rpc-test.cc
+++ b/connection/rpc-test.cc
@@ -88,6 +88,36 @@ std::shared_ptr<RpcClient> CreateRpcClient(std::shared_ptr<Configuration> conf,
}
/**
+* test ping
+*/
+TEST_F(RpcTest, Ping) {
+ auto conf = CreateConf();
+ auto server = CreateRpcServer();
+ auto server_addr = GetRpcServerAddress(server);
+ auto client = CreateRpcClient(conf);
+
+ auto method = "ping";
+ auto request = std::make_unique<Request>(std::make_shared<EmptyRequestProto>(),
+ std::make_shared<EmptyResponseProto>(), method);
+
+ /* sending out request */
+ client
+ ->AsyncCall(server_addr->getAddressStr(), server_addr->getPort(), std::move(request),
+ hbase::security::User::defaultUser())
+ .then([&](std::unique_ptr<Response> response) {
+ auto pb_resp = std::static_pointer_cast<EmptyResponseProto>(response->resp_msg());
+ EXPECT_TRUE(pb_resp != nullptr);
+ VLOG(1) << folly::sformat(FLAGS_result_format, method, "");
+ })
+ .onError([&](const folly::exception_wrapper& ew) {
+ FAIL() << folly::sformat(FLAGS_fail_format, method);
+ });
+
+ server->stop();
+ server->join();
+}
+
+/**
* test echo
*/
TEST_F(RpcTest, Echo) {
[hbase] 67/137: HBASE-17226 [C++] Filter and Query classes
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit c117c361b394442eb59f411340a5577465bda092
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Fri Feb 24 14:18:18 2017 -0800
HBASE-17226 [C++] Filter and Query classes
---
core/BUCK | 12 ++
core/filter-test.cc | 136 +++++++++++++
core/filter.h | 481 ++++++++++++++++++++++++++++++++++++++++++++++
core/get.h | 3 +-
core/query.h | 41 ++++
core/request_converter.cc | 8 +
core/scan.h | 2 +-
7 files changed, 681 insertions(+), 2 deletions(-)
diff --git a/core/BUCK b/core/BUCK
index d8d15a9..ce0c733 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -22,6 +22,8 @@ cxx_library(
"client.h",
"cell.h",
"hbase_macros.h",
+ "filter.h",
+ "query.h",
"keyvalue-codec.h",
"region-location.h",
"location-cache.h",
@@ -78,6 +80,16 @@ cxx_test(
deps=[":core",],
run_test_separately=True,)
cxx_test(
+ name="filter-test",
+ srcs=["filter-test.cc",],
+ deps=[
+ ":core",
+ "//if:if",
+ "//serde:serde",
+ "//test-util:test-util",
+ ],
+ run_test_separately=True,)
+cxx_test(
name="get-test",
srcs=["get-test.cc",],
deps=[":core",],
diff --git a/core/filter-test.cc b/core/filter-test.cc
new file mode 100644
index 0000000..ff683b6
--- /dev/null
+++ b/core/filter-test.cc
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <gtest/gtest.h>
+#include "core/client.h"
+#include "core/configuration.h"
+#include "core/get.h"
+#include "core/result.h"
+#include "core/table.h"
+#include "if/Comparator.pb.h"
+#include "if/HBase.pb.h"
+#include "serde/table-name.h"
+#include "test-util/test-util.h"
+
+using hbase::Configuration;
+using hbase::Get;
+using hbase::FilterFactory;
+using hbase::Table;
+using hbase::TestUtil;
+using hbase::pb::CompareType;
+using hbase::ComparatorFactory;
+using hbase::Comparator;
+
+class FilterTest : public ::testing::Test {
+ protected:
+ static void SetUpTestCase() { test_util_ = std::make_unique<TestUtil>(); }
+
+ static void TearDownTestCase() { test_util_.release(); }
+
+ virtual void SetUp() {}
+ virtual void TearDown() {}
+
+ static std::unique_ptr<TestUtil> test_util_;
+};
+
+std::unique_ptr<TestUtil> FilterTest::test_util_ = nullptr;
+
+TEST_F(FilterTest, GetWithColumnPrefixFilter) {
+ // write row1 with 3 columns (column_1, column_2, and foo_column)
+ FilterTest::test_util_->RunShellCmd(
+ "create 't', 'd'; put 't', 'row1', 'd:column_1', 'value1'; put 't', 'row1', 'd:column_2', "
+ "'value2'; put 't', 'row1', 'd:foo_column', 'value3'");
+
+ // Create TableName and Row to be fetched from HBase
+ auto tn = folly::to<hbase::pb::TableName>("t");
+ auto row = "row1";
+
+ // Gets to be performed on above HBase Table
+ Get get_all(row); // expected to return all 3 columns
+ Get get_one(row); // expected to return 1 column
+ Get get_two(row); // expected to return 2 column
+
+ get_one.SetFilter(FilterFactory::ColumnPrefixFilter("foo_"));
+ get_two.SetFilter(FilterFactory::ColumnPrefixFilter("column_"));
+
+ // Create a client
+ hbase::Client client(Configuration{});
+
+ // Get connection to HBase Table
+ auto table = client.Table(tn);
+ ASSERT_TRUE(table) << "Unable to get connection to Table.";
+
+ // Perform the Get
+ auto result_all = table->Get(get_all);
+ auto result_one = table->Get(get_one);
+ auto result_two = table->Get(get_two);
+
+ table->Close();
+ client.Close();
+
+ // Test the values
+ ASSERT_TRUE(!result_one->IsEmpty()) << "Result shouldn't be empty.";
+ ASSERT_TRUE(!result_two->IsEmpty()) << "Result shouldn't be empty.";
+ ASSERT_TRUE(!result_all->IsEmpty()) << "Result shouldn't be empty.";
+ EXPECT_EQ(row, result_one->Row());
+ EXPECT_EQ(row, result_two->Row());
+ EXPECT_EQ(row, result_all->Row());
+ EXPECT_EQ(1, result_one->Size());
+ EXPECT_EQ(2, result_two->Size());
+ EXPECT_EQ(3, result_all->Size());
+ EXPECT_EQ("value3", *(result_one->Value("d", "foo_column")));
+ EXPECT_EQ("value1", *(result_two->Value("d", "column_1")));
+ EXPECT_EQ("value2", *(result_two->Value("d", "column_2")));
+}
+
+TEST_F(FilterTest, GetWithQualifierFilter) {
+ // write row1 with 3 columns (a,b,c)
+ FilterTest::test_util_->RunShellCmd(
+ "create 't1', 'd'; put 't1', 'row1', 'd:a', 'value1'; put 't1', 'row1', 'd:b', "
+ "'value2'; put 't1', 'row1', 'd:c', 'value3'");
+
+ // Create TableName and Row to be fetched from HBase
+ auto tn = folly::to<hbase::pb::TableName>("t1");
+ auto row = "row1";
+
+ // Gets to be performed on above HBase Table
+ Get get(row);
+ get.SetFilter(FilterFactory::QualifierFilter(CompareType::GREATER_OR_EQUAL,
+ *ComparatorFactory::BinaryComparator("b")));
+
+ // Create a client
+ hbase::Client client(Configuration{});
+
+ // Get connection to HBase Table
+ auto table = client.Table(tn);
+ ASSERT_TRUE(table) << "Unable to get connection to Table.";
+
+ // Perform the Get
+ auto result = table->Get(get);
+
+ table->Close();
+ client.Close();
+
+ // Test the values
+ ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty.";
+ EXPECT_EQ(row, result->Row());
+ EXPECT_EQ(2, result->Size());
+ EXPECT_EQ("value2", *(result->Value("d", "b")));
+ EXPECT_EQ("value3", *(result->Value("d", "c")));
+}
diff --git a/core/filter.h b/core/filter.h
new file mode 100644
index 0000000..b5b7133
--- /dev/null
+++ b/core/filter.h
@@ -0,0 +1,481 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <utility>
+#include <set>
+#include <vector>
+
+#include "if/Comparator.pb.h"
+#include "if/Filter.pb.h"
+#include "if/HBase.pb.h"
+
+using google::protobuf::Message;
+
+namespace hbase {
+
+/**
+ * In C++ Client, Filter is a thin wrapper for calling filters defined as a Java class. The actual
+ * filtering logic is not implemented here, but this class provides a mechanism to call
+ * pre-existing Filter classes (like KeyOnlyFilter, SingleColumnValueFilter, etc) with your Get or
+ * Scan RPCs. This class can also be used to call custom Filters defined as a Java class, or
+ * pre-existing Filters not defined below. Some of the interfaces depends on protobuf classes
+ * defined in HBase.proto, Filter.proto and Comparator.proto.
+ *
+ * Consult the Java class docs for learning about the various filters and how they work (and filter
+ * arguments).
+ *
+ * Pre-existing Filters can be used like this:
+ *
+ * Get get(row);
+ * get.SetFilter(FilterFactory::ColumnPrefixFilter("foo_"));
+ *
+ * Custom filters can be invoked like this:
+ * Get get(row);
+ * std::string filter_java_class_name = "foo.bar.baz";
+ * auto filter_data = std::make_unique<pb::MyFilter>();
+ * filter_data->set_foo(foo);
+ * get.SetFilter(std::make_unique<Filter>(filter_java_class_name, filter_data));
+ *
+ */
+class Filter {
+ public:
+ Filter(std::string java_class_name, std::unique_ptr<Message> data)
+ : java_class_name_(java_class_name), data_(std::move(data)) {}
+ virtual ~Filter() {}
+
+ const std::string java_class_name() const { return java_class_name_; }
+
+ const Message& data() const { return *data_; }
+ /**
+ * Serialize the filter data to the given buffer. Does protobuf encoding by default.
+ * Can be overriden if Filter does not use protobuf.
+ */
+ virtual void Serialize(std::string* buf) const {
+ if (data_ != nullptr) {
+ data_->SerializeToString(buf);
+ }
+ }
+
+ /** Internal method */
+ static std::unique_ptr<pb::Filter> ToProto(const Filter& filter) {
+ auto pb_filter = std::make_unique<pb::Filter>();
+ pb_filter->set_name(filter.java_class_name());
+ filter.Serialize(pb_filter->mutable_serialized_filter());
+ return std::move(pb_filter);
+ }
+
+ private:
+ std::unique_ptr<Message> data_;
+ std::string java_class_name_;
+};
+
+/**
+ * Comparator for filters. See ByteArrayComparable documentation in Java.
+ */
+class Comparator {
+ public:
+ Comparator(std::string java_class_name, std::unique_ptr<Message> data)
+ : java_class_name_(java_class_name), data_(std::move(data)) {}
+ virtual ~Comparator() {}
+
+ const std::string java_class_name() const { return java_class_name_; }
+
+ /**
+ * Serialize the Comparator data to the given buffer. Does protobuf encoding by default.
+ * Can be overriden if Comparator does not use protobuf.
+ */
+ virtual void Serialize(std::string* buf) const {
+ if (data_ != nullptr) {
+ data_->SerializeToString(buf);
+ }
+ }
+
+ /** Internal method */
+ static std::unique_ptr<pb::Comparator> ToProto(const Comparator& comparator) {
+ auto pb_comparator = std::make_unique<pb::Comparator>();
+ pb_comparator->set_name(comparator.java_class_name());
+ comparator.Serialize(pb_comparator->mutable_serialized_comparator());
+ return std::move(pb_comparator);
+ }
+
+ private:
+ std::unique_ptr<Message> data_;
+ std::string java_class_name_;
+};
+
+/**
+ * Used in row range filters
+ */
+struct RowRange {
+ std::string start_row;
+ bool start_row_inclusive;
+ std::string stop_row;
+ bool stop_row_inclusive;
+};
+
+/**
+ * Factory for creating pre-defined filters.
+ */
+class FilterFactory {
+ public:
+ static std::unique_ptr<Filter> ColumnCountGetFilter(uint32_t limit) noexcept {
+ auto data = std::make_unique<pb::ColumnCountGetFilter>();
+ data->set_limit(limit);
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.ColumnCountGetFilter",
+ std::move(data));
+ }
+
+ static std::unique_ptr<Filter> ColumnPaginationFilter(uint32_t limit, uint32_t offset) noexcept {
+ auto data = std::make_unique<pb::ColumnPaginationFilter>();
+ data->set_limit(limit);
+ data->set_offset(offset);
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.ColumnPaginationFilter",
+ std::move(data));
+ }
+
+ static std::unique_ptr<Filter> ColumnPaginationFilter(uint32_t limit,
+ const std::string& column_offset) noexcept {
+ auto data = std::make_unique<pb::ColumnPaginationFilter>();
+ data->set_limit(limit);
+ data->set_column_offset(column_offset);
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.ColumnPaginationFilter",
+ std::move(data));
+ }
+
+ static std::unique_ptr<Filter> ColumnPrefixFilter(const std::string& prefix) noexcept {
+ auto data = std::make_unique<pb::ColumnPrefixFilter>();
+ data->set_prefix(prefix);
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.ColumnPrefixFilter",
+ std::move(data));
+ }
+
+ static std::unique_ptr<Filter> ColumnRangeFilter(const std::string& min_column,
+ bool min_column_inclusive,
+ const std::string& max_column,
+ bool max_column_inclusive) noexcept {
+ auto data = std::make_unique<pb::ColumnRangeFilter>();
+ data->set_min_column(min_column);
+ data->set_min_column_inclusive(min_column_inclusive);
+ data->set_max_column(max_column);
+ data->set_max_column_inclusive(max_column_inclusive);
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.ColumnRangeFilter",
+ std::move(data));
+ }
+
+ static std::unique_ptr<pb::CompareFilter> CompareFilter(pb::CompareType compare_op,
+ const Comparator& comparator) noexcept {
+ auto data = std::make_unique<pb::CompareFilter>();
+ data->set_compare_op(compare_op);
+ data->set_allocated_comparator(Comparator::ToProto(comparator).release());
+ return std::move(data);
+ }
+
+ /**
+ * Build a dependent column filter with value checking
+ * dependent column varies will be compared using the supplied
+ * compareOp and comparator, for usage of which
+ * refer to {@link CompareFilter}
+ *
+ * @param family dependent column family
+ * @param qualifier dependent column qualifier
+ * @param drop_dependent_column whether the column should be discarded after
+ * @param compare_op comparison op
+ * @param comparator comparator
+ */
+ static std::unique_ptr<Filter> DependentColumnFilter(const std::string& family,
+ const std::string& qualifier,
+ bool drop_dependent_column,
+ pb::CompareType compare_op,
+ const Comparator& comparator) noexcept {
+ auto data = std::make_unique<pb::DependentColumnFilter>();
+ data->set_column_family(family);
+ data->set_column_qualifier(qualifier);
+ data->set_drop_dependent_column(drop_dependent_column);
+ data->set_allocated_compare_filter(CompareFilter(compare_op, comparator).release());
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.DependentColumnFilter",
+ std::move(data));
+ }
+
+ static std::unique_ptr<Filter> FamilyFilter(pb::CompareType compare_op,
+ const Comparator& comparator) noexcept {
+ auto data = std::make_unique<pb::FamilyFilter>();
+ data->set_allocated_compare_filter(CompareFilter(compare_op, comparator).release());
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.FamilyFilter", std::move(data));
+ }
+
+ static std::unique_ptr<Filter> FilterAllFilter() noexcept {
+ auto data = std::make_unique<pb::FilterAllFilter>();
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.FilterAllFilter",
+ std::move(data));
+ }
+
+ static std::unique_ptr<Filter> FilterList(
+ pb::FilterList_Operator op, const std::vector<std::unique_ptr<Filter>>& filters) noexcept {
+ auto data = std::make_unique<pb::FilterList>();
+ data->set_operator_(op);
+ for (auto const& f : filters) {
+ data->mutable_filters()->AddAllocated(Filter::ToProto(*f).release());
+ }
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.FilterList", std::move(data));
+ }
+
+ static std::unique_ptr<Filter> FirstKeyOnlyFilter() noexcept {
+ auto data = std::make_unique<pb::FirstKeyOnlyFilter>();
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter",
+ std::move(data));
+ }
+
+ static std::unique_ptr<Filter> FirstKeyValueMatchingQualifiersFilter(
+ const std::set<std::string>& qualifiers) noexcept {
+ auto data = std::make_unique<pb::FirstKeyValueMatchingQualifiersFilter>();
+ for (auto q : qualifiers) {
+ data->add_qualifiers(q);
+ }
+ return std::make_unique<Filter>(
+ "org.apache.hadoop.hbase.filter.FirstKeyValueMatchingQualifiersFilter", std::move(data));
+ }
+
+ static std::unique_ptr<Filter> FuzzyRowFilter(
+ const std::vector<std::pair<std::string, std::string>>& fuzzy_keys_data) noexcept {
+ auto data = std::make_unique<pb::FuzzyRowFilter>();
+ for (auto q : fuzzy_keys_data) {
+ auto p = data->add_fuzzy_keys_data();
+ p->set_first(q.first);
+ p->set_second(q.second);
+ }
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.FuzzyRowFilter",
+ std::move(data));
+ }
+
+ static std::unique_ptr<Filter> InclusiveStopFilter(const std::string& stop_row_key) noexcept {
+ auto data = std::make_unique<pb::InclusiveStopFilter>();
+ data->set_stop_row_key(stop_row_key);
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.InclusiveStopFilter",
+ std::move(data));
+ }
+
+ static std::unique_ptr<Filter> KeyOnlyFilter(bool len_as_val) noexcept {
+ auto data = std::make_unique<pb::KeyOnlyFilter>();
+ data->set_len_as_val(len_as_val);
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.KeyOnlyFilter",
+ std::move(data));
+ }
+
+ static std::unique_ptr<Filter> MultipleColumnPrefixFilter(
+ const std::vector<std::string>& sorted_prefixes) noexcept {
+ auto data = std::make_unique<pb::MultipleColumnPrefixFilter>();
+ for (auto p : sorted_prefixes) {
+ data->add_sorted_prefixes(p);
+ }
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.MultipleColumnPrefixFilter",
+ std::move(data));
+ }
+
+ static std::unique_ptr<Filter> MultiRowRangeFilter(
+ const std::vector<RowRange>& row_ranges) noexcept {
+ auto data = std::make_unique<pb::MultiRowRangeFilter>();
+ for (auto r : row_ranges) {
+ auto range = data->add_row_range_list();
+ range->set_start_row(r.start_row);
+ range->set_start_row_inclusive(r.start_row_inclusive);
+ range->set_stop_row(r.stop_row);
+ range->set_stop_row_inclusive(r.stop_row_inclusive);
+ }
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.MultiRowRangeFilter",
+ std::move(data));
+ }
+
+ static std::unique_ptr<Filter> PageFilter(uint64_t page_size) noexcept {
+ auto data = std::make_unique<pb::PageFilter>();
+ data->set_page_size(page_size);
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.PageFilter", std::move(data));
+ }
+
+ static std::unique_ptr<Filter> PrefixFilter(const std::string& prefix) noexcept {
+ auto data = std::make_unique<pb::PrefixFilter>();
+ data->set_prefix(prefix);
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.PrefixFilter", std::move(data));
+ }
+
+ static std::unique_ptr<Filter> QualifierFilter(pb::CompareType compare_op,
+ const Comparator& comparator) noexcept {
+ auto data = std::make_unique<pb::QualifierFilter>();
+ data->set_allocated_compare_filter(CompareFilter(compare_op, comparator).release());
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.QualifierFilter",
+ std::move(data));
+ }
+
+ static std::unique_ptr<Filter> RandomRowFilter(float chance) noexcept {
+ auto data = std::make_unique<pb::RandomRowFilter>();
+ data->set_chance(chance);
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.RandomRowFilter",
+ std::move(data));
+ }
+
+ static std::unique_ptr<Filter> RowFilter(pb::CompareType compare_op,
+ const Comparator& comparator) noexcept {
+ auto data = std::make_unique<pb::RowFilter>();
+ data->set_allocated_compare_filter(CompareFilter(compare_op, comparator).release());
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.RowFilter", std::move(data));
+ }
+
+ static std::unique_ptr<Filter> SingleColumnValueExcludeFilter(
+ const std::string& family, const std::string& qualifier, bool filter_if_missing,
+ bool latest_version_only, pb::CompareType compare_op, const Comparator& comparator) noexcept {
+ auto data = std::make_unique<pb::SingleColumnValueExcludeFilter>();
+ auto f = SingleColumnValueFilterProto(family, qualifier, filter_if_missing, latest_version_only,
+ compare_op, comparator);
+ data->set_allocated_single_column_value_filter(f.release());
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.SingleColumnValueExcludeFilter",
+ std::move(data));
+ }
+
+ static std::unique_ptr<pb::SingleColumnValueFilter> SingleColumnValueFilterProto(
+ const std::string& family, const std::string& qualifier, bool filter_if_missing,
+ bool latest_version_only, pb::CompareType compare_op, const Comparator& comparator) noexcept {
+ auto data = std::make_unique<pb::SingleColumnValueFilter>();
+ data->set_column_family(family);
+ data->set_column_qualifier(qualifier);
+ data->set_compare_op(compare_op);
+ data->set_filter_if_missing(filter_if_missing);
+ data->set_latest_version_only(latest_version_only);
+ data->set_allocated_comparator(Comparator::ToProto(comparator).release());
+ return data;
+ }
+
+ static std::unique_ptr<Filter> SingleColumnValueFilter(
+ const std::string& family, const std::string& qualifier, bool filter_if_missing,
+ bool latest_version_only, pb::CompareType compare_op, const Comparator& comparator) noexcept {
+ auto data = SingleColumnValueFilterProto(family, qualifier, filter_if_missing,
+ latest_version_only, compare_op, comparator);
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.SingleColumnValueFilter",
+ std::move(data));
+ }
+
+ static std::unique_ptr<Filter> SkipFilter(const Filter& filter) noexcept {
+ auto data = std::make_unique<pb::SkipFilter>();
+ data->set_allocated_filter(Filter::ToProto(filter).release());
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.SkipFilter", std::move(data));
+ }
+
+ static std::unique_ptr<Filter> TimestampsFilter(std::vector<uint64_t> timestamps,
+ bool can_hint) noexcept {
+ auto data = std::make_unique<pb::TimestampsFilter>();
+ for (auto t : timestamps) {
+ data->add_timestamps(t);
+ }
+ data->set_can_hint(can_hint);
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.TimestampsFilter",
+ std::move(data));
+ }
+
+ static std::unique_ptr<Filter> ValueFilter(pb::CompareType compare_op,
+ const Comparator& comparator) noexcept {
+ auto data = std::make_unique<pb::ValueFilter>();
+ data->set_allocated_compare_filter(CompareFilter(compare_op, comparator).release());
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.ValueFilter", std::move(data));
+ }
+
+ static std::unique_ptr<Filter> WhileMatchFilter(const Filter& filter) noexcept {
+ auto data = std::make_unique<pb::WhileMatchFilter>();
+ data->set_allocated_filter(Filter::ToProto(filter).release());
+ return std::make_unique<Filter>("org.apache.hadoop.hbase.filter.WhileMatchFilter",
+ std::move(data));
+ }
+};
+
+/**
+ * Factory for creating pre-defined Comparators.
+ */
+class ComparatorFactory {
+ public:
+ static std::unique_ptr<pb::ByteArrayComparable> ByteArrayComparable(
+ const std::string& value) noexcept {
+ auto data = std::make_unique<pb::ByteArrayComparable>();
+ data->set_value(value);
+ return std::move(data);
+ }
+
+ static std::unique_ptr<Comparator> BinaryComparator(const std::string& value) noexcept {
+ auto data = std::make_unique<pb::BinaryComparator>();
+ data->set_allocated_comparable(ByteArrayComparable(value).release());
+ return std::make_unique<Comparator>("org.apache.hadoop.hbase.filter.BinaryComparator",
+ std::move(data));
+ }
+
+ static std::unique_ptr<Comparator> LongComparator(const std::string& value) noexcept {
+ // TODO: this should take a uint64_t argument, not a byte array.
+ auto data = std::make_unique<pb::LongComparator>();
+ data->set_allocated_comparable(ByteArrayComparable(value).release());
+ return std::make_unique<Comparator>("org.apache.hadoop.hbase.filter.LongComparator",
+ std::move(data));
+ }
+
+ static std::unique_ptr<Comparator> BinaryPrefixComparator(const std::string& value) noexcept {
+ auto data = std::make_unique<pb::BinaryPrefixComparator>();
+ data->set_allocated_comparable(ByteArrayComparable(value).release());
+ return std::make_unique<Comparator>("org.apache.hadoop.hbase.filter.BinaryPrefixComparator",
+ std::move(data));
+ }
+
+ static std::unique_ptr<Comparator> BitComparator(const std::string& value,
+ pb::BitComparator_BitwiseOp bit_op) noexcept {
+ auto data = std::make_unique<pb::BitComparator>();
+ data->set_allocated_comparable(ByteArrayComparable(value).release());
+ data->set_bitwise_op(bit_op);
+ return std::make_unique<Comparator>("org.apache.hadoop.hbase.filter.BitComparator",
+ std::move(data));
+ }
+
+ static std::unique_ptr<Comparator> NullComparator() noexcept {
+ auto data = std::make_unique<pb::NullComparator>();
+ return std::make_unique<Comparator>("org.apache.hadoop.hbase.filter.NullComparator",
+ std::move(data));
+ }
+
+ /**
+ * @param pattern a valid regular expression
+ * @param pattern_flags java.util.regex.Pattern flags
+ * @param charset the charset name
+ * @param engine engine implementation type, either JAVA or JONI
+ */
+ static std::unique_ptr<Comparator> RegexStringComparator(
+ const std::string& pattern, int32_t pattern_flags, const std::string& charset = "UTF-8",
+ const std::string& engine = "JAVA") noexcept {
+ auto data = std::make_unique<pb::RegexStringComparator>();
+ data->set_pattern(pattern);
+ data->set_pattern_flags(pattern_flags);
+ data->set_charset(charset);
+ data->set_engine(engine);
+ return std::make_unique<Comparator>("org.apache.hadoop.hbase.filter.RegexStringComparator",
+ std::move(data));
+ }
+
+ static std::unique_ptr<Comparator> SubstringComparator(const std::string& substr) noexcept {
+ auto data = std::make_unique<pb::SubstringComparator>();
+ data->set_substr(substr);
+ return std::make_unique<Comparator>("org.apache.hadoop.hbase.filter.SubstringComparator",
+ std::move(data));
+ }
+};
+} // namespace hbase
diff --git a/core/get.h b/core/get.h
index f79c633..92d1fee 100644
--- a/core/get.h
+++ b/core/get.h
@@ -24,6 +24,7 @@
#include <memory>
#include <string>
#include <vector>
+#include "core/query.h"
#include "core/time_range.h"
#include "if/Client.pb.h"
@@ -35,7 +36,7 @@ namespace hbase {
*/
using FamilyMap = std::map<std::string, std::vector<std::string>>;
-class Get {
+class Get : public Query {
public:
/**
* Constructors
diff --git a/core/query.h b/core/query.h
new file mode 100644
index 0000000..b706303
--- /dev/null
+++ b/core/query.h
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include "core/filter.h"
+
+namespace hbase {
+
+/**
+ * Base class for read RPC calls (Get / Scan).
+ */
+class Query {
+ public:
+ virtual ~Query() {}
+
+ void SetFilter(std::unique_ptr<Filter> filter) { filter_ = std::move(filter); }
+
+ const std::unique_ptr<Filter>& filter() const { return filter_; }
+
+ protected:
+ std::unique_ptr<Filter> filter_ = nullptr;
+};
+
+} // namespace hbase
diff --git a/core/request_converter.cc b/core/request_converter.cc
index eba07df..149202e 100644
--- a/core/request_converter.cc
+++ b/core/request_converter.cc
@@ -68,6 +68,10 @@ std::unique_ptr<Request> RequestConverter::ToGetRequest(const Get &get,
}
}
+ if (get.filter() != nullptr) {
+ pb_get->set_allocated_filter(Filter::ToProto(*(get.filter())).release());
+ }
+
return pb_req;
}
@@ -108,6 +112,10 @@ std::unique_ptr<Request> RequestConverter::ToScanRequest(const Scan &scan,
}
}
+ if (scan.filter() != nullptr) {
+ pb_scan->set_allocated_filter(Filter::ToProto(*(scan.filter())).release());
+ }
+
// TODO We will change this later.
pb_msg->set_client_handles_partials(false);
pb_msg->set_client_handles_heartbeats(false);
diff --git a/core/scan.h b/core/scan.h
index e2e7f1a..7e8c7bd 100644
--- a/core/scan.h
+++ b/core/scan.h
@@ -36,7 +36,7 @@ namespace hbase {
*/
using FamilyMap = std::map<std::string, std::vector<std::string>>;
-class Scan {
+class Scan : public Query {
public:
/**
* @brief Constructors. Create a Scan operation across all rows.
[hbase] 75/137: HBASE-17754 [C++] RawAsyncTable
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 0458058afd71da1215befa4e4abb218c3423dd9f
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Fri Mar 17 19:23:09 2017 -0700
HBASE-17754 [C++] RawAsyncTable
---
core/BUCK | 9 +-
core/{client.cc => async-connection.cc} | 40 ++------
core/async-connection.h | 111 +++++++++++++++++++++
core/async-region-locator.h | 62 ++++++++++++
core/async-rpc-retrying-caller-factory.h | 61 ++++++++----
core/async-rpc-retrying-caller.h | 49 +++++-----
core/async-rpc-retrying-test.cc | 160 +++++++++++++++----------------
core/client-test.cc | 2 +
core/client.cc | 51 ++--------
core/client.h | 38 ++------
core/connection-configuration.h | 16 +++-
core/location-cache-test.cc | 1 +
core/location-cache.cc | 10 +-
core/location-cache.h | 15 ++-
core/meta-utils.cc | 2 +-
core/raw-async-table.cc | 80 ++++++++++++++++
core/raw-async-table.h | 77 +++++++++++++++
core/response-converter.cc | 20 ++--
core/response-converter.h | 6 +-
core/table.cc | 39 +++-----
core/table.h | 22 +++--
security/user.h | 1 +
utils/time-util.h | 27 +++++-
23 files changed, 612 insertions(+), 287 deletions(-)
diff --git a/core/BUCK b/core/BUCK
index 3df48d6..2d77f2d 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -19,6 +19,8 @@
cxx_library(
name="core",
exported_headers=[
+ "async-connection.h",
+ "async-region-locator.h",
"client.h",
"cell.h",
"hbase-macros.h",
@@ -38,12 +40,14 @@ cxx_library(
"request-converter.h",
"response-converter.h",
"table.h",
+ "raw-async-table.h",
"async-rpc-retrying-caller-factory.h",
"async-rpc-retrying-caller.h",
"hbase-rpc-controller.h",
"zk-util.h",
],
srcs=[
+ "async-connection.cc",
"cell.cc",
"client.cc",
"keyvalue-codec.cc",
@@ -52,6 +56,7 @@ cxx_library(
"get.cc",
"time-range.cc",
"scan.cc",
+ "raw-async-table.cc",
"result.cc",
"request-converter.cc",
"response-converter.cc",
@@ -70,9 +75,7 @@ cxx_library(
"//third-party:zookeeper_mt",
],
compiler_flags=['-Weffc++', '-ggdb'],
- visibility=[
- 'PUBLIC',
- ],)
+ visibility=['PUBLIC',],)
cxx_library(
name="conf",
exported_headers=[
diff --git a/core/client.cc b/core/async-connection.cc
similarity index 65%
copy from core/client.cc
copy to core/async-connection.cc
index 3f889d4..b945e38 100644
--- a/core/client.cc
+++ b/core/async-connection.cc
@@ -17,32 +17,13 @@
*
*/
-#include "core/client.h"
-
-#include <glog/logging.h>
-#include <chrono>
-#include <exception>
-#include <utility>
+#include "core/async-connection.h"
+#include "core/async-rpc-retrying-caller-factory.h"
namespace hbase {
-Client::Client() {
- HBaseConfigurationLoader loader;
- auto conf = loader.LoadDefaultResources();
- if (!conf) {
- LOG(ERROR) << "Unable to create default Configuration object. Either hbase-default.xml or "
- "hbase-site.xml is absent in the search path or problems in XML parsing";
- throw std::runtime_error("Configuration object not present.");
- }
- init(conf.value());
-}
-
-Client::Client(const hbase::Configuration &conf) { init(conf); }
-
-void Client::init(const hbase::Configuration &conf) {
- conf_ = std::make_shared<hbase::Configuration>(conf);
-
- conn_conf_ = std::make_shared<hbase::ConnectionConfiguration>(*conf_);
+void AsyncConnectionImpl::Init() {
+ connection_conf_ = std::make_shared<hbase::ConnectionConfiguration>(*conf_);
// start thread pools
auto io_threads = conf_->GetInt(kClientIoThreadPoolSize, sysconf(_SC_NPROCESSORS_ONLN));
auto cpu_threads = conf_->GetInt(kClientCpuThreadPoolSize, 2 * sysconf(_SC_NPROCESSORS_ONLN));
@@ -50,31 +31,28 @@ void Client::init(const hbase::Configuration &conf) {
io_executor_ = std::make_shared<wangle::IOThreadPoolExecutor>(io_threads);
std::shared_ptr<Codec> codec = nullptr;
- if (conf.Get(kRpcCodec, hbase::KeyValueCodec::kJavaClassName) ==
+ if (conf_->Get(kRpcCodec, hbase::KeyValueCodec::kJavaClassName) ==
std::string(KeyValueCodec::kJavaClassName)) {
codec = std::make_shared<hbase::KeyValueCodec>();
} else {
LOG(WARNING) << "Not using RPC Cell Codec";
}
rpc_client_ =
- std::make_shared<hbase::RpcClient>(io_executor_, codec, conn_conf_->connect_timeout());
+ std::make_shared<hbase::RpcClient>(io_executor_, codec, connection_conf_->connect_timeout());
location_cache_ =
std::make_shared<hbase::LocationCache>(conf_, cpu_executor_, rpc_client_->connection_pool());
+ caller_factory_ = std::make_shared<AsyncRpcRetryingCallerFactory>(shared_from_this());
}
// We can't have the threads continue running after everything is done
// that leads to an error.
-Client::~Client() {
+AsyncConnectionImpl::~AsyncConnectionImpl() {
cpu_executor_->stop();
io_executor_->stop();
if (rpc_client_.get()) rpc_client_->Close();
}
-std::unique_ptr<hbase::Table> Client::Table(const TableName &table_name) {
- return std::make_unique<hbase::Table>(table_name, location_cache_, rpc_client_, conf_);
-}
-
-void Client::Close() {
+void AsyncConnectionImpl::Close() {
if (is_closed_) return;
cpu_executor_->stop();
diff --git a/core/async-connection.h b/core/async-connection.h
new file mode 100644
index 0000000..6a61124
--- /dev/null
+++ b/core/async-connection.h
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <folly/futures/Future.h>
+#include <folly/io/IOBuf.h>
+#include <wangle/concurrent/CPUThreadPoolExecutor.h>
+#include <wangle/concurrent/IOThreadPoolExecutor.h>
+
+#include <memory>
+#include <string>
+
+#include "connection/rpc-client.h"
+#include "core/async-region-locator.h"
+#include "core/configuration.h"
+#include "core/connection-configuration.h"
+#include "core/connection-configuration.h"
+#include "core/hbase-configuration-loader.h"
+#include "core/hbase-rpc-controller.h"
+#include "core/keyvalue-codec.h"
+#include "core/location-cache.h"
+#include "if/Cell.pb.h"
+#include "serde/table-name.h"
+
+namespace hbase {
+
+class AsyncRpcRetryingCallerFactory;
+
+class AsyncConnection {
+ public:
+ AsyncConnection(){};
+ virtual ~AsyncConnection(){};
+ virtual std::shared_ptr<Configuration> conf() = 0;
+ virtual std::shared_ptr<ConnectionConfiguration> connection_conf() = 0;
+ virtual std::shared_ptr<AsyncRpcRetryingCallerFactory> caller_factory() = 0;
+ virtual std::shared_ptr<RpcClient> rpc_client() = 0;
+ virtual std::shared_ptr<AsyncRegionLocator> region_locator() = 0;
+ virtual std::shared_ptr<HBaseRpcController> CreateRpcController() = 0;
+ virtual void Close() = 0;
+};
+
+class AsyncConnectionImpl : public AsyncConnection,
+ public std::enable_shared_from_this<AsyncConnectionImpl> {
+ public:
+ virtual ~AsyncConnectionImpl();
+
+ // See https://mortoray.com/2013/08/02/safely-using-enable_shared_from_this/
+ template <typename... T>
+ static std::shared_ptr<AsyncConnectionImpl> Create(T&&... all) {
+ auto conn =
+ std::shared_ptr<AsyncConnectionImpl>(new AsyncConnectionImpl(std::forward<T>(all)...));
+ conn->Init();
+ return conn;
+ }
+
+ std::shared_ptr<Configuration> conf() override { return conf_; }
+ std::shared_ptr<ConnectionConfiguration> connection_conf() override { return connection_conf_; }
+ std::shared_ptr<AsyncRpcRetryingCallerFactory> caller_factory() override {
+ return caller_factory_;
+ }
+ std::shared_ptr<RpcClient> rpc_client() override { return rpc_client_; }
+ std::shared_ptr<LocationCache> location_cache() { return location_cache_; }
+ std::shared_ptr<AsyncRegionLocator> region_locator() override { return location_cache_; }
+ std::shared_ptr<HBaseRpcController> CreateRpcController() override {
+ return std::make_shared<HBaseRpcController>();
+ }
+
+ virtual void Close() override;
+
+ protected:
+ AsyncConnectionImpl() {}
+
+ private:
+ /** Parameter name for HBase client IO thread pool size. Defaults to num cpus */
+ static constexpr const char* kClientIoThreadPoolSize = "hbase.client.io.thread.pool.size";
+ /** Parameter name for HBase client CPU thread pool size. Defaults to (2 * num cpus) */
+ static constexpr const char* kClientCpuThreadPoolSize = "hbase.client.cpu.thread.pool.size";
+ /** The RPC codec to encode cells. For now it is KeyValueCodec */
+ static constexpr const char* kRpcCodec = "hbase.client.rpc.codec";
+
+ std::shared_ptr<Configuration> conf_;
+ std::shared_ptr<ConnectionConfiguration> connection_conf_;
+ std::shared_ptr<AsyncRpcRetryingCallerFactory> caller_factory_;
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_;
+ std::shared_ptr<LocationCache> location_cache_;
+ std::shared_ptr<RpcClient> rpc_client_;
+ bool is_closed_ = false;
+
+ private:
+ AsyncConnectionImpl(std::shared_ptr<Configuration> conf) : conf_(conf) {}
+ void Init();
+};
+} // namespace hbase
diff --git a/core/async-region-locator.h b/core/async-region-locator.h
new file mode 100644
index 0000000..b0019e0
--- /dev/null
+++ b/core/async-region-locator.h
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <folly/futures/Future.h>
+#include <string>
+
+#include "core/region-location.h"
+#include "if/Client.pb.h"
+#include "serde/region-info.h"
+#include "serde/server-name.h"
+#include "serde/table-name.h"
+
+namespace hbase {
+
+class AsyncRegionLocator {
+ public:
+ AsyncRegionLocator() {}
+ virtual ~AsyncRegionLocator() = default;
+
+ /**
+ * The only method clients should use for meta lookups. If corresponding
+ * location is cached, it's returned from the cache, otherwise lookup
+ * in meta table is done, location is cached and then returned.
+ * It's expected that tiny fraction of invocations incurs meta scan.
+ * This method is to look up non-meta regions; use LocateMeta() to get the
+ * location of hbase:meta region.
+ *
+ * @param tn Table name of the table to look up. This object must live until
+ * after the future is returned
+ *
+ * @param row of the table to look up. This object must live until after the
+ * future is returned
+ */
+ virtual folly::Future<std::shared_ptr<RegionLocation>> LocateRegion(
+ const hbase::pb::TableName &tn, const std::string &row,
+ const RegionLocateType locate_type = RegionLocateType::kCurrent,
+ const int64_t locate_ns = 0) = 0;
+ /**
+ * Update cached region location, possibly using the information from exception.
+ */
+ virtual void UpdateCachedLocation(const RegionLocation &loc, const std::exception &error) = 0;
+};
+
+} // namespace hbase
diff --git a/core/async-rpc-retrying-caller-factory.h b/core/async-rpc-retrying-caller-factory.h
index 3342e29..5bcad6c 100644
--- a/core/async-rpc-retrying-caller-factory.h
+++ b/core/async-rpc-retrying-caller-factory.h
@@ -35,20 +35,25 @@ using std::chrono::nanoseconds;
namespace hbase {
-template <typename CONN, typename RESP, typename RPC_CLIENT>
+class AsyncConnection;
+
+template <typename RESP>
class SingleRequestCallerBuilder
- : public std::enable_shared_from_this<SingleRequestCallerBuilder<CONN, RESP, RPC_CLIENT>> {
+ : public std::enable_shared_from_this<SingleRequestCallerBuilder<RESP>> {
public:
- explicit SingleRequestCallerBuilder(std::shared_ptr<CONN> conn)
+ explicit SingleRequestCallerBuilder(std::shared_ptr<AsyncConnection> conn)
: conn_(conn),
table_name_(nullptr),
- rpc_timeout_nanos_(0),
- operation_timeout_nanos_(0),
+ rpc_timeout_nanos_(conn->connection_conf()->rpc_timeout()),
+ pause_(conn->connection_conf()->pause()),
+ operation_timeout_nanos_(conn->connection_conf()->operation_timeout()),
+ max_retries_(conn->connection_conf()->max_retries()),
+ start_log_errors_count_(conn->connection_conf()->start_log_errors_count()),
locate_type_(RegionLocateType::kCurrent) {}
virtual ~SingleRequestCallerBuilder() = default;
- typedef SingleRequestCallerBuilder<CONN, RESP, RPC_CLIENT> GenenericThisType;
+ typedef SingleRequestCallerBuilder<RESP> GenenericThisType;
typedef std::shared_ptr<GenenericThisType> SharedThisPtr;
SharedThisPtr table(std::shared_ptr<TableName> table_name) {
@@ -66,6 +71,21 @@ class SingleRequestCallerBuilder
return shared_this();
}
+ SharedThisPtr pause(nanoseconds pause) {
+ pause_ = pause;
+ return shared_this();
+ }
+
+ SharedThisPtr max_retries(uint32_t max_retries) {
+ max_retries_ = max_retries;
+ return shared_this();
+ }
+
+ SharedThisPtr start_log_errors_count(uint32_t start_log_errors_count) {
+ start_log_errors_count_ = start_log_errors_count;
+ return shared_this();
+ }
+
SharedThisPtr row(const std::string& row) {
row_ = row;
return shared_this();
@@ -76,18 +96,17 @@ class SingleRequestCallerBuilder
return shared_this();
}
- SharedThisPtr action(Callable<RESP, RPC_CLIENT> callable) {
+ SharedThisPtr action(Callable<RESP> callable) {
callable_ = callable;
return shared_this();
}
folly::Future<RESP> Call() { return Build()->Call(); }
- std::shared_ptr<AsyncSingleRequestRpcRetryingCaller<CONN, RESP, RPC_CLIENT>> Build() {
- return std::make_shared<AsyncSingleRequestRpcRetryingCaller<CONN, RESP, RPC_CLIENT>>(
- conn_, table_name_, row_, locate_type_, callable_, conn_->get_conn_conf()->GetPauseNs(),
- conn_->get_conn_conf()->GetMaxRetries(), operation_timeout_nanos_, rpc_timeout_nanos_,
- conn_->get_conn_conf()->GetStartLogErrorsCount());
+ std::shared_ptr<AsyncSingleRequestRpcRetryingCaller<RESP>> Build() {
+ return std::make_shared<AsyncSingleRequestRpcRetryingCaller<RESP>>(
+ conn_, table_name_, row_, locate_type_, callable_, pause_, max_retries_,
+ operation_timeout_nanos_, rpc_timeout_nanos_, start_log_errors_count_);
}
private:
@@ -96,28 +115,30 @@ class SingleRequestCallerBuilder
}
private:
- std::shared_ptr<CONN> conn_;
+ std::shared_ptr<AsyncConnection> conn_;
std::shared_ptr<TableName> table_name_;
nanoseconds rpc_timeout_nanos_;
nanoseconds operation_timeout_nanos_;
+ nanoseconds pause_;
+ uint32_t max_retries_;
+ uint32_t start_log_errors_count_;
std::string row_;
RegionLocateType locate_type_;
- Callable<RESP, RPC_CLIENT> callable_;
+ Callable<RESP> callable_;
}; // end of SingleRequestCallerBuilder
-template <typename CONN>
class AsyncRpcRetryingCallerFactory {
private:
- std::shared_ptr<CONN> conn_;
+ std::shared_ptr<AsyncConnection> conn_;
public:
- explicit AsyncRpcRetryingCallerFactory(std::shared_ptr<CONN> conn) : conn_(conn) {}
+ explicit AsyncRpcRetryingCallerFactory(std::shared_ptr<AsyncConnection> conn) : conn_(conn) {}
virtual ~AsyncRpcRetryingCallerFactory() = default;
- template <typename RESP, typename RPC_CLIENT = hbase::RpcClient>
- std::shared_ptr<SingleRequestCallerBuilder<CONN, RESP, RPC_CLIENT>> Single() {
- return std::make_shared<SingleRequestCallerBuilder<CONN, RESP, RPC_CLIENT>>(conn_);
+ template <typename RESP>
+ std::shared_ptr<SingleRequestCallerBuilder<RESP>> Single() {
+ return std::make_shared<SingleRequestCallerBuilder<RESP>>(conn_);
}
};
diff --git a/core/async-rpc-retrying-caller.h b/core/async-rpc-retrying-caller.h
index f7a1523..6503301 100644
--- a/core/async-rpc-retrying-caller.h
+++ b/core/async-rpc-retrying-caller.h
@@ -32,6 +32,7 @@
#include <utility>
#include <vector>
#include "connection/rpc-client.h"
+#include "core/async-connection.h"
#include "core/hbase-rpc-controller.h"
#include "core/region-location.h"
#include "exceptions/exception.h"
@@ -60,23 +61,23 @@ using RespConverter = std::function<R(const S&)>;
template <typename RESP>
using RpcCallback = std::function<void(const RESP&)>;
-template <typename REQ, typename RESP, typename RPC_CLIENT = hbase::RpcClient>
+template <typename REQ, typename RESP>
using RpcCall = std::function<folly::Future<std::unique_ptr<RESP>>(
- std::shared_ptr<RPC_CLIENT>, std::shared_ptr<RegionLocation>,
+ std::shared_ptr<RpcClient>, std::shared_ptr<RegionLocation>,
std::shared_ptr<HBaseRpcController>, std::unique_ptr<REQ>)>;
-template <typename RESP, typename RPC_CLIENT = hbase::RpcClient>
-using Callable = std::function<folly::Future<RESP>(std::shared_ptr<HBaseRpcController>,
- std::shared_ptr<RegionLocation>,
- std::shared_ptr<RPC_CLIENT>)>;
+template <typename RESP>
+using Callable =
+ std::function<folly::Future<RESP>(std::shared_ptr<HBaseRpcController>,
+ std::shared_ptr<RegionLocation>, std::shared_ptr<RpcClient>)>;
-template <typename CONN, typename RESP, typename RPC_CLIENT = hbase::RpcClient>
+template <typename RESP>
class AsyncSingleRequestRpcRetryingCaller {
public:
- AsyncSingleRequestRpcRetryingCaller(std::shared_ptr<CONN> conn,
+ AsyncSingleRequestRpcRetryingCaller(std::shared_ptr<AsyncConnection> conn,
std::shared_ptr<hbase::pb::TableName> table_name,
const std::string& row, RegionLocateType locate_type,
- Callable<RESP, RPC_CLIENT> callable, nanoseconds pause_ns,
+ Callable<RESP> callable, nanoseconds pause,
uint32_t max_retries, nanoseconds operation_timeout_nanos,
nanoseconds rpc_timeout_nanos,
uint32_t start_log_errors_count)
@@ -85,14 +86,14 @@ class AsyncSingleRequestRpcRetryingCaller {
row_(row),
locate_type_(locate_type),
callable_(callable),
- pause_ns_(pause_ns),
+ pause_(pause),
max_retries_(max_retries),
operation_timeout_nanos_(operation_timeout_nanos),
rpc_timeout_nanos_(rpc_timeout_nanos),
start_log_errors_count_(start_log_errors_count),
promise_(std::make_shared<folly::Promise<RESP>>()),
tries_(1) {
- controller_ = conn_->get_rpc_controller_factory()->NewController();
+ controller_ = conn_->CreateRpcController();
start_ns_ = TimeUtil::GetNowNanos();
max_attempts_ = ConnectionUtils::Retries2Attempts(max_retries);
exceptions_ = std::make_shared<std::vector<ThrowableWithExtraContext>>();
@@ -120,9 +121,9 @@ class AsyncSingleRequestRpcRetryingCaller {
locate_timeout_ns = -1L;
}
- conn_->get_locator()
- ->GetRegionLocation(table_name_, row_, locate_type_, locate_timeout_ns)
- .then([this](RegionLocation& loc) { Call(loc); })
+ conn_->region_locator()
+ ->LocateRegion(*table_name_, row_, locate_type_, locate_timeout_ns)
+ .then([this](std::shared_ptr<RegionLocation> loc) { Call(*loc); })
.onError([this](const std::exception& e) {
OnError(e,
[this]() -> std::string {
@@ -155,10 +156,9 @@ class AsyncSingleRequestRpcRetryingCaller {
CompleteExceptionally();
return;
}
- delay_ns =
- std::min(max_delay_ns, ConnectionUtils::GetPauseTime(pause_ns_.count(), tries_ - 1));
+ delay_ns = std::min(max_delay_ns, ConnectionUtils::GetPauseTime(pause_.count(), tries_ - 1));
} else {
- delay_ns = ConnectionUtils::GetPauseTime(pause_ns_.count(), tries_ - 1);
+ delay_ns = ConnectionUtils::GetPauseTime(pause_.count(), tries_ - 1);
}
update_cached_location(error);
tries_++;
@@ -179,9 +179,10 @@ class AsyncSingleRequestRpcRetryingCaller {
call_timeout_ns = rpc_timeout_nanos_.count();
}
- std::shared_ptr<RPC_CLIENT> rpc_client;
+ std::shared_ptr<RpcClient> rpc_client;
try {
- rpc_client = conn_->GetRpcClient();
+ // TODO: There is no connection attempt happening here, no need to try-catch.
+ rpc_client = conn_->rpc_client();
} catch (const IOException& e) {
OnError(e,
[&, this]() -> std::string {
@@ -196,7 +197,7 @@ class AsyncSingleRequestRpcRetryingCaller {
" ms, time elapsed = " + TimeUtil::ElapsedMillisStr(this->start_ns_) + " ms";
},
[&, this](const std::exception& error) {
- conn_->get_locator()->UpdateCachedLocation(loc, error);
+ conn_->region_locator()->UpdateCachedLocation(loc, error);
});
return;
}
@@ -219,7 +220,7 @@ class AsyncSingleRequestRpcRetryingCaller {
" ms";
},
[&, this](const std::exception& error) {
- conn_->get_locator()->UpdateCachedLocation(loc, error);
+ conn_->region_locator()->UpdateCachedLocation(loc, error);
});
return;
});
@@ -244,12 +245,12 @@ class AsyncSingleRequestRpcRetryingCaller {
private:
folly::HHWheelTimer::UniquePtr retry_timer_;
- std::shared_ptr<CONN> conn_;
+ std::shared_ptr<AsyncConnection> conn_;
std::shared_ptr<hbase::pb::TableName> table_name_;
std::string row_;
RegionLocateType locate_type_;
- Callable<RESP, RPC_CLIENT> callable_;
- nanoseconds pause_ns_;
+ Callable<RESP> callable_;
+ nanoseconds pause_;
uint32_t max_retries_;
nanoseconds operation_timeout_nanos_;
nanoseconds rpc_timeout_nanos_;
diff --git a/core/async-rpc-retrying-test.cc b/core/async-rpc-retrying-test.cc
index 81d726f..d0c7921 100644
--- a/core/async-rpc-retrying-test.cc
+++ b/core/async-rpc-retrying-test.cc
@@ -30,9 +30,11 @@
#include "connection/request.h"
#include "connection/response.h"
#include "connection/rpc-client.h"
+#include "core/async-connection.h"
#include "core/async-rpc-retrying-caller-factory.h"
#include "core/async-rpc-retrying-caller.h"
#include "core/client.h"
+#include "core/connection-configuration.h"
#include "core/hbase-rpc-controller.h"
#include "core/keyvalue-codec.h"
#include "core/region-location.h"
@@ -43,6 +45,7 @@
#include "if/Client.pb.h"
#include "if/HBase.pb.h"
#include "test-util/test-util.h"
+#include "utils/time-util.h"
using namespace google::protobuf;
using namespace hbase;
@@ -53,61 +56,73 @@ using ::testing::Return;
using ::testing::_;
using std::chrono::nanoseconds;
-class MockRpcControllerFactory {
+class MockAsyncRegionLocator : public AsyncRegionLocator {
public:
- MOCK_METHOD0(NewController, std::shared_ptr<HBaseRpcController>());
-};
-
-class MockAsyncConnectionConfiguration {
- public:
- MOCK_METHOD0(GetPauseNs, nanoseconds());
- MOCK_METHOD0(GetMaxRetries, int32_t());
- MOCK_METHOD0(GetStartLogErrorsCount, int32_t());
- MOCK_METHOD0(GetReadRpcTimeoutNs, nanoseconds());
- MOCK_METHOD0(GetOperationTimeoutNs, nanoseconds());
-};
-
-class AsyncRegionLocator {
- public:
- explicit AsyncRegionLocator(std::shared_ptr<RegionLocation> region_location)
+ explicit MockAsyncRegionLocator(std::shared_ptr<RegionLocation> region_location)
: region_location_(region_location) {}
- ~AsyncRegionLocator() = default;
-
- folly::Future<RegionLocation> GetRegionLocation(std::shared_ptr<hbase::pb::TableName>,
- const std::string&, RegionLocateType, int64_t) {
- folly::Promise<RegionLocation> promise;
- promise.setValue(*region_location_);
+ ~MockAsyncRegionLocator() = default;
+
+ folly::Future<std::shared_ptr<RegionLocation>> LocateRegion(const hbase::pb::TableName&,
+ const std::string&,
+ const RegionLocateType,
+ const int64_t) override {
+ folly::Promise<std::shared_ptr<RegionLocation>> promise;
+ promise.setValue(region_location_);
return promise.getFuture();
}
- void UpdateCachedLocation(const RegionLocation&, const std::exception&) {}
+ void UpdateCachedLocation(const RegionLocation&, const std::exception&) override {}
private:
std::shared_ptr<RegionLocation> region_location_;
};
-class MockAsyncConnection {
+class MockAsyncConnection : public AsyncConnection,
+ public std::enable_shared_from_this<MockAsyncConnection> {
public:
- MOCK_METHOD0(get_conn_conf, std::shared_ptr<MockAsyncConnectionConfiguration>());
- MOCK_METHOD0(get_rpc_controller_factory, std::shared_ptr<MockRpcControllerFactory>());
- MOCK_METHOD0(get_locator, std::shared_ptr<AsyncRegionLocator>());
- MOCK_METHOD0(GetRpcClient, std::shared_ptr<hbase::RpcClient>());
+ MockAsyncConnection(std::shared_ptr<ConnectionConfiguration> conn_conf,
+ std::shared_ptr<RpcClient> rpc_client,
+ std::shared_ptr<AsyncRegionLocator> region_locator)
+ : conn_conf_(conn_conf), rpc_client_(rpc_client), region_locator_(region_locator) {}
+ ~MockAsyncConnection() {}
+ void Init() {
+ caller_factory_ = std::make_shared<AsyncRpcRetryingCallerFactory>(shared_from_this());
+ }
+
+ std::shared_ptr<Configuration> conf() override { return nullptr; }
+ std::shared_ptr<ConnectionConfiguration> connection_conf() override { return conn_conf_; }
+ std::shared_ptr<AsyncRpcRetryingCallerFactory> caller_factory() override {
+ return caller_factory_;
+ }
+ std::shared_ptr<RpcClient> rpc_client() override { return rpc_client_; }
+ std::shared_ptr<AsyncRegionLocator> region_locator() override { return region_locator_; }
+
+ void Close() override {}
+ std::shared_ptr<HBaseRpcController> CreateRpcController() override {
+ return std::make_shared<HBaseRpcController>();
+ }
+
+ private:
+ std::shared_ptr<ConnectionConfiguration> conn_conf_;
+ std::shared_ptr<AsyncRpcRetryingCallerFactory> caller_factory_;
+ std::shared_ptr<RpcClient> rpc_client_;
+ std::shared_ptr<AsyncRegionLocator> region_locator_;
};
template <typename CONN>
class MockRawAsyncTableImpl {
public:
explicit MockRawAsyncTableImpl(std::shared_ptr<CONN> conn)
- : conn_(conn), promise_(std::make_shared<folly::Promise<hbase::Result>>()) {}
+ : conn_(conn), promise_(std::make_shared<folly::Promise<std::shared_ptr<hbase::Result>>>()) {}
virtual ~MockRawAsyncTableImpl() = default;
/* implement this in real RawAsyncTableImpl. */
/* in real RawAsyncTableImpl, this should be private. */
- folly::Future<hbase::Result> GetCall(std::shared_ptr<hbase::RpcClient> rpc_client,
- std::shared_ptr<HBaseRpcController> controller,
- std::shared_ptr<RegionLocation> loc, const hbase::Get& get) {
- hbase::RpcCall<hbase::Request, hbase::Response, hbase::RpcClient> rpc_call = [](
+ folly::Future<std::shared_ptr<hbase::Result>> GetCall(
+ std::shared_ptr<hbase::RpcClient> rpc_client, std::shared_ptr<HBaseRpcController> controller,
+ std::shared_ptr<RegionLocation> loc, const hbase::Get& get) {
+ hbase::RpcCall<hbase::Request, hbase::Response> rpc_call = [](
std::shared_ptr<hbase::RpcClient> rpc_client, std::shared_ptr<RegionLocation> loc,
std::shared_ptr<HBaseRpcController> controller,
std::unique_ptr<hbase::Request> preq) -> folly::Future<std::unique_ptr<hbase::Response>> {
@@ -115,7 +130,7 @@ class MockRawAsyncTableImpl {
std::move(preq), User::defaultUser(), "ClientService");
};
- return Call<hbase::Get, hbase::Request, hbase::Response, hbase::Result>(
+ return Call<hbase::Get, hbase::Request, hbase::Response, std::shared_ptr<hbase::Result>>(
rpc_client, controller, loc, get, &hbase::RequestConverter::ToGetRequest, rpc_call,
&hbase::ResponseConverter::FromGetResponse);
}
@@ -126,12 +141,12 @@ class MockRawAsyncTableImpl {
std::shared_ptr<hbase::RpcClient> rpc_client, std::shared_ptr<HBaseRpcController> controller,
std::shared_ptr<RegionLocation> loc, const REQ& req,
const ReqConverter<std::unique_ptr<PREQ>, REQ, std::string>& req_converter,
- const hbase::RpcCall<PREQ, PRESP, hbase::RpcClient>& rpc_call,
- const RespConverter<std::unique_ptr<RESP>, PRESP>& resp_converter) {
+ const hbase::RpcCall<PREQ, PRESP>& rpc_call,
+ const RespConverter<RESP, PRESP>& resp_converter) {
rpc_call(rpc_client, loc, controller, std::move(req_converter(req, loc->region_name())))
.then([&, this](std::unique_ptr<PRESP> presp) {
- std::unique_ptr<hbase::Result> result = hbase::ResponseConverter::FromGetResponse(*presp);
- promise_->setValue(std::move(*result));
+ std::shared_ptr<hbase::Result> result = hbase::ResponseConverter::FromGetResponse(*presp);
+ promise_->setValue(result);
})
.onError([this](const std::exception& e) { promise_->setException(e); });
return promise_->getFuture();
@@ -139,7 +154,7 @@ class MockRawAsyncTableImpl {
private:
std::shared_ptr<CONN> conn_;
- std::shared_ptr<folly::Promise<hbase::Result>> promise_;
+ std::shared_ptr<folly::Promise<std::shared_ptr<hbase::Result>>> promise_;
};
TEST(AsyncRpcRetryTest, TestGetBasic) {
@@ -172,69 +187,50 @@ TEST(AsyncRpcRetryTest, TestGetBasic) {
auto codec = std::make_shared<hbase::KeyValueCodec>();
auto rpc_client = std::make_shared<RpcClient>(io_executor_, codec);
- /* init rpc controller */
- auto controller = std::make_shared<HBaseRpcController>();
-
- /* init rpc controller factory */
- auto controller_factory = std::make_shared<MockRpcControllerFactory>();
- EXPECT_CALL((*controller_factory), NewController()).Times(1).WillRepeatedly(Return(controller));
-
/* init connection configuration */
- auto connection_conf = std::make_shared<MockAsyncConnectionConfiguration>();
- EXPECT_CALL((*connection_conf), GetPauseNs())
- .Times(1)
- .WillRepeatedly(Return(nanoseconds(100000000)));
- EXPECT_CALL((*connection_conf), GetMaxRetries()).Times(1).WillRepeatedly(Return(31));
- EXPECT_CALL((*connection_conf), GetStartLogErrorsCount()).Times(1).WillRepeatedly(Return(9));
- EXPECT_CALL((*connection_conf), GetReadRpcTimeoutNs())
- .Times(1)
- .WillRepeatedly(Return(nanoseconds(60000000000)));
- EXPECT_CALL((*connection_conf), GetOperationTimeoutNs())
- .Times(1)
- .WillRepeatedly(Return(nanoseconds(1200000000000)));
+ auto connection_conf = std::make_shared<ConnectionConfiguration>(
+ TimeUtil::SecondsToNanos(20), // connect_timeout
+ TimeUtil::SecondsToNanos(1200), // operation_timeout
+ TimeUtil::SecondsToNanos(60), // rpc_timeout
+ TimeUtil::MillisToNanos(100), // pause
+ 31, // max retries
+ 9); // start log errors count
/* init region locator */
- auto region_locator = std::make_shared<AsyncRegionLocator>(region_location);
+ auto region_locator = std::make_shared<MockAsyncRegionLocator>(region_location);
/* init hbase client connection */
- auto conn = std::make_shared<MockAsyncConnection>();
- EXPECT_CALL((*conn), get_conn_conf()).Times(AtLeast(1)).WillRepeatedly(Return(connection_conf));
- EXPECT_CALL((*conn), get_rpc_controller_factory())
- .Times(AtLeast(1))
- .WillRepeatedly(Return(controller_factory));
- EXPECT_CALL((*conn), get_locator()).Times(AtLeast(1)).WillRepeatedly(Return(region_locator));
- EXPECT_CALL((*conn), GetRpcClient()).Times(AtLeast(1)).WillRepeatedly(Return(rpc_client));
+ auto conn = std::make_shared<MockAsyncConnection>(connection_conf, rpc_client, region_locator);
+ conn->Init();
/* init retry caller factory */
auto tableImpl = std::make_shared<MockRawAsyncTableImpl<MockAsyncConnection>>(conn);
- AsyncRpcRetryingCallerFactory<MockAsyncConnection> caller_factory(conn);
/* init request caller builder */
- auto builder = caller_factory.Single<hbase::Result>();
+ auto builder = conn->caller_factory()->Single<std::shared_ptr<hbase::Result>>();
/* call with retry to get result */
try {
auto async_caller =
builder->table(std::make_shared<TableName>(tn))
->row(row)
- ->rpc_timeout(conn->get_conn_conf()->GetReadRpcTimeoutNs())
- ->operation_timeout(conn->get_conn_conf()->GetOperationTimeoutNs())
- ->action(
- [=, &get](
- std::shared_ptr<hbase::HBaseRpcController> controller,
- std::shared_ptr<hbase::RegionLocation> loc,
- std::shared_ptr<hbase::RpcClient> rpc_client) -> folly::Future<hbase::Result> {
- return tableImpl->GetCall(rpc_client, controller, loc, get);
- })
+ ->rpc_timeout(conn->connection_conf()->read_rpc_timeout())
+ ->operation_timeout(conn->connection_conf()->operation_timeout())
+ ->action([=, &get](std::shared_ptr<hbase::HBaseRpcController> controller,
+ std::shared_ptr<hbase::RegionLocation> loc,
+ std::shared_ptr<hbase::RpcClient> rpc_client)
+ -> folly::Future<std::shared_ptr<hbase::Result>> {
+ return tableImpl->GetCall(rpc_client, controller, loc, get);
+ })
->Build();
- hbase::Result result = async_caller->Call().get();
+ auto result = async_caller->Call().get();
// Test the values, should be same as in put executed on hbase shell
- ASSERT_TRUE(!result.IsEmpty()) << "Result shouldn't be empty.";
- EXPECT_EQ("test2", result.Row());
- EXPECT_EQ("value2", *(result.Value("d", "2")));
- EXPECT_EQ("value for extra", *(result.Value("d", "extra")));
+ ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty.";
+ EXPECT_EQ("test2", result->Row());
+ EXPECT_EQ("value2", *(result->Value("d", "2")));
+ EXPECT_EQ("value for extra", *(result->Value("d", "extra")));
} catch (std::exception& e) {
LOG(ERROR) << e.what();
throw e;
diff --git a/core/client-test.cc b/core/client-test.cc
index 72680c6..184a6a7 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -47,6 +47,7 @@ class ClientTest : public ::testing::Test {
const std::string xml_data) {
// Remove temp file always
boost::filesystem::remove((dir + file).c_str());
+ boost::filesystem::create_directories(dir.c_str());
WriteDataToFile((dir + file), xml_data);
}
@@ -59,6 +60,7 @@ class ClientTest : public ::testing::Test {
static std::unique_ptr<hbase::TestUtil> test_util;
static void SetUpTestCase() {
+ google::InstallFailureSignalHandler();
test_util = std::make_unique<hbase::TestUtil>();
test_util->StartMiniCluster(2);
}
diff --git a/core/client.cc b/core/client.cc
index 3f889d4..0053e19 100644
--- a/core/client.cc
+++ b/core/client.cc
@@ -22,6 +22,7 @@
#include <glog/logging.h>
#include <chrono>
#include <exception>
+#include <memory>
#include <utility>
namespace hbase {
@@ -34,53 +35,19 @@ Client::Client() {
"hbase-site.xml is absent in the search path or problems in XML parsing";
throw std::runtime_error("Configuration object not present.");
}
- init(conf.value());
+ Init(conf.value());
}
-Client::Client(const hbase::Configuration &conf) { init(conf); }
+Client::Client(const Configuration &conf) { Init(conf); }
-void Client::init(const hbase::Configuration &conf) {
- conf_ = std::make_shared<hbase::Configuration>(conf);
-
- conn_conf_ = std::make_shared<hbase::ConnectionConfiguration>(*conf_);
- // start thread pools
- auto io_threads = conf_->GetInt(kClientIoThreadPoolSize, sysconf(_SC_NPROCESSORS_ONLN));
- auto cpu_threads = conf_->GetInt(kClientCpuThreadPoolSize, 2 * sysconf(_SC_NPROCESSORS_ONLN));
- cpu_executor_ = std::make_shared<wangle::CPUThreadPoolExecutor>(cpu_threads);
- io_executor_ = std::make_shared<wangle::IOThreadPoolExecutor>(io_threads);
-
- std::shared_ptr<Codec> codec = nullptr;
- if (conf.Get(kRpcCodec, hbase::KeyValueCodec::kJavaClassName) ==
- std::string(KeyValueCodec::kJavaClassName)) {
- codec = std::make_shared<hbase::KeyValueCodec>();
- } else {
- LOG(WARNING) << "Not using RPC Cell Codec";
- }
- rpc_client_ =
- std::make_shared<hbase::RpcClient>(io_executor_, codec, conn_conf_->connect_timeout());
- location_cache_ =
- std::make_shared<hbase::LocationCache>(conf_, cpu_executor_, rpc_client_->connection_pool());
-}
-
-// We can't have the threads continue running after everything is done
-// that leads to an error.
-Client::~Client() {
- cpu_executor_->stop();
- io_executor_->stop();
- if (rpc_client_.get()) rpc_client_->Close();
-}
-
-std::unique_ptr<hbase::Table> Client::Table(const TableName &table_name) {
- return std::make_unique<hbase::Table>(table_name, location_cache_, rpc_client_, conf_);
+void Client::Init(const Configuration &conf) {
+ auto conf_ = std::make_shared<Configuration>(conf);
+ async_connection_ = AsyncConnectionImpl::Create(conf_);
}
-void Client::Close() {
- if (is_closed_) return;
-
- cpu_executor_->stop();
- io_executor_->stop();
- if (rpc_client_.get()) rpc_client_->Close();
- is_closed_ = true;
+std::unique_ptr<Table> Client::Table(const TableName &table_name) {
+ return std::make_unique<hbase::Table>(table_name, async_connection_);
}
+void Client::Close() { async_connection_->Close(); }
} // namespace hbase
diff --git a/core/client.h b/core/client.h
index 040bea0..0e11278 100644
--- a/core/client.h
+++ b/core/client.h
@@ -19,22 +19,14 @@
#pragma once
-#include <folly/futures/Future.h>
-#include <folly/io/IOBuf.h>
-#include <wangle/concurrent/CPUThreadPoolExecutor.h>
-#include <wangle/concurrent/IOThreadPoolExecutor.h>
-
#include <memory>
#include <string>
#include "connection/rpc-client.h"
+#include "core/async-connection.h"
#include "core/configuration.h"
-#include "core/connection-configuration.h"
-#include "core/hbase-configuration-loader.h"
-#include "core/keyvalue-codec.h"
-#include "core/location-cache.h"
+
#include "core/table.h"
-#include "if/Cell.pb.h"
#include "serde/table-name.h"
using hbase::pb::TableName;
@@ -55,13 +47,14 @@ class Client {
* @param quorum_spec Where to connect to get Zookeeper bootstrap information.
*/
Client();
- explicit Client(const hbase::Configuration& conf);
- ~Client();
+ explicit Client(const Configuration& conf);
+ ~Client() = default;
+
/**
* @brief Retrieve a Table implementation for accessing a table.
* @param - table_name
*/
- std::unique_ptr<hbase::Table> Table(const TableName& table_name);
+ std::unique_ptr<::hbase::Table> Table(const TableName& table_name);
/**
* @brief Close the Client connection.
@@ -69,25 +62,12 @@ class Client {
void Close();
private:
- /** Constants */
- /** Parameter name for HBase client IO thread pool size. Defaults to num cpus */
- static constexpr const char* kClientIoThreadPoolSize = "hbase.client.io.thread.pool.size";
- /** Parameter name for HBase client CPU thread pool size. Defaults to (2 * num cpus) */
- static constexpr const char* kClientCpuThreadPoolSize = "hbase.client.cpu.thread.pool.size";
- /** The RPC codec to encode cells. For now it is KeyValueCodec */
- static constexpr const char* kRpcCodec = "hbase.client.rpc.codec";
-
/** Data */
- std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
- std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_;
- std::shared_ptr<hbase::LocationCache> location_cache_;
- std::shared_ptr<hbase::RpcClient> rpc_client_;
- std::shared_ptr<hbase::Configuration> conf_;
- std::shared_ptr<hbase::ConnectionConfiguration> conn_conf_;
- bool is_closed_ = false;
+ std::shared_ptr<AsyncConnectionImpl> async_connection_;
+ private:
/** Methods */
- void init(const hbase::Configuration& conf);
+ void Init(const Configuration& conf);
};
} // namespace hbase
diff --git a/core/connection-configuration.h b/core/connection-configuration.h
index e1e9f87..c6d1d60 100644
--- a/core/connection-configuration.h
+++ b/core/connection-configuration.h
@@ -56,6 +56,20 @@ class ConnectionConfiguration {
conf.GetLong(kClientScannerMaxResultsSize, kDefaultClientScannerMaxResultsSize);
}
+ // Used by tests
+ ConnectionConfiguration(nanoseconds connect_timeout, nanoseconds operation_timeout,
+ nanoseconds rpc_timeout, nanoseconds pause, uint32_t max_retries,
+ uint32_t start_log_errors_count)
+ : connect_timeout_(connect_timeout),
+ operation_timeout_(operation_timeout),
+ meta_operation_timeout_(operation_timeout),
+ rpc_timeout_(rpc_timeout),
+ read_rpc_timeout_(rpc_timeout),
+ write_rpc_timeout_(rpc_timeout),
+ pause_(pause),
+ max_retries_(max_retries),
+ start_log_errors_count_(start_log_errors_count) {}
+
nanoseconds connect_timeout() const { return connect_timeout_; }
nanoseconds meta_operation_timeout() const { return meta_operation_timeout_; }
@@ -74,7 +88,7 @@ class ConnectionConfiguration {
// timeout for each write rpc request
nanoseconds write_rpc_timeout() const { return write_rpc_timeout_; }
- nanoseconds pause_nanos() const { return pause_; }
+ nanoseconds pause() const { return pause_; }
uint32_t max_retries() const { return max_retries_; }
diff --git a/core/location-cache-test.cc b/core/location-cache-test.cc
index 4159af4..8d1ac5f 100644
--- a/core/location-cache-test.cc
+++ b/core/location-cache-test.cc
@@ -33,6 +33,7 @@ using namespace std::chrono;
class LocationCacheTest : public ::testing::Test {
protected:
static void SetUpTestCase() {
+ google::InstallFailureSignalHandler();
test_util_ = std::make_unique<TestUtil>();
test_util_->StartMiniCluster(2);
}
diff --git a/core/location-cache.cc b/core/location-cache.cc
index 505f48c..07c3d61 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -158,7 +158,10 @@ Future<std::shared_ptr<RegionLocation>> LocationCache::LocateFromMeta(const Tabl
}
Future<shared_ptr<RegionLocation>> LocationCache::LocateRegion(const hbase::pb::TableName &tn,
- const std::string &row) {
+ const std::string &row,
+ const RegionLocateType locate_type,
+ const int64_t locate_ns) {
+ // TODO: implement region locate type and timeout
auto cached_loc = this->GetCachedLocation(tn, row);
if (cached_loc != nullptr) {
return cached_loc;
@@ -280,3 +283,8 @@ void LocationCache::ClearCachedLocation(const hbase::pb::TableName &tn, const st
unique_lock<folly::SharedMutexWritePriority> lock(locations_lock_);
table_locs->erase(row);
}
+
+void LocationCache::UpdateCachedLocation(const RegionLocation &loc, const std::exception &error) {
+ // TODO: just clears the location for now. We can inspect RegionMovedExceptions, etc later.
+ ClearCachedLocation(loc.region_info().table_name(), loc.region_info().start_key());
+}
diff --git a/core/location-cache.h b/core/location-cache.h
index c1c5f62..5e79213 100644
--- a/core/location-cache.h
+++ b/core/location-cache.h
@@ -32,6 +32,7 @@
#include <string>
#include "connection/connection-pool.h"
+#include "core/async-region-locator.h"
#include "core/configuration.h"
#include "core/meta-utils.h"
#include "core/region-location.h"
@@ -75,7 +76,7 @@ typedef std::unordered_map<hbase::pb::TableName, std::shared_ptr<PerTableLocatio
/**
* Class that can look up and cache locations.
*/
-class LocationCache {
+class LocationCache : public AsyncRegionLocator {
public:
/**
* Constructor.
@@ -127,8 +128,10 @@ class LocationCache {
* @param row of the table to look up. This object must live until after the
* future is returned
*/
- folly::Future<std::shared_ptr<RegionLocation>> LocateRegion(const hbase::pb::TableName &tn,
- const std::string &row);
+ virtual folly::Future<std::shared_ptr<RegionLocation>> LocateRegion(
+ const hbase::pb::TableName &tn, const std::string &row,
+ const RegionLocateType locate_type = RegionLocateType::kCurrent,
+ const int64_t locate_ns = 0) override;
/**
* Remove the cached location of meta.
@@ -173,6 +176,12 @@ class LocationCache {
*/
void ClearCachedLocation(const hbase::pb::TableName &tn, const std::string &row);
+ /**
+ * Update cached region location, possibly using the information from exception.
+ */
+ virtual void UpdateCachedLocation(const RegionLocation &loc,
+ const std::exception &error) override;
+
const std::string &zk_quorum() { return zk_quorum_; }
private:
diff --git a/core/meta-utils.cc b/core/meta-utils.cc
index 24a0360..119520f 100644
--- a/core/meta-utils.cc
+++ b/core/meta-utils.cc
@@ -89,7 +89,7 @@ std::unique_ptr<Request> MetaUtil::MetaRequest(const TableName tn, const std::st
}
std::shared_ptr<RegionLocation> MetaUtil::CreateLocation(const Response &resp) {
- std::vector<std::unique_ptr<Result>> results = ResponseConverter::FromScanResponse(resp);
+ std::vector<std::shared_ptr<Result>> results = ResponseConverter::FromScanResponse(resp);
if (results.size() != 1) {
throw std::runtime_error("Was expecting exactly 1 result in meta scan response, got:" +
std::to_string(results.size()));
diff --git a/core/raw-async-table.cc b/core/raw-async-table.cc
new file mode 100644
index 0000000..641f3c8
--- /dev/null
+++ b/core/raw-async-table.cc
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/raw-async-table.h"
+#include "core/request-converter.h"
+#include "core/response-converter.h"
+
+namespace hbase {
+
+template <typename RESP>
+std::shared_ptr<SingleRequestCallerBuilder<RESP>> RawAsyncTable::CreateCallerBuilder(
+ std::string row, nanoseconds rpc_timeout) {
+ return connection_->caller_factory()
+ ->Single<RESP>()
+ ->table(table_name_)
+ ->row(row)
+ ->rpc_timeout(rpc_timeout)
+ ->operation_timeout(connection_conf_->operation_timeout())
+ ->pause(connection_conf_->pause())
+ ->max_retries(connection_conf_->max_retries())
+ ->start_log_errors_count(connection_conf_->start_log_errors_count());
+}
+
+template <typename REQ, typename PREQ, typename PRESP, typename RESP>
+folly::Future<RESP> RawAsyncTable::Call(
+ std::shared_ptr<RpcClient> rpc_client, std::shared_ptr<HBaseRpcController> controller,
+ std::shared_ptr<RegionLocation> loc, const REQ& req,
+ const ReqConverter<std::unique_ptr<PREQ>, REQ, std::string>& req_converter,
+ const RespConverter<RESP, PRESP>& resp_converter) {
+ std::unique_ptr<PREQ> preq = req_converter(req, loc->region_name());
+
+ // No need to make take a callable argument, it is always the same
+ return rpc_client
+ ->AsyncCall(loc->server_name().host_name(), loc->server_name().port(), std::move(preq),
+ User::defaultUser(), "ClientService")
+ .then([&](const std::unique_ptr<Response>& presp) {
+ return ResponseConverter::FromGetResponse(*presp);
+ // return resp_converter(*presp); // TODO this is causing SEGFAULT, figure out why
+ });
+}
+
+Future<std::shared_ptr<Result>> RawAsyncTable::Get(const hbase::Get& get) {
+ auto caller =
+ CreateCallerBuilder<std::shared_ptr<Result>>(get.Row(), connection_conf_->read_rpc_timeout())
+ ->action([=, &get](std::shared_ptr<hbase::HBaseRpcController> controller,
+ std::shared_ptr<hbase::RegionLocation> loc,
+ std::shared_ptr<hbase::RpcClient> rpc_client)
+ -> folly::Future<std::shared_ptr<hbase::Result>> {
+ return Call<hbase::Get, hbase::Request, hbase::Response,
+ std::shared_ptr<hbase::Result>>(
+ rpc_client, controller, loc, get,
+ &hbase::RequestConverter::ToGetRequest,
+ &hbase::ResponseConverter::FromGetResponse);
+ })
+ ->Build();
+
+ // Return the Future we obtain from the call(). However, we do not want the Caller to go out of
+ // context and get deallocated since the caller injects a lot of closures which capture [this, &]
+ // which is use-after-free. We are just passing an identity closure capturing caller by value to
+ // ensure that the lifecycle of the Caller object is longer than the retry lambdas.
+ return caller->Call().then([caller](const auto r) { return r; });
+}
+
+} /* namespace hbase */
diff --git a/core/raw-async-table.h b/core/raw-async-table.h
new file mode 100644
index 0000000..527c7be
--- /dev/null
+++ b/core/raw-async-table.h
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <chrono>
+#include <memory>
+#include <string>
+
+#include <folly/futures/Future.h>
+
+#include "core/async-connection.h"
+#include "core/async-rpc-retrying-caller-factory.h"
+#include "core/async-rpc-retrying-caller.h"
+#include "core/connection-configuration.h"
+#include "core/get.h"
+#include "core/result.h"
+
+using folly::Future;
+using hbase::pb::TableName;
+using std::chrono::nanoseconds;
+using std::chrono::milliseconds;
+
+namespace hbase {
+
+/**
+ * A low level asynchronous table that should not be used by user applications.The implementation
+ * is required to be thread safe.
+ */
+class RawAsyncTable {
+ public:
+ RawAsyncTable(std::shared_ptr<TableName> table_name, std::shared_ptr<AsyncConnection> connection)
+ : connection_(connection),
+ connection_conf_(connection->connection_conf()),
+ table_name_(table_name),
+ rpc_client_(connection->rpc_client()) {}
+ virtual ~RawAsyncTable() = default;
+
+ Future<std::shared_ptr<Result>> Get(const hbase::Get& get);
+ void Close() {}
+
+ private:
+ /* Data */
+ std::shared_ptr<AsyncConnection> connection_;
+ std::shared_ptr<ConnectionConfiguration> connection_conf_;
+ std::shared_ptr<TableName> table_name_;
+ std::shared_ptr<RpcClient> rpc_client_;
+
+ /* Methods */
+ template <typename REQ, typename PREQ, typename PRESP, typename RESP>
+ folly::Future<RESP> Call(
+ std::shared_ptr<RpcClient> rpc_client, std::shared_ptr<HBaseRpcController> controller,
+ std::shared_ptr<RegionLocation> loc, const REQ& req,
+ const ReqConverter<std::unique_ptr<PREQ>, REQ, std::string>& req_converter,
+ const RespConverter<RESP, PRESP>& resp_converter);
+
+ template <typename RESP>
+ std::shared_ptr<SingleRequestCallerBuilder<RESP>> CreateCallerBuilder(std::string row,
+ nanoseconds rpc_timeout);
+};
+
+} // namespace hbase
diff --git a/core/response-converter.cc b/core/response-converter.cc
index 7bb5e5d..b11856c 100644
--- a/core/response-converter.cc
+++ b/core/response-converter.cc
@@ -33,14 +33,17 @@ ResponseConverter::ResponseConverter() {}
ResponseConverter::~ResponseConverter() {}
-std::unique_ptr<Result> ResponseConverter::FromGetResponse(const Response& resp) {
+// impl note: we are returning shared_ptr's instead of unique_ptr's because these
+// go inside folly::Future's, making the move semantics extremely tricky.
+std::shared_ptr<Result> ResponseConverter::FromGetResponse(const Response& resp) {
+ LOG(INFO) << "FromGetResponse";
auto get_resp = std::static_pointer_cast<GetResponse>(resp.resp_msg());
-
return ToResult(get_resp->result(), resp.cell_scanner());
}
-std::unique_ptr<Result> ResponseConverter::ToResult(
+std::shared_ptr<Result> ResponseConverter::ToResult(
const hbase::pb::Result& result, const std::unique_ptr<CellScanner>& cell_scanner) {
+ LOG(INFO) << "ToResult";
std::vector<std::shared_ptr<Cell>> vcells;
for (auto cell : result.cell()) {
std::shared_ptr<Cell> pcell =
@@ -56,16 +59,17 @@ std::unique_ptr<Result> ResponseConverter::ToResult(
}
// TODO: check associated cell count?
}
- return std::make_unique<Result>(vcells, result.exists(), result.stale(), result.partial());
+ LOG(INFO) << "Returning Result";
+ return std::make_shared<Result>(vcells, result.exists(), result.stale(), result.partial());
}
-std::vector<std::unique_ptr<Result>> ResponseConverter::FromScanResponse(const Response& resp) {
+std::vector<std::shared_ptr<Result>> ResponseConverter::FromScanResponse(const Response& resp) {
auto scan_resp = std::static_pointer_cast<ScanResponse>(resp.resp_msg());
- VLOG(3) << "FromScanResponse:" << scan_resp->ShortDebugString();
+ LOG(INFO) << "FromScanResponse:" << scan_resp->ShortDebugString();
int num_results = resp.cell_scanner() != nullptr ? scan_resp->cells_per_result_size()
: scan_resp->results_size();
- std::vector<std::unique_ptr<Result>> results{static_cast<size_t>(num_results)};
+ std::vector<std::shared_ptr<Result>> results{static_cast<size_t>(num_results)};
for (int i = 0; i < num_results; i++) {
if (resp.cell_scanner() != nullptr) {
// Cells are out in cellblocks. Group them up again as Results. How many to read at a
@@ -86,7 +90,7 @@ std::vector<std::unique_ptr<Result>> ResponseConverter::FromScanResponse(const R
throw std::runtime_error(msg);
}
// TODO: handle partial results per Result by checking partial_flag_per_result
- results[i] = std::make_unique<Result>(vcells, false, scan_resp->stale(), false);
+ results[i] = std::make_shared<Result>(vcells, false, scan_resp->stale(), false);
} else {
results[i] = ToResult(scan_resp->results(i), resp.cell_scanner());
}
diff --git a/core/response-converter.h b/core/response-converter.h
index 759b1ce..743c14b 100644
--- a/core/response-converter.h
+++ b/core/response-converter.h
@@ -36,16 +36,16 @@ class ResponseConverter {
public:
~ResponseConverter();
- static std::unique_ptr<Result> ToResult(const hbase::pb::Result& result,
+ static std::shared_ptr<Result> ToResult(const hbase::pb::Result& result,
const std::unique_ptr<CellScanner>& cell_scanner);
/**
* @brief Returns a Result object created by PB Message in passed Response object.
* @param resp - Response object having the PB message.
*/
- static std::unique_ptr<hbase::Result> FromGetResponse(const Response& resp);
+ static std::shared_ptr<hbase::Result> FromGetResponse(const Response& resp);
- static std::vector<std::unique_ptr<Result>> FromScanResponse(const Response& resp);
+ static std::vector<std::shared_ptr<Result>> FromScanResponse(const Response& resp);
private:
// Constructor not required. We have all static methods to extract response from PB messages.
diff --git a/core/table.cc b/core/table.cc
index 2ce8fcd..3c54d78 100644
--- a/core/table.cc
+++ b/core/table.cc
@@ -25,11 +25,13 @@
#include <utility>
#include <vector>
+#include "core/async-connection.h"
#include "core/request-converter.h"
#include "core/response-converter.h"
#include "if/Client.pb.h"
#include "security/user.h"
#include "serde/server-name.h"
+#include "utils/time-util.h"
using folly::Future;
using hbase::pb::TableName;
@@ -38,41 +40,28 @@ using std::chrono::milliseconds;
namespace hbase {
-Table::Table(const TableName &table_name,
- const std::shared_ptr<hbase::LocationCache> &location_cache,
- const std::shared_ptr<hbase::RpcClient> &rpc_client,
- const std::shared_ptr<hbase::Configuration> &conf)
+Table::Table(const TableName &table_name, std::shared_ptr<AsyncConnection> async_connection)
: table_name_(std::make_shared<TableName>(table_name)),
- location_cache_(location_cache),
- rpc_client_(rpc_client),
- conf_(conf) {
- client_retries_ = (conf_) ? conf_->GetInt("hbase.client.retries", client_retries_) : 5;
+ async_connection_(async_connection),
+ conf_(async_connection->conf()) {
+ async_table_ = std::make_unique<RawAsyncTable>(table_name_, async_connection);
}
Table::~Table() {}
-std::unique_ptr<hbase::Result> Table::Get(const hbase::Get &get) {
- auto loc = location_cache_->LocateFromMeta(*table_name_, get.Row()).get(milliseconds(1000));
- auto req = hbase::RequestConverter::ToGetRequest(get, loc->region_name());
- auto user = User::defaultUser(); // TODO: make User::current() similar to UserUtil
-
- Future<std::unique_ptr<Response>> f =
- rpc_client_->AsyncCall(loc->server_name().host_name(), loc->server_name().port(),
- std::move(req), user, "ClientService");
- auto resp = f.get();
-
- return hbase::ResponseConverter::FromGetResponse(*resp);
+std::shared_ptr<hbase::Result> Table::Get(const hbase::Get &get) {
+ auto context = async_table_->Get(get);
+ return context.get(operation_timeout());
}
-void Table::Close() {
- if (is_closed_) return;
-
- if (rpc_client_.get()) rpc_client_->Close();
- is_closed_ = true;
+milliseconds Table::operation_timeout() const {
+ return TimeUtil::ToMillis(async_connection_->connection_conf()->operation_timeout());
}
+void Table::Close() { async_table_->Close(); }
+
std::shared_ptr<RegionLocation> Table::GetRegionLocation(const std::string &row) {
- return location_cache_->LocateRegion(*table_name_, row).get();
+ return async_connection_->region_locator()->LocateRegion(*table_name_, row).get();
}
} /* namespace hbase */
diff --git a/core/table.h b/core/table.h
index f82382e..93ab91b 100644
--- a/core/table.h
+++ b/core/table.h
@@ -19,15 +19,18 @@
#pragma once
+#include <chrono>
#include <memory>
#include <string>
#include <vector>
#include "connection/rpc-client.h"
+#include "core/async-connection.h"
#include "core/client.h"
#include "core/configuration.h"
#include "core/get.h"
#include "core/location-cache.h"
+#include "core/raw-async-table.h"
#include "core/result.h"
#include "serde/table-name.h"
@@ -41,16 +44,17 @@ class Table {
/**
* Constructors
*/
- Table(const TableName &table_name, const std::shared_ptr<hbase::LocationCache> &location_cache,
- const std::shared_ptr<hbase::RpcClient> &rpc_client,
- const std::shared_ptr<hbase::Configuration> &conf);
+ Table(const TableName &table_name, std::shared_ptr<AsyncConnection> async_connection);
~Table();
/**
* @brief - Returns a Result object for the constructed Get.
* @param - get Get object to perform HBase Get operation.
*/
- std::unique_ptr<hbase::Result> Get(const hbase::Get &get);
+ std::shared_ptr<hbase::Result> Get(const hbase::Get &get);
+
+ // TODO: next jira
+ // std::vector<std::unique_ptr<hbase::Result>> Get(const std::vector<hbase::Get> &gets);
/**
* @brief - Close the client connection.
@@ -64,11 +68,11 @@ class Table {
private:
std::shared_ptr<TableName> table_name_;
- std::shared_ptr<hbase::LocationCache> location_cache_;
- std::shared_ptr<hbase::RpcClient> rpc_client_;
+ std::shared_ptr<AsyncConnection> async_connection_;
std::shared_ptr<hbase::Configuration> conf_;
- bool is_closed_ = false;
- // default 5 retries. over-ridden in constructor.
- int client_retries_ = 5;
+ std::unique_ptr<RawAsyncTable> async_table_;
+
+ private:
+ milliseconds operation_timeout() const;
};
} /* namespace hbase */
diff --git a/security/user.h b/security/user.h
index 372ab44..035af31 100644
--- a/security/user.h
+++ b/security/user.h
@@ -37,6 +37,7 @@ class User {
static bool IsSecurityEnabled(const Configuration& conf) {
return conf.Get("hbase.security.authentication", "").compare(kKerberos) == 0;
}
+
private:
std::string user_name_;
};
diff --git a/utils/time-util.h b/utils/time-util.h
index bbc3b35..183260b 100644
--- a/utils/time-util.h
+++ b/utils/time-util.h
@@ -23,28 +23,45 @@
#include <string>
using std::chrono::nanoseconds;
using std::chrono::milliseconds;
+using std::chrono::seconds;
namespace hbase {
class TimeUtil {
public:
- static int64_t ToMillis(const int64_t& nanos) {
+ static inline int64_t ToMillis(const int64_t& nanos) {
return std::chrono::duration_cast<milliseconds>(nanoseconds(nanos)).count();
}
- static std::string ToMillisStr(const nanoseconds& nanos) {
+ static inline milliseconds ToMillis(const nanoseconds& nanos) {
+ return std::chrono::duration_cast<milliseconds>(nanoseconds(nanos));
+ }
+
+ static inline nanoseconds ToNanos(const milliseconds& millis) {
+ return std::chrono::duration_cast<nanoseconds>(millis);
+ }
+
+ static inline nanoseconds MillisToNanos(const int64_t& millis) {
+ return std::chrono::duration_cast<nanoseconds>(milliseconds(millis));
+ }
+
+ static inline nanoseconds SecondsToNanos(const int64_t& secs) {
+ return std::chrono::duration_cast<nanoseconds>(seconds(secs));
+ }
+
+ static inline std::string ToMillisStr(const nanoseconds& nanos) {
return std::to_string(std::chrono::duration_cast<milliseconds>(nanos).count());
}
- static int64_t GetNowNanos() {
+ static inline int64_t GetNowNanos() {
auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
return std::chrono::duration_cast<nanoseconds>(duration).count();
}
- static int64_t ElapsedMillis(const int64_t& start_ns) {
+ static inline int64_t ElapsedMillis(const int64_t& start_ns) {
return std::chrono::duration_cast<milliseconds>(nanoseconds(GetNowNanos() - start_ns)).count();
}
- static std::string ElapsedMillisStr(const int64_t& start_ns) {
+ static inline std::string ElapsedMillisStr(const int64_t& start_ns) {
return std::to_string(
std::chrono::duration_cast<milliseconds>(nanoseconds(GetNowNanos() - start_ns)).count());
}
[hbase] 08/137: HBASE-14854 Read meta location from zk
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 66e226de0ac64ed29f9e74f4f06db56d85768140
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Sat Mar 5 00:09:08 2016 -0800
HBASE-14854 Read meta location from zk
---
Dockerfile | 20 +--
bin/start-docker.sh | 5 +-
core/BUCK | 106 +++++++++-------
core/location-cache-test.cc | 14 +++
core/location-cache.cc | 67 ++++++++++
core/location-cache.h | 35 ++++++
...eClientTestEnv.cc => native-client-test-env.cc} | 4 +-
...eClientTest.cc => simple-native-client-test.cc} | 0
core/test_env.h | 2 +
if/BUCK | 1 +
third-party/BUCK | 138 +++++++++++----------
11 files changed, 271 insertions(+), 121 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 1364d22..36959a5 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM cpp_update
+FROM pjameson/buck-folly-watchman
ARG CC=/usr/bin/gcc-5
ARG CXX=/usr/bin/g++-5
@@ -25,20 +25,26 @@ ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 -p
RUN apt-get install -y clang-format-3.7 vim maven inetutils-ping
RUN git clone --depth 1 --branch v2.6.1 https://github.com/google/protobuf.git /usr/src/protobuf && \
cd /usr/src/protobuf/ && \
+ ldconfig && \
./autogen.sh && \
- ./configure --disable-shared && \
+ ./configure && \
make && \
- make check && \
- make install
+ make install && \
+ make clean && \
+ rm -rf .git
+
RUN cd /usr/src && \
wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && \
tar zxf zookeeper-3.4.8.tar.gz && \
rm -rf zookeeper-3.4.8.tar.gz && \
cd zookeeper-3.4.8 && \
cd src/c && \
- ./configure --disable-shared && \
+ ldconfig && \
+ ./configure && \
make && \
make install && \
- make clean
+ make clean
+
+RUN ldconfig
-WORKDIR /usr/local/src/hbase/hbase-native-client
+WORKDIR /usr/src/hbase/hbase-native-client
diff --git a/bin/start-docker.sh b/bin/start-docker.sh
index 4426705..725ed6a 100755
--- a/bin/start-docker.sh
+++ b/bin/start-docker.sh
@@ -19,8 +19,11 @@
set -e
set -x
+# Try out some standard docker machine names that could work
eval "$(docker-machine env docker-vm)"
eval "$(docker-machine env dinghy)"
+
+# Build the image
docker build -t hbase_native .
@@ -36,6 +39,6 @@ fi;
docker run -p 16010:16010/tcp \
-e "JAVA_HOME=/usr/lib/jvm/java-8-oracle" \
- -v ${PWD}/..:/usr/local/src/hbase \
+ -v ${PWD}/..:/usr/src/hbase \
-v ~/.m2:/root/.m2 \
-it hbase_native /bin/bash
diff --git a/core/BUCK b/core/BUCK
index ef027a1..817b5a0 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -15,52 +15,62 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+cxx_library(name="core",
+ headers=[
+ "admin.h",
+ "client.h",
+ "connection.h",
+ "connection_attr.h",
+ "delete.h",
+ "get.h",
+ "hbase_macros.h",
+ "mutation.h",
+ "put.h",
+ "scanner.h",
+ "location-cache.h",
+ ],
+ srcs=[
+ "admin.cc",
+ "client.cc",
+ "connection.cc",
+ "get.cc",
+ "mutation.cc",
+ "put.cc",
+ "delete.cc",
+ "scanner.cc",
+ "location-cache.cc",
+ ],
+ deps=[
+ "//if:if",
+ "//third-party:zookeeper_mt",
+ "//third-party:folly",
+ "//third-party:wangle",
+ ],
+ visibility=[
+ 'PUBLIC',
+ ], )
-cxx_binary(
- name = "core",
- headers = [
- "admin.h",
- "client.h",
- "connection.h",
- "connection_attr.h",
- "delete.h",
- "get.h",
- "hbase_macros.h",
- "mutation.h",
- "put.h",
- "scanner.h",
- ],
- srcs = [
- "admin.cc",
- "client.cc",
- "connection.cc",
- "get.cc",
- "mutation.cc",
- "put.cc",
- "delete.cc",
- "scanner.cc",
- ],
- deps = [
- "//if:if",
- "//third-party:folly",
- "//third-party:wangle",
- ],
- visibility = [
- 'PUBLIC',
- ],
-)
-
-cxx_test(
- name = "core_test",
- headers = [
- "test_env.h",
- ],
- srcs = [
- "HBaseNativeClientTestEnv.cc",
- "SampleNativeClientTest.cc",
- ],
- deps = [
- ":core",
- ],
- run_test_separately = True,
-)
+cxx_test(name="simple-test",
+ headers=[
+ "test_env.h",
+ ],
+ srcs=[
+ "native-client-test-env.cc",
+ "simple-native-client-test.cc",
+ ],
+ deps=[
+ ":core",
+ ],
+ run_test_separately=True, )
+cxx_test(name="location-cache-test",
+ headers=[
+ "test_env.h",
+ ],
+ srcs=[
+ "native-client-test-env.cc",
+ "location-cache-test.cc",
+ ],
+ deps=[
+ ":core",
+ ],
+ run_test_separately=True, )
diff --git a/core/location-cache-test.cc b/core/location-cache-test.cc
new file mode 100644
index 0000000..3106e36
--- /dev/null
+++ b/core/location-cache-test.cc
@@ -0,0 +1,14 @@
+#include <gtest/gtest.h>
+#include <folly/Memory.h>
+#include <wangle/concurrent/GlobalExecutor.h>
+
+#include "location-cache.h"
+using namespace hbase;
+
+TEST(LocationCacheTest, TestGetMetaNodeContents) {
+ // TODO(elliott): need to make a test utility for this.
+ LocationCache cache{"localhost:2181", wangle::getCPUExecutor()};
+ auto result = cache.LocateMeta();
+ result.wait();
+ ASSERT_FALSE(result.hasException());
+}
diff --git a/core/location-cache.cc b/core/location-cache.cc
new file mode 100644
index 0000000..cf61e24
--- /dev/null
+++ b/core/location-cache.cc
@@ -0,0 +1,67 @@
+#include "location-cache.h"
+
+#include <folly/Logging.h>
+
+#include "if/ZooKeeper.pb.h"
+
+using namespace std;
+using namespace folly;
+using namespace hbase::pb;
+
+namespace hbase {
+
+// TODO(elliott): make this configurable on client creation
+const static string META_LOCATION = "/hbase/meta-region-server";
+
+LocationCache::LocationCache(string quorum_spec,
+ shared_ptr<folly::Executor> executor)
+ : quorum_spec_(quorum_spec), executor_(executor), meta_promise_(nullptr) {
+ zk_ = zookeeper_init(quorum_spec.c_str(), nullptr, 1000, 0, 0, 0);
+}
+
+LocationCache::~LocationCache() {
+ zookeeper_close(zk_);
+ zk_ = nullptr;
+ LOG(INFO) << "Closed connection to ZooKeeper.";
+}
+
+Future<ServerName> LocationCache::LocateMeta() {
+ lock_guard<mutex> g(meta_lock_);
+ if (meta_promise_ == nullptr) {
+ this->RefreshMetaLocation();
+ }
+ return meta_promise_->getFuture();
+}
+
+void LocationCache::InvalidateMeta() {
+ if (meta_promise_ != nullptr) {
+ lock_guard<mutex> g(meta_lock_);
+ meta_promise_ = nullptr;
+ }
+}
+
+/// MUST hold the meta_lock_
+void LocationCache::RefreshMetaLocation() {
+ meta_promise_ = make_unique<SharedPromise<ServerName>>();
+ executor_->add([&] {
+ meta_promise_->setWith([&] { return this->ReadMetaLocation(); });
+ });
+}
+
+ServerName LocationCache::ReadMetaLocation() {
+ char contents[4096];
+ int len = sizeof(contents);
+ // TODO(elliott): handle disconnects/reconntion as needed.
+ int zk_result =
+ zoo_get(this->zk_, META_LOCATION.c_str(), 0, contents, &len, nullptr);
+
+ if (zk_result != ZOK) {
+ LOG(ERROR) << "Error getting meta location.";
+ throw runtime_error("Error getting meta location");
+ }
+
+ MetaRegionServer mrs;
+ mrs.ParseFromArray(contents, len);
+ return mrs.server();
+}
+}
diff --git a/core/location-cache.h b/core/location-cache.h
new file mode 100644
index 0000000..8dc2760
--- /dev/null
+++ b/core/location-cache.h
@@ -0,0 +1,35 @@
+#pragma once
+
+#include <memory>
+#include <mutex>
+
+#include <zookeeper/zookeeper.h>
+#include <folly/futures/Future.h>
+#include <folly/futures/SharedPromise.h>
+
+#include <folly/Executor.h>
+#include "if/HBase.pb.h"
+
+namespace hbase {
+class LocationCache {
+public:
+ explicit LocationCache(std::string quorum_spec,
+ std::shared_ptr<folly::Executor> executor);
+ ~LocationCache();
+ // Meta Related Methods.
+ // These are only public until testing is complete
+ folly::Future<hbase::pb::ServerName> LocateMeta();
+ void InvalidateMeta();
+
+private:
+ void RefreshMetaLocation();
+ hbase::pb::ServerName ReadMetaLocation();
+
+ std::string quorum_spec_;
+ std::shared_ptr<folly::Executor> executor_;
+ std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> meta_promise_;
+ std::mutex meta_lock_;
+
+ zhandle_t *zk_;
+};
+} // hbase
diff --git a/core/HBaseNativeClientTestEnv.cc b/core/native-client-test-env.cc
similarity index 89%
rename from core/HBaseNativeClientTestEnv.cc
rename to core/native-client-test-env.cc
index b8cb8db..a86961f 100644
--- a/core/HBaseNativeClientTestEnv.cc
+++ b/core/native-client-test-env.cc
@@ -22,7 +22,7 @@
namespace {
-class HBaseNativeClientTestEnv : public ::testing::Environment {
+class NativeClientTestEnv : public ::testing::Environment {
public:
void SetUp() override {
init_test_env();
@@ -37,6 +37,6 @@ class HBaseNativeClientTestEnv : public ::testing::Environment {
int main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
- ::testing::AddGlobalTestEnvironment(new HBaseNativeClientTestEnv());
+ ::testing::AddGlobalTestEnvironment(new NativeClientTestEnv());
return RUN_ALL_TESTS();
}
diff --git a/core/SampleNativeClientTest.cc b/core/simple-native-client-test.cc
similarity index 100%
rename from core/SampleNativeClientTest.cc
rename to core/simple-native-client-test.cc
diff --git a/core/test_env.h b/core/test_env.h
index 5796ae1..79bdbec 100644
--- a/core/test_env.h
+++ b/core/test_env.h
@@ -17,6 +17,8 @@
*
*/
+#pragma once
+
#include <cstdlib>
inline void init_test_env() {
diff --git a/if/BUCK b/if/BUCK
index 3490a05..9b989b5 100644
--- a/if/BUCK
+++ b/if/BUCK
@@ -33,4 +33,5 @@ cxx_library(
+ [':' + x for x in CC_FILENAMES]
+ [ ':' + x for x in HEADER_FILENAMES ],
visibility = [ 'PUBLIC', ],
+ exported_deps = ['//third-party:protobuf']
)
diff --git a/third-party/BUCK b/third-party/BUCK
index e577a5f..6548695 100644
--- a/third-party/BUCK
+++ b/third-party/BUCK
@@ -15,85 +15,97 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-def add_system_libs(names = [], lib_dir = "/usr/lib/x86_64-linux-gnu", deps = [], exported_linker_flags = []):
- rules = []
- for name in names:
- gen_rule_name = "gen_lib{}".format(name)
- genrule(
- name = gen_rule_name,
- out = gen_rule_name,
- bash = "mkdir -p $OUT && cp {}/lib{}.* $OUT".format(lib_dir,name),
- )
- prebuilt_cxx_library(
- name = name,
- lib_name = name,
- lib_dir = '$(location :{})'.format(gen_rule_name),
- force_static = True,
- deps = deps,
- visibility = [ 'PUBLIC' ],
- exported_linker_flags = exported_linker_flags,
- )
- rules.append(":" + name)
- return rules
-system_libs = [
- "unwind",
- "lzma",
-]
+def add_system_libs(names=[],
+ lib_dir="/usr/lib/x86_64-linux-gnu",
+ deps=[],
+ exported_deps=[],
+ exported_linker_flags=[]):
+ rules = []
+ for name in names:
+ gen_rule_name = "gen_lib{}".format(name)
+ genrule(name=gen_rule_name,
+ out=gen_rule_name,
+ bash="mkdir -p $OUT && cp {}/lib{}.a $OUT".format(lib_dir,
+ name), )
+ prebuilt_cxx_library(name=name,
+ lib_name=name,
+ lib_dir='$(location :{})'.format(gen_rule_name),
+ deps=deps,
+ force_static = True,
+ exported_deps=exported_deps,
+ visibility=['PUBLIC'],
+ exported_linker_flags=exported_linker_flags, )
+ rules.append(":" + name)
+ return rules
+
+
+system_libs = ["unwind", "lzma", "event", ]
local_libs = [
- "double-conversion",
- "glog",
- "gflags",
- "protobuf",
- "zookeeper_mt",
- "boost_regex",
+ "double-conversion",
+ "boost_regex",
+ "boost_context",
+ "boost_thread",
+ "boost_system",
+ "boost_filesystem",
+ "boost_program_options",
+ "boost_chrono",
+ "gflags",
+ "glog",
+ "protobuf",
]
+
+
tp_dep_rules = add_system_libs(system_libs) \
- + add_system_libs(local_libs, lib_dir = "/usr/local/lib")
-folly = add_system_libs(
- ['folly'],
- lib_dir = '/usr/local/lib',
- deps = tp_dep_rules,
- exported_linker_flags = [ "-pthread", "-lstdc++",]
- )
-folly_bench = add_system_libs(
- ['follybenchmark'],
- lib_dir = '/usr/local/lib',
- deps = tp_dep_rules + folly,
- exported_linker_flags = [ "-pthread", "-lstdc++",]
- )
-wangle = add_system_libs(
- ['wangle'],
- lib_dir = '/usr/local/lib',
- deps = tp_dep_rules + folly,
- exported_linker_flags = [ "-pthread", "-lstdc++",]
- )
+ + add_system_libs(local_libs, lib_dir = "/usr/local/lib")
+
+zookeeper = add_system_libs(["zookeeper_mt"], lib_dir = "/usr/local/lib")
+folly = add_system_libs(['folly'],
+ lib_dir='/usr/local/lib',
+ exported_deps=tp_dep_rules,
+ exported_linker_flags=["-pthread",
+ "-lstdc++", ])
+folly_bench = add_system_libs(['follybenchmark'],
+ lib_dir='/usr/local/lib',
+ exported_deps=tp_dep_rules + folly,
+ exported_linker_flags=["-pthread",
+ "-lstdc++", ])
+wangle = add_system_libs(['wangle'],
+ lib_dir='/usr/local/lib',
+ exported_deps=tp_dep_rules + folly,
+ exported_linker_flags=["-pthread",
+ "-lstdc++", ])
+
+
+genrule(
+name = "gen_zk",
+out = "gen_zk",
+bash = "mkdir -p $OUT && wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && tar zxf zookeeper-3.4.8.tar.gz && rm -rf zookeeper-3.4.8.tar.gz && cd zookeeper-3.4.8 && cd src/c && ./configure --prefix=$OUT && make && make install && cd $OUT && rm -rf zookeeper-3.4.8*"
+)
cxx_library(
- name = 'google-test',
- srcs = [
+ name = 'google-test',
+ srcs = [
'googletest/googletest/src/gtest-all.cc',
'googletest/googlemock/src/gmock-all.cc',
'googletest/googlemock/src/gmock_main.cc',
- ],
- header_namespace = '',
- exported_headers = subdir_glob([
+ ],
+ header_namespace = '',
+ exported_headers = subdir_glob([
('googletest/googletest/include', '**/*.h'),
('googletest/googlemock/include', '**/*.h'),
- ]),
- headers = subdir_glob([
+ ]),
+ headers = subdir_glob([
('googletest/googletest', 'src/*.h'),
('googletest/googletest', 'src/*.cc'),
('googletest/googlemock', 'src/*.h'),
('googletest/googlemock', 'src/*.cc'),
- ]),
- exported_linker_flags = [
+ ]),
+ exported_linker_flags = [
"-pthread",
"-lstdc++",
- ],
- visibility = [
+ ],
+ visibility = [
'PUBLIC',
- ],
- deps = [
- ]
+ ],
)
[hbase] 10/137: HBASE-15604 Add a good readme on the build.
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit f68267e03bc511b86ed9561b1f7b9e5da345e792
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Fri Apr 8 13:44:45 2016 -0700
HBASE-15604 Add a good readme on the build.
---
BUILDING.md | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 63 insertions(+)
diff --git a/BUILDING.md b/BUILDING.md
new file mode 100644
index 0000000..20ef2a0
--- /dev/null
+++ b/BUILDING.md
@@ -0,0 +1,63 @@
+<!---
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+#Building HBase native client
+
+The HBase native client build using buck and produces a linux library.
+
+
+# Dependencies
+
+The easiest way to build hbase-native-client is to
+use [Docker](https://www.docker.com/). This will mean that any platform
+with docker can be used to build the hbase-native-client. It will also
+mean that you're building with the same versions of all dependencies that
+the client is tested with.
+
+On OSX the current boot2docker solution will work however it's pretty
+slow. If things get too slow using [dinghy](https://github.com/codekitchen/dinghy)
+can help speed things up by using nfs.
+
+If possible pairing virtual box with dinghy will result in the fastest,
+most stable docker environment. However none of it is needed.
+
+# Building using docker
+
+To start with make sure that you have built the java project using
+`mvn package -DskipTests`. That will allow all tests to spin up a standalone
+hbase instance from the jar's created.
+
+Then go into the hbase-native-client directory and run `./bin/start-docker.sh`
+that will build the docker development environment and when complete will
+drop you into a shell on a linux vm with all the tools needed installed.
+
+# Buck
+
+From then on we can use [buck](https://buckbuild.com/) to build everything.
+For example:
+```
+buck build //core:core
+buck test --all
+buck build //core:simple-client
+```
+
+That will build the library, then build and test everything, then build
+the simple-client binary. Buck will find all modules used, and compile
+them in parallel, caching the results. Output from buck is in the buck-out
+foulder. Generated binaries are in buck-out/gen logs are in buck-out/logs
[hbase] 92/137: HBASE-18126 Increment class - addendum creates
TestToStringToInt64 in bytes-util-test
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 39f4b71b2167375ecc2f6b9d934adbd5b0e79355
Author: tedyu <yu...@gmail.com>
AuthorDate: Tue Jun 6 11:32:36 2017 -0700
HBASE-18126 Increment class - addendum creates TestToStringToInt64 in bytes-util-test
---
utils/bytes-util-test.cc | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/utils/bytes-util-test.cc b/utils/bytes-util-test.cc
index 676b441..ca64a21 100644
--- a/utils/bytes-util-test.cc
+++ b/utils/bytes-util-test.cc
@@ -46,7 +46,8 @@ TEST(TestBytesUtil, TestToStringBinary) {
EXPECT_EQ("foo_\\x00\\xFF_bar",
BytesUtil::ToStringBinary("foo_" + std::string{zero} + std::string{max} + "_bar"));
-
+}
+TEST(TestBytesUtil, TestToStringToInt64) {
int64_t num = 761235;
EXPECT_EQ(num, BytesUtil::ToInt64(BytesUtil::ToString(num)));
[hbase] 07/137: HBASE-15401 Add Zookeeper to third party
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit de7a347444e2dc3dab2aaf4622262cda1785de36
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Mon Mar 7 14:58:21 2016 -0800
HBASE-15401 Add Zookeeper to third party
---
Dockerfile | 26 +++++++++++++--
third-party/BUCK | 96 ++++++++++++++++++++++++++------------------------------
2 files changed, 68 insertions(+), 54 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 5f17f04..1364d22 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -15,8 +15,30 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM pjameson/buck-folly-watchman
+FROM cpp_update
-RUN apt-get install -y libprotobuf-dev protobuf-compiler clang-format-3.7 vim maven inetutils-ping
+ARG CC=/usr/bin/gcc-5
+ARG CXX=/usr/bin/g++-5
+ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 -pthread"
+ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 -pthread"
+
+RUN apt-get install -y clang-format-3.7 vim maven inetutils-ping
+RUN git clone --depth 1 --branch v2.6.1 https://github.com/google/protobuf.git /usr/src/protobuf && \
+ cd /usr/src/protobuf/ && \
+ ./autogen.sh && \
+ ./configure --disable-shared && \
+ make && \
+ make check && \
+ make install
+RUN cd /usr/src && \
+ wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && \
+ tar zxf zookeeper-3.4.8.tar.gz && \
+ rm -rf zookeeper-3.4.8.tar.gz && \
+ cd zookeeper-3.4.8 && \
+ cd src/c && \
+ ./configure --disable-shared && \
+ make && \
+ make install && \
+ make clean
WORKDIR /usr/local/src/hbase/hbase-native-client
diff --git a/third-party/BUCK b/third-party/BUCK
index b7baa86..e577a5f 100644
--- a/third-party/BUCK
+++ b/third-party/BUCK
@@ -15,68 +15,60 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-def add_system_libs(names = []):
+def add_system_libs(names = [], lib_dir = "/usr/lib/x86_64-linux-gnu", deps = [], exported_linker_flags = []):
rules = []
for name in names:
- prebuilt_cxx_library(
- name = name,
- lib_name = name,
- lib_dir = "/usr/lib/x86_64-linux-gnu",
- visibility = [ 'PUBLIC', ],
- )
- rules.append(":" + name)
-
+ gen_rule_name = "gen_lib{}".format(name)
+ genrule(
+ name = gen_rule_name,
+ out = gen_rule_name,
+ bash = "mkdir -p $OUT && cp {}/lib{}.* $OUT".format(lib_dir,name),
+ )
+ prebuilt_cxx_library(
+ name = name,
+ lib_name = name,
+ lib_dir = '$(location :{})'.format(gen_rule_name),
+ force_static = True,
+ deps = deps,
+ visibility = [ 'PUBLIC' ],
+ exported_linker_flags = exported_linker_flags,
+ )
+ rules.append(":" + name)
return rules
system_libs = [
+ "unwind",
+ "lzma",
+]
+local_libs = [
"double-conversion",
"glog",
- "protobuf",
"gflags",
- "unwind",
- "lzma",
+ "protobuf",
+ "zookeeper_mt",
"boost_regex",
]
-tp_dep_rules = add_system_libs(system_libs)
-prebuilt_cxx_library(
- name = "folly",
- lib_name = "folly",
- lib_dir = "/usr/local/lib",
- deps = tp_dep_rules,
- exported_linker_flags = [
- "-pthread",
- "-lstdc++",
- ],
- visibility = [
- 'PUBLIC',
- ]
-)
-prebuilt_cxx_library(
- name = "follybenchmark",
- lib_name = "follybenchmark",
- lib_dir = "/usr/local/lib",
- deps = tp_dep_rules + [":folly"],
- exported_linker_flags = [
- "-pthread",
- "-lstdc++",
- ],
- visibility = [
- 'PUBLIC',
- ]
-)
-prebuilt_cxx_library(
- name = "wangle",
- lib_name = "wangle",
- lib_dir = "/usr/local/lib",
- deps = tp_dep_rules + [":folly"],
- exported_linker_flags = [
- "-pthread",
- "-lstdc++",
- ],
- visibility = [
- 'PUBLIC',
- ]
-)
+
+tp_dep_rules = add_system_libs(system_libs) \
+ + add_system_libs(local_libs, lib_dir = "/usr/local/lib")
+folly = add_system_libs(
+ ['folly'],
+ lib_dir = '/usr/local/lib',
+ deps = tp_dep_rules,
+ exported_linker_flags = [ "-pthread", "-lstdc++",]
+ )
+folly_bench = add_system_libs(
+ ['follybenchmark'],
+ lib_dir = '/usr/local/lib',
+ deps = tp_dep_rules + folly,
+ exported_linker_flags = [ "-pthread", "-lstdc++",]
+ )
+wangle = add_system_libs(
+ ['wangle'],
+ lib_dir = '/usr/local/lib',
+ deps = tp_dep_rules + folly,
+ exported_linker_flags = [ "-pthread", "-lstdc++",]
+ )
cxx_library(
name = 'google-test',
srcs = [
[hbase] 115/137: HBASE-18564 [C++] Problems compiling with GCC
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 4bc225d6ee32a5fc53ad0f75fd5cec71c6d01471
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Fri Aug 11 10:47:56 2017 -0700
HBASE-18564 [C++] Problems compiling with GCC
---
Makefile | 2 +-
core/async-batch-rpc-retrying-test.cc | 1 -
core/async-rpc-retrying-test.cc | 1 -
core/client-test.cc | 27 ++++++++++++++-------------
core/filter-test.cc | 2 --
core/hbase-configuration-test.cc | 35 ++++++++++++++++++-----------------
core/location-cache-retry-test.cc | 2 +-
core/result-test.cc | 3 ++-
8 files changed, 36 insertions(+), 37 deletions(-)
diff --git a/Makefile b/Makefile
index 143c00c..708e907 100644
--- a/Makefile
+++ b/Makefile
@@ -38,7 +38,7 @@ LINKFLAG := -shared
#define list of source files and object files
ALLSRC := $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc))
-EXCLUDE_SRC := $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*-test.cc)) core/simple-client.cc
+EXCLUDE_SRC := $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*-test.cc)) core/simple-client.cc core/load-client.cc
SRC := $(filter-out $(EXCLUDE_SRC), $(ALLSRC))
PROTOSRC := $(patsubst %.proto, $(addprefix $(BUILD_PATH)/,%.pb.cc),$(wildcard if/*.proto))
PROTOHDR := $(patsubst %.proto, $(addprefix $(BUILD_PATH)/,%.pb.h),$(wildcard if/*.proto))
diff --git a/core/async-batch-rpc-retrying-test.cc b/core/async-batch-rpc-retrying-test.cc
index c186276..0d186b4 100644
--- a/core/async-batch-rpc-retrying-test.cc
+++ b/core/async-batch-rpc-retrying-test.cc
@@ -296,7 +296,6 @@ void runMultiTest(std::shared_ptr<AsyncRegionLocatorBase> region_locator,
// Get connection to HBase Table
auto table = client.Table(tn);
- ASSERT_TRUE(table) << "Unable to get connection to Table.";
for (uint64_t i = 0; i < num_rows; i++) {
table->Put(Put{"test" + std::to_string(i)}.AddColumn("d", std::to_string(i),
diff --git a/core/async-rpc-retrying-test.cc b/core/async-rpc-retrying-test.cc
index f887815..95b7143 100644
--- a/core/async-rpc-retrying-test.cc
+++ b/core/async-rpc-retrying-test.cc
@@ -304,7 +304,6 @@ void runTest(std::shared_ptr<AsyncRegionLocatorBase> region_locator, std::string
// Get connection to HBase Table
auto table = client.Table(tn);
- ASSERT_TRUE(table) << "Unable to get connection to Table.";
table->Put(Put{"test2"}.AddColumn("d", "2", "value2"));
table->Put(Put{"test2"}.AddColumn("d", "extra", "value for extra"));
diff --git a/core/client-test.cc b/core/client-test.cc
index 9efe0b6..1c9b709 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -42,6 +42,7 @@ using hbase::RetriesExhaustedException;
using hbase::Put;
using hbase::Table;
using hbase::TestUtil;
+using std::experimental::nullopt;
class ClientTest : public ::testing::Test {
public:
@@ -134,7 +135,7 @@ TEST_F(ClientTest, Append) {
// Get connection to HBase Table
auto table = client.Table(tn);
- ASSERT_TRUE(table) << "Unable to get connection to Table.";
+ ASSERT_TRUE(table != nullptr) << "Unable to get connection to Table.";
std::string val1 = "a";
auto result = table->Append(hbase::Append{row}.Add("d", "1", val1));
@@ -164,7 +165,7 @@ TEST_F(ClientTest, PutGetDelete) {
// Get connection to HBase Table
auto table = client.Table(tn);
- ASSERT_TRUE(table) << "Unable to get connection to Table.";
+ ASSERT_TRUE(table != nullptr) << "Unable to get connection to Table.";
// Perform Puts
std::string valExtra = "value for extra";
@@ -201,7 +202,7 @@ TEST_F(ClientTest, PutGetDelete) {
result = table->Get(get);
ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty.";
ASSERT_FALSE(result->Value("d", "1")) << "Column 1 should be gone";
- ASSERT_TRUE(result->Value("d", "extra")) << "Column extra should have value";
+ ASSERT_TRUE(result->Value("d", "extra") != nullopt) << "Column extra should have value";
EXPECT_EQ(valExt, *(result->Value("d", "ext"))) << "Column ext should have value";
// delete all cells from "extra" column
@@ -234,7 +235,7 @@ TEST_F(ClientTest, Increment) {
// Get connection to HBase Table
auto table = client.Table(tn);
- ASSERT_TRUE(table) << "Unable to get connection to Table.";
+ ASSERT_TRUE(table != nullptr) << "Unable to get connection to Table.";
int64_t incr1 = 1235;
auto result = table->Increment(hbase::Increment{row}.AddColumn("d", "1", incr1));
EXPECT_EQ(row, result->Row());
@@ -262,7 +263,7 @@ TEST_F(ClientTest, CheckAndPut) {
// Get connection to HBase Table
auto table = client.Table(tn);
- ASSERT_TRUE(table) << "Unable to get connection to Table.";
+ ASSERT_TRUE(table != nullptr) << "Unable to get connection to Table.";
// Perform Puts
table->Put(Put{row}.AddColumn("d", "1", "value1"));
@@ -291,7 +292,7 @@ TEST_F(ClientTest, CheckAndDelete) {
// Get connection to HBase Table
auto table = client.Table(tn);
- ASSERT_TRUE(table) << "Unable to get connection to Table.";
+ ASSERT_TRUE(table != nullptr) << "Unable to get connection to Table.";
auto val1 = "value1";
@@ -321,7 +322,7 @@ TEST_F(ClientTest, PutGet) {
// Get connection to HBase Table
auto table = client.Table(tn);
- ASSERT_TRUE(table) << "Unable to get connection to Table.";
+ ASSERT_TRUE(table != nullptr) << "Unable to get connection to Table.";
// Perform Puts
table->Put(Put{"test1"}.AddColumn("d", "1", "value1"));
@@ -355,7 +356,7 @@ TEST_F(ClientTest, GetForNonExistentTable) {
// Get connection to HBase Table
auto table = client.Table(tn);
- ASSERT_TRUE(table) << "Unable to get connection to Table.";
+ ASSERT_TRUE(table != nullptr) << "Unable to get connection to Table.";
// Perform the Get
try {
@@ -387,7 +388,7 @@ TEST_F(ClientTest, GetForNonExistentRow) {
// Get connection to HBase Table
auto table = client.Table(tn);
- ASSERT_TRUE(table) << "Unable to get connection to Table.";
+ ASSERT_TRUE(table != nullptr) << "Unable to get connection to Table.";
// Perform the Get
auto result = table->Get(get);
@@ -410,7 +411,7 @@ TEST_F(ClientTest, PutsWithTimestamp) {
// Get connection to HBase Table
auto table = client.Table(tn);
- ASSERT_TRUE(table) << "Unable to get connection to Table.";
+ ASSERT_TRUE(table != nullptr) << "Unable to get connection to Table.";
int64_t ts = 42;
// Perform Puts
@@ -446,7 +447,7 @@ void PerformPuts(uint64_t num_rows, std::shared_ptr<hbase::Client> client,
const std::string &table_name) {
auto tn = folly::to<hbase::pb::TableName>(table_name);
auto table = client->Table(tn);
- ASSERT_TRUE(table) << "Unable to get connection to Table.";
+ ASSERT_TRUE(table != nullptr) << "Unable to get connection to Table.";
// Perform Puts
for (uint64_t i = 0; i < num_rows; i++) {
table->Put(Put{"test" + std::to_string(i)}.AddColumn("d", std::to_string(i),
@@ -501,7 +502,7 @@ TEST_F(ClientTest, MultiGets) {
// Get connection to HBase Table
auto table = client.Table(tn);
- ASSERT_TRUE(table) << "Unable to get connection to Table.";
+ ASSERT_TRUE(table != nullptr) << "Unable to get connection to Table.";
std::vector<hbase::Get> gets;
MakeGets(num_rows, "test", gets);
@@ -534,7 +535,7 @@ TEST_F(ClientTest, MultiGetsWithRegionSplits) {
// Get connection to HBase Table
auto table = client.Table(tn);
- ASSERT_TRUE(table) << "Unable to get connection to Table.";
+ ASSERT_TRUE(table != nullptr) << "Unable to get connection to Table.";
std::vector<hbase::Get> gets;
MakeGets(num_rows, "test", gets);
diff --git a/core/filter-test.cc b/core/filter-test.cc
index f401698..7276dfb 100644
--- a/core/filter-test.cc
+++ b/core/filter-test.cc
@@ -75,7 +75,6 @@ TEST_F(FilterTest, GetWithColumnPrefixFilter) {
// Create a client
hbase::Client client(*(FilterTest::test_util_->conf()));
auto table = client.Table(tn);
- ASSERT_TRUE(table) << "Unable to get connection to Table.";
table->Put(Put{"row1"}.AddColumn("d", "column_1", "value1"));
table->Put(Put{"row1"}.AddColumn("d", "column_2", "value2"));
@@ -122,7 +121,6 @@ TEST_F(FilterTest, GetWithQualifierFilter) {
// Get connection to HBase Table
auto table = client.Table(tn);
- ASSERT_TRUE(table) << "Unable to get connection to Table.";
table->Put(Put{"row1"}.AddColumn("d", "a", "value1"));
table->Put(Put{"row1"}.AddColumn("d", "b", "value2"));
diff --git a/core/hbase-configuration-test.cc b/core/hbase-configuration-test.cc
index ebf0817..56b4463 100644
--- a/core/hbase-configuration-test.cc
+++ b/core/hbase-configuration-test.cc
@@ -27,6 +27,7 @@
#include "core/hbase-configuration-loader.h"
using namespace hbase;
+using std::experimental::nullopt;
const std::string kDefHBaseConfPath("./build/test-data/hbase-configuration-test/conf/");
const std::string kHBaseConfPath("./build/test-data/hbase-configuration-test/custom-conf/");
@@ -149,7 +150,7 @@ TEST(Configuration, LoadConfFromDefaultLocation) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf) << "No configuration object present.";
+ ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
EXPECT_STREQ((*conf).Get("custom-prop", "Set this value").c_str(), "custom-value");
EXPECT_STREQ((*conf).Get("default-prop", "Set this value").c_str(), "default-value");
}
@@ -166,7 +167,7 @@ TEST(Configuration, LoadConfFromCustomLocation) {
HBaseConfigurationLoader loader;
std::vector<std::string> resources{kHBaseSiteXml};
hbase::optional<Configuration> conf = loader.LoadResources(kHBaseConfPath, resources);
- ASSERT_TRUE(conf) << "No configuration object present.";
+ ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
EXPECT_STREQ((*conf).Get("custom-prop", "").c_str(), "custom-value");
EXPECT_STRNE((*conf).Get("custom-prop", "").c_str(), "some-value");
}
@@ -187,7 +188,7 @@ TEST(Configuration, LoadConfFromMultipleLocatons) {
std::string conf_paths = kDefHBaseConfPath + ":" + kHBaseConfPath;
std::vector<std::string> resources{kHBaseDefaultXml, kHBaseSiteXml};
hbase::optional<Configuration> conf = loader.LoadResources(conf_paths, resources);
- ASSERT_TRUE(conf) << "No configuration object present.";
+ ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
EXPECT_STREQ((*conf).Get("default-prop", "From hbase-default.xml").c_str(), "default-value");
EXPECT_STREQ((*conf).Get("custom-prop", "").c_str(), "custom-value");
EXPECT_STRNE((*conf).Get("custom-prop", "").c_str(), "some-value");
@@ -206,7 +207,7 @@ TEST(Configuration, DefaultValues) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf) << "No configuration object present.";
+ ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
EXPECT_STREQ((*conf).Get("default-prop", "Set this value.").c_str(), "default-value");
EXPECT_STREQ((*conf).Get("custom-prop", "Set this value.").c_str(), "custom-value");
}
@@ -218,7 +219,7 @@ TEST(Configuration, FinalValues) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf) << "No configuration object present.";
+ ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
EXPECT_STREQ((*conf).Get("hbase.rootdir", "").c_str(), "/root/hbase-docker/apps/hbase/data");
EXPECT_STREQ((*conf).Get("hbase.zookeeper.property.datadir", "").c_str(),
"/root/hbase-docker/zookeeper");
@@ -240,7 +241,7 @@ TEST(Configuration, EnvVars) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf) << "No configuration object present.";
+ ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
EXPECT_STREQ((*conf).Get("hbase-client.user.name", "").c_str(), "${user.name}");
EXPECT_STRNE((*conf).Get("hbase-client.user.name", "root").c_str(), "test-user");
}
@@ -252,7 +253,7 @@ TEST(Configuration, SelfRef) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf) << "No configuration object present.";
+ ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
EXPECT_STREQ((*conf).Get("selfRef", "${selfRef}").c_str(), "${selfRef}");
}
@@ -263,7 +264,7 @@ TEST(Configuration, VarExpansion) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf) << "No configuration object present.";
+ ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
EXPECT_STREQ((*conf).Get("foo.substs", "foo-value").c_str(),
"bar-value,bar-value,bar-value,bar-value,bar-value,bar-value,"
"bar-value,bar-value,bar-value,bar-value,");
@@ -277,7 +278,7 @@ TEST(Configuration, VarExpansionException) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf) << "No configuration object present.";
+ ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
ASSERT_THROW((*conf).Get("foo.substs.exception", "foo-value").c_str(), std::runtime_error);
}
@@ -288,7 +289,7 @@ TEST(Configuration, GetInt) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf) << "No configuration object present.";
+ ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
EXPECT_EQ(16000, (*conf).GetInt("int", 0));
EXPECT_EQ(2147483646, (*conf).GetInt("int.largevalue", 0));
}
@@ -300,7 +301,7 @@ TEST(Configuration, GetLong) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf) << "No configuration object present.";
+ ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
EXPECT_EQ(2147483850, (*conf).GetLong("long", 0));
EXPECT_EQ(9223372036854775807, (*conf).GetLong("long.largevalue", 0));
}
@@ -312,7 +313,7 @@ TEST(Configuration, GetDouble) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf) << "No configuration object present.";
+ ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
EXPECT_DOUBLE_EQ(17.9769e+100, (*conf).GetDouble("double", 0.0));
EXPECT_DOUBLE_EQ(170.769e+200, (*conf).GetDouble("double.largevalue", 0.0));
}
@@ -324,7 +325,7 @@ TEST(Configuration, GetBool) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf) << "No configuration object present.";
+ ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
EXPECT_EQ(true, (*conf).GetBool("bool.true", true));
EXPECT_EQ(false, (*conf).GetBool("bool.false", false));
}
@@ -336,7 +337,7 @@ TEST(Configuration, GetIntException) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf) << "No configuration object present.";
+ ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
ASSERT_THROW((*conf).GetInt("int.exception", 0), std::runtime_error);
}
@@ -347,7 +348,7 @@ TEST(Configuration, GetLongException) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf) << "No configuration object present.";
+ ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
ASSERT_THROW((*conf).GetLong("long.exception", 0), std::runtime_error);
}
@@ -358,7 +359,7 @@ TEST(Configuration, GetDoubleException) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf) << "No configuration object present.";
+ ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
ASSERT_THROW((*conf).GetDouble("double.exception", 0), std::runtime_error);
}
@@ -369,6 +370,6 @@ TEST(Configuration, GetBoolException) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
- ASSERT_TRUE(conf) << "No configuration object present.";
+ ASSERT_TRUE(conf != nullopt) << "No configuration object present.";
ASSERT_THROW((*conf).GetBool("bool.exception", false), std::runtime_error);
}
diff --git a/core/location-cache-retry-test.cc b/core/location-cache-retry-test.cc
index 988f994..f154b69 100644
--- a/core/location-cache-retry-test.cc
+++ b/core/location-cache-retry-test.cc
@@ -45,7 +45,7 @@ using hbase::Put;
using hbase::Table;
using hbase::TestUtil;
-using std::chrono_literals::operator""s;
+using std::chrono_literals::operator"" s;
class LocationCacheRetryTest : public ::testing::Test {
public:
diff --git a/core/result-test.cc b/core/result-test.cc
index 3299ffc..dd60aeb 100644
--- a/core/result-test.cc
+++ b/core/result-test.cc
@@ -30,6 +30,7 @@
using hbase::Cell;
using hbase::CellType;
using hbase::Result;
+using std::experimental::nullopt;
void PopulateCells(std::vector<std::shared_ptr<Cell> > &cells) {
// Populate some Results
@@ -111,7 +112,7 @@ TEST(Result, FilledResult) {
// Value will be nullptr as no such family and qualifier is present
ASSERT_FALSE(result.Value("family-4", "qualifier"));
// Value will be present as family and qualifier is present
- ASSERT_TRUE(result.Value("family-4", "column-4"));
+ ASSERT_TRUE(result.Value("family-4", "column-4") != nullopt);
// Value should be present and match.
EXPECT_EQ(latest_cell->Value(), (*result.ColumnLatestCell("family-4", "column-4")).Value());
EXPECT_EQ("value-5", (*result.ColumnLatestCell("family-5", "column-5")).Value());
[hbase] 15/137: HBASE-15620 Add on Call serialization
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 928278990c5bebcd368e4c1a455418b6f477b001
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Tue Apr 12 12:03:52 2016 -0700
HBASE-15620 Add on Call serialization
Summary: Add on delimited serialization so that request headers and request payloads can be serialized.
Test Plan: Add a unit test.
Differential Revision: https://reviews.facebook.net/D56757
---
Dockerfile | 19 +--
...ocal_hbase_and_wait.sh => start-local-hbase.sh} | 0
...local_hbase_and_wait.sh => stop-local-hbase.sh} | 0
connection/BUCK | 43 +++++++
{core => connection}/client-dispatcher.cc | 4 +-
{core => connection}/client-dispatcher.h | 12 +-
connection/client-handler.cc | 80 ++++++++++++
.../client-handler.h | 27 ++--
{core => connection}/connection-factory.cc | 35 +++---
{core => connection}/connection-factory.h | 15 ++-
{core => connection}/pipeline.cc | 12 +-
{core => connection}/pipeline.h | 11 +-
{core => connection}/request.h | 11 +-
core/request.h => connection/response.h | 19 ++-
{core => connection}/service.h | 6 +-
core/BUCK | 41 +-----
core/admin.cc | 20 ---
core/admin.h | 22 ----
core/client-serialize-handler.cc | 104 ---------------
core/client.h | 2 +-
core/connection_attr.h | 24 ----
core/delete.cc | 21 ----
core/delete.h | 27 ----
core/get-request.h | 2 +-
core/get-result.h | 2 +-
core/location-cache.cc | 3 +-
core/location-cache.h | 2 +-
core/mutation.cc | 41 ------
core/mutation.h | 58 ---------
core/scanner.cc | 20 ---
core/scanner.h | 22 ----
core/simple-client.cc | 45 ++++++-
core/simple-native-client-test.cc | 25 ----
core/table-name.h | 2 +-
core/{native-client-test-env.cc => test-env.cc} | 4 +-
if/BUCK | 53 ++++----
serde/BUCK | 54 ++++++++
serde/client-deserializer-test.cc | 67 ++++++++++
serde/client-deserializer.cc | 68 ++++++++++
core/response.h => serde/client-deserializer.h | 22 ++--
serde/client-serializer-test.cc | 75 +++++++++++
serde/client-serializer.cc | 139 +++++++++++++++++++++
.../client-serializer.h | 43 ++++---
43 files changed, 747 insertions(+), 555 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 36959a5..9ccc0d3 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -22,7 +22,14 @@ ARG CXX=/usr/bin/g++-5
ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 -pthread"
ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 -pthread"
-RUN apt-get install -y clang-format-3.7 vim maven inetutils-ping
+RUN apt-get install -y clang-format-3.7 vim maven inetutils-ping python-pip && \
+ pip install yapf && \
+ ln -sf /usr/bin/clang-format-3.7 /usr/bin/clang-format && \
+ apt-get -qq clean && \
+ apt-get -y -qq autoremove && \
+ rm -rf /var/lib/{apt,dpkg,cache,log}/ && \
+ rm -rf /tmp/*
+
RUN git clone --depth 1 --branch v2.6.1 https://github.com/google/protobuf.git /usr/src/protobuf && \
cd /usr/src/protobuf/ && \
ldconfig && \
@@ -31,9 +38,8 @@ RUN git clone --depth 1 --branch v2.6.1 https://github.com/google/protobuf.git /
make && \
make install && \
make clean && \
- rm -rf .git
-
-RUN cd /usr/src && \
+ rm -rf .git && \
+ cd /usr/src && \
wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && \
tar zxf zookeeper-3.4.8.tar.gz && \
rm -rf zookeeper-3.4.8.tar.gz && \
@@ -43,8 +49,7 @@ RUN cd /usr/src && \
./configure && \
make && \
make install && \
- make clean
-
-RUN ldconfig
+ make clean && \
+ ldconfig
WORKDIR /usr/src/hbase/hbase-native-client
diff --git a/bin/start_local_hbase_and_wait.sh b/bin/start-local-hbase.sh
similarity index 100%
rename from bin/start_local_hbase_and_wait.sh
rename to bin/start-local-hbase.sh
diff --git a/bin/stop_local_hbase_and_wait.sh b/bin/stop-local-hbase.sh
similarity index 100%
rename from bin/stop_local_hbase_and_wait.sh
rename to bin/stop-local-hbase.sh
diff --git a/connection/BUCK b/connection/BUCK
new file mode 100644
index 0000000..5067708
--- /dev/null
+++ b/connection/BUCK
@@ -0,0 +1,43 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is the library dealing with a single connection
+# to a single server.
+cxx_library(name="connection",
+ exported_headers=[
+ "client-dispatcher.h",
+ "client-handler.h",
+ "connection-factory.h",
+ "pipeline.h",
+ "request.h",
+ "response.h",
+ "service.h",
+ ],
+ srcs=[
+ "client-dispatcher.cc",
+ "client-handler.cc",
+ "connection-factory.cc",
+ "pipeline.cc",
+ ],
+ deps=[
+ "//if:if",
+ "//utils:utils",
+ "//serde:serde",
+ "//third-party:folly",
+ "//third-party:wangle",
+ ],
+ visibility=['//core/...', ], )
diff --git a/core/client-dispatcher.cc b/connection/client-dispatcher.cc
similarity index 97%
rename from core/client-dispatcher.cc
rename to connection/client-dispatcher.cc
index d356759..25cff7d 100644
--- a/core/client-dispatcher.cc
+++ b/connection/client-dispatcher.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*
*/
-#include "core/client-dispatcher.h"
+#include "connection/client-dispatcher.h"
using namespace folly;
using namespace hbase;
@@ -27,6 +27,7 @@ void ClientDispatcher::read(Context *ctx, Response in) {
auto search = requests_.find(call_id);
CHECK(search != requests_.end());
auto p = std::move(search->second);
+
requests_.erase(call_id);
// TODO(eclark): check if the response
@@ -36,6 +37,7 @@ void ClientDispatcher::read(Context *ctx, Response in) {
Future<Response> ClientDispatcher::operator()(Request arg) {
auto call_id = ++current_call_id_;
+
arg.set_call_id(call_id);
auto &p = requests_[call_id];
auto f = p.getFuture();
diff --git a/core/client-dispatcher.h b/connection/client-dispatcher.h
similarity index 85%
rename from core/client-dispatcher.h
rename to connection/client-dispatcher.h
index 4b9d35a..c3987c9 100644
--- a/core/client-dispatcher.h
+++ b/connection/client-dispatcher.h
@@ -21,9 +21,9 @@
#include <wangle/service/ClientDispatcher.h>
-#include "core/pipeline.h"
-#include "core/request.h"
-#include "core/response.h"
+#include "connection/pipeline.h"
+#include "connection/request.h"
+#include "connection/response.h"
namespace hbase {
class ClientDispatcher
@@ -36,7 +36,7 @@ public:
folly::Future<folly::Unit> close() override;
private:
- std::unordered_map<int32_t, folly::Promise<Response>> requests_;
- uint32_t current_call_id_ = 1;
+ std::unordered_map<uint32_t, folly::Promise<Response>> requests_;
+ uint32_t current_call_id_ = 10;
};
-} // namespace hbase
+} // namespace hbase
diff --git a/connection/client-handler.cc b/connection/client-handler.cc
new file mode 100644
index 0000000..74b23ef
--- /dev/null
+++ b/connection/client-handler.cc
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "connection/client-handler.h"
+
+#include <glog/logging.h>
+#include <folly/Likely.h>
+
+#include <string>
+
+#include "connection/request.h"
+#include "connection/response.h"
+#include "if/RPC.pb.h"
+#include "if/Client.pb.h"
+
+using namespace hbase;
+using namespace folly;
+using namespace wangle;
+using hbase::pb::ResponseHeader;
+using hbase::pb::GetResponse;
+
+ClientHandler::ClientHandler(std::string user_name) : user_name_(user_name) {}
+
+void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
+ if (LIKELY(buf != nullptr)) {
+ buf->coalesce();
+ Response received;
+ ResponseHeader header;
+
+ int used_bytes = deser_.parse_delimited(buf.get(), &header);
+ LOG(INFO) << "Read ResponseHeader size=" << used_bytes
+ << " call_id=" << header.call_id()
+ << " has_exception=" << header.has_exception();
+ received.set_call_id(header.call_id());
+
+ if (header.has_exception() == false) {
+ buf->trimStart(used_bytes);
+ // For now assume that everything was a get.
+ // We'll need to set this up later.
+ received.set_response(std::make_shared<GetResponse>());
+ used_bytes = deser_.parse_delimited(buf.get(), received.response().get());
+ }
+ ctx->fireRead(std::move(received));
+ }
+}
+
+Future<Unit> ClientHandler::write(Context *ctx, Request r) {
+ // Keep track of if we have sent the header.
+ if (UNLIKELY(need_send_header_)) {
+ need_send_header_ = false;
+
+ // Should we be sending just one fireWrite?
+ // Right now we're sending one for the header
+ // and one for the request.
+ //
+ // That doesn't seem like too bad, but who knows.
+ auto pre = ser_.preamble();
+ auto header = ser_.header(user_name_);
+ pre->appendChain(std::move(header));
+ ctx->fireWrite(std::move(pre));
+ }
+
+ return ctx->fireWrite(ser_.request(r.call_id(), r.method(), r.msg()));
+}
diff --git a/core/client-serialize-handler.h b/connection/client-handler.h
similarity index 75%
copy from core/client-serialize-handler.h
copy to connection/client-handler.h
index 961a03b..38c5725 100644
--- a/core/client-serialize-handler.h
+++ b/connection/client-handler.h
@@ -20,25 +20,30 @@
#include <wangle/channel/Handler.h>
-#include "if/HBase.pb.h"
-#include "if/RPC.pb.h"
-#include "core/request.h"
-#include "core/response.h"
+#include <string>
+
+#include "serde/client-serializer.h"
+#include "serde/client-deserializer.h"
+
+// Forward decs.
+namespace hbase {
+class Request;
+class Response;
+}
namespace hbase {
-class ClientSerializeHandler
+class ClientHandler
: public wangle::Handler<std::unique_ptr<folly::IOBuf>, Response, Request,
std::unique_ptr<folly::IOBuf>> {
public:
+ ClientHandler(std::string user_name);
void read(Context *ctx, std::unique_ptr<folly::IOBuf> msg) override;
folly::Future<folly::Unit> write(Context *ctx, Request r) override;
private:
- folly::Future<folly::Unit> write_preamble(Context *ctx);
- folly::Future<folly::Unit> write_header(Context *ctx);
- // Our own simple version of LengthFieldPrepender
- std::unique_ptr<folly::IOBuf>
- prepend_length(std::unique_ptr<folly::IOBuf> msg);
bool need_send_header_ = true;
+ std::string user_name_;
+ ClientSerializer ser_;
+ ClientDeserializer deser_;
};
-} // namespace hbase
+} // namespace hbase
diff --git a/core/connection-factory.cc b/connection/connection-factory.cc
similarity index 66%
rename from core/connection-factory.cc
rename to connection/connection-factory.cc
index 785b239..5d1b0da 100644
--- a/core/connection-factory.cc
+++ b/connection/connection-factory.cc
@@ -17,41 +17,42 @@
*
*/
-#include "core/connection-factory.h"
+#include "connection/connection-factory.h"
+#include <folly/futures/Future.h>
+#include <wangle/bootstrap/ClientBootstrap.h>
#include <wangle/channel/AsyncSocketHandler.h>
#include <wangle/channel/EventBaseHandler.h>
#include <wangle/channel/OutputBufferingHandler.h>
#include <wangle/service/ClientDispatcher.h>
+#include <wangle/service/CloseOnReleaseFilter.h>
#include <wangle/service/ExpiringFilter.h>
-#include <folly/futures/Future.h>
#include <string>
-#include "core/client-dispatcher.h"
-#include "core/pipeline.h"
-#include "core/request.h"
-#include "core/response.h"
-#include "core/service.h"
+#include "connection/client-dispatcher.h"
+#include "connection/pipeline.h"
+#include "connection/request.h"
+#include "connection/response.h"
+#include "connection/service.h"
using namespace folly;
using namespace hbase;
using namespace wangle;
ConnectionFactory::ConnectionFactory() {
- bootstrap_.group(std::make_shared<wangle::IOThreadPoolExecutor>(2));
+ bootstrap_.group(std::make_shared<wangle::IOThreadPoolExecutor>(1));
bootstrap_.pipelineFactory(std::make_shared<RpcPipelineFactory>());
}
-Future<ClientDispatcher> ConnectionFactory::make_connection(std::string host,
- int port) {
+std::shared_ptr<Service<Request, Response>>
+ConnectionFactory::make_connection(std::string host, int port) {
// Connect to a given server
// Then when connected create a ClientDispactcher.
- auto srv = bootstrap_.connect(SocketAddress(host, port, true))
- .then([](SerializePipeline *pipeline) {
- ClientDispatcher dispatcher;
- dispatcher.setPipeline(pipeline);
- return dispatcher;
- });
- return srv;
+ auto pipeline = bootstrap_.connect(SocketAddress(host, port, true)).get();
+ auto dispatcher = std::make_shared<ClientDispatcher>();
+ dispatcher->setPipeline(pipeline);
+ auto service =
+ std::make_shared<CloseOnReleaseFilter<Request, Response>>(dispatcher);
+ return service;
}
diff --git a/core/connection-factory.h b/connection/connection-factory.h
similarity index 78%
rename from core/connection-factory.h
rename to connection/connection-factory.h
index 6f450c2..73ac032 100644
--- a/core/connection-factory.h
+++ b/connection/connection-factory.h
@@ -18,24 +18,23 @@
*/
#pragma once
-#include <wangle/bootstrap/ClientBootstrap.h>
#include <wangle/service/Service.h>
#include <string>
-#include "core/service.h"
-#include "core/pipeline.h"
-#include "core/client-dispatcher.h"
-#include "core/request.h"
-#include "core/response.h"
+#include "connection/pipeline.h"
+#include "connection/request.h"
+#include "connection/response.h"
+#include "connection/service.h"
namespace hbase {
class ConnectionFactory {
public:
ConnectionFactory();
- folly::Future<ClientDispatcher> make_connection(std::string host, int port);
+ std::shared_ptr<wangle::Service<Request, Response>>
+ make_connection(std::string host, int port);
private:
wangle::ClientBootstrap<SerializePipeline> bootstrap_;
};
-} // namespace hbase
+} // namespace hbase
diff --git a/core/pipeline.cc b/connection/pipeline.cc
similarity index 82%
rename from core/pipeline.cc
rename to connection/pipeline.cc
index 30d14ff..b9f5e0b 100644
--- a/core/pipeline.cc
+++ b/connection/pipeline.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*
*/
-#include "core/pipeline.h"
+#include "connection/pipeline.h"
#include <folly/Logging.h>
#include <wangle/channel/AsyncSocketHandler.h>
@@ -24,7 +24,7 @@
#include <wangle/channel/OutputBufferingHandler.h>
#include <wangle/codec/LengthFieldBasedFrameDecoder.h>
-#include "core/client-serialize-handler.h"
+#include "connection/client-handler.h"
using namespace folly;
using namespace hbase;
@@ -33,10 +33,10 @@ using namespace wangle;
SerializePipeline::Ptr
RpcPipelineFactory::newPipeline(std::shared_ptr<AsyncTransportWrapper> sock) {
auto pipeline = SerializePipeline::create();
- pipeline->addBack(AsyncSocketHandler(sock));
- pipeline->addBack(EventBaseHandler());
- pipeline->addBack(LengthFieldBasedFrameDecoder());
- pipeline->addBack(ClientSerializeHandler());
+ pipeline->addBack(AsyncSocketHandler{sock});
+ pipeline->addBack(EventBaseHandler{});
+ pipeline->addBack(LengthFieldBasedFrameDecoder{});
+ pipeline->addBack(ClientHandler{user_util_.user_name()});
pipeline->finalize();
return pipeline;
}
diff --git a/core/pipeline.h b/connection/pipeline.h
similarity index 88%
rename from core/pipeline.h
rename to connection/pipeline.h
index d199d08..68ade48 100644
--- a/core/pipeline.h
+++ b/connection/pipeline.h
@@ -20,8 +20,10 @@
#include <wangle/service/Service.h>
#include <folly/io/IOBufQueue.h>
-#include "core/request.h"
-#include "core/response.h"
+
+#include "connection/request.h"
+#include "connection/response.h"
+#include "utils/user-util.h"
namespace hbase {
using SerializePipeline = wangle::Pipeline<folly::IOBufQueue &, Request>;
@@ -30,5 +32,8 @@ class RpcPipelineFactory : public wangle::PipelineFactory<SerializePipeline> {
public:
SerializePipeline::Ptr
newPipeline(std::shared_ptr<folly::AsyncTransportWrapper> sock) override;
+
+private:
+ UserUtil user_util_;
};
-} // namespace hbase
+} // namespace hbase
diff --git a/core/request.h b/connection/request.h
similarity index 71%
copy from core/request.h
copy to connection/request.h
index 39083ed..e9e3e88 100644
--- a/core/request.h
+++ b/connection/request.h
@@ -18,7 +18,10 @@
*/
#pragma once
+#include <google/protobuf/message.h>
+
#include <cstdint>
+#include <string>
namespace hbase {
class Request {
@@ -26,8 +29,14 @@ public:
Request() : call_id_(0) {}
uint32_t call_id() { return call_id_; }
void set_call_id(uint32_t call_id) { call_id_ = call_id; }
+ google::protobuf::Message *msg() { return msg_.get(); }
+ void set_msg(std::shared_ptr<google::protobuf::Message> msg) { msg_ = msg; }
+ std::string method() { return method_; }
+ void set_method(std::string method) { method_ = method; }
private:
uint32_t call_id_;
+ std::shared_ptr<google::protobuf::Message> msg_ = nullptr;
+ std::string method_ = "Get";
};
-} // namespace hbase
+} // namespace hbase
diff --git a/core/request.h b/connection/response.h
similarity index 72%
rename from core/request.h
rename to connection/response.h
index 39083ed..a7f7939 100644
--- a/core/request.h
+++ b/connection/response.h
@@ -20,14 +20,27 @@
#include <cstdint>
+// Forward
+namespace google {
+namespace protobuf {
+class Message;
+}
+}
+
namespace hbase {
-class Request {
+
+class Response {
public:
- Request() : call_id_(0) {}
+ Response() : call_id_(0) {}
uint32_t call_id() { return call_id_; }
void set_call_id(uint32_t call_id) { call_id_ = call_id; }
+ std::shared_ptr<google::protobuf::Message> response() { return response_; }
+ void set_response(std::shared_ptr<google::protobuf::Message> response) {
+ response_ = std::move(response);
+ }
private:
uint32_t call_id_;
+ std::shared_ptr<google::protobuf::Message> response_;
};
-} // namespace hbase
+} // namespace hbase
diff --git a/core/service.h b/connection/service.h
similarity index 91%
rename from core/service.h
rename to connection/service.h
index 880e65f..feb14ec 100644
--- a/core/service.h
+++ b/connection/service.h
@@ -18,9 +18,9 @@
*/
#pragma once
-#include "core/request.h"
-#include "core/response.h"
+#include "connection/request.h"
+#include "connection/response.h"
namespace hbase {
using HBaseService = wangle::Service<Request, Response>;
-} // namespace hbase
+} // namespace hbase
diff --git a/core/BUCK b/core/BUCK
index 2b00d66..195fc5c 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -15,50 +15,28 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# This is the main library.
cxx_library(name="core",
exported_headers=[
- "admin.h",
- "client-dispatcher.h",
- "client-serialize-handler.h",
"client.h",
- "connection-factory.h",
"connection.h",
- "connection_attr.h",
- "delete.h",
"get-request.h",
"get-result.h",
- "get.h",
"hbase_macros.h",
"location-cache.h",
- "mutation.h",
- "pipeline.h",
- "put.h",
- "request.h",
- "response.h",
- "scanner.h",
- "service.h",
"table-name.h",
],
srcs=[
- "admin.cc",
- "client-dispatcher.cc",
- "client-serialize-handler.cc",
"client.cc",
- "connection-factory.cc",
- "connection.cc",
- "delete.cc",
"get-request.cc",
"get-result.cc",
- "get.cc",
"location-cache.cc",
- "mutation.cc",
- "pipeline.cc",
- "put.cc",
- "scanner.cc",
"table-name.cc",
],
deps=[
+ "//connection:connection",
"//if:if",
+ "//serde:serde",
"//third-party:folly",
"//third-party:wangle",
"//third-party:zookeeper_mt",
@@ -67,18 +45,9 @@ cxx_library(name="core",
'PUBLIC',
], )
-cxx_test(name="simple-test",
- srcs=[
- "native-client-test-env.cc",
- "simple-native-client-test.cc",
- ],
- deps=[
- ":core",
- ],
- run_test_separately=True, )
cxx_test(name="location-cache-test",
srcs=[
- "native-client-test-env.cc",
+ "test-env.cc",
"location-cache-test.cc",
],
deps=[
@@ -87,4 +56,4 @@ cxx_test(name="location-cache-test",
run_test_separately=True, )
cxx_binary(name="simple-client",
srcs=["simple-client.cc", ],
- deps=[":core", ], )
+ deps=[":core", "//connection:connection"], )
diff --git a/core/admin.cc b/core/admin.cc
deleted file mode 100644
index 897e6bf..0000000
--- a/core/admin.cc
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "core/admin.h"
diff --git a/core/admin.h b/core/admin.h
deleted file mode 100644
index 775181c..0000000
--- a/core/admin.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#pragma once
-
-class Admin {};
diff --git a/core/client-serialize-handler.cc b/core/client-serialize-handler.cc
deleted file mode 100644
index cad1308..0000000
--- a/core/client-serialize-handler.cc
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "core/client-serialize-handler.h"
-
-#include <string>
-
-using namespace hbase;
-using namespace folly;
-using namespace wangle;
-
-static const std::string PREAMBLE = "HBas";
-static const std::string INTERFACE = "ClientService";
-static const uint8_t RPC_VERSION = 0;
-static const uint8_t AUTH_TYPE = 80;
-
-// TODO(eclark): Make this actually do ANYTHING.
-void ClientSerializeHandler::read(Context *ctx, std::unique_ptr<IOBuf> msg) {
- Response received;
- ctx->fireRead(received);
-}
-
-Future<Unit> ClientSerializeHandler::write(Context *ctx, Request r) {
- // Keep track of if we have sent the header.
- if (need_send_header_) {
- need_send_header_ = false;
-
- // Should this be replacing the IOBuf rather than
- // sending several different calls?
- write_preamble(ctx);
- write_header(ctx);
- }
-
- // Send out the actual request and not just a test string.
- std::string out{"test"};
- return ctx->fireWrite(prepend_length(IOBuf::copyBuffer(out)));
-}
-
-Future<Unit> ClientSerializeHandler::write_preamble(Context *ctx) {
- auto magic = IOBuf::copyBuffer(PREAMBLE);
- auto buf = IOBuf::create(2);
- buf->append(2);
- folly::io::RWPrivateCursor c(buf.get());
-
- // Version
- c.write(RPC_VERSION);
- // Standard security aka Please don't lie to me.
- c.write(AUTH_TYPE);
- magic->appendChain(std::move(buf));
- return ctx->fireWrite(std::move(magic));
-}
-
-Future<Unit> ClientSerializeHandler::write_header(Context *ctx) {
- pb::ConnectionHeader h;
-
- // TODO(eclark): Make this not a total lie.
- h.mutable_user_info()->set_effective_user("elliott");
- // The service name that we want to talk to.
- //
- // Right now we're completely ignoring the service interface.
- // That may or may not be the correct thing to do.
- // It worked for a while with the java client; until it
- // didn't.
- h.set_service_name(INTERFACE);
- // TODO(eclark): Make this 1 copy.
- auto msg = IOBuf::copyBuffer(h.SerializeAsString());
- return ctx->fireWrite(prepend_length(std::move(msg)));
-}
-
-// Our own simple version of LengthFieldPrepender
-std::unique_ptr<IOBuf>
-ClientSerializeHandler::prepend_length(std::unique_ptr<IOBuf> msg) {
- // Java ints are 4 long. So create a buffer that large
- auto len_buf = IOBuf::create(4);
- // Then make those bytes visible.
- len_buf->append(4);
-
- io::RWPrivateCursor c(len_buf.get());
- // Get the size of the data to be pushed out the network.
- auto size = msg->computeChainDataLength();
-
- // Write the length to this IOBuf.
- c.writeBE(static_cast<uint32_t>(size));
-
- // Then attach the origional to the back of len_buf
- len_buf->appendChain(std::move(msg));
- return len_buf;
-}
diff --git a/core/client.h b/core/client.h
index 818bc6b..c2dc226 100644
--- a/core/client.h
+++ b/core/client.h
@@ -39,4 +39,4 @@ private:
LocationCache location_cache;
};
-} // namespace hbase
+} // namespace hbase
diff --git a/core/connection_attr.h b/core/connection_attr.h
deleted file mode 100644
index a312005..0000000
--- a/core/connection_attr.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#pragma once
-
-#include "core/hbase_macros.h"
-
-class ConnectionAttr {};
diff --git a/core/delete.cc b/core/delete.cc
deleted file mode 100644
index 57030be..0000000
--- a/core/delete.cc
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "core/delete.h"
-
-Delete::~Delete() {}
diff --git a/core/delete.h b/core/delete.h
deleted file mode 100644
index 34f6a6c..0000000
--- a/core/delete.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#pragma once
-
-#include "core/mutation.h"
-
-class Delete : public Mutation {
-public:
- ~Delete();
-};
diff --git a/core/get-request.h b/core/get-request.h
index c9113ad..bb755c5 100644
--- a/core/get-request.h
+++ b/core/get-request.h
@@ -32,4 +32,4 @@ private:
TableName table_name_;
std::string key_;
};
-} // namespace hbase
+} // namespace hbase
diff --git a/core/get-result.h b/core/get-result.h
index e021316..a49ad98 100644
--- a/core/get-result.h
+++ b/core/get-result.h
@@ -29,4 +29,4 @@ public:
private:
std::string key_;
};
-} // namespace hbase
+} // namespace hbase
diff --git a/core/location-cache.cc b/core/location-cache.cc
index 34e3236..52e86e3 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -70,8 +70,7 @@ ServerName LocationCache::ReadMetaLocation() {
// This needs to be int rather than size_t as that's what ZK expects.
int len = sizeof(contents);
// TODO(elliott): handle disconnects/reconntion as needed.
- int zk_result =
- zoo_get(this->zk_, META_LOCATION, 0, contents, &len, nullptr);
+ int zk_result = zoo_get(this->zk_, META_LOCATION, 0, contents, &len, nullptr);
if (zk_result != ZOK || len < 9) {
LOG(ERROR) << "Error getting meta location.";
throw runtime_error("Error getting meta location");
diff --git a/core/location-cache.h b/core/location-cache.h
index efcfde5..28a1ee1 100644
--- a/core/location-cache.h
+++ b/core/location-cache.h
@@ -51,4 +51,4 @@ private:
zhandle_t *zk_;
};
-} // namespace hbase
+} // namespace hbase
diff --git a/core/mutation.cc b/core/mutation.cc
deleted file mode 100644
index 52910d5..0000000
--- a/core/mutation.cc
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "core/mutation.h"
-
-void Mutation::set_namespace(char *name_space, size_t name_space_length) {
- this->name_space = name_space;
- this->name_space_length = name_space_length;
-}
-
-void Mutation::set_table(char *table, size_t table_length) {
- this->table = table;
- this->table_length = table_length;
-}
-
-void Mutation::set_row(unsigned char *row, size_t row_length) {
- this->row = row;
- this->row_length = row_length;
-}
-
-void Mutation::set_durability(durability_type durability) {
- this->durability = durability;
-}
-
-Mutation::~Mutation() {}
diff --git a/core/mutation.h b/core/mutation.h
deleted file mode 100644
index 1880571..0000000
--- a/core/mutation.h
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#pragma once
-
-#include <stdlib.h>
-
-typedef enum {
- DELETE_ONE_VERSION,
- DELETE_MULTIPLE_VERSIONS,
- DELETE_FAMILY,
- DELETE_FAMILY_VERSION
-} delete_type;
-
-typedef enum {
- USE_DEFAULT,
- SKIP_WAL,
- ASYNC_WAL,
- SYNC_WAL,
- HSYNC_WAL
-} durability_type;
-
-class Mutation {
- char *name_space;
- size_t name_space_length;
-
- char *table;
- size_t table_length;
-
- unsigned char *row;
- size_t row_length;
-
- durability_type durability;
-
-public:
- void set_namespace(char *name_space, size_t name_space_length);
- void set_table(char *table, size_t table_length);
- void set_row(unsigned char *row, size_t row_length);
- void set_durability(durability_type durability);
-
- virtual ~Mutation();
-};
diff --git a/core/scanner.cc b/core/scanner.cc
deleted file mode 100644
index a10e444..0000000
--- a/core/scanner.cc
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "core/scanner.h"
diff --git a/core/scanner.h b/core/scanner.h
deleted file mode 100644
index 180865a..0000000
--- a/core/scanner.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#pragma once
-
-class Scanner {};
diff --git a/core/simple-client.cc b/core/simple-client.cc
index 08e886a..065f70b 100644
--- a/core/simple-client.cc
+++ b/core/simple-client.cc
@@ -20,21 +20,30 @@
#include <folly/Logging.h>
#include <folly/Random.h>
#include <gflags/gflags.h>
-#include <glog/logging.h>
#include <wangle/concurrent/GlobalExecutor.h>
#include <iostream>
+#include <chrono>
#include "core/client.h"
-#include "core/connection-factory.h"
+#include "connection/connection-factory.h"
#include "if/ZooKeeper.pb.h"
+#include "if/Client.pb.h"
using namespace folly;
using namespace std;
+using namespace std::chrono;
using namespace hbase;
using namespace hbase::pb;
+using namespace google::protobuf;
+
+// TODO(eclark): remove the need for this.
+DEFINE_string(region, "1588230740", "What region to send a get to");
+DEFINE_string(row, "test", "What row to get");
int main(int argc, char *argv[]) {
+ google::SetUsageMessage(
+ "Simple client to get a single row from HBase on the comamnd line");
google::ParseCommandLineFlags(&argc, &argv, true);
google::InitGoogleLogging(argv[0]);
@@ -44,14 +53,40 @@ int main(int argc, char *argv[]) {
LocationCache cache{"localhost:2181", wangle::getCPUExecutor()};
auto result = cache.LocateMeta().get();
- cout << "ServerName = " << result.host_name() << ":" << result.port() << endl;
// Create a connection to the local host
- auto conn = cf.make_connection(result.host_name(), result.port()).get();
+ auto conn = cf.make_connection(result.host_name(), result.port());
// Send the request
Request r;
- conn(r).get();
+
+ // This is a get request so make that
+ auto msg = make_shared<hbase::pb::GetRequest>();
+
+ // Set what region
+ msg->mutable_region()->set_value(FLAGS_region);
+ // It's always this.
+ msg->mutable_region()->set_type(
+ RegionSpecifier_RegionSpecifierType::
+ RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
+ // What row.
+ msg->mutable_get()->set_row(FLAGS_row);
+ // Send it.
+ r.set_msg(std::move(msg));
+ auto resp = (*conn)(r).get(milliseconds(5000));
+
+ auto get_resp = std::static_pointer_cast<GetResponse>(resp.response());
+ cout << "GetResponse has_result = " << get_resp->has_result() << '\n';
+ if (get_resp->has_result()) {
+ auto &r = get_resp->result();
+ cout << "Result cell_size = " << r.cell_size() << endl;
+ for (auto &cell : r.cell()) {
+ cout << "\trow = " << cell.row() << " family = " << cell.family()
+ << " qualifier = " << cell.qualifier()
+ << " timestamp = " << cell.timestamp() << " value = " << cell.value()
+ << endl;
+ }
+ }
return 0;
}
diff --git a/core/simple-native-client-test.cc b/core/simple-native-client-test.cc
deleted file mode 100644
index ee39986..0000000
--- a/core/simple-native-client-test.cc
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "gtest/gtest.h"
-
-/**
- * Sample test.
- */
-TEST(SampleTest, sample) { EXPECT_TRUE(true); }
diff --git a/core/table-name.h b/core/table-name.h
index 796115b..37c3461 100644
--- a/core/table-name.h
+++ b/core/table-name.h
@@ -29,4 +29,4 @@ public:
explicit TableName(std::string tableName);
explicit TableName(std::string namespaceName, std::string tableName);
};
-} // namespace hbase
+} // namespace hbase
diff --git a/core/native-client-test-env.cc b/core/test-env.cc
similarity index 91%
rename from core/native-client-test-env.cc
rename to core/test-env.cc
index 0269a43..277abd9 100644
--- a/core/native-client-test-env.cc
+++ b/core/test-env.cc
@@ -25,13 +25,13 @@ class NativeClientTestEnv : public ::testing::Environment {
public:
void SetUp() override {
// start local HBase cluster to be reused by all tests
- auto result = system("bin/start_local_hbase_and_wait.sh");
+ auto result = system("bin/start-local-hbase.sh");
ASSERT_EQ(0, result);
}
void TearDown() override {
// shutdown local HBase cluster
- auto result = system("bin/stop_local_hbase_and_wait.sh");
+ auto result = system("bin/stop-local-hbase.sh");
ASSERT_EQ(0, result);
}
};
diff --git a/if/BUCK b/if/BUCK
index 5ff617d..1a9721d 100644
--- a/if/BUCK
+++ b/if/BUCK
@@ -15,39 +15,34 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
PROTO_SRCS = glob(['*.proto'])
-HEADER_FILENAMES = [ x.replace('.proto','.pb.h') for x in PROTO_SRCS]
-CC_FILENAMES = [ x.replace('.proto', '.pb.cc') for x in PROTO_SRCS]
+HEADER_FILENAMES = [x.replace('.proto', '.pb.h') for x in PROTO_SRCS]
+CC_FILENAMES = [x.replace('.proto', '.pb.cc') for x in PROTO_SRCS]
genrule(
- name = 'generate-proto-sources',
- srcs = PROTO_SRCS,
- cmd = 'mkdir -p $OUT && pwd && protoc --proto_path=. --cpp_out=$OUT *.proto',
- out = 'output',
-)
+ name='generate-proto-sources',
+ srcs=PROTO_SRCS,
+ cmd='mkdir -p $OUT && pwd && protoc --proto_path=. --cpp_out=$OUT *.proto',
+ out='output', )
for header_filename in HEADER_FILENAMES:
- genrule(
- name = header_filename,
- cmd = 'mkdir -p `dirname $OUT` '
- ' && cp $(location :generate-proto-sources)/{} $OUT'.format(header_filename),
- out = header_filename,
- )
+ genrule(name=header_filename,
+ cmd='mkdir -p `dirname $OUT` '
+ ' && cp $(location :generate-proto-sources)/{} $OUT'.format(
+ header_filename),
+ out=header_filename, )
for cc_filename in CC_FILENAMES:
- genrule(
- name = cc_filename,
- cmd = 'mkdir -p `dirname $OUT` '
- ' && cp $(location :generate-proto-sources)/*.cc `dirname $OUT` '
- ' && cp $(location :generate-proto-sources)/*.h `dirname $OUT`'.format(cc_filename),
- out = cc_filename,
- )
+ genrule(
+ name=cc_filename,
+ cmd='mkdir -p `dirname $OUT` '
+ ' && cp $(location :generate-proto-sources)/*.cc `dirname $OUT` '
+ ' && cp $(location :generate-proto-sources)/*.h `dirname $OUT`'.format(
+ cc_filename),
+ out=cc_filename, )
-cxx_library(
- name = 'if',
- exported_headers = [':' + x for x in HEADER_FILENAMES],
- srcs = [':' + x for x in CC_FILENAMES],
- deps = [ '//third-party:protobuf'],
- visibility = [ 'PUBLIC', ],
- exported_deps = ['//third-party:protobuf']
-)
+cxx_library(name='if',
+ exported_headers=[':' + x for x in HEADER_FILENAMES],
+ srcs=[':' + x for x in CC_FILENAMES],
+ deps=['//third-party:protobuf'],
+ visibility=['PUBLIC', ],
+ exported_deps=['//third-party:protobuf'])
diff --git a/serde/BUCK b/serde/BUCK
new file mode 100644
index 0000000..207607f
--- /dev/null
+++ b/serde/BUCK
@@ -0,0 +1,54 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cxx_library(name="serde",
+ exported_headers=[
+ "client-serializer.h",
+ "client-deserializer.h",
+ ],
+ srcs=[
+ "client-serializer.cc",
+ "client-deserializer.cc",
+ ],
+ deps=[
+ "//if:if",
+ "//third-party:folly",
+ ],
+ tests=[
+ ":client-serializer-test",
+ ":client-deserializer-test",
+ ],
+ visibility=[
+ 'PUBLIC',
+ ], )
+
+cxx_test(name="client-serializer-test",
+ srcs=[
+ "client-serializer-test.cc",
+ ],
+ deps=[
+ ":serde",
+ "//if:if",
+ ], )
+cxx_test(name="client-deserializer-test",
+ srcs=[
+ "client-deserializer-test.cc",
+ ],
+ deps=[
+ ":serde",
+ "//if:if",
+ ], )
diff --git a/serde/client-deserializer-test.cc b/serde/client-deserializer-test.cc
new file mode 100644
index 0000000..bb57e50
--- /dev/null
+++ b/serde/client-deserializer-test.cc
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <gtest/gtest.h>
+#include <folly/io/IOBuf.h>
+
+#include "serde/client-deserializer.h"
+#include "serde/client-serializer.h"
+#include "if/Client.pb.h"
+
+using namespace hbase;
+using folly::IOBuf;
+using hbase::pb::GetRequest;
+using hbase::pb::RegionSpecifier;
+using hbase::pb::RegionSpecifier_RegionSpecifierType;
+
+TEST(TestClientDeserializer, TestReturnFalseOnNullPtr) {
+ ClientDeserializer deser;
+ ASSERT_LT(deser.parse_delimited(nullptr, nullptr), 0);
+}
+
+TEST(TestClientDeserializer, TestReturnFalseOnBadInput) {
+ ClientDeserializer deser;
+ auto buf = IOBuf::copyBuffer("test");
+ GetRequest gr;
+
+ ASSERT_LT(deser.parse_delimited(buf.get(), &gr), 0);
+}
+
+TEST(TestClientDeserializer, TestGoodGetRequestFullRoundTrip) {
+ GetRequest in;
+ ClientSerializer ser;
+ ClientDeserializer deser;
+
+ // fill up the GetRequest.
+ in.mutable_region()->set_value("test_region_id");
+ in.mutable_region()->set_type(
+ RegionSpecifier_RegionSpecifierType::
+ RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
+ in.mutable_get()->set_row("test_row");
+
+ // Create the buffer
+ auto buf = ser.serialize_delimited(in);
+
+ GetRequest out;
+
+ int used_bytes = deser.parse_delimited(buf.get(), &out);
+
+ ASSERT_GT(used_bytes, 0);
+ ASSERT_EQ(used_bytes, buf->length());
+}
diff --git a/serde/client-deserializer.cc b/serde/client-deserializer.cc
new file mode 100644
index 0000000..118b0d1
--- /dev/null
+++ b/serde/client-deserializer.cc
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "serde/client-deserializer.h"
+
+#include <google/protobuf/message.h>
+#include <google/protobuf/io/coded_stream.h>
+#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
+#include <folly/Logging.h>
+
+using namespace hbase;
+
+using folly::IOBuf;
+using google::protobuf::Message;
+using google::protobuf::io::ArrayInputStream;
+using google::protobuf::io::CodedInputStream;
+
+int ClientDeserializer::parse_delimited(const IOBuf *buf, Message *msg) {
+ if (buf == nullptr || msg == nullptr) {
+ return -2;
+ }
+
+ DCHECK(!buf->isChained());
+
+ ArrayInputStream ais{buf->data(), static_cast<int>(buf->length())};
+ CodedInputStream coded_stream{&ais};
+
+ uint32_t msg_size;
+
+ // Try and read the varint.
+ if (coded_stream.ReadVarint32(&msg_size) == false) {
+ FB_LOG_EVERY_MS(ERROR, 1000) << "Unable to read a var uint32_t";
+ return -3;
+ }
+
+ coded_stream.PushLimit(msg_size);
+ // Parse the message.
+ if (msg->MergeFromCodedStream(&coded_stream) == false) {
+ FB_LOG_EVERY_MS(ERROR, 1000)
+ << "Unable to read a protobuf message from data.";
+ return -4;
+ }
+
+ // Make sure all the data was consumed.
+ if (coded_stream.ConsumedEntireMessage() == false) {
+ FB_LOG_EVERY_MS(ERROR, 1000)
+ << "Orphaned data left after reading protobuf message";
+ return -5;
+ }
+
+ return coded_stream.CurrentPosition();
+}
diff --git a/core/response.h b/serde/client-deserializer.h
similarity index 79%
rename from core/response.h
rename to serde/client-deserializer.h
index 34a284d..b9664b0 100644
--- a/core/response.h
+++ b/serde/client-deserializer.h
@@ -18,17 +18,19 @@
*/
#pragma once
-#include <cstdint>
+#include <folly/io/IOBuf.h>
-namespace hbase {
+// Forward
+namespace google {
+namespace protobuf {
+class Message;
+}
+}
-class Response {
+namespace hbase {
+class ClientDeserializer {
public:
- Response() : call_id_(0) {}
- uint32_t call_id() { return call_id_; }
- void set_call_id(uint32_t call_id) { call_id_ = call_id; }
-
-private:
- uint32_t call_id_;
+ int parse_delimited(const folly::IOBuf *buf, google::protobuf::Message *msg);
};
-} // namespace hbase
+
+} // namespace hbase
diff --git a/serde/client-serializer-test.cc b/serde/client-serializer-test.cc
new file mode 100644
index 0000000..b32b55d
--- /dev/null
+++ b/serde/client-serializer-test.cc
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include <gtest/gtest.h>
+
+#include <folly/io/Cursor.h>
+
+#include <string>
+
+#include "serde/client-serializer.h"
+#include "if/HBase.pb.h"
+#include "if/RPC.pb.h"
+
+using namespace hbase;
+using namespace hbase::pb;
+using namespace folly;
+using namespace folly::io;
+
+TEST(ClientSerializerTest, PreambleIncludesHBas) {
+ ClientSerializer ser;
+ auto buf = ser.preamble();
+ const char *p = reinterpret_cast<const char *>(buf->data());
+ // Take the first for chars and make sure they are the
+ // magic string
+ EXPECT_EQ("HBas", std::string(p, 4));
+
+ EXPECT_EQ(6, buf->computeChainDataLength());
+}
+
+TEST(ClientSerializerTest, PreambleIncludesVersion) {
+ ClientSerializer ser;
+ auto buf = ser.preamble();
+ EXPECT_EQ(0, static_cast<const uint8_t *>(buf->data())[4]);
+ EXPECT_EQ(80, static_cast<const uint8_t *>(buf->data())[5]);
+}
+
+TEST(ClientSerializerTest, TestHeaderLengthPrefixed) {
+ ClientSerializer ser;
+ auto header = ser.header("elliott");
+
+ // The header should be prefixed by 4 bytes of length.
+ EXPECT_EQ(4, header->length());
+ EXPECT_TRUE(header->length() < header->computeChainDataLength());
+ EXPECT_TRUE(header->isChained());
+
+ // Now make sure the length is correct.
+ Cursor cursor(header.get());
+ auto prefixed_len = cursor.readBE<uint32_t>();
+ EXPECT_EQ(prefixed_len, header->next()->length());
+}
+
+TEST(ClientSerializerTest, TestHeaderDecode) {
+ ClientSerializer ser;
+ auto buf = ser.header("elliott");
+ auto header_buf = buf->next();
+ ConnectionHeader h;
+
+ EXPECT_TRUE(h.ParseFromArray(header_buf->data(), header_buf->length()));
+ EXPECT_EQ("elliott", h.user_info().effective_user());
+}
diff --git a/serde/client-serializer.cc b/serde/client-serializer.cc
new file mode 100644
index 0000000..881b6e4
--- /dev/null
+++ b/serde/client-serializer.cc
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include "serde/client-serializer.h"
+
+#include <folly/io/Cursor.h>
+#include <folly/Logging.h>
+#include <google/protobuf/io/coded_stream.h>
+#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
+
+#include "if/HBase.pb.h"
+#include "if/RPC.pb.h"
+
+using namespace hbase;
+
+using folly::IOBuf;
+using folly::io::RWPrivateCursor;
+using google::protobuf::Message;
+using google::protobuf::io::ArrayOutputStream;
+using google::protobuf::io::CodedOutputStream;
+using google::protobuf::io::ZeroCopyOutputStream;
+using std::string;
+using std::unique_ptr;
+
+static const std::string PREAMBLE = "HBas";
+static const std::string INTERFACE = "ClientService";
+static const uint8_t RPC_VERSION = 0;
+static const uint8_t DEFAULT_AUTH_TYPE = 80;
+
+ClientSerializer::ClientSerializer() : auth_type_(DEFAULT_AUTH_TYPE) {}
+
+unique_ptr<IOBuf> ClientSerializer::preamble() {
+ auto magic = IOBuf::copyBuffer(PREAMBLE, 0, 2);
+ magic->append(2);
+ RWPrivateCursor c(magic.get());
+ c.skip(4);
+ // Version
+ c.write(RPC_VERSION);
+ // Standard security aka Please don't lie to me.
+ c.write(auth_type_);
+ return magic;
+}
+
+unique_ptr<IOBuf> ClientSerializer::header(const string &user) {
+ pb::ConnectionHeader h;
+
+ // TODO(eclark): Make this not a total lie.
+ h.mutable_user_info()->set_effective_user(user);
+ // The service name that we want to talk to.
+ //
+ // Right now we're completely ignoring the service interface.
+ // That may or may not be the correct thing to do.
+ // It worked for a while with the java client; until it
+ // didn't.
+ h.set_service_name(INTERFACE);
+ return prepend_length(serialize_message(h));
+}
+
+unique_ptr<IOBuf> ClientSerializer::request(const uint32_t call_id,
+ const string &method,
+ const Message *msg) {
+ pb::RequestHeader rq;
+ rq.set_method_name(method);
+ rq.set_call_id(call_id);
+ rq.set_request_param(msg != nullptr);
+ auto ser_header = serialize_delimited(rq);
+ if (msg != nullptr) {
+ auto ser_req = serialize_delimited(*msg);
+ ser_header->appendChain(std::move(ser_req));
+ }
+
+ return prepend_length(std::move(ser_header));
+}
+
+unique_ptr<IOBuf> ClientSerializer::prepend_length(unique_ptr<IOBuf> msg) {
+ // Java ints are 4 long. So create a buffer that large
+ auto len_buf = IOBuf::create(4);
+ // Then make those bytes visible.
+ len_buf->append(4);
+
+ RWPrivateCursor c(len_buf.get());
+ // Get the size of the data to be pushed out the network.
+ auto size = msg->computeChainDataLength();
+
+ // Write the length to this IOBuf.
+ c.writeBE(static_cast<uint32_t>(size));
+
+ // Then attach the origional to the back of len_buf
+ len_buf->appendChain(std::move(msg));
+ return len_buf;
+}
+
+unique_ptr<IOBuf> ClientSerializer::serialize_delimited(const Message &msg) {
+ // Get the buffer size needed for just the message.
+ int msg_size = msg.ByteSize();
+ int buf_size = CodedOutputStream::VarintSize32(msg_size) + msg_size;
+
+ // Create a buffer big enough to hold the varint and the object.
+ auto buf = IOBuf::create(buf_size);
+ buf->append(buf_size);
+
+ // Create the array output stream.
+ ArrayOutputStream aos{buf->writableData(), static_cast<int>(buf->length())};
+ // Wrap the ArrayOuputStream in the coded output stream to allow writing
+ // Varint32
+ CodedOutputStream cos{&aos};
+
+ // Write out the size.
+ cos.WriteVarint32(msg_size);
+
+ // Now write the rest out.
+ // We're using the protobuf output streams here to keep track
+ // of where in the output array we are rather than IOBuf.
+ msg.SerializeWithCachedSizesToArray(
+ cos.GetDirectBufferForNBytesAndAdvance(msg_size));
+
+ // Return the buffer.
+ return buf;
+}
+// TODO(eclark): Make this 1 copy.
+unique_ptr<IOBuf> ClientSerializer::serialize_message(const Message &msg) {
+ auto buf = IOBuf::copyBuffer(msg.SerializeAsString());
+ return buf;
+}
diff --git a/core/client-serialize-handler.h b/serde/client-serializer.h
similarity index 55%
rename from core/client-serialize-handler.h
rename to serde/client-serializer.h
index 961a03b..685095d 100644
--- a/core/client-serialize-handler.h
+++ b/serde/client-serializer.h
@@ -18,27 +18,38 @@
*/
#pragma once
-#include <wangle/channel/Handler.h>
+#include <folly/io/IOBuf.h>
+#include <string>
+#include <cstdint>
-#include "if/HBase.pb.h"
-#include "if/RPC.pb.h"
-#include "core/request.h"
-#include "core/response.h"
+// Forward
+namespace google {
+namespace protobuf {
+class Message;
+}
+}
+namespace hbase {
+class Request;
+}
namespace hbase {
-class ClientSerializeHandler
- : public wangle::Handler<std::unique_ptr<folly::IOBuf>, Response, Request,
- std::unique_ptr<folly::IOBuf>> {
+class ClientSerializer {
public:
- void read(Context *ctx, std::unique_ptr<folly::IOBuf> msg) override;
- folly::Future<folly::Unit> write(Context *ctx, Request r) override;
+ ClientSerializer();
+ std::unique_ptr<folly::IOBuf> preamble();
+ std::unique_ptr<folly::IOBuf> header(const std::string &user);
+ std::unique_ptr<folly::IOBuf> request(const uint32_t call_id,
+ const std::string &method,
+ const google::protobuf::Message *msg);
+ std::unique_ptr<folly::IOBuf>
+ serialize_delimited(const google::protobuf::Message &msg);
+
+ std::unique_ptr<folly::IOBuf>
+ serialize_message(const google::protobuf::Message &msg);
-private:
- folly::Future<folly::Unit> write_preamble(Context *ctx);
- folly::Future<folly::Unit> write_header(Context *ctx);
- // Our own simple version of LengthFieldPrepender
std::unique_ptr<folly::IOBuf>
prepend_length(std::unique_ptr<folly::IOBuf> msg);
- bool need_send_header_ = true;
+
+ uint8_t auth_type_;
};
-} // namespace hbase
+} // namespace hbase
[hbase] 03/137: HBASE-14087 Ensure correct ASF headers for docs/code
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 89dda63e04b3ced210cee71b1773c17d9f6d9768
Author: Sean Busbey <bu...@apache.org>
AuthorDate: Wed Jul 15 05:22:53 2015 -0500
HBASE-14087 Ensure correct ASF headers for docs/code
* adds missing headers
* corrects some old-form headers to current
* restores proper header for guava copied LimitInputStream
* adds proper header for protobuf copied read/writeRawVarint
---
CMakeLists.txt | 24 ++++++++++++++----------
README.md | 19 +++++++++++++++++++
cmake_modules/FindGTest.cmake | 24 ++++++++++++++----------
cmake_modules/FindLibEv.cmake | 24 ++++++++++++++----------
src/async/CMakeLists.txt | 23 ++++++++++++++---------
src/core/CMakeLists.txt | 23 ++++++++++++++---------
src/rpc/CMakeLists.txt | 24 ++++++++++++++----------
src/sync/CMakeLists.txt | 23 ++++++++++++++---------
8 files changed, 117 insertions(+), 67 deletions(-)
diff --git a/CMakeLists.txt b/CMakeLists.txt
index be872df..819f178 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -1,15 +1,19 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
cmake_minimum_required(VERSION 2.6)
diff --git a/README.md b/README.md
index a959eca..0bb4d48 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,22 @@
+<!---
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
# hbase-native-client
Native client for HBase 0.96
diff --git a/cmake_modules/FindGTest.cmake b/cmake_modules/FindGTest.cmake
index 1da6892..8d27876 100644
--- a/cmake_modules/FindGTest.cmake
+++ b/cmake_modules/FindGTest.cmake
@@ -1,15 +1,19 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
set( GTEST_INCLUDE_SEARCH
${CMAKE_SOURCE_DIR}/thirdparty/gtest/include
diff --git a/cmake_modules/FindLibEv.cmake b/cmake_modules/FindLibEv.cmake
index 9a40c17..5fd3e86 100644
--- a/cmake_modules/FindLibEv.cmake
+++ b/cmake_modules/FindLibEv.cmake
@@ -1,15 +1,19 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
set( LIBEV_INCLUDE_SEARCH
${CMAKE_SOURCE_DIR}/thirdparty/installed/include
diff --git a/src/async/CMakeLists.txt b/src/async/CMakeLists.txt
index c11537c..b2ed461 100644
--- a/src/async/CMakeLists.txt
+++ b/src/async/CMakeLists.txt
@@ -1,14 +1,19 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
set( ASYNC_SRC
hbase_admin.cc
diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt
index c15f649..49f93b9 100644
--- a/src/core/CMakeLists.txt
+++ b/src/core/CMakeLists.txt
@@ -1,14 +1,19 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
set( CORE_SRC
admin.cc
diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt
index d9c0695..2456923 100644
--- a/src/rpc/CMakeLists.txt
+++ b/src/rpc/CMakeLists.txt
@@ -1,13 +1,17 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/src/sync/CMakeLists.txt b/src/sync/CMakeLists.txt
index ddd2212..bfb7e6c 100644
--- a/src/sync/CMakeLists.txt
+++ b/src/sync/CMakeLists.txt
@@ -1,14 +1,19 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
#
-# http://www.apache.org/licenses/LICENSE-2.0
+# http://www.apache.org/licenses/LICENSE-2.0
#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
set( SYNC_SRC
hbase_connection.cc
[hbase] 09/137: HBASE-15418 Clean up un-used warning in test util
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 881dd5173f04fbf4fae2e82e95fa144e148f436a
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Fri Mar 25 15:44:06 2016 -0700
HBASE-15418 Clean up un-used warning in test util
---
bin/start_local_hbase_and_wait.sh | 9 ++++++++-
bin/stop_local_hbase_and_wait.sh | 2 +-
core/BUCK | 6 ------
core/native-client-test-env.cc | 9 ++++++---
core/test_env.h | 32 --------------------------------
5 files changed, 15 insertions(+), 43 deletions(-)
diff --git a/bin/start_local_hbase_and_wait.sh b/bin/start_local_hbase_and_wait.sh
index 64d0b68..cfc71f9 100755
--- a/bin/start_local_hbase_and_wait.sh
+++ b/bin/start_local_hbase_and_wait.sh
@@ -17,10 +17,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# Clean up from any other tests.
+rm -rf /tmp/hbase-*
+
+# Start the master/regionservers.
$PWD/../bin/start-hbase.sh
-until [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010) == "200" ]
+until [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010/jmx) == "200" ]
do
printf "Waiting for local HBase cluster to start\n"
sleep 1
done
+
+# This sucks, but master can easily be up and meta not be assigned yet.
+sleep 30
diff --git a/bin/stop_local_hbase_and_wait.sh b/bin/stop_local_hbase_and_wait.sh
index 4e89334..761412a 100755
--- a/bin/stop_local_hbase_and_wait.sh
+++ b/bin/stop_local_hbase_and_wait.sh
@@ -17,7 +17,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-$PWD/../bin/stop-hbase.sh
+ps aux | grep proc_master | awk '{print $2}' | xargs kill -9
while [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010) == "200" ]
do
diff --git a/core/BUCK b/core/BUCK
index 817b5a0..d1e89d1 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -51,9 +51,6 @@ cxx_library(name="core",
], )
cxx_test(name="simple-test",
- headers=[
- "test_env.h",
- ],
srcs=[
"native-client-test-env.cc",
"simple-native-client-test.cc",
@@ -63,9 +60,6 @@ cxx_test(name="simple-test",
],
run_test_separately=True, )
cxx_test(name="location-cache-test",
- headers=[
- "test_env.h",
- ],
srcs=[
"native-client-test-env.cc",
"location-cache-test.cc",
diff --git a/core/native-client-test-env.cc b/core/native-client-test-env.cc
index a86961f..07f30a6 100644
--- a/core/native-client-test-env.cc
+++ b/core/native-client-test-env.cc
@@ -18,18 +18,21 @@
*/
#include <gtest/gtest.h>
-#include <core/test_env.h>
namespace {
class NativeClientTestEnv : public ::testing::Environment {
public:
void SetUp() override {
- init_test_env();
+ // start local HBase cluster to be reused by all tests
+ auto result = system("bin/start_local_hbase_and_wait.sh");
+ ASSERT_EQ(0, result);
}
void TearDown() override {
- clean_test_env();
+ // shutdown local HBase cluster
+ auto result = system("bin/stop_local_hbase_and_wait.sh");
+ ASSERT_EQ(0, result);
}
};
diff --git a/core/test_env.h b/core/test_env.h
deleted file mode 100644
index 79bdbec..0000000
--- a/core/test_env.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#pragma once
-
-#include <cstdlib>
-
-inline void init_test_env() {
- // start local HBase cluster to be reused by all tests
- system("scripts/start_local_hbase_and_wait.sh");
-}
-
-inline void clean_test_env() {
- // shutdown local HBase cluster
- system("scripts/stop_local_hbase_and_wait.sh");
-}
[hbase] 70/137: HBASE-17680 Run mini cluster through JNI in tests
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit ad85594098d367c5409a44e4c7f9364f3242c9dd
Author: tedyu <yu...@gmail.com>
AuthorDate: Mon Mar 6 14:59:39 2017 -0800
HBASE-17680 Run mini cluster through JNI in tests
---
Makefile | 4 +-
connection/BUCK | 12 +-
core/BUCK | 30 +++-
core/async-rpc-retrying-test.cc | 11 +-
core/client-test.cc | 49 ++----
core/filter-test.cc | 23 +--
core/location-cache-test.cc | 42 +++--
core/location-cache.cc | 3 +-
security/BUCK | 4 +-
serde/BUCK | 52 ++++--
test-util/BUCK | 24 ++-
test-util/mini-cluster.cc | 345 ++++++++++++++++++++++++++++++++++++++++
test-util/mini-cluster.h | 84 ++++++++++
test-util/test-util.cc | 36 +++--
test-util/test-util.h | 30 ++--
third-party/BUCK | 4 +-
16 files changed, 635 insertions(+), 118 deletions(-)
diff --git a/Makefile b/Makefile
index b926220..166c6ab 100644
--- a/Makefile
+++ b/Makefile
@@ -26,13 +26,13 @@ MODULES = connection core serde test-util utils security exceptions
SRC_DIR = $(MODULES)
DEBUG_BUILD_DIR = $(addprefix $(DEBUG_PATH)/,$(MODULES))
RELEASE_BUILD_DIR = $(addprefix $(RELEASE_PATH)/,$(MODULES))
-INCLUDE_DIR = . build/
+INCLUDE_DIR = . build/ $(JAVA_HOME)/include/ $(JAVA_HOME)/include/linux
#flags to pass to the CPP compiler & linker
CPPFLAGS_DEBUG = -D_GLIBCXX_USE_CXX11_ABI=0 -g -Wall -std=c++14 -pedantic -fPIC
CPPFLAGS_RELEASE = -D_GLIBCXX_USE_CXX11_ABI=0 -DNDEBUG -O2 -Wall -std=c++14 -pedantic -fPIC
LDFLAGS = -lprotobuf -lzookeeper_mt -lsasl2 -lfolly -lwangle
-LINKFLAG = -shared
+LINKFLAG = -shared -L$(JAVA_HOME)/jre/lib/amd64/server -ljvm
#define list of source files and object files
ALLSRC = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc))
diff --git a/connection/BUCK b/connection/BUCK
index 19536d5..bc05be0 100644
--- a/connection/BUCK
+++ b/connection/BUCK
@@ -50,8 +50,14 @@ cxx_library(
"//third-party:wangle",
],
compiler_flags=['-Weffc++'],
- visibility=['//core/...',],)
+ visibility=[
+ '//core/...',
+ ],)
cxx_test(
name="connection-pool-test",
- srcs=["connection-pool-test.cc",],
- deps=[":connection",],)
+ srcs=[
+ "connection-pool-test.cc",
+ ],
+ deps=[
+ ":connection",
+ ],)
diff --git a/core/BUCK b/core/BUCK
index 2f4f6c1..95fb8b5 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -70,11 +70,15 @@ cxx_library(
"//third-party:wangle",
"//third-party:zookeeper_mt",
],
- compiler_flags=['-Weffc++'],
- visibility=['PUBLIC',],)
+ compiler_flags=['-Weffc++', '-ggdb'],
+ visibility=[
+ 'PUBLIC',
+ ],)
cxx_test(
name="location-cache-test",
- srcs=["location-cache-test.cc",],
+ srcs=[
+ "location-cache-test.cc",
+ ],
deps=[
":core",
"//test-util:test-util",
@@ -82,12 +86,18 @@ cxx_test(
run_test_separately=True,)
cxx_test(
name="cell-test",
- srcs=["cell-test.cc",],
- deps=[":core",],
+ srcs=[
+ "cell-test.cc",
+ ],
+ deps=[
+ ":core",
+ ],
run_test_separately=True,)
cxx_test(
name="filter-test",
- srcs=["filter-test.cc",],
+ srcs=[
+ "filter-test.cc",
+ ],
deps=[
":core",
"//if:if",
@@ -145,7 +155,9 @@ cxx_test(
run_test_separately=True,)
cxx_test(
name="client-test",
- srcs=["client-test.cc",],
+ srcs=[
+ "client-test.cc",
+ ],
deps=[
":core",
"//if:if",
@@ -155,5 +167,7 @@ cxx_test(
run_test_separately=True,)
cxx_binary(
name="simple-client",
- srcs=["simple-client.cc",],
+ srcs=[
+ "simple-client.cc",
+ ],
deps=[":core", "//connection:connection"],)
diff --git a/core/async-rpc-retrying-test.cc b/core/async-rpc-retrying-test.cc
index a9b0017..d2435b9 100644
--- a/core/async-rpc-retrying-test.cc
+++ b/core/async-rpc-retrying-test.cc
@@ -148,9 +148,9 @@ TEST(AsyncRpcRetryTest, TestGetBasic) {
// Using TestUtil to populate test data
hbase::TestUtil* test_util = new hbase::TestUtil();
- test_util->RunShellCmd("create 't', 'd'");
- test_util->RunShellCmd("put 't', 'test2', 'd:2', 'value2'");
- test_util->RunShellCmd("put 't', 'test2', 'd:extra', 'value for extra'");
+ test_util->CreateTable("t", "d");
+ test_util->TablePut("t", "test2", "d", "2", "value2");
+ test_util->TablePut("t", "test2", "d", "extra", "value for extra");
// Create TableName and Row to be fetched from HBase
auto tn = folly::to<hbase::pb::TableName>("t");
@@ -159,11 +159,8 @@ TEST(AsyncRpcRetryTest, TestGetBasic) {
// Get to be performed on above HBase Table
hbase::Get get(row);
- // Create Configuration
- hbase::Configuration conf;
-
// Create a client
- Client client(conf);
+ Client client(*(test_util->conf()));
// Get connection to HBase Table
auto table = client.Table(tn);
diff --git a/core/client-test.cc b/core/client-test.cc
index 0a45fff..afca847 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -27,7 +27,7 @@
#include "serde/table-name.h"
#include "test-util/test-util.h"
-class ClientTest {
+class ClientTest : public ::testing::Test {
public:
const static std::string kDefHBaseConfPath;
@@ -45,10 +45,6 @@ class ClientTest {
static void CreateHBaseConf(const std::string &dir, const std::string &file,
const std::string xml_data) {
- // Directory will be created if not present
- if (!boost::filesystem::exists(dir)) {
- boost::filesystem::create_directories(dir);
- }
// Remove temp file always
boost::filesystem::remove((dir + file).c_str());
WriteDataToFile((dir + file), xml_data);
@@ -57,10 +53,16 @@ class ClientTest {
static void CreateHBaseConfWithEnv() {
// Creating Empty Config Files so that we dont get a Configuration exception @Client
CreateHBaseConf(kDefHBaseConfPath, kHBaseDefaultXml, kHBaseXmlData);
- CreateHBaseConf(kDefHBaseConfPath, kHBaseSiteXml, kHBaseXmlData);
+ // the hbase-site.xml would be persisted by MiniCluster
setenv("HBASE_CONF", kDefHBaseConfPath.c_str(), 1);
}
+ static std::unique_ptr<hbase::TestUtil> test_util;
+
+ static void SetUpTestCase() {
+ test_util = std::make_unique<hbase::TestUtil>(2, ClientTest::kDefHBaseConfPath.c_str());
+ }
};
+std::unique_ptr<hbase::TestUtil> ClientTest::test_util = nullptr;
const std::string ClientTest::kDefHBaseConfPath("./build/test-data/client-test/conf/");
@@ -109,16 +111,15 @@ TEST(Client, DefaultConfiguration) {
client.Close();
}
-TEST(Client, Get) {
+TEST_F(ClientTest, Get) {
// Remove already configured env if present.
unsetenv("HBASE_CONF");
ClientTest::CreateHBaseConfWithEnv();
// Using TestUtil to populate test data
- hbase::TestUtil *test_util = new hbase::TestUtil();
- test_util->RunShellCmd(
- "create 't', 'd'; put 't', 'test2', 'd:2', 'value2'; put 't', 'test2', 'd:extra', 'value for "
- "extra'");
+ ClientTest::test_util->CreateTable("t", "d");
+ ClientTest::test_util->TablePut("t", "test2", "d", "2", "value2");
+ ClientTest::test_util->TablePut("t", "test2", "d", "extra", "value for extra");
// Create TableName and Row to be fetched from HBase
auto tn = folly::to<hbase::pb::TableName>("t");
@@ -141,12 +142,6 @@ TEST(Client, Get) {
// Perform the Get
auto result = table->Get(get);
- // Stopping the connection as we are getting segfault due to some folly issue
- // The connection stays open and we don't want that.
- // So we are stopping the connection.
- // We can remove this once we have fixed the folly part
- delete test_util;
-
// Test the values, should be same as in put executed on hbase shell
ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty.";
EXPECT_EQ("test2", result->Row());
@@ -162,9 +157,6 @@ TEST(Client, GetForNonExistentTable) {
unsetenv("HBASE_CONF");
ClientTest::CreateHBaseConfWithEnv();
- // Using TestUtil to populate test data
- hbase::TestUtil *test_util = new hbase::TestUtil();
-
// Create TableName and Row to be fetched from HBase
auto tn = folly::to<hbase::pb::TableName>("t_not_exists");
auto row = "test2";
@@ -186,24 +178,17 @@ TEST(Client, GetForNonExistentTable) {
// Perform the Get
ASSERT_ANY_THROW(table->Get(get)) << "Table does not exist. We should get an exception";
- // Stopping the connection as we are getting segfault due to some folly issue
- // The connection stays open and we don't want that.
- // So we are stopping the connection.
- // We can remove this once we have fixed the folly part
- delete test_util;
-
table->Close();
client.Close();
}
-TEST(Client, GetForNonExistentRow) {
+TEST_F(ClientTest, GetForNonExistentRow) {
// Remove already configured env if present.
unsetenv("HBASE_CONF");
ClientTest::CreateHBaseConfWithEnv();
// Using TestUtil to populate test data
- hbase::TestUtil *test_util = new hbase::TestUtil();
- test_util->RunShellCmd("create 't_exists', 'd'");
+ ClientTest::test_util->CreateTable("t_exists", "d");
// Create TableName and Row to be fetched from HBase
auto tn = folly::to<hbase::pb::TableName>("t_exists");
@@ -227,12 +212,6 @@ TEST(Client, GetForNonExistentRow) {
auto result = table->Get(get);
ASSERT_TRUE(result->IsEmpty()) << "Result should be empty.";
- // Stopping the connection as we are getting segfault due to some folly issue
- // The connection stays open and we don't want that.
- // So we are stopping the connection.
- // We can remove this once we have fixed the folly part
- delete test_util;
-
table->Close();
client.Close();
}
diff --git a/core/filter-test.cc b/core/filter-test.cc
index ff683b6..6529b5a 100644
--- a/core/filter-test.cc
+++ b/core/filter-test.cc
@@ -39,7 +39,9 @@ using hbase::Comparator;
class FilterTest : public ::testing::Test {
protected:
- static void SetUpTestCase() { test_util_ = std::make_unique<TestUtil>(); }
+ static void SetUpTestCase() {
+ test_util_ = std::make_unique<TestUtil>();
+ }
static void TearDownTestCase() { test_util_.release(); }
@@ -49,13 +51,15 @@ class FilterTest : public ::testing::Test {
static std::unique_ptr<TestUtil> test_util_;
};
+
std::unique_ptr<TestUtil> FilterTest::test_util_ = nullptr;
TEST_F(FilterTest, GetWithColumnPrefixFilter) {
// write row1 with 3 columns (column_1, column_2, and foo_column)
- FilterTest::test_util_->RunShellCmd(
- "create 't', 'd'; put 't', 'row1', 'd:column_1', 'value1'; put 't', 'row1', 'd:column_2', "
- "'value2'; put 't', 'row1', 'd:foo_column', 'value3'");
+ FilterTest::test_util_->CreateTable("t", "d");
+ FilterTest::test_util_->TablePut("t", "row1", "d", "column_1", "value1");
+ FilterTest::test_util_->TablePut("t", "row1", "d", "column_2", "value2");
+ FilterTest::test_util_->TablePut("t", "row1", "d", "foo_column", "value3");
// Create TableName and Row to be fetched from HBase
auto tn = folly::to<hbase::pb::TableName>("t");
@@ -70,7 +74,7 @@ TEST_F(FilterTest, GetWithColumnPrefixFilter) {
get_two.SetFilter(FilterFactory::ColumnPrefixFilter("column_"));
// Create a client
- hbase::Client client(Configuration{});
+ hbase::Client client(*(FilterTest::test_util_->conf()));
// Get connection to HBase Table
auto table = client.Table(tn);
@@ -101,9 +105,10 @@ TEST_F(FilterTest, GetWithColumnPrefixFilter) {
TEST_F(FilterTest, GetWithQualifierFilter) {
// write row1 with 3 columns (a,b,c)
- FilterTest::test_util_->RunShellCmd(
- "create 't1', 'd'; put 't1', 'row1', 'd:a', 'value1'; put 't1', 'row1', 'd:b', "
- "'value2'; put 't1', 'row1', 'd:c', 'value3'");
+ FilterTest::test_util_->CreateTable("t1", "d");
+ FilterTest::test_util_->TablePut("t1", "row1", "d", "a", "value1");
+ FilterTest::test_util_->TablePut("t1", "row1", "d", "b", "value2");
+ FilterTest::test_util_->TablePut("t1", "row1", "d", "c", "value3");
// Create TableName and Row to be fetched from HBase
auto tn = folly::to<hbase::pb::TableName>("t1");
@@ -115,7 +120,7 @@ TEST_F(FilterTest, GetWithQualifierFilter) {
*ComparatorFactory::BinaryComparator("b")));
// Create a client
- hbase::Client client(Configuration{});
+ hbase::Client client(*(FilterTest::test_util_->conf()));
// Get connection to HBase Table
auto table = client.Table(tn);
diff --git a/core/location-cache-test.cc b/core/location-cache-test.cc
index 9a778c6..75a56d1 100644
--- a/core/location-cache-test.cc
+++ b/core/location-cache-test.cc
@@ -30,13 +30,29 @@
using namespace hbase;
using namespace std::chrono;
-TEST(LocationCacheTest, TestGetMetaNodeContents) {
- TestUtil test_util{};
+class LocationCacheTest : public ::testing::Test {
+ protected:
+ static void SetUpTestCase() {
+ test_util_ = std::make_unique<TestUtil>();
+ }
+ static void TearDownTestCase() { test_util_.release(); }
+
+ virtual void SetUp() {}
+ virtual void TearDown() {}
+
+ public:
+ static std::unique_ptr<TestUtil> test_util_;
+};
+
+std::unique_ptr<TestUtil> LocationCacheTest::test_util_ = nullptr;
+
+
+TEST_F(LocationCacheTest, TestGetMetaNodeContents) {
auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
auto codec = std::make_shared<KeyValueCodec>();
auto cp = std::make_shared<ConnectionPool>(io, codec);
- LocationCache cache{test_util.conf(), cpu, cp};
+ LocationCache cache{LocationCacheTest::test_util_->conf(), cpu, cp};
auto f = cache.LocateMeta();
auto result = f.get();
ASSERT_FALSE(f.hasException());
@@ -47,19 +63,18 @@ TEST(LocationCacheTest, TestGetMetaNodeContents) {
cp->Close();
}
-TEST(LocationCacheTest, TestGetRegionLocation) {
- TestUtil test_util{};
+TEST_F(LocationCacheTest, TestGetRegionLocation) {
auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
auto codec = std::make_shared<KeyValueCodec>();
auto cp = std::make_shared<ConnectionPool>(io, codec);
- LocationCache cache{test_util.conf(), cpu, cp};
+ LocationCache cache{LocationCacheTest::test_util_->conf(), cpu, cp};
// If there is no table this should throw an exception
auto tn = folly::to<hbase::pb::TableName>("t");
auto row = "test";
ASSERT_ANY_THROW(cache.LocateFromMeta(tn, row).get(milliseconds(1000)));
- test_util.RunShellCmd("create 't', 'd'");
+ LocationCacheTest::test_util_->CreateTable("t", "d");
auto loc = cache.LocateFromMeta(tn, row).get(milliseconds(1000));
ASSERT_TRUE(loc != nullptr);
cpu->stop();
@@ -67,13 +82,12 @@ TEST(LocationCacheTest, TestGetRegionLocation) {
cp->Close();
}
-TEST(LocationCacheTest, TestCaching) {
- TestUtil test_util{};
+TEST_F(LocationCacheTest, TestCaching) {
auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
auto codec = std::make_shared<KeyValueCodec>();
auto cp = std::make_shared<ConnectionPool>(io, codec);
- LocationCache cache{test_util.conf(), cpu, cp};
+ LocationCache cache{LocationCacheTest::test_util_->conf(), cpu, cp};
auto tn_1 = folly::to<hbase::pb::TableName>("t1");
auto tn_2 = folly::to<hbase::pb::TableName>("t2");
@@ -83,8 +97,7 @@ TEST(LocationCacheTest, TestCaching) {
// test location pulled from meta gets cached
ASSERT_ANY_THROW(cache.LocateRegion(tn_1, row_a).get(milliseconds(1000)));
ASSERT_ANY_THROW(cache.LocateFromMeta(tn_1, row_a).get(milliseconds(1000)));
-
- test_util.RunShellCmd("create 't1', 'd'");
+ LocationCacheTest::test_util_->CreateTable("t1", "d");
ASSERT_FALSE(cache.IsLocationCached(tn_1, row_a));
auto loc = cache.LocateRegion(tn_1, row_a).get(milliseconds(1000));
@@ -92,7 +105,8 @@ TEST(LocationCacheTest, TestCaching) {
ASSERT_EQ(loc, cache.GetCachedLocation(tn_1, row_a));
// test with two regions
- test_util.RunShellCmd("create 't2', 'd', SPLITS => ['b']");
+ std::string empty;
+ LocationCacheTest::test_util_->CreateTable("t2", "d", "b", empty);
ASSERT_FALSE(cache.IsLocationCached(tn_2, "a"));
loc = cache.LocateRegion(tn_2, "a").get(milliseconds(1000));
@@ -107,7 +121,7 @@ TEST(LocationCacheTest, TestCaching) {
ASSERT_EQ(loc, cache.GetCachedLocation(tn_2, "ba"));
// test with three regions
- test_util.RunShellCmd("create 't3', 'd', SPLITS => ['b', 'c']");
+ LocationCacheTest::test_util_->CreateTable("t3", "d", "b", "c");
ASSERT_FALSE(cache.IsLocationCached(tn_3, "c"));
ASSERT_FALSE(cache.IsLocationCached(tn_3, "ca"));
diff --git a/core/location-cache.cc b/core/location-cache.cc
index 17032fe..9550614 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -107,7 +107,8 @@ ServerName LocationCache::ReadMetaLocation() {
reinterpret_cast<char *>(buf->writableData()), &len, nullptr);
if (zk_result != ZOK || len < 9) {
LOG(ERROR) << "Error getting meta location.";
- throw runtime_error("Error getting meta location");
+ throw runtime_error("Error getting meta location. Quorum " +
+ conf_->Get(kHBaseZookeeperQuorum_, ""));
}
buf->append(len);
diff --git a/security/BUCK b/security/BUCK
index e176c90..d602ff3 100644
--- a/security/BUCK
+++ b/security/BUCK
@@ -19,7 +19,9 @@
# to a single server.
cxx_library(
name="security",
- exported_headers=["user.h",],
+ exported_headers=[
+ "user.h",
+ ],
srcs=[],
deps=[],
compiler_flags=['-Weffc++'],
diff --git a/serde/BUCK b/serde/BUCK
index 38e7b4d..76d1c72 100644
--- a/serde/BUCK
+++ b/serde/BUCK
@@ -41,28 +41,54 @@ cxx_library(
":region-info-deserializer-test",
],
compiler_flags=['-Weffc++'],
- visibility=['PUBLIC',],)
+ visibility=[
+ 'PUBLIC',
+ ],)
cxx_test(
name="table-name-test",
- srcs=["table-name-test.cc",],
- deps=[":serde",],)
+ srcs=[
+ "table-name-test.cc",
+ ],
+ deps=[
+ ":serde",
+ ],)
cxx_test(
name="server-name-test",
- srcs=["server-name-test.cc",],
- deps=[":serde",],)
+ srcs=[
+ "server-name-test.cc",
+ ],
+ deps=[
+ ":serde",
+ ],)
cxx_test(
name="client-serializer-test",
- srcs=["client-serializer-test.cc",],
- deps=[":serde",],)
+ srcs=[
+ "client-serializer-test.cc",
+ ],
+ deps=[
+ ":serde",
+ ],)
cxx_test(
name="client-deserializer-test",
- srcs=["client-deserializer-test.cc",],
- deps=[":serde",],)
+ srcs=[
+ "client-deserializer-test.cc",
+ ],
+ deps=[
+ ":serde",
+ ],)
cxx_test(
name="zk-deserializer-test",
- srcs=["zk-deserializer-test.cc",],
- deps=[":serde",],)
+ srcs=[
+ "zk-deserializer-test.cc",
+ ],
+ deps=[
+ ":serde",
+ ],)
cxx_test(
name="region-info-deserializer-test",
- srcs=["region-info-deserializer-test.cc",],
- deps=[":serde",],)
+ srcs=[
+ "region-info-deserializer-test.cc",
+ ],
+ deps=[
+ ":serde",
+ ],)
diff --git a/test-util/BUCK b/test-util/BUCK
index c6e41f1..4998614 100644
--- a/test-util/BUCK
+++ b/test-util/BUCK
@@ -14,16 +14,34 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+import os
cxx_library(name="test-util",
exported_headers=[
- "test-util.h",
+ "test-util.h","mini-cluster.h"
],
- srcs=["test-util.cc"],
+ srcs=["test-util.cc","mini-cluster.cc"],
deps=[
"//third-party:folly",
- "//core:core"
+ "//core:core",
],
+ preprocessor_flags= [
+ '-I' + os.environ['JAVA_HOME'] + '/include',
+ '-I' + os.environ['JAVA_HOME'] + '/include/darwin',
+ '-I' + os.environ['JAVA_HOME'] + '/include/linux'],
+ exported_preprocessor_flags= [
+ '-I' + os.environ['JAVA_HOME'] + '/include',
+ '-I' + os.environ['JAVA_HOME'] + '/include/darwin',
+ '-I' + os.environ['JAVA_HOME'] + '/include/linux'],
+ compiler_flags = [
+ '-I' + os.environ['JAVA_HOME'] + '/include',
+ '-I' + os.environ['JAVA_HOME'] + '/include/darwin',
+ '-I' + os.environ['JAVA_HOME'] + '/include/linux', '-ggdb'],
+ linker_flags = ['-ljvm',
+ '-L' + os.environ['JAVA_HOME'] + '/jre/lib/amd64/server', '-ggdb'],
+ exported_linker_flags = ['-ljvm',
+ '-L' + os.environ['JAVA_HOME'] + '/jre/lib/amd64/server',
+ '-Wl,-rpath=' + os.environ['JAVA_HOME'] + '/jre/lib/amd64/server'],
visibility=[
'PUBLIC',
],
diff --git a/test-util/mini-cluster.cc b/test-util/mini-cluster.cc
new file mode 100644
index 0000000..b40b689
--- /dev/null
+++ b/test-util/mini-cluster.cc
@@ -0,0 +1,345 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "test-util/mini-cluster.h"
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+#include <boost/filesystem/operations.hpp>
+#include <boost/filesystem/fstream.hpp>
+#include <fstream>
+#include "core/client.h"
+#include "core/configuration.h"
+#include "core/get.h"
+#include "core/hbase_configuration_loader.h"
+#include "core/result.h"
+#include "core/table.h"
+#include "serde/table-name.h"
+
+using hbase::MiniCluster;
+
+JNIEnv *MiniCluster::CreateVM(JavaVM **jvm) {
+ JavaVMInitArgs args;
+ JavaVMOption jvm_options;
+ args.version = JNI_VERSION_1_6;
+ args.nOptions = 1;
+ char *classpath = getenv("CLASSPATH");
+ std::string clspath;
+ if (classpath == NULL || strstr(classpath, "-tests.jar") == NULL) {
+ std::string clsPathFilePath("../target/cached_classpath.txt");
+ std::ifstream fd(clsPathFilePath);
+ std::string prefix("");
+ if (fd.is_open()) {
+ if (classpath == NULL) {
+ LOG(INFO) << "got empty classpath";
+ } else {
+ // prefix bootstrapper.jar
+ prefix.assign(classpath);
+ }
+ std::string line;
+ if (getline(fd, line)) {
+ clspath = prefix + ":" + line;
+ int ret = setenv("CLASSPATH", clspath.c_str(), 1);
+ LOG(INFO) << "set clspath " << ret;
+ } else {
+ LOG(INFO) << "nothing read from " << clsPathFilePath;
+ exit(-1);
+ }
+ } else {
+ LOG(INFO) << "nothing read from " << clsPathFilePath;
+ exit(-1);
+ }
+ fd.close();
+ }
+ auto options = std::string {"-Djava.class.path="} + clspath;
+ jvm_options.optionString = const_cast<char *>(options.c_str());
+ args.options = &jvm_options;
+ args.ignoreUnrecognized = 0;
+ int rv;
+ rv = JNI_CreateJavaVM(jvm, reinterpret_cast<void **>(&env_), &args);
+ if (rv < 0 || !env_) {
+ LOG(INFO) << "Unable to Launch JVM " << rv;
+ } else {
+ LOG(INFO) << "Launched JVM! " << options;
+ }
+ return env_;
+}
+
+void MiniCluster::WriteConf(jobject conf, const std::string& filepath) {
+ jclass class_fdesc = env_->FindClass("java/io/FileDescriptor");
+ // construct a new FileDescriptor
+ jmethodID const_fdesc = env_->GetMethodID(class_fdesc, "<init>", "()V");
+
+ jobject file = env_->NewObject(class_fdesc, const_fdesc);
+ jfieldID field_fd = env_->GetFieldID(class_fdesc, "fd", "I");
+
+ int fd = open(filepath.c_str(), O_RDWR | O_NONBLOCK | O_CREAT, S_IRWXU);
+ if (fd < 0) {
+ LOG(INFO) << "Couldn't open file " << filepath.c_str();
+ exit(-1);
+ }
+ env_->SetIntField(file, field_fd, fd);
+
+ jclass cls_outstream = env_->FindClass("java/io/FileOutputStream");
+ jmethodID ctor_stream = env_->GetMethodID(cls_outstream, "<init>", "(Ljava/io/FileDescriptor;)V");
+ if (ctor_stream == NULL) {
+ LOG(INFO) << "Couldn't get ctor for FileOutputStream";
+ exit(-1);
+ }
+ jobject file_outstream = env_->NewObject(cls_outstream, ctor_stream, file);
+ if (file_outstream == NULL) {
+ LOG(INFO) << "Couldn't create FileOutputStream";
+ exit(-1);
+ }
+ jmethodID writeXmlMid = env_->GetMethodID(conf_class_, "writeXml", "(Ljava/io/OutputStream;)V");
+ env_->CallObjectMethod(conf, writeXmlMid, file_outstream);
+}
+void MiniCluster::Setup() {
+ jmethodID constructor;
+ pthread_mutex_lock(&count_mutex_);
+ if (env_ == NULL) {
+ env_ = CreateVM(&jvm);
+ if (env_ == NULL) {
+ exit(-1);
+ }
+ testing_util_class_ = env_->FindClass("org/apache/hadoop/hbase/HBaseTestingUtility");
+ // this should be converted to a globalref I think to avoid the underlying java obj getting
+ // GC'ed
+ if (testing_util_class_ == NULL) {
+ LOG(INFO) << "Couldn't find class HBaseTestingUtility";
+ exit(-1);
+ }
+ jmethodID mid = env_->GetStaticMethodID(testing_util_class_, "createLocalHTU",
+ "()Lorg/apache/hadoop/hbase/HBaseTestingUtility;");
+ htu_ = env_->CallStaticObjectMethod(testing_util_class_, mid);
+ // this should be converted to a globalref I think to avoid the underlying java obj getting
+ // GC'ed
+ if (htu_ == NULL) {
+ LOG(INFO) << "Couldn't invoke method createLocalHTU in HBaseTestingUtility";
+ exit(-1);
+ }
+ get_conn_mid_ = env_->GetMethodID(testing_util_class_, "getConnection",
+ "()Lorg/apache/hadoop/hbase/client/Connection;");
+ jclass connClass = env_->FindClass("org/apache/hadoop/hbase/client/Connection");
+ get_admin_mid_ =
+ env_->GetMethodID(connClass, "getAdmin", "()Lorg/apache/hadoop/hbase/client/Admin;");
+ get_table_mid_ = env_->GetMethodID(
+ connClass, "getTable",
+ "(Lorg/apache/hadoop/hbase/TableName;)Lorg/apache/hadoop/hbase/client/Table;");
+ if (get_table_mid_ == NULL) {
+ LOG(INFO) << "Couldn't find getConnection";
+ exit(-1);
+ }
+ jclass adminClass = env_->FindClass("org/apache/hadoop/hbase/client/Admin");
+ move_mid_ = env_->GetMethodID(adminClass, "move", "([B[B)V");
+ if (move_mid_ == NULL) {
+ LOG(INFO) << "Couldn't find move";
+ exit(-1);
+ }
+ create_table_mid_ = env_->GetMethodID(testing_util_class_, "createTable",
+ "(Lorg/apache/hadoop/hbase/TableName;Ljava/lang/String;)Lorg/"
+ "apache/hadoop/hbase/client/Table;");
+ create_table_with_split_mid_ = env_->GetMethodID(
+ testing_util_class_, "createTable",
+ "(Lorg/apache/hadoop/hbase/TableName;[[B[[B)Lorg/apache/hadoop/hbase/client/Table;");
+ if (create_table_with_split_mid_ == NULL) {
+ LOG(INFO) << "Couldn't find method createTable with split";
+ exit(-1);
+ }
+
+ table_name_class_ = env_->FindClass("org/apache/hadoop/hbase/TableName");
+ tbl_name_value_of_mid_ = env_->GetStaticMethodID(
+ table_name_class_, "valueOf", "(Ljava/lang/String;)Lorg/apache/hadoop/hbase/TableName;");
+ if (tbl_name_value_of_mid_ == NULL) {
+ LOG(INFO) << "Couldn't find method valueOf in TableName";
+ exit(-1);
+ }
+ jclass hbaseMiniClusterClass = env_->FindClass("org/apache/hadoop/hbase/MiniHBaseCluster");
+ stop_rs_mid_ =
+ env_->GetMethodID(hbaseMiniClusterClass, "stopRegionServer",
+ "(I)Lorg/apache/hadoop/hbase/util/JVMClusterUtil$RegionServerThread;");
+ get_conf_mid_ = env_->GetMethodID(hbaseMiniClusterClass, "getConfiguration",
+ "()Lorg/apache/hadoop/conf/Configuration;");
+
+ conf_class_ = env_->FindClass("org/apache/hadoop/conf/Configuration");
+ set_conf_mid_ = env_->GetMethodID(conf_class_, "set", "(Ljava/lang/String;Ljava/lang/String;)V");
+ if (set_conf_mid_ == NULL) {
+ LOG(INFO) << "Couldn't find method getConf in MiniHBaseCluster";
+ exit(-1);
+ }
+ conf_get_mid_ = env_->GetMethodID(conf_class_, "get", "(Ljava/lang/String;)Ljava/lang/String;");
+
+ jclass tableClass = env_->FindClass("org/apache/hadoop/hbase/client/Table");
+ put_mid_ = env_->GetMethodID(tableClass, "put", "(Lorg/apache/hadoop/hbase/client/Put;)V");
+ jclass connFactoryClass = env_->FindClass("org/apache/hadoop/hbase/client/ConnectionFactory");
+ create_conn_mid_ = env_->GetStaticMethodID(connFactoryClass, "createConnection",
+ "()Lorg/apache/hadoop/hbase/client/Connection;");
+ if (create_conn_mid_ == NULL) {
+ LOG(INFO) << "Couldn't find createConnection";
+ exit(-1);
+ }
+ put_class_ = env_->FindClass("org/apache/hadoop/hbase/client/Put");
+ put_ctor_ = env_->GetMethodID(put_class_, "<init>", "([B)V");
+ add_col_mid_ =
+ env_->GetMethodID(put_class_, "addColumn", "([B[B[B)Lorg/apache/hadoop/hbase/client/Put;");
+ if (add_col_mid_ == NULL) {
+ LOG(INFO) << "Couldn't find method addColumn";
+ exit(-1);
+ }
+ }
+ pthread_mutex_unlock(&count_mutex_);
+}
+
+jobject MiniCluster::htu() {
+ Setup();
+ return htu_;
+}
+
+JNIEnv *MiniCluster::env() {
+ Setup();
+ return env_;
+}
+// converts C char* to Java byte[]
+jbyteArray MiniCluster::StrToByteChar(const std::string& str) {
+ char *p = const_cast<char*>(str.c_str());
+ int n = 0;
+ while (*p++) {
+ n++;
+ }
+ if (n == NULL) return NULL;
+ jbyteArray arr = env_->NewByteArray(n);
+ env_->SetByteArrayRegion(arr, 0, n, reinterpret_cast<const jbyte *>(str.c_str()));
+ return arr;
+}
+
+jobject MiniCluster::CreateTable(std::string tblNam, std::string familyName) {
+ jstring tblNameStr = env_->NewStringUTF(tblNam.c_str());
+ jobject tblName = env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_, tblNameStr);
+ jstring famStr = env_->NewStringUTF(familyName.c_str());
+ jobject tbl = env_->CallObjectMethod(htu_, create_table_mid_, tblName, famStr);
+ return tbl;
+}
+jobject MiniCluster::CreateTable(std::string tblNam, std::string familyName, std::string key1,
+ std::string key2) {
+ jstring tblNameStr = env_->NewStringUTF(tblNam.c_str());
+ jobject tblName = env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_, tblNameStr);
+ jclass arrayElemType = env_->FindClass("[B");
+
+ jobjectArray famArray = env_->NewObjectArray(1, arrayElemType, env_->NewByteArray(1));
+ env_->SetObjectArrayElement(famArray, 0, StrToByteChar(familyName));
+
+ int len = 2;
+ if (key2.empty()) len = 1;
+ jobjectArray keyArray = env_->NewObjectArray(len, arrayElemType, env_->NewByteArray(1));
+
+ env_->SetObjectArrayElement(keyArray, 0, StrToByteChar(key1));
+ if (!key2.empty()) {
+ env_->SetObjectArrayElement(keyArray, 1, StrToByteChar(key2));
+ }
+ jobject tbl = env_->CallObjectMethod(htu_, create_table_with_split_mid_, tblName, famArray, keyArray);
+ return tbl;
+}
+
+jobject MiniCluster::StopRegionServer(jobject cluster, int idx) {
+ env();
+ return env_->CallObjectMethod(cluster, stop_rs_mid_, (jint)idx);
+}
+
+// returns the Configuration for the cluster
+jobject MiniCluster::GetConf() {
+ env();
+ return env_->CallObjectMethod(cluster_, get_conf_mid_);
+}
+// return the Admin instance for the local cluster
+jobject MiniCluster::admin() {
+ env();
+ jobject conn = env_->CallObjectMethod(htu(), get_conn_mid_);
+ jobject admin = env_->CallObjectMethod(conn, get_admin_mid_);
+ return admin;
+}
+
+jobject MiniCluster::TablePut(const std::string table, const std::string row, const std::string fam,
+ const std::string col, const std::string value) {
+ env();
+ jobject conn = env_->CallObjectMethod(htu(), get_conn_mid_);
+ jobject put = env_->NewObject(put_class_, put_ctor_, StrToByteChar(row));
+ if (put == NULL) {
+ LOG(INFO) << "Couldn't create Put";
+ exit(-1);
+ }
+ env_->CallObjectMethod(put, add_col_mid_, StrToByteChar(fam), StrToByteChar(col),
+ StrToByteChar(value));
+ jobject tblName =
+ env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_,
+ env_->NewStringUTF(table.c_str()));
+ jobject tableObj = env_->CallObjectMethod(conn, get_table_mid_, tblName);
+ env_->CallObjectMethod(tableObj, put_mid_, put);
+ return tableObj;
+}
+
+// moves region to server
+void MiniCluster::MoveRegion(std::string region, std::string server) {
+ jobject admin_ = admin();
+ env_->CallObjectMethod(admin_, move_mid_, StrToByteChar(region),
+ StrToByteChar(server));
+}
+
+jobject MiniCluster::StartCluster(int numRegionServers, std::string conf_path) {
+ env();
+ jmethodID mid = env_->GetMethodID(testing_util_class_, "startMiniCluster",
+ "(I)Lorg/apache/hadoop/hbase/MiniHBaseCluster;");
+ if (mid == NULL) {
+ LOG(INFO) << "Couldn't find method startMiniCluster in the class HBaseTestingUtility";
+ exit(-1);
+ }
+ cluster_ = env_->CallObjectMethod(htu(), mid, (jint)numRegionServers);
+ jobject conf = GetConf();
+ jstring jport = (jstring)env_->CallObjectMethod(
+ conf, conf_get_mid_, env_->NewStringUTF("hbase.zookeeper.property.clientPort"));
+ const char *port = env_->GetStringUTFChars(jport, 0);
+ LOG(INFO) << "retrieved port " << port;
+ std::string quorum("localhost:");
+ env_->CallObjectMethod(conf, set_conf_mid_, env_->NewStringUTF("hbase.zookeeper.quorum"),
+ env_->NewStringUTF((quorum + port).c_str()));
+ if (!conf_path.empty()) {
+ // Directory will be created if not present
+ if (!boost::filesystem::exists(conf_path)) {
+ boost::filesystem::create_directories(conf_path);
+ }
+ WriteConf(conf, conf_path + "/hbase-site.xml");
+ }
+ return cluster_;
+}
+
+void MiniCluster::StopCluster() {
+ env();
+ jmethodID mid = env_->GetMethodID(testing_util_class_, "shutdownMiniCluster", "()V");
+ env_->CallVoidMethod(htu(), mid);
+ if (jvm != NULL) {
+ jvm->DestroyJavaVM();
+ jvm = NULL;
+ }
+}
+
+const std::string MiniCluster::GetConfValue(std::string key) {
+ jobject conf = GetConf();
+ jstring jval = (jstring)env_->CallObjectMethod(conf, conf_get_mid_,
+ env_->NewStringUTF(key.c_str()));
+ const char *val = env_->GetStringUTFChars(jval, 0);
+ return val;
+}
diff --git a/test-util/mini-cluster.h b/test-util/mini-cluster.h
new file mode 100644
index 0000000..9b8ca92
--- /dev/null
+++ b/test-util/mini-cluster.h
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <gtest/gtest.h>
+#include <string>
+#include "core/client.h"
+#include "core/configuration.h"
+#include "core/get.h"
+#include "core/hbase_configuration_loader.h"
+#include "core/result.h"
+#include "core/table.h"
+#include "jni.h"
+#include "serde/table-name.h"
+namespace hbase {
+class MiniCluster {
+ public:
+ jobject StartCluster(int numRegionServers, std::string conf_path);
+ void StopCluster();
+ jobject CreateTable(std::string tblNam, std::string familyName);
+ jobject CreateTable(std::string tblNam, std::string familyName, std::string key1, std::string k2);
+ jobject StopRegionServer(jobject cluster, int idx);
+
+ // moves region to server
+ void MoveRegion(std::string region, std::string server);
+ // returns the Configuration instance for the cluster
+ jobject GetConf();
+ // returns the value for config key retrieved from cluster
+ const std::string GetConfValue(std::string key);
+ // Does Put into table for family fam, qualifier col with value
+ jobject TablePut(const std::string table, const std::string row, const std::string fam,
+ const std::string col, const std::string value);
+
+ private:
+ JNIEnv *env_;
+ jclass testing_util_class_;
+ jclass table_name_class_;
+ jclass put_class_;
+ jclass conf_class_;
+ jmethodID stop_rs_mid_;
+ jmethodID get_conf_mid_;
+ jmethodID set_conf_mid_;
+ jmethodID tbl_name_value_of_mid_;
+ jmethodID create_table_mid_;
+ jmethodID create_table_with_split_mid_;
+ jmethodID put_mid_;
+ jmethodID put_ctor_;
+ jmethodID add_col_mid_;
+ jmethodID create_conn_mid_;
+ jmethodID get_conn_mid_;
+ jmethodID get_table_mid_;
+ jmethodID conf_get_mid_;
+ jmethodID get_admin_mid_;
+ jmethodID move_mid_;
+ jmethodID str_ctor_mid_;
+ jobject htu_;
+ jobject cluster_;
+ pthread_mutex_t count_mutex_;
+ JavaVM *jvm;
+ JNIEnv *CreateVM(JavaVM **jvm);
+ void WriteConf(jobject conf, const std::string& filepath);
+ void Setup();
+ jobject htu();
+ JNIEnv *env();
+ jbyteArray StrToByteChar(const std::string& str);
+ jobject admin();
+};
+} /*namespace hbase*/
diff --git a/test-util/test-util.cc b/test-util/test-util.cc
index e355bdf..13c4ec4 100644
--- a/test-util/test-util.cc
+++ b/test-util/test-util.cc
@@ -18,6 +18,7 @@
*/
#include "test-util/test-util.h"
+#include <string.h>
#include <folly/Format.h>
@@ -41,20 +42,33 @@ std::string TestUtil::RandString(int len) {
return s;
}
-TestUtil::TestUtil() : temp_dir_(TestUtil::RandString()) {
+TestUtil::TestUtil() : TestUtil::TestUtil(2, "") {}
+
+TestUtil::TestUtil(int servers, const std::string& confPath)
+ : temp_dir_(TestUtil::RandString()), numRegionServers(servers), conf_path(confPath) {
auto p = temp_dir_.path().string();
- auto cmd = std::string{"bin/start-local-hbase.sh " + p};
- auto res_code = std::system(cmd.c_str());
- CHECK_EQ(res_code, 0);
+ StartMiniCluster(2);
+ std::string quorum("localhost:");
+ const std::string port = mini->GetConfValue("hbase.zookeeper.property.clientPort");
+ conf()->Set("hbase.zookeeper.quorum", quorum + port);
}
-TestUtil::~TestUtil() {
- auto res_code = std::system("bin/stop-local-hbase.sh");
- CHECK_EQ(res_code, 0);
+TestUtil::~TestUtil() { StopMiniCluster(); }
+
+void TestUtil::StartMiniCluster(int num_region_servers) {
+ mini = std::make_unique<MiniCluster>();
+ mini->StartCluster(num_region_servers, conf_path);
}
+void TestUtil::StopMiniCluster() { mini->StopCluster(); }
-void TestUtil::RunShellCmd(const std::string &command) {
- auto cmd_string = folly::sformat("echo \"{}\" | ../bin/hbase shell", command);
- auto res_code = std::system(cmd_string.c_str());
- CHECK_EQ(res_code, 0);
+void TestUtil::CreateTable(std::string tblNam, std::string familyName) {
+ mini->CreateTable(tblNam, familyName);
+}
+void TestUtil::CreateTable(std::string tblNam, std::string familyName, std::string key1,
+ std::string k2) {
+ mini->CreateTable(tblNam, familyName, key1, k2);
+}
+void TestUtil::TablePut(std::string table, std::string row, std::string fam, std::string col,
+ std::string value) {
+ mini->TablePut(table, row, fam, col, value);
}
diff --git a/test-util/test-util.h b/test-util/test-util.h
index cc35511..1734870 100644
--- a/test-util/test-util.h
+++ b/test-util/test-util.h
@@ -22,9 +22,10 @@
#include <folly/experimental/TestUtil.h>
#include <cstdlib>
+#include <memory>
#include <string>
-
#include "core/configuration.h"
+#include "test-util/mini-cluster.h"
namespace hbase {
/**
@@ -36,20 +37,15 @@ class TestUtil {
* Creating a TestUtil will spin up a cluster.
*/
TestUtil();
-
/**
- * Destroying a TestUtil will spin down a cluster and remove the test dir.
+ * Creating a TestUtil will spin up a cluster with numRegionServers region servers.
*/
- ~TestUtil();
+ TestUtil(int numRegionServers, const std::string& confPath);
/**
- * Run a command in the hbase shell. Command should not include any double
- * quotes.
- *
- * This should only be used until there is a good Admin support from the
- * native client
+ * Destroying a TestUtil will spin down a cluster and remove the test dir.
*/
- void RunShellCmd(const std::string &command);
+ ~TestUtil();
/**
* Create a random string. This random string is all letters, as such it is
@@ -62,8 +58,22 @@ class TestUtil {
*/
std::shared_ptr<Configuration> conf() const { return conf_; }
+ /**
+ * Starts mini hbase cluster with specified number of region servers
+ */
+ void StartMiniCluster(int num_region_servers);
+
+ void StopMiniCluster();
+ void CreateTable(std::string tblNam, std::string familyName);
+ void CreateTable(std::string tblNam, std::string familyName, std::string key1, std::string k2);
+ void TablePut(std::string table, std::string row, std::string fam, std::string col,
+ std::string value);
+
private:
+ std::unique_ptr<MiniCluster> mini;
folly::test::TemporaryDirectory temp_dir_;
+ int numRegionServers = 2;
+ std::string conf_path;
std::shared_ptr<Configuration> conf_ = std::make_shared<Configuration>();
};
} // namespace hbase
diff --git a/third-party/BUCK b/third-party/BUCK
index f37eb4e..a55a6fb 100644
--- a/third-party/BUCK
+++ b/third-party/BUCK
@@ -112,4 +112,6 @@ cxx_library(
('googletest/googlemock', 'src/*.cc'),
]),
exported_deps=dynamic_rules,
- visibility=['PUBLIC',],)
+ visibility=[
+ 'PUBLIC',
+ ],)
[hbase] 65/137: HBASE-17585 [C++] Use KVCodec in the RPC
request/response
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 608c192893de3869adf3a39aad298354cda050d4
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Mon Feb 13 18:50:36 2017 -0800
HBASE-17585 [C++] Use KVCodec in the RPC request/response
---
connection/client-dispatcher.cc | 10 +++----
connection/client-dispatcher.h | 9 +++---
connection/client-handler.cc | 36 ++++++++++++++++++------
connection/client-handler.h | 7 +++--
connection/connection-factory.cc | 5 ++--
connection/connection-factory.h | 3 +-
connection/connection-pool-test.cc | 10 +++----
connection/connection-pool.cc | 11 ++++----
connection/connection-pool.h | 3 +-
connection/pipeline.cc | 5 ++--
connection/pipeline.h | 4 ++-
connection/response.h | 11 +++++++-
connection/rpc-client.cc | 32 ++++++++++++----------
connection/rpc-client.h | 21 +++++++-------
connection/rpc-connection.h | 2 +-
connection/service.h | 2 +-
core/client-test.cc | 6 ++--
core/client.cc | 9 +++++-
core/client.h | 2 ++
core/location-cache-test.cc | 10 +++++--
core/location-cache.cc | 20 ++------------
core/meta-utils.cc | 34 +++++++++++++++++++++++
core/meta-utils.h | 7 +++++
core/response_converter.cc | 56 ++++++++++++++++++++++++++++++++++----
core/response_converter.h | 9 +++++-
core/result.cc | 2 +-
core/result.h | 2 +-
core/simple-client.cc | 7 +++--
core/table.cc | 4 +--
serde/client-deserializer-test.cc | 8 +++---
serde/client-serializer-test.cc | 8 +++---
serde/rpc.cc | 14 +++++++++-
serde/rpc.h | 13 ++++++++-
33 files changed, 270 insertions(+), 112 deletions(-)
diff --git a/connection/client-dispatcher.cc b/connection/client-dispatcher.cc
index 1ace99c..626fc76 100644
--- a/connection/client-dispatcher.cc
+++ b/connection/client-dispatcher.cc
@@ -26,8 +26,8 @@ using namespace wangle;
ClientDispatcher::ClientDispatcher() : requests_(5000), current_call_id_(9) {}
-void ClientDispatcher::read(Context *ctx, Response in) {
- auto call_id = in.call_id();
+void ClientDispatcher::read(Context *ctx, std::unique_ptr<Response> in) {
+ auto call_id = in->call_id();
auto search = requests_.find(call_id);
CHECK(search != requests_.end());
@@ -37,13 +37,13 @@ void ClientDispatcher::read(Context *ctx, Response in) {
// TODO(eclark): check if the response
// is an exception. If it is then set that.
- p.setValue(in);
+ p.setValue(std::move(in));
}
-Future<Response> ClientDispatcher::operator()(std::unique_ptr<Request> arg) {
+Future<std::unique_ptr<Response>> ClientDispatcher::operator()(std::unique_ptr<Request> arg) {
auto call_id = current_call_id_++;
arg->set_call_id(call_id);
- requests_.insert(call_id, Promise<Response>{});
+ requests_.insert(call_id, Promise<std::unique_ptr<Response>>{});
auto &p = requests_.find(call_id)->second;
auto f = p.getFuture();
p.setInterruptHandler([call_id, this](const folly::exception_wrapper &e) {
diff --git a/connection/client-dispatcher.h b/connection/client-dispatcher.h
index 0489717..857042c 100644
--- a/connection/client-dispatcher.h
+++ b/connection/client-dispatcher.h
@@ -36,21 +36,22 @@ namespace hbase {
* future.
*/
class ClientDispatcher
- : public wangle::ClientDispatcherBase<SerializePipeline, std::unique_ptr<Request>, Response> {
+ : public wangle::ClientDispatcherBase<SerializePipeline, std::unique_ptr<Request>,
+ std::unique_ptr<Response>> {
public:
/** Create a new ClientDispatcher */
ClientDispatcher();
/** Read a response off the pipeline. */
- void read(Context *ctx, Response in) override;
+ void read(Context *ctx, std::unique_ptr<Response> in) override;
/** Take a request as a call and send it down the pipeline. */
- folly::Future<Response> operator()(std::unique_ptr<Request> arg) override;
+ folly::Future<std::unique_ptr<Response>> operator()(std::unique_ptr<Request> arg) override;
/** Close the dispatcher and the associated pipeline. */
folly::Future<folly::Unit> close(Context *ctx) override;
/** Close the dispatcher and the associated pipeline. */
folly::Future<folly::Unit> close() override;
private:
- folly::AtomicHashMap<uint32_t, folly::Promise<Response>> requests_;
+ folly::AtomicHashMap<uint32_t, folly::Promise<std::unique_ptr<Response>>> requests_;
// Start at some number way above what could
// be there for un-initialized call id counters.
//
diff --git a/connection/client-handler.cc b/connection/client-handler.cc
index 5a6dce2..af84572 100644
--- a/connection/client-handler.cc
+++ b/connection/client-handler.cc
@@ -36,9 +36,9 @@ using hbase::pb::ResponseHeader;
using hbase::pb::GetResponse;
using google::protobuf::Message;
-ClientHandler::ClientHandler(std::string user_name)
+ClientHandler::ClientHandler(std::string user_name, std::shared_ptr<Codec> codec)
: user_name_(user_name),
- serde_(),
+ serde_(codec),
once_flag_(std::make_unique<std::once_flag>()),
resp_msgs_(
make_unique<folly::AtomicHashMap<uint32_t, std::shared_ptr<google::protobuf::Message>>>(
@@ -47,12 +47,12 @@ ClientHandler::ClientHandler(std::string user_name)
void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
if (LIKELY(buf != nullptr)) {
buf->coalesce();
- Response received;
+ auto received = std::make_unique<Response>();
ResponseHeader header;
int used_bytes = serde_.ParseDelimited(buf.get(), &header);
- LOG(INFO) << "Read ResponseHeader size=" << used_bytes << " call_id=" << header.call_id()
- << " has_exception=" << header.has_exception();
+ VLOG(1) << "Read RPC ResponseHeader size=" << used_bytes << " call_id=" << header.call_id()
+ << " has_exception=" << header.has_exception();
// Get the response protobuf from the map
auto search = resp_msgs_->find(header.call_id());
@@ -67,17 +67,34 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
// set the call_id.
// This will be used to by the dispatcher to match up
// the promise with the response.
- received.set_call_id(header.call_id());
+ received->set_call_id(header.call_id());
// If there was an exception then there's no
// data left on the wire.
if (header.has_exception() == false) {
buf->trimStart(used_bytes);
+
+ int cell_block_length = 0;
used_bytes = serde_.ParseDelimited(buf.get(), resp_msg.get());
+ if (header.has_cell_block_meta() && header.cell_block_meta().has_length()) {
+ cell_block_length = header.cell_block_meta().length();
+ }
+
+ VLOG(3) << "Read RPCResponse, buf length:" << buf->length()
+ << ", header PB length:" << used_bytes << ", cell_block length:" << cell_block_length;
+
// Make sure that bytes were parsed.
- CHECK(used_bytes == buf->length());
- received.set_resp_msg(resp_msg);
+ CHECK((used_bytes + cell_block_length) == buf->length());
+
+ if (cell_block_length > 0) {
+ auto cell_scanner = serde_.CreateCellScanner(std::move(buf), used_bytes, cell_block_length);
+ received->set_cell_scanner(std::move(cell_scanner));
+ }
+
+ received->set_resp_msg(resp_msg);
}
+ // TODO: set exception in Response here
+
ctx->fireRead(std::move(received));
}
}
@@ -94,6 +111,9 @@ Future<Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
// Now store the call id to response.
resp_msgs_->insert(r->call_id(), r->resp_msg());
+
+ VLOG(1) << "Writing RPC Request with call_id:" << r->call_id();
+
// Send the data down the pipeline.
return ctx->fireWrite(serde_.Request(r->call_id(), r->method(), r->req_msg().get()));
}
diff --git a/connection/client-handler.h b/connection/client-handler.h
index d860cc1..afb8e62 100644
--- a/connection/client-handler.h
+++ b/connection/client-handler.h
@@ -27,6 +27,7 @@
#include <string>
#include <utility>
+#include "serde/codec.h"
#include "serde/rpc.h"
// Forward decs.
@@ -51,14 +52,14 @@ namespace hbase {
* on first request.
*/
class ClientHandler
- : public wangle::Handler<std::unique_ptr<folly::IOBuf>, Response, std::unique_ptr<Request>,
- std::unique_ptr<folly::IOBuf>> {
+ : public wangle::Handler<std::unique_ptr<folly::IOBuf>, std::unique_ptr<Response>,
+ std::unique_ptr<Request>, std::unique_ptr<folly::IOBuf>> {
public:
/**
* Create the handler
* @param user_name the user name of the user running this process.
*/
- explicit ClientHandler(std::string user_name);
+ explicit ClientHandler(std::string user_name, std::shared_ptr<Codec> codec);
/**
* Get bytes from the wire.
diff --git a/connection/connection-factory.cc b/connection/connection-factory.cc
index ff83212..6aba351 100644
--- a/connection/connection-factory.cc
+++ b/connection/connection-factory.cc
@@ -26,8 +26,9 @@
using namespace folly;
using namespace hbase;
-ConnectionFactory::ConnectionFactory(std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool)
- : io_pool_(io_pool), pipeline_factory_(std::make_shared<RpcPipelineFactory>()) {}
+ConnectionFactory::ConnectionFactory(std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool,
+ std::shared_ptr<Codec> codec)
+ : io_pool_(io_pool), pipeline_factory_(std::make_shared<RpcPipelineFactory>(codec)) {}
std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> ConnectionFactory::MakeBootstrap() {
auto client = std::make_shared<wangle::ClientBootstrap<SerializePipeline>>();
diff --git a/connection/connection-factory.h b/connection/connection-factory.h
index da44c35..0d1e0d0 100644
--- a/connection/connection-factory.h
+++ b/connection/connection-factory.h
@@ -40,7 +40,8 @@ class ConnectionFactory {
* Constructor.
* There should only be one ConnectionFactory per client.
*/
- explicit ConnectionFactory(std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool);
+ ConnectionFactory(std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool,
+ std::shared_ptr<Codec> codec);
/** Default Desctructor */
virtual ~ConnectionFactory() = default;
diff --git a/connection/connection-pool-test.cc b/connection/connection-pool-test.cc
index 0930095..623ce3c 100644
--- a/connection/connection-pool-test.cc
+++ b/connection/connection-pool-test.cc
@@ -36,7 +36,7 @@ using hbase::ConnectionId;
class MockConnectionFactory : public ConnectionFactory {
public:
- MockConnectionFactory() : ConnectionFactory(nullptr) {}
+ MockConnectionFactory() : ConnectionFactory(nullptr, nullptr) {}
MOCK_METHOD0(MakeBootstrap, std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>());
MOCK_METHOD3(Connect, std::shared_ptr<HBaseService>(
std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>,
@@ -47,17 +47,17 @@ class MockBootstrap : public wangle::ClientBootstrap<SerializePipeline> {};
class MockServiceBase : public HBaseService {
public:
- folly::Future<Response> operator()(std::unique_ptr<Request> req) override {
+ folly::Future<std::unique_ptr<Response>> operator()(std::unique_ptr<Request> req) override {
return do_operation(req.get());
}
- virtual folly::Future<Response> do_operation(Request *req) {
- return folly::makeFuture<Response>(Response{});
+ virtual folly::Future<std::unique_ptr<Response>> do_operation(Request *req) {
+ return folly::makeFuture<std::unique_ptr<Response>>(std::make_unique<Response>());
}
};
class MockService : public MockServiceBase {
public:
- MOCK_METHOD1(do_operation, folly::Future<Response>(Request *));
+ MOCK_METHOD1(do_operation, folly::Future<std::unique_ptr<Response>>(Request *));
};
TEST(TestConnectionPool, TestOnlyCreateOnce) {
diff --git a/connection/connection-pool.cc b/connection/connection-pool.cc
index 15dd64e..6635a6d 100644
--- a/connection/connection-pool.cc
+++ b/connection/connection-pool.cc
@@ -33,8 +33,9 @@ using hbase::HBaseService;
using folly::SharedMutexWritePriority;
using folly::SocketAddress;
-ConnectionPool::ConnectionPool(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor)
- : cf_(std::make_shared<ConnectionFactory>(io_executor)),
+ConnectionPool::ConnectionPool(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
+ std::shared_ptr<Codec> codec)
+ : cf_(std::make_shared<ConnectionFactory>(io_executor, codec)),
clients_(),
connections_(),
map_mutex_() {}
@@ -88,12 +89,12 @@ std::shared_ptr<RpcConnection> ConnectionPool::GetNewConnection(
auto clientBootstrap = cf_->MakeBootstrap();
auto dispatcher = cf_->Connect(clientBootstrap, remote_id->host(), remote_id->port());
- auto conneciton = std::make_shared<RpcConnection>(remote_id, dispatcher);
+ auto connection = std::make_shared<RpcConnection>(remote_id, dispatcher);
- connections_.insert(std::make_pair(remote_id, conneciton));
+ connections_.insert(std::make_pair(remote_id, connection));
clients_.insert(std::make_pair(remote_id, clientBootstrap));
- return conneciton;
+ return connection;
}
}
diff --git a/connection/connection-pool.h b/connection/connection-pool.h
index 1f2a182..23e5e9a 100644
--- a/connection/connection-pool.h
+++ b/connection/connection-pool.h
@@ -46,7 +46,8 @@ namespace hbase {
class ConnectionPool {
public:
/** Create connection pool wit default connection factory */
- explicit ConnectionPool(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor);
+ ConnectionPool(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
+ std::shared_ptr<Codec> codec);
/**
* Desctructor.
diff --git a/connection/pipeline.cc b/connection/pipeline.cc
index 14ad73c..00dc05c 100644
--- a/connection/pipeline.cc
+++ b/connection/pipeline.cc
@@ -30,7 +30,8 @@ using namespace folly;
using namespace hbase;
using namespace wangle;
-RpcPipelineFactory::RpcPipelineFactory() : user_util_() {}
+RpcPipelineFactory::RpcPipelineFactory(std::shared_ptr<Codec> codec)
+ : user_util_(), codec_(codec) {}
SerializePipeline::Ptr RpcPipelineFactory::newPipeline(
std::shared_ptr<AsyncTransportWrapper> sock) {
@@ -38,7 +39,7 @@ SerializePipeline::Ptr RpcPipelineFactory::newPipeline(
pipeline->addBack(AsyncSocketHandler{sock});
pipeline->addBack(EventBaseHandler{});
pipeline->addBack(LengthFieldBasedFrameDecoder{});
- pipeline->addBack(ClientHandler{user_util_.user_name()});
+ pipeline->addBack(ClientHandler{user_util_.user_name(), codec_});
pipeline->finalize();
return pipeline;
}
diff --git a/connection/pipeline.h b/connection/pipeline.h
index 343219d..ea40cfd 100644
--- a/connection/pipeline.h
+++ b/connection/pipeline.h
@@ -25,6 +25,7 @@
#include "connection/request.h"
#include "connection/response.h"
+#include "serde/codec.h"
#include "utils/user-util.h"
namespace hbase {
@@ -40,7 +41,7 @@ class RpcPipelineFactory : public wangle::PipelineFactory<SerializePipeline> {
/**
* Constructor. This will create user util.
*/
- RpcPipelineFactory();
+ explicit RpcPipelineFactory(std::shared_ptr<Codec> codec);
/**
* Create a new pipeline.
@@ -55,5 +56,6 @@ class RpcPipelineFactory : public wangle::PipelineFactory<SerializePipeline> {
private:
UserUtil user_util_;
+ std::shared_ptr<Codec> codec_;
};
} // namespace hbase
diff --git a/connection/response.h b/connection/response.h
index 560387c..1d60fed 100644
--- a/connection/response.h
+++ b/connection/response.h
@@ -22,6 +22,8 @@
#include <memory>
#include <utility>
+#include "serde/cell-scanner.h"
+
// Forward
namespace google {
namespace protobuf {
@@ -42,7 +44,7 @@ class Response {
* Constructor.
* Initinalizes the call id to 0. 0 should never be a valid call id.
*/
- Response() : call_id_(0), resp_msg_(nullptr) {}
+ Response() : call_id_(0), resp_msg_(nullptr), cell_scanner_(nullptr) {}
/** Get the call_id */
uint32_t call_id() { return call_id_; }
@@ -62,8 +64,15 @@ class Response {
resp_msg_ = std::move(response);
}
+ void set_cell_scanner(std::unique_ptr<CellScanner> cell_scanner) {
+ cell_scanner_ = std::move(cell_scanner);
+ }
+
+ const std::unique_ptr<CellScanner>& cell_scanner() const { return cell_scanner_; }
+
private:
uint32_t call_id_;
std::shared_ptr<google::protobuf::Message> resp_msg_;
+ std::unique_ptr<CellScanner> cell_scanner_;
};
} // namespace hbase
diff --git a/connection/rpc-client.cc b/connection/rpc-client.cc
index 7621193..cfbda3a 100644
--- a/connection/rpc-client.cc
+++ b/connection/rpc-client.cc
@@ -39,38 +39,40 @@ class RpcChannelImplementation : public AbstractRpcChannel {
};
} // namespace hbase
-RpcClient::RpcClient(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor)
+RpcClient::RpcClient(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
+ std::shared_ptr<Codec> codec)
: io_executor_(io_executor) {
- cp_ = std::make_shared<ConnectionPool>(io_executor_);
+ cp_ = std::make_shared<ConnectionPool>(io_executor_, codec);
}
void RpcClient::Close() { io_executor_->stop(); }
-std::shared_ptr<Response> RpcClient::SyncCall(const std::string& host, uint16_t port,
+std::unique_ptr<Response> RpcClient::SyncCall(const std::string& host, uint16_t port,
std::unique_ptr<Request> req,
std::shared_ptr<User> ticket) {
- return std::make_shared<Response>(AsyncCall(host, port, std::move(req), ticket).get());
+ return AsyncCall(host, port, std::move(req), ticket).get();
}
-std::shared_ptr<Response> RpcClient::SyncCall(const std::string& host, uint16_t port,
+std::unique_ptr<Response> RpcClient::SyncCall(const std::string& host, uint16_t port,
std::unique_ptr<Request> req,
std::shared_ptr<User> ticket,
const std::string& service_name) {
- return std::make_shared<Response>(
- AsyncCall(host, port, std::move(req), ticket, service_name).get());
+ return AsyncCall(host, port, std::move(req), ticket, service_name).get();
}
-folly::Future<Response> RpcClient::AsyncCall(const std::string& host, uint16_t port,
- std::unique_ptr<Request> req,
- std::shared_ptr<User> ticket) {
+folly::Future<std::unique_ptr<Response>> RpcClient::AsyncCall(const std::string& host,
+ uint16_t port,
+ std::unique_ptr<Request> req,
+ std::shared_ptr<User> ticket) {
auto remote_id = std::make_shared<ConnectionId>(host, port, ticket);
return GetConnection(remote_id)->SendRequest(std::move(req));
}
-folly::Future<Response> RpcClient::AsyncCall(const std::string& host, uint16_t port,
- std::unique_ptr<Request> req,
- std::shared_ptr<User> ticket,
- const std::string& service_name) {
+folly::Future<std::unique_ptr<Response>> RpcClient::AsyncCall(const std::string& host,
+ uint16_t port,
+ std::unique_ptr<Request> req,
+ std::shared_ptr<User> ticket,
+ const std::string& service_name) {
auto remote_id = std::make_shared<ConnectionId>(host, port, ticket, service_name);
return GetConnection(remote_id)->SendRequest(std::move(req));
}
@@ -99,5 +101,5 @@ void RpcClient::CallMethod(const MethodDescriptor* method, RpcController* contro
std::unique_ptr<Request> req = std::make_unique<Request>(shared_req, shared_resp, method->name());
AsyncCall(host, port, std::move(req), ticket, method->service()->name())
- .then([done, this](Response resp) { done->Run(); });
+ .then([done, this](std::unique_ptr<Response> resp) { done->Run(); });
}
diff --git a/connection/rpc-client.h b/connection/rpc-client.h
index aeb9b56..f4645a0 100644
--- a/connection/rpc-client.h
+++ b/connection/rpc-client.h
@@ -51,27 +51,28 @@ class RpcClient : public std::enable_shared_from_this<RpcClient> {
friend class RpcChannelImplementation;
public:
- RpcClient(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor);
+ RpcClient(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
+ std::shared_ptr<Codec> codec);
virtual ~RpcClient() { Close(); }
- virtual std::shared_ptr<Response> SyncCall(const std::string &host, uint16_t port,
+ virtual std::unique_ptr<Response> SyncCall(const std::string &host, uint16_t port,
std::unique_ptr<Request> req,
std::shared_ptr<User> ticket);
- virtual std::shared_ptr<Response> SyncCall(const std::string &host, uint16_t port,
+ virtual std::unique_ptr<Response> SyncCall(const std::string &host, uint16_t port,
std::unique_ptr<Request> req,
std::shared_ptr<User> ticket,
const std::string &service_name);
- virtual folly::Future<Response> AsyncCall(const std::string &host, uint16_t port,
- std::unique_ptr<Request> req,
- std::shared_ptr<User> ticket);
+ virtual folly::Future<std::unique_ptr<Response>> AsyncCall(const std::string &host, uint16_t port,
+ std::unique_ptr<Request> req,
+ std::shared_ptr<User> ticket);
- virtual folly::Future<Response> AsyncCall(const std::string &host, uint16_t port,
- std::unique_ptr<Request> req,
- std::shared_ptr<User> ticket,
- const std::string &service_name);
+ virtual folly::Future<std::unique_ptr<Response>> AsyncCall(const std::string &host, uint16_t port,
+ std::unique_ptr<Request> req,
+ std::shared_ptr<User> ticket,
+ const std::string &service_name);
virtual void Close();
diff --git a/connection/rpc-connection.h b/connection/rpc-connection.h
index e2500b2..c37b1e0 100644
--- a/connection/rpc-connection.h
+++ b/connection/rpc-connection.h
@@ -41,7 +41,7 @@ class RpcConnection {
virtual std::shared_ptr<HBaseService> get_service() const { return hbase_service_; }
- virtual folly::Future<Response> SendRequest(std::unique_ptr<Request> req) {
+ virtual folly::Future<std::unique_ptr<Response>> SendRequest(std::unique_ptr<Request> req) {
return (*hbase_service_)(std::move(req));
}
diff --git a/connection/service.h b/connection/service.h
index 0d2e258..64d4f07 100644
--- a/connection/service.h
+++ b/connection/service.h
@@ -26,5 +26,5 @@
#include "connection/response.h"
namespace hbase {
-using HBaseService = wangle::Service<std::unique_ptr<Request>, Response>;
+using HBaseService = wangle::Service<std::unique_ptr<Request>, std::unique_ptr<Response>>;
} // namespace hbase
diff --git a/core/client-test.cc b/core/client-test.cc
index 28eec6f..0a45fff 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -116,9 +116,9 @@ TEST(Client, Get) {
// Using TestUtil to populate test data
hbase::TestUtil *test_util = new hbase::TestUtil();
- test_util->RunShellCmd("create 't', 'd'");
- test_util->RunShellCmd("put 't', 'test2', 'd:2', 'value2'");
- test_util->RunShellCmd("put 't', 'test2', 'd:extra', 'value for extra'");
+ test_util->RunShellCmd(
+ "create 't', 'd'; put 't', 'test2', 'd:2', 'value2'; put 't', 'test2', 'd:extra', 'value for "
+ "extra'");
// Create TableName and Row to be fetched from HBase
auto tn = folly::to<hbase::pb::TableName>("t");
diff --git a/core/client.cc b/core/client.cc
index c1efd8b..685524f 100644
--- a/core/client.cc
+++ b/core/client.cc
@@ -46,7 +46,14 @@ void Client::init(const hbase::Configuration &conf) {
std::make_shared<wangle::CPUThreadPoolExecutor>(4); // TODO: read num threads from conf
io_executor_ = std::make_shared<wangle::IOThreadPoolExecutor>(sysconf(_SC_NPROCESSORS_ONLN));
- rpc_client_ = std::make_shared<hbase::RpcClient>(io_executor_);
+ std::shared_ptr<Codec> codec = nullptr;
+ if (conf.Get(kRpcCodec, hbase::KeyValueCodec::kJavaClassName) ==
+ std::string(KeyValueCodec::kJavaClassName)) {
+ codec = std::make_shared<hbase::KeyValueCodec>();
+ } else {
+ LOG(WARNING) << "Not using RPC Cell Codec";
+ }
+ rpc_client_ = std::make_shared<hbase::RpcClient>(io_executor_, codec);
location_cache_ =
std::make_shared<hbase::LocationCache>(conf_, cpu_executor_, rpc_client_->connection_pool());
}
diff --git a/core/client.h b/core/client.h
index 730981d..0e436ba 100644
--- a/core/client.h
+++ b/core/client.h
@@ -30,6 +30,7 @@
#include "connection/rpc-client.h"
#include "core/configuration.h"
#include "core/hbase_configuration_loader.h"
+#include "core/keyvalue-codec.h"
#include "core/location-cache.h"
#include "core/table.h"
#include "if/Cell.pb.h"
@@ -70,6 +71,7 @@ class Client {
void init(const hbase::Configuration &conf);
const std::string kHBaseZookeeperQuorum_ = "hbase.zookeeper.quorum";
const std::string kDefHBaseZookeeperQuorum_ = "localhost:2181";
+ const std::string kRpcCodec = "hbase.client.rpc.codec";
std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_;
std::shared_ptr<hbase::LocationCache> location_cache_;
diff --git a/core/location-cache-test.cc b/core/location-cache-test.cc
index 1ad6c65..9a778c6 100644
--- a/core/location-cache-test.cc
+++ b/core/location-cache-test.cc
@@ -23,6 +23,7 @@
#include <chrono>
+#include "core/keyvalue-codec.h"
#include "if/HBase.pb.h"
#include "serde/table-name.h"
#include "test-util/test-util.h"
@@ -33,7 +34,8 @@ TEST(LocationCacheTest, TestGetMetaNodeContents) {
TestUtil test_util{};
auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
- auto cp = std::make_shared<ConnectionPool>(io);
+ auto codec = std::make_shared<KeyValueCodec>();
+ auto cp = std::make_shared<ConnectionPool>(io, codec);
LocationCache cache{test_util.conf(), cpu, cp};
auto f = cache.LocateMeta();
auto result = f.get();
@@ -49,7 +51,8 @@ TEST(LocationCacheTest, TestGetRegionLocation) {
TestUtil test_util{};
auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
- auto cp = std::make_shared<ConnectionPool>(io);
+ auto codec = std::make_shared<KeyValueCodec>();
+ auto cp = std::make_shared<ConnectionPool>(io, codec);
LocationCache cache{test_util.conf(), cpu, cp};
// If there is no table this should throw an exception
@@ -68,7 +71,8 @@ TEST(LocationCacheTest, TestCaching) {
TestUtil test_util{};
auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
- auto cp = std::make_shared<ConnectionPool>(io);
+ auto codec = std::make_shared<KeyValueCodec>();
+ auto cp = std::make_shared<ConnectionPool>(io, codec);
LocationCache cache{test_util.conf(), cpu, cp};
auto tn_1 = folly::to<hbase::pb::TableName>("t1");
diff --git a/core/location-cache.cc b/core/location-cache.cc
index dab5deb..da9f64a 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -128,10 +128,10 @@ Future<std::shared_ptr<RegionLocation>> LocationCache::LocateFromMeta(const Tabl
.then([tn, row, this](std::shared_ptr<RpcConnection> rpc_connection) {
return (*rpc_connection->get_service())(std::move(meta_util_.MetaRequest(tn, row)));
})
- .then([this](Response resp) {
+ .then([this](std::unique_ptr<Response> resp) {
// take the protobuf response and make it into
// a region location.
- return this->CreateLocation(std::move(resp));
+ return meta_util_.CreateLocation(std::move(*resp));
})
.then([tn, this](std::shared_ptr<RegionLocation> rl) {
// Make sure that the correct location was found.
@@ -166,22 +166,6 @@ Future<shared_ptr<RegionLocation>> LocationCache::LocateRegion(const hbase::pb::
}
}
-std::shared_ptr<RegionLocation> LocationCache::CreateLocation(const Response &resp) {
- auto resp_msg = static_pointer_cast<ScanResponse>(resp.resp_msg());
- auto &results = resp_msg->results().Get(0);
- auto &cells = results.cell();
-
- // TODO(eclark): There should probably be some better error
- // handling around this.
- auto cell_zero = cells.Get(0).value();
- auto cell_one = cells.Get(1).value();
- auto row = cells.Get(0).row();
-
- auto region_info = folly::to<RegionInfo>(cell_zero);
- auto server_name = folly::to<ServerName>(cell_one);
- return std::make_shared<RegionLocation>(row, std::move(region_info), server_name, nullptr);
-}
-
// must hold shared lock on locations_lock_
shared_ptr<RegionLocation> LocationCache::GetCachedLocation(const hbase::pb::TableName &tn,
const std::string &row) {
diff --git a/core/meta-utils.cc b/core/meta-utils.cc
index f92300c..bd26607 100644
--- a/core/meta-utils.cc
+++ b/core/meta-utils.cc
@@ -24,18 +24,26 @@
#include "connection/request.h"
#include "connection/response.h"
+#include "core/response_converter.h"
#include "if/Client.pb.h"
+#include "serde/region-info.h"
+#include "serde/server-name.h"
#include "serde/table-name.h"
using hbase::pb::TableName;
using hbase::MetaUtil;
using hbase::Request;
using hbase::Response;
+using hbase::RegionLocation;
+using hbase::pb::RegionInfo;
using hbase::pb::ScanRequest;
using hbase::pb::ServerName;
using hbase::pb::RegionSpecifier_RegionSpecifierType;
static const std::string META_REGION = "1588230740";
+static const std::string CATALOG_FAMILY = "info";
+static const std::string REGION_INFO_COLUMN = "regioninfo";
+static const std::string SERVER_COLUMN = "server";
std::string MetaUtil::RegionLookupRowkey(const TableName &tn, const std::string &row) const {
return folly::to<std::string>(tn, ",", row, ",", "999999999999999999");
@@ -79,3 +87,29 @@ std::unique_ptr<Request> MetaUtil::MetaRequest(const TableName tn, const std::st
scan->set_start_row(RegionLookupRowkey(tn, row));
return request;
}
+
+std::shared_ptr<RegionLocation> MetaUtil::CreateLocation(const Response &resp) {
+ std::vector<std::unique_ptr<Result>> results = ResponseConverter::FromScanResponse(resp);
+ if (results.size() != 1) {
+ throw std::runtime_error("Was expecting exactly 1 result in meta scan response, got:" +
+ std::to_string(results.size()));
+ }
+
+ auto result = *results[0];
+ // VLOG(1) << "Creating RegionLocation from received Response " << *result; TODO
+
+ std::shared_ptr<std::string> region_info_str = result.Value(CATALOG_FAMILY, REGION_INFO_COLUMN);
+ std::shared_ptr<std::string> server_str = result.Value(CATALOG_FAMILY, SERVER_COLUMN);
+
+ if (region_info_str == nullptr) {
+ throw std::runtime_error("regioninfo column null for location");
+ }
+ if (server_str == nullptr) {
+ throw std::runtime_error("server column null for location");
+ }
+
+ auto row = result.Row();
+ auto region_info = folly::to<RegionInfo>(*region_info_str);
+ auto server_name = folly::to<ServerName>(*server_str);
+ return std::make_shared<RegionLocation>(row, std::move(region_info), server_name, nullptr);
+}
diff --git a/core/meta-utils.h b/core/meta-utils.h
index 075215e..d67f32d 100644
--- a/core/meta-utils.h
+++ b/core/meta-utils.h
@@ -22,6 +22,8 @@
#include <string>
#include "connection/request.h"
+#include "connection/response.h"
+#include "core/region-location.h"
#include "if/HBase.pb.h"
#include "serde/table-name.h"
@@ -43,5 +45,10 @@ class MetaUtil {
* location.
*/
std::unique_ptr<Request> MetaRequest(const hbase::pb::TableName tn, const std::string &row) const;
+
+ /**
+ * Return a RegionLocation from the parsed Response
+ */
+ std::shared_ptr<RegionLocation> CreateLocation(const Response &resp);
};
} // namespace hbase
diff --git a/core/response_converter.cc b/core/response_converter.cc
index 3fe2ba9..19a3554 100644
--- a/core/response_converter.cc
+++ b/core/response_converter.cc
@@ -22,9 +22,9 @@
#include <vector>
#include "core/cell.h"
-#include "if/Client.pb.h"
using hbase::pb::GetResponse;
+using hbase::pb::ScanResponse;
namespace hbase {
@@ -32,19 +32,65 @@ ResponseConverter::ResponseConverter() {}
ResponseConverter::~ResponseConverter() {}
-std::unique_ptr<hbase::Result> ResponseConverter::FromGetResponse(const Response& resp) {
+std::unique_ptr<Result> ResponseConverter::FromGetResponse(const Response& resp) {
auto get_resp = std::static_pointer_cast<GetResponse>(resp.resp_msg());
+ return ToResult(get_resp->result(), resp.cell_scanner());
+}
+
+std::unique_ptr<Result> ResponseConverter::ToResult(
+ const hbase::pb::Result& result, const std::unique_ptr<CellScanner>& cell_scanner) {
std::vector<std::shared_ptr<Cell>> vcells;
- for (auto cell : get_resp->result().cell()) {
+ for (auto cell : result.cell()) {
std::shared_ptr<Cell> pcell =
std::make_shared<Cell>(cell.row(), cell.family(), cell.qualifier(), cell.timestamp(),
cell.value(), static_cast<hbase::CellType>(cell.cell_type()));
vcells.push_back(pcell);
}
- return std::make_unique<hbase::Result>(vcells, get_resp->result().exists(),
- get_resp->result().stale(), get_resp->result().partial());
+ // iterate over the cells coming from rpc codec
+ if (cell_scanner != nullptr) {
+ while (cell_scanner->Advance()) {
+ vcells.push_back(cell_scanner->Current());
+ }
+ // TODO: check associated cell count?
+ }
+ return std::make_unique<Result>(vcells, result.exists(), result.stale(), result.partial());
}
+std::vector<std::unique_ptr<Result>> ResponseConverter::FromScanResponse(const Response& resp) {
+ auto scan_resp = std::static_pointer_cast<ScanResponse>(resp.resp_msg());
+ VLOG(3) << "FromScanResponse:" << scan_resp->ShortDebugString();
+ int num_results = resp.cell_scanner() != nullptr ? scan_resp->cells_per_result_size()
+ : scan_resp->results_size();
+
+ std::vector<std::unique_ptr<Result>> results{static_cast<size_t>(num_results)};
+ for (int i = 0; i < num_results; i++) {
+ if (resp.cell_scanner() != nullptr) {
+ // Cells are out in cellblocks. Group them up again as Results. How many to read at a
+ // time will be found in getCellsLength -- length here is how many Cells in the i'th Result
+ int num_cells = scan_resp->cells_per_result(i);
+
+ std::vector<std::shared_ptr<Cell>> vcells;
+ while (resp.cell_scanner()->Advance()) {
+ vcells.push_back(resp.cell_scanner()->Current());
+ }
+ // TODO: check associated cell count?
+
+ if (vcells.size() != num_cells) {
+ std::string msg = "Results sent from server=" + std::to_string(num_results) +
+ ". But only got " + std::to_string(i) +
+ " results completely at client. Resetting the scanner to scan again.";
+ LOG(ERROR) << msg;
+ throw std::runtime_error(msg);
+ }
+ // TODO: handle partial results per Result by checking partial_flag_per_result
+ results[i] = std::make_unique<Result>(vcells, false, scan_resp->stale(), false);
+ } else {
+ results[i] = ToResult(scan_resp->results(i), resp.cell_scanner());
+ }
+ }
+
+ return results;
+}
} /* namespace hbase */
diff --git a/core/response_converter.h b/core/response_converter.h
index 86fb632..859644b 100644
--- a/core/response_converter.h
+++ b/core/response_converter.h
@@ -22,6 +22,8 @@
#include <memory>
#include "connection/response.h"
#include "core/result.h"
+#include "if/Client.pb.h"
+#include "serde/cell-scanner.h"
namespace hbase {
@@ -33,11 +35,16 @@ class ResponseConverter {
public:
~ResponseConverter();
+ static std::unique_ptr<Result> ToResult(const hbase::pb::Result& result,
+ const std::unique_ptr<CellScanner>& cell_scanner);
+
/**
* @brief Returns a Result object created by PB Message in passed Response object.
* @param resp - Response object having the PB message.
*/
- static std::unique_ptr<hbase::Result> FromGetResponse(const Response &resp);
+ static std::unique_ptr<hbase::Result> FromGetResponse(const Response& resp);
+
+ static std::vector<std::unique_ptr<Result>> FromScanResponse(const Response& resp);
private:
// Constructor not required. We have all static methods to extract response from PB messages.
diff --git a/core/result.cc b/core/result.cc
index d73a5b2..4db3fca 100644
--- a/core/result.cc
+++ b/core/result.cc
@@ -87,7 +87,7 @@ bool Result::IsEmpty() const { return cells_.empty(); }
const std::string &Result::Row() const { return row_; }
-const int Result::Size() const { return cells_.size(); }
+int Result::Size() const { return cells_.size(); }
const ResultMap &Result::Map() const { return result_map_; }
diff --git a/core/result.h b/core/result.h
index cd41cf0..8ff4311 100644
--- a/core/result.h
+++ b/core/result.h
@@ -100,7 +100,7 @@ class Result {
/**
* @brief Returns the size of the underlying Cell vector
*/
- const int Size() const;
+ int Size() const;
/**
* @brief Map of families to all versions of its qualifiers and values.
diff --git a/core/simple-client.cc b/core/simple-client.cc
index 3cd0a93..4b1144c 100644
--- a/core/simple-client.cc
+++ b/core/simple-client.cc
@@ -31,6 +31,7 @@
#include "connection/connection-pool.h"
#include "core/client.h"
+#include "core/keyvalue-codec.h"
#include "if/Client.pb.h"
#include "if/ZooKeeper.pb.h"
#include "serde/server-name.h"
@@ -43,6 +44,7 @@ using hbase::Configuration;
using hbase::Response;
using hbase::Request;
using hbase::HBaseService;
+using hbase::KeyValueCodec;
using hbase::LocationCache;
using hbase::ConnectionPool;
using hbase::ConnectionFactory;
@@ -88,7 +90,8 @@ int main(int argc, char *argv[]) {
// Set up thread pools.
auto cpu_pool = std::make_shared<wangle::CPUThreadPoolExecutor>(FLAGS_threads);
auto io_pool = std::make_shared<wangle::IOThreadPoolExecutor>(5);
- auto cp = std::make_shared<ConnectionPool>(io_pool);
+ auto codec = std::make_shared<KeyValueCodec>();
+ auto cp = std::make_shared<ConnectionPool>(io_pool, codec);
// Configuration
auto conf = std::make_shared<Configuration>();
@@ -105,7 +108,7 @@ int main(int argc, char *argv[]) {
auto num_puts = FLAGS_columns;
- auto results = std::vector<Future<Response>>{};
+ auto results = std::vector<Future<std::unique_ptr<Response>>>{};
auto col = uint64_t{0};
for (; col < num_puts; col++) {
results.push_back(
diff --git a/core/table.cc b/core/table.cc
index 58125f9..4e30d4b 100644
--- a/core/table.cc
+++ b/core/table.cc
@@ -56,12 +56,12 @@ std::unique_ptr<hbase::Result> Table::Get(const hbase::Get &get) {
auto req = hbase::RequestConverter::ToGetRequest(get, loc->region_name());
auto user = User::defaultUser(); // TODO: make User::current() similar to UserUtil
- Future<Response> f =
+ Future<std::unique_ptr<Response>> f =
rpc_client_->AsyncCall(loc->server_name().host_name(), loc->server_name().port(),
std::move(req), user, "ClientService");
auto resp = f.get();
- return hbase::ResponseConverter::FromGetResponse(resp);
+ return hbase::ResponseConverter::FromGetResponse(*resp);
}
void Table::Close() {
diff --git a/serde/client-deserializer-test.cc b/serde/client-deserializer-test.cc
index a30f904..054684d 100644
--- a/serde/client-deserializer-test.cc
+++ b/serde/client-deserializer-test.cc
@@ -30,12 +30,12 @@ using hbase::pb::RegionSpecifier;
using hbase::pb::RegionSpecifier_RegionSpecifierType;
TEST(TestRpcSerde, TestReturnFalseOnNullPtr) {
- RpcSerde deser;
+ RpcSerde deser{nullptr};
ASSERT_LT(deser.ParseDelimited(nullptr, nullptr), 0);
}
TEST(TestRpcSerde, TestReturnFalseOnBadInput) {
- RpcSerde deser;
+ RpcSerde deser{nullptr};
auto buf = IOBuf::copyBuffer("test");
GetRequest gr;
@@ -44,8 +44,8 @@ TEST(TestRpcSerde, TestReturnFalseOnBadInput) {
TEST(TestRpcSerde, TestGoodGetRequestFullRoundTrip) {
GetRequest in;
- RpcSerde ser;
- RpcSerde deser;
+ RpcSerde ser{nullptr};
+ RpcSerde deser{nullptr};
// fill up the GetRequest.
in.mutable_region()->set_value("test_region_id");
diff --git a/serde/client-serializer-test.cc b/serde/client-serializer-test.cc
index 2bd17fb..33c48f3 100644
--- a/serde/client-serializer-test.cc
+++ b/serde/client-serializer-test.cc
@@ -32,7 +32,7 @@ using namespace folly;
using namespace folly::io;
TEST(RpcSerdeTest, PreambleIncludesHBas) {
- RpcSerde ser;
+ RpcSerde ser{nullptr};
auto buf = ser.Preamble();
const char *p = reinterpret_cast<const char *>(buf->data());
// Take the first for chars and make sure they are the
@@ -43,14 +43,14 @@ TEST(RpcSerdeTest, PreambleIncludesHBas) {
}
TEST(RpcSerdeTest, PreambleIncludesVersion) {
- RpcSerde ser;
+ RpcSerde ser{nullptr};
auto buf = ser.Preamble();
EXPECT_EQ(0, static_cast<const uint8_t *>(buf->data())[4]);
EXPECT_EQ(80, static_cast<const uint8_t *>(buf->data())[5]);
}
TEST(RpcSerdeTest, TestHeaderLengthPrefixed) {
- RpcSerde ser;
+ RpcSerde ser{nullptr};
auto header = ser.Header("elliott");
// The header should be prefixed by 4 bytes of length.
@@ -65,7 +65,7 @@ TEST(RpcSerdeTest, TestHeaderLengthPrefixed) {
}
TEST(RpcSerdeTest, TestHeaderDecode) {
- RpcSerde ser;
+ RpcSerde ser{nullptr};
auto buf = ser.Header("elliott");
auto header_buf = buf->next();
ConnectionHeader h;
diff --git a/serde/rpc.cc b/serde/rpc.cc
index 3bdb489..d5bca62 100644
--- a/serde/rpc.cc
+++ b/serde/rpc.cc
@@ -83,7 +83,7 @@ int RpcSerde::ParseDelimited(const IOBuf *buf, Message *msg) {
return coded_stream.CurrentPosition();
}
-RpcSerde::RpcSerde() : auth_type_(DEFAULT_AUTH_TYPE) {}
+RpcSerde::RpcSerde(std::shared_ptr<Codec> codec) : auth_type_(DEFAULT_AUTH_TYPE), codec_(codec) {}
unique_ptr<IOBuf> RpcSerde::Preamble() {
auto magic = IOBuf::copyBuffer(PREAMBLE, 0, 2);
@@ -110,6 +110,10 @@ unique_ptr<IOBuf> RpcSerde::Header(const string &user) {
// didn't.
// TODO: send the service name and user from the RpcClient
h.set_service_name(INTERFACE);
+
+ if (codec_ != nullptr) {
+ h.set_cell_block_codec_class(codec_->java_class_name());
+ }
return PrependLength(SerializeMessage(h));
}
@@ -128,6 +132,14 @@ unique_ptr<IOBuf> RpcSerde::Request(const uint32_t call_id, const string &method
return PrependLength(std::move(ser_header));
}
+std::unique_ptr<CellScanner> RpcSerde::CreateCellScanner(std::unique_ptr<folly::IOBuf> buf,
+ uint32_t offset, uint32_t length) {
+ if (codec_ == nullptr) {
+ return nullptr;
+ }
+ return codec_->CreateDecoder(std::move(buf), offset, length);
+}
+
unique_ptr<IOBuf> RpcSerde::PrependLength(unique_ptr<IOBuf> msg) {
// Java ints are 4 long. So create a buffer that large
auto len_buf = IOBuf::create(4);
diff --git a/serde/rpc.h b/serde/rpc.h
index 7d060c7..c59f903 100644
--- a/serde/rpc.h
+++ b/serde/rpc.h
@@ -21,6 +21,9 @@
#include <memory>
#include <string>
+#include "serde/cell-scanner.h"
+#include "serde/codec.h"
+
// Forward
namespace folly {
class IOBuf;
@@ -44,7 +47,7 @@ class RpcSerde {
/**
* Constructor assumes the default auth type.
*/
- RpcSerde();
+ RpcSerde(std::shared_ptr<Codec> codec);
/**
* Destructor. This is provided just for testing purposes.
@@ -76,6 +79,13 @@ class RpcSerde {
std::unique_ptr<folly::IOBuf> Header(const std::string &user);
/**
+ * Take ownership of the passed buffer, and create a CellScanner using the
+ * Codec class to parse Cells out of the wire.
+ */
+ std::unique_ptr<CellScanner> CreateCellScanner(std::unique_ptr<folly::IOBuf> buf, uint32_t offset,
+ uint32_t length);
+
+ /**
* Serialize a request message into a protobuf.
* Request consists of:
*
@@ -109,5 +119,6 @@ class RpcSerde {
private:
/* data */
uint8_t auth_type_;
+ std::shared_ptr<Codec> codec_;
};
} // namespace hbase
[hbase] 23/137: HBASE-15748 Don't link in static libunwind.
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit f21799dbc1e029a489ff6c5ca10cfb153550a8be
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Mon May 2 11:41:02 2016 -0700
HBASE-15748 Don't link in static libunwind.
---
third-party/BUCK | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/third-party/BUCK b/third-party/BUCK
index 4327530..6561b2a 100644
--- a/third-party/BUCK
+++ b/third-party/BUCK
@@ -51,7 +51,7 @@ def add_dynamic_libs(names=[]):
return rules
-system_libs = ["unwind", "lzma", "event", ]
+system_libs = ["lzma", "event", ]
local_libs = [
"double-conversion",
"boost_regex",
@@ -65,7 +65,7 @@ local_libs = [
"glog",
"protobuf",
]
-dynamic_libs = ["stdc++", "pthread", "ssl", "crypto", "dl", "atomic", ]
+dynamic_libs = ["stdc++", "pthread", "ssl", "crypto", "dl", "atomic", "unwind"]
dynamic_rules = add_dynamic_libs(dynamic_libs)
tp_dep_rules = add_system_libs(system_libs,) \
+ add_system_libs(local_libs, lib_dir = "/usr/local/lib") \
[hbase] 44/137: HBASE-17217 Docker should bind to a different port
than 16010
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 84a20fabbe7119a29a80e47b0053d8faaf6fac8d
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Wed Nov 30 12:02:02 2016 -0800
HBASE-17217 Docker should bind to a different port than 16010
---
bin/start-docker.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bin/start-docker.sh b/bin/start-docker.sh
index 5c03eb9..38affa0 100755
--- a/bin/start-docker.sh
+++ b/bin/start-docker.sh
@@ -52,7 +52,7 @@ fi;
docker build -t hbase_native .
# After the image is built run the thing
-docker run -p 16010:16010/tcp \
+docker run -p 16050:16050/tcp \
-v ${BASE_DIR}/..:/usr/src/hbase \
-v ~/.m2:/root/.m2 \
-it hbase_native /bin/bash
[hbase] 84/137: HBASE-17860 Implement secure native client
connection
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit e51f4a3f27ed11054a30432cd8bf90d700ee6d0a
Author: tedyu <yu...@gmail.com>
AuthorDate: Tue May 30 14:31:23 2017 -0700
HBASE-17860 Implement secure native client connection
---
bin/start-docker.sh | 6 +-
connection/BUCK | 18 ++-
connection/client-handler.cc | 6 +-
connection/connection-factory.cc | 10 +-
connection/connection-factory.h | 5 +-
connection/connection-pool-test.cc | 2 +-
connection/connection-pool.cc | 8 +-
connection/connection-pool.h | 4 +-
connection/pipeline.cc | 10 +-
connection/pipeline.h | 4 +-
connection/rpc-client.cc | 7 +-
connection/rpc-client.h | 3 +-
connection/sasl-handler.cc | 224 ++++++++++++++++++++++++++++++
connection/sasl-handler.h | 78 +++++++++++
connection/sasl-util.cc | 92 ++++++++++++
security/user.h => connection/sasl-util.h | 35 ++---
core/BUCK | 104 ++++++++++----
core/async-connection.cc | 4 +-
core/async-rpc-retrying-test.cc | 3 +-
core/location-cache-test.cc | 6 +-
docker-files/Dockerfile | 88 ++++++++++++
docker-files/krb5.conf | 20 +++
security/BUCK | 8 +-
security/user.h | 1 +
serde/BUCK | 56 ++++++--
serde/client-serializer-test.cc | 4 +-
serde/rpc.cc | 13 +-
serde/rpc.h | 3 +-
third-party/BUCK | 7 +-
utils/BUCK | 26 +++-
utils/user-util-test.cc | 2 +-
utils/user-util.cc | 37 ++++-
utils/user-util.h | 4 +-
33 files changed, 784 insertions(+), 114 deletions(-)
diff --git a/bin/start-docker.sh b/bin/start-docker.sh
index 8b017a0..53325c1 100755
--- a/bin/start-docker.sh
+++ b/bin/start-docker.sh
@@ -52,11 +52,11 @@ fi;
# Build the image
#
# This shouldn't be needed after the development environment is a little more stable.
-docker build -t hbase_native .
+docker build -t hbase_native -f docker-files/Dockerfile .
# After the image is built run the thing
-docker run -p 16050:16050/tcp \
+docker run -h="securecluster" -p 16050:16050/tcp \
-v ${BASE_DIR}/..:/usr/src/hbase \
-v ~/.m2:/root/.m2 \
- -it hbase_native /bin/bash
+ -it hbase_native /bin/bash
popd
diff --git a/connection/BUCK b/connection/BUCK
index 36111f8..c3119eb 100644
--- a/connection/BUCK
+++ b/connection/BUCK
@@ -22,6 +22,7 @@ cxx_library(
exported_headers=[
"client-dispatcher.h",
"client-handler.h",
+ "sasl-handler.h",
"connection-factory.h",
"connection-pool.h",
"connection-id.h",
@@ -31,6 +32,7 @@ cxx_library(
"response.h",
"service.h",
"rpc-client.h",
+ "sasl-util.h",
],
srcs=[
"client-dispatcher.cc",
@@ -40,6 +42,8 @@ cxx_library(
"pipeline.cc",
"request.cc",
"rpc-client.cc",
+ "sasl-handler.cc",
+ "sasl-util.cc",
],
deps=[
"//if:if",
@@ -51,8 +55,16 @@ cxx_library(
"//exceptions:exceptions",
],
compiler_flags=['-Weffc++'],
- visibility=['//core/...',],)
+ linker_flags=['-L/usr/local/lib', '-lsasl2', '-lkrb5'],
+ exported_linker_flags=['-L/usr/local/lib', '-lsasl2', '-lkrb5'],
+ visibility=[
+ '//core/...',
+ ],)
cxx_test(
name="connection-pool-test",
- srcs=["connection-pool-test.cc",],
- deps=[":connection",],)
+ srcs=[
+ "connection-pool-test.cc",
+ ],
+ deps=[
+ ":connection",
+ ],)
diff --git a/connection/client-handler.cc b/connection/client-handler.cc
index 113ebd0..e60382d 100644
--- a/connection/client-handler.cc
+++ b/connection/client-handler.cc
@@ -128,11 +128,9 @@ Future<Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
// We need to send the header once.
// So use call_once to make sure that only one thread wins this.
std::call_once((*once_flag_), [ctx, this]() {
- VLOG(3) << "Writing RPC connection Preamble and Header to server: " << server_;
- auto pre = serde_.Preamble();
+ VLOG(3) << "Writing RPC Header to server: " << server_;
auto header = serde_.Header(user_name_);
- pre->appendChain(std::move(header));
- ctx->fireWrite(std::move(pre));
+ ctx->fireWrite(std::move(header));
});
VLOG(3) << "Writing RPC Request with call_id:"
diff --git a/connection/connection-factory.cc b/connection/connection-factory.cc
index afa227d..9b06c84 100644
--- a/connection/connection-factory.cc
+++ b/connection/connection-factory.cc
@@ -18,9 +18,12 @@
*/
#include "connection/connection-factory.h"
+#include "connection/sasl-handler.h"
#include <chrono>
+#include <glog/logging.h>
+#include <wangle/channel/Handler.h>
#include "connection/client-dispatcher.h"
#include "connection/pipeline.h"
#include "connection/service.h"
@@ -31,10 +34,13 @@ using std::chrono::milliseconds;
using std::chrono::nanoseconds;
ConnectionFactory::ConnectionFactory(std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool,
- std::shared_ptr<Codec> codec, nanoseconds connect_timeout)
+ std::shared_ptr<Codec> codec,
+ std::shared_ptr<Configuration> conf,
+ nanoseconds connect_timeout)
: connect_timeout_(connect_timeout),
io_pool_(io_pool),
- pipeline_factory_(std::make_shared<RpcPipelineFactory>(codec)) {}
+ conf_(conf),
+ pipeline_factory_(std::make_shared<RpcPipelineFactory>(codec, conf)) {}
std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> ConnectionFactory::MakeBootstrap() {
auto client = std::make_shared<wangle::ClientBootstrap<SerializePipeline>>();
diff --git a/connection/connection-factory.h b/connection/connection-factory.h
index 1e75571..e1d7f6c 100644
--- a/connection/connection-factory.h
+++ b/connection/connection-factory.h
@@ -28,6 +28,7 @@
#include "connection/request.h"
#include "connection/response.h"
#include "connection/service.h"
+#include "security/user.h"
using std::chrono::nanoseconds;
@@ -44,7 +45,8 @@ class ConnectionFactory {
* There should only be one ConnectionFactory per client.
*/
ConnectionFactory(std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool,
- std::shared_ptr<Codec> codec, nanoseconds connect_timeout = nanoseconds(0));
+ std::shared_ptr<Codec> codec, std::shared_ptr<Configuration> conf,
+ nanoseconds connect_timeout = nanoseconds(0));
/** Default Destructor */
virtual ~ConnectionFactory() = default;
@@ -65,6 +67,7 @@ class ConnectionFactory {
private:
nanoseconds connect_timeout_;
+ std::shared_ptr<Configuration> conf_;
std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool_;
std::shared_ptr<RpcPipelineFactory> pipeline_factory_;
};
diff --git a/connection/connection-pool-test.cc b/connection/connection-pool-test.cc
index 8ecdf29..04ec7f1 100644
--- a/connection/connection-pool-test.cc
+++ b/connection/connection-pool-test.cc
@@ -36,7 +36,7 @@ using hbase::ConnectionId;
class MockConnectionFactory : public ConnectionFactory {
public:
- MockConnectionFactory() : ConnectionFactory(nullptr, nullptr) {}
+ MockConnectionFactory() : ConnectionFactory(nullptr, nullptr, nullptr) {}
MOCK_METHOD0(MakeBootstrap, std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>());
MOCK_METHOD3(Connect, std::shared_ptr<HBaseService>(
std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>,
diff --git a/connection/connection-pool.cc b/connection/connection-pool.cc
index 3121294..d3ac3c2 100644
--- a/connection/connection-pool.cc
+++ b/connection/connection-pool.cc
@@ -36,11 +36,13 @@ using folly::SharedMutexWritePriority;
using folly::SocketAddress;
ConnectionPool::ConnectionPool(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
- std::shared_ptr<Codec> codec, nanoseconds connect_timeout)
- : cf_(std::make_shared<ConnectionFactory>(io_executor, codec, connect_timeout)),
+ std::shared_ptr<Codec> codec, std::shared_ptr<Configuration> conf,
+ nanoseconds connect_timeout)
+ : cf_(std::make_shared<ConnectionFactory>(io_executor, codec, conf, connect_timeout)),
clients_(),
connections_(),
- map_mutex_() {}
+ map_mutex_(),
+ conf_(conf) {}
ConnectionPool::ConnectionPool(std::shared_ptr<ConnectionFactory> cf)
: cf_(cf), clients_(), connections_(), map_mutex_() {}
diff --git a/connection/connection-pool.h b/connection/connection-pool.h
index 2a8f195..0582d9b 100644
--- a/connection/connection-pool.h
+++ b/connection/connection-pool.h
@@ -50,7 +50,8 @@ class ConnectionPool {
public:
/** Create connection pool wit default connection factory */
ConnectionPool(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
- std::shared_ptr<Codec> codec, nanoseconds connect_timeout = nanoseconds(0));
+ std::shared_ptr<Codec> codec, std::shared_ptr<Configuration> conf,
+ nanoseconds connect_timeout = nanoseconds(0));
/**
* Constructor that allows specifiying the connetion factory.
@@ -93,6 +94,7 @@ class ConnectionPool {
clients_;
folly::SharedMutexWritePriority map_mutex_;
std::shared_ptr<ConnectionFactory> cf_;
+ std::shared_ptr<Configuration> conf_;
};
} // namespace hbase
diff --git a/connection/pipeline.cc b/connection/pipeline.cc
index edada52..d27c849 100644
--- a/connection/pipeline.cc
+++ b/connection/pipeline.cc
@@ -25,13 +25,15 @@
#include <wangle/codec/LengthFieldBasedFrameDecoder.h>
#include "connection/client-handler.h"
+#include "connection/sasl-handler.h"
using namespace folly;
using namespace hbase;
using namespace wangle;
-RpcPipelineFactory::RpcPipelineFactory(std::shared_ptr<Codec> codec)
- : user_util_(), codec_(codec) {}
+RpcPipelineFactory::RpcPipelineFactory(std::shared_ptr<Codec> codec,
+ std::shared_ptr<Configuration> conf)
+ : user_util_(), codec_(codec), conf_(conf) {}
SerializePipeline::Ptr RpcPipelineFactory::newPipeline(
std::shared_ptr<AsyncTransportWrapper> sock) {
@@ -41,8 +43,10 @@ SerializePipeline::Ptr RpcPipelineFactory::newPipeline(
auto pipeline = SerializePipeline::create();
pipeline->addBack(AsyncSocketHandler{sock});
pipeline->addBack(EventBaseHandler{});
+ auto secure = security::User::IsSecurityEnabled(*conf_);
+ pipeline->addBack(SaslHandler{user_util_.user_name(secure), conf_});
pipeline->addBack(LengthFieldBasedFrameDecoder{});
- pipeline->addBack(ClientHandler{user_util_.user_name(), codec_, addr.describe()});
+ pipeline->addBack(ClientHandler{user_util_.user_name(secure), codec_, addr.describe()});
pipeline->finalize();
return pipeline;
}
diff --git a/connection/pipeline.h b/connection/pipeline.h
index ea40cfd..add7fe5 100644
--- a/connection/pipeline.h
+++ b/connection/pipeline.h
@@ -25,6 +25,7 @@
#include "connection/request.h"
#include "connection/response.h"
+#include "core/configuration.h"
#include "serde/codec.h"
#include "utils/user-util.h"
@@ -41,7 +42,7 @@ class RpcPipelineFactory : public wangle::PipelineFactory<SerializePipeline> {
/**
* Constructor. This will create user util.
*/
- explicit RpcPipelineFactory(std::shared_ptr<Codec> codec);
+ explicit RpcPipelineFactory(std::shared_ptr<Codec> codec, std::shared_ptr<Configuration> conf);
/**
* Create a new pipeline.
@@ -57,5 +58,6 @@ class RpcPipelineFactory : public wangle::PipelineFactory<SerializePipeline> {
private:
UserUtil user_util_;
std::shared_ptr<Codec> codec_;
+ std::shared_ptr<Configuration> conf_;
};
} // namespace hbase
diff --git a/connection/rpc-client.cc b/connection/rpc-client.cc
index 5fa1138..57df66d 100644
--- a/connection/rpc-client.cc
+++ b/connection/rpc-client.cc
@@ -30,9 +30,10 @@ using hbase::RpcClient;
namespace hbase {
RpcClient::RpcClient(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
- std::shared_ptr<Codec> codec, nanoseconds connect_timeout)
- : io_executor_(io_executor) {
- cp_ = std::make_shared<ConnectionPool>(io_executor_, codec, connect_timeout);
+ std::shared_ptr<Codec> codec, std::shared_ptr<Configuration> conf,
+ nanoseconds connect_timeout)
+ : io_executor_(io_executor), conf_(conf) {
+ cp_ = std::make_shared<ConnectionPool>(io_executor_, codec, conf, connect_timeout);
}
void RpcClient::Close() { io_executor_->stop(); }
diff --git a/connection/rpc-client.h b/connection/rpc-client.h
index d416ceb..fbb773a 100644
--- a/connection/rpc-client.h
+++ b/connection/rpc-client.h
@@ -46,7 +46,7 @@ namespace hbase {
class RpcClient {
public:
RpcClient(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor, std::shared_ptr<Codec> codec,
- nanoseconds connect_timeout = nanoseconds(0));
+ std::shared_ptr<Configuration> conf, nanoseconds connect_timeout = nanoseconds(0));
virtual ~RpcClient() { Close(); }
@@ -78,5 +78,6 @@ class RpcClient {
private:
std::shared_ptr<ConnectionPool> cp_;
std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_;
+ std::shared_ptr<Configuration> conf_;
};
} // namespace hbase
diff --git a/connection/sasl-handler.cc b/connection/sasl-handler.cc
new file mode 100644
index 0000000..02cddce
--- /dev/null
+++ b/connection/sasl-handler.cc
@@ -0,0 +1,224 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include "connection/sasl-handler.h"
+
+#include <glog/logging.h>
+#include <sasl/sasl.h>
+#include <sasl/saslplug.h>
+#include <sasl/saslutil.h>
+
+#include <google/protobuf/io/coded_stream.h>
+#include <google/protobuf/io/zero_copy_stream_impl.h>
+#include <wangle/channel/Handler.h>
+
+#include <condition_variable>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <utility>
+
+#include "connection/service.h"
+#include "security/user.h"
+using hbase::security::User;
+
+using std::chrono::nanoseconds;
+using namespace folly;
+using namespace wangle;
+using namespace hbase;
+
+SaslHandler::SaslHandler(std::string user_name, std::shared_ptr<Configuration> conf)
+ : user_name_(user_name) {
+ host_name_.clear();
+ secure_ = User::IsSecurityEnabled(*conf);
+ service_name_ = SaslUtil::ParseServiceName(conf, secure_);
+ sasl_connection_setup_started_.clear();
+ sasl_connection_setup_in_progress_.store(true);
+}
+
+SaslHandler::SaslHandler(const SaslHandler &hdlr) {
+ user_name_ = hdlr.user_name_;
+ service_name_ = hdlr.service_name_;
+ secure_ = hdlr.secure_;
+ host_name_ = hdlr.host_name_;
+ // copy-constructor sets the flags below to their initial state as opposed to getting them
+ // from the object this class is constructed from. That way, this instance is ready to do
+ // sasl stuff without issues, right from the SaslInit. Sharing a sasl session is not useful
+ // between two handler instances.
+ sasl_connection_setup_started_.clear();
+ sasl_connection_setup_in_progress_.store(true);
+ sconn_ = nullptr;
+}
+
+SaslHandler::~SaslHandler() {
+ if (nullptr != sconn_) {
+ sasl_dispose(&sconn_);
+ }
+ sconn_ = nullptr;
+}
+
+void SaslHandler::transportActive(Context *ctx) {
+ // assign hostname; needed for the sasl handshake if secure
+ folly::SocketAddress address;
+ ctx->getTransport()->getPeerAddress(&address);
+ host_name_ = address.getHostStr();
+
+ // now init the sasl library; this is once per process
+ if (secure_) {
+ sasl_util_.InitializeSaslLib();
+ }
+ // write the preamble to kick off the RPC handshake
+ VLOG(3) << "Writing RPC connection Preamble to server: " << host_name_;
+ auto preamble = RpcSerde::Preamble(secure_);
+ ctx->fireWrite(std::move(preamble));
+}
+
+void SaslHandler::read(Context *ctx, folly::IOBufQueue &buf) {
+ // if security is not on, or in case of security-on, if secure connection setup not in progress,
+ // pass it up without touching
+ if (!secure_ || !sasl_connection_setup_in_progress_.load()) {
+ ctx->fireRead(buf);
+ } else {
+ // message is for this handler; process it appropriately
+ ContinueSaslNegotiation(ctx, &buf);
+ }
+}
+
+folly::Future<folly::Unit> SaslHandler::write(Context *ctx, std::unique_ptr<folly::IOBuf> buf) {
+ // if security is on, and if secure connection setup in progress,
+ // this message is for this handler to process and respond
+ if (secure_ && sasl_connection_setup_in_progress_.load()) {
+ // store IOBuf which is to be sent to server after SASL handshake
+ iobuf_.push_back(std::move(buf));
+ if (!sasl_connection_setup_started_.test_and_set()) {
+ // for the first incoming RPC from the higher layer, trigger sasl initialization
+ return SaslInit(ctx);
+ } else {
+ // for the subsequent incoming RPCs from the higher layer, just return empty future
+ folly::Promise<folly::Unit> p_;
+ return p_.getFuture();
+ }
+ }
+ // pass the bytes recieved down without touching it
+ return ctx->fireWrite(std::move(buf));
+}
+
+folly::Future<folly::Unit> SaslHandler::WriteSaslOutput(Context *ctx, const char *out,
+ unsigned int outlen) {
+ int buffer_size = outlen + 4;
+ auto iob = IOBuf::create(buffer_size);
+ iob->append(buffer_size);
+ // Create the array output stream.
+ google::protobuf::io::ArrayOutputStream aos{iob->writableData(), buffer_size};
+ std::unique_ptr<google::protobuf::io::CodedOutputStream> coded_output =
+ std::make_unique<google::protobuf::io::CodedOutputStream>(&aos);
+ uint32_t total_size = outlen;
+ total_size = ntohl(total_size);
+ coded_output->WriteRaw(&total_size, 4);
+ coded_output->WriteRaw(out, outlen);
+ return ctx->fireWrite(std::move(iob));
+}
+
+void SaslHandler::FinishAuth(Context *ctx, folly::IOBufQueue* bufQueue) {
+ std::unique_ptr<folly::IOBuf> iob;
+ if (!bufQueue->empty()) {
+ iob = bufQueue->pop_front();
+ throw std::runtime_error("Error in the final step of handshake " +
+ std::string(reinterpret_cast<const char *>(iob->data())));
+ } else {
+ sasl_connection_setup_in_progress_.store(false);
+ // write what we buffered
+ for (size_t i = 0; i < iobuf_.size(); i++) {
+ iob = std::move(iobuf_.at(i));
+ ctx->fireWrite(std::move(iob));
+ }
+ }
+}
+
+folly::Future<folly::Unit> SaslHandler::SaslInit(Context *ctx) {
+ int rc;
+ const char *mechusing, *mechlist = "GSSAPI";
+ const char *out;
+ unsigned int outlen;
+
+ rc = sasl_client_new(service_name_.c_str(), /* The service we are using*/
+ host_name_.c_str(), NULL,
+ NULL, /* Local and remote IP address strings
+ (NULL disables mechanisms which require this info)*/
+ NULL, /*connection-specific callbacks*/
+ 0 /*security flags*/, &sconn_);
+ if (rc != SASL_OK) {
+ LOG(FATAL) << "Cannot create client (" << rc << ") ";
+ throw std::runtime_error("Cannot create client");
+ }
+ int curr_rc;
+ do {
+ curr_rc = sasl_client_start(sconn_, /* the same context from above */
+ mechlist, /* the list of mechanisms from the server */
+ NULL, /* filled in if an interaction is needed */
+ &out, /* filled in on success */
+ &outlen, /* filled in on success */
+ &mechusing);
+ } while (curr_rc == SASL_INTERACT); /* the mechanism may ask us to fill
+ in things many times. result is SASL_CONTINUE on success */
+ if (curr_rc != SASL_CONTINUE) {
+ throw std::runtime_error("Cannot start client (" + std::to_string(curr_rc) + ")");
+ }
+ folly::Future<folly::Unit> fut = WriteSaslOutput(ctx, out, outlen);
+ return fut;
+}
+
+void SaslHandler::ContinueSaslNegotiation(Context *ctx, folly::IOBufQueue* bufQueue) {
+ const char *out;
+ unsigned int outlen;
+
+ int bytes_sent = 0;
+ int bytes_received = 0;
+
+ std::unique_ptr<folly::IOBuf> iob = bufQueue->pop_front();
+ bytes_received = iob->length();
+ if (bytes_received == 0) {
+ throw std::runtime_error("Error in sasl handshake");
+ }
+ folly::io::RWPrivateCursor c(iob.get());
+ std::uint32_t status = c.readBE<std::uint32_t>();
+ std::uint32_t sz = c.readBE<std::uint32_t>();
+
+ if (status != 0 /*Status 0 is success*/) {
+ // Assumption here is that the response from server is not more than 8 * 1024
+ throw std::runtime_error("Error in sasl handshake " +
+ std::string(reinterpret_cast<char *>(c.writableData())));
+ }
+ out = nullptr;
+ outlen = 0;
+
+ int curr_rc =
+ sasl_client_step(sconn_, /* our context */
+ reinterpret_cast<char *>(c.writableData()), /* the data from the server */
+ sz, /* its length */
+ NULL, /* this should be unallocated and NULL */
+ &out, /* filled in on success */
+ &outlen); /* filled in on success */
+
+ if (curr_rc == SASL_OK || curr_rc == SASL_CONTINUE) {
+ WriteSaslOutput(ctx, out, outlen);
+ }
+ if (curr_rc == SASL_OK) {
+ FinishAuth(ctx, bufQueue);
+ }
+}
diff --git a/connection/sasl-handler.h b/connection/sasl-handler.h
new file mode 100644
index 0000000..f606a23
--- /dev/null
+++ b/connection/sasl-handler.h
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <glog/logging.h>
+#include <sasl/sasl.h>
+#include <sasl/saslplug.h>
+#include <sasl/saslutil.h>
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "connection/sasl-util.h"
+#include "connection/service.h"
+#include "security/user.h"
+#include "serde/rpc.h"
+
+namespace hbase {
+
+/**
+ * Class to perform SASL handshake with server (currently works with regionserver principals only)
+ * It is inserted between EventBaseHandler and LengthFieldBasedFrameDecoder in the pipeline
+ * SaslHandler would intercept writes to server by buffering the IOBuf's and start the handshake
+ * process
+ * (via sasl_client_XX calls provided by Cyrus)
+ * After handshake is complete, SaslHandler would send the buffered IOBuf's to server and
+ * act as pass-thru from then on
+ */
+class SaslHandler
+ : public wangle::HandlerAdapter<folly::IOBufQueue&, std::unique_ptr<folly::IOBuf>> {
+ public:
+ explicit SaslHandler(std::string user_name, std::shared_ptr<Configuration> conf);
+ SaslHandler(const SaslHandler& hdlr);
+ ~SaslHandler();
+
+ // from HandlerAdapter
+ void read(Context* ctx, folly::IOBufQueue& buf) override;
+ folly::Future<folly::Unit> write(Context* ctx, std::unique_ptr<folly::IOBuf> buf) override;
+ void transportActive(Context* ctx) override;
+
+ private:
+ // used by Cyrus
+ sasl_conn_t* sconn_ = nullptr;
+ std::string user_name_;
+ std::string service_name_;
+ std::string host_name_;
+ bool secure_;
+ std::atomic_flag sasl_connection_setup_started_;
+ std::atomic<bool> sasl_connection_setup_in_progress_{true};
+ // vector of folly::IOBuf which buffers client writes before handshake is complete
+ std::vector<std::unique_ptr<folly::IOBuf>> iobuf_;
+ SaslUtil sasl_util_;
+
+ // writes the output returned by sasl_client_XX to server
+ folly::Future<folly::Unit> WriteSaslOutput(Context* ctx, const char* out, unsigned int outlen);
+ folly::Future<folly::Unit> SaslInit(Context* ctx);
+ void FinishAuth(Context* ctx, folly::IOBufQueue* bufQueue);
+ void ContinueSaslNegotiation(Context* ctx, folly::IOBufQueue* buf);
+ std::string ParseServiceName(std::shared_ptr<Configuration> conf, bool secure);
+};
+} // namespace hbase
diff --git a/connection/sasl-util.cc b/connection/sasl-util.cc
new file mode 100644
index 0000000..ecaf015
--- /dev/null
+++ b/connection/sasl-util.cc
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include "connection/sasl-util.h"
+
+#include <glog/logging.h>
+#include <sasl/sasl.h>
+#include <sasl/saslplug.h>
+#include <sasl/saslutil.h>
+
+#include <string>
+
+int SaslUtil::GetPluginPath(void *context __attribute__((unused)), const char **path) {
+ *path = getenv("SASL_PATH");
+
+ if (*path == NULL) {
+ *path = kDefaultPluginDir;
+ }
+ return SASL_OK;
+}
+
+void *SaslUtil::MutexNew(void) {
+ auto m = new std::mutex();
+ return m;
+}
+
+int SaslUtil::MutexLock(void *m) {
+ (reinterpret_cast<std::mutex *>(m))->lock();
+ return SASL_OK;
+}
+
+int SaslUtil::MutexUnlock(void *m) {
+ (reinterpret_cast<std::mutex *>(m))->unlock();
+ return SASL_OK;
+}
+
+void SaslUtil::MutexDispose(void *m) {
+ std::mutex *mutex = reinterpret_cast<std::mutex *>(m);
+ delete mutex;
+}
+
+std::once_flag SaslUtil::library_inited_;
+
+void SaslUtil::InitializeSaslLib() {
+ std::call_once(library_inited_, []() {
+ sasl_set_mutex(reinterpret_cast<sasl_mutex_alloc_t *>(&SaslUtil::MutexNew),
+ reinterpret_cast<sasl_mutex_lock_t *>(&SaslUtil::MutexLock),
+ reinterpret_cast<sasl_mutex_unlock_t *>(&SaslUtil::MutexUnlock),
+ reinterpret_cast<sasl_mutex_free_t *>(&SaslUtil::MutexDispose));
+ static sasl_callback_t callbacks[] = {
+ {SASL_CB_GETPATH, (sasl_callback_ft)&SaslUtil::GetPluginPath, NULL},
+ {SASL_CB_LIST_END, NULL, NULL}};
+ int rc = sasl_client_init(callbacks);
+ if (rc != SASL_OK) {
+ throw std::runtime_error("Cannot initialize client " + std::to_string(rc));
+ }
+ });
+}
+
+std::string SaslUtil::ParseServiceName(std::shared_ptr<hbase::Configuration> conf, bool secure) {
+ if (!secure) {
+ return std::string();
+ }
+ std::string svrPrincipal = conf->Get(kServerPrincipalConfKey, "");
+ // principal is of this form: hbase/23a03935850c@EXAMPLE.COM
+ // where 23a03935850c is the host (optional)
+ std::size_t pos = svrPrincipal.find("/");
+ if (pos == std::string::npos && svrPrincipal.find("@") != std::string::npos) {
+ pos = svrPrincipal.find("@");
+ }
+ if (pos == std::string::npos) {
+ throw std::runtime_error("Couldn't retrieve service principal from conf");
+ }
+ VLOG(1) << "pos " << pos << " " << svrPrincipal;
+ std::string service_name = svrPrincipal.substr(0, pos);
+ return service_name;
+}
diff --git a/security/user.h b/connection/sasl-util.h
similarity index 53%
copy from security/user.h
copy to connection/sasl-util.h
index 035af31..4d58d9ee 100644
--- a/security/user.h
+++ b/connection/sasl-util.h
@@ -18,28 +18,29 @@
*/
#pragma once
+#include <memory>
#include <mutex>
#include <string>
+
#include "core/configuration.h"
-namespace hbase {
-namespace security {
-static constexpr const char* kKerberos = "kerberos";
-class User {
+class SaslUtil {
public:
- explicit User(const std::string& user_name) : user_name_(user_name) {}
- virtual ~User() = default;
-
- std::string user_name() { return user_name_; }
-
- static std::shared_ptr<User> defaultUser() { return std::make_shared<User>("__drwho"); }
-
- static bool IsSecurityEnabled(const Configuration& conf) {
- return conf.Get("hbase.security.authentication", "").compare(kKerberos) == 0;
- }
+ void InitializeSaslLib(void);
+ static std::string ParseServiceName(std::shared_ptr<hbase::Configuration> conf, bool secure);
private:
- std::string user_name_;
+ static constexpr const char *kDefaultPluginDir = "/usr/lib/sasl2";
+ // for now the sasl handler is hardcoded to work against the regionservers only. In the future, if
+ // we
+ // need the master rpc to work, we could have a map of service names to principals to use (similar
+ // to the Java implementation)
+ static constexpr const char *kServerPrincipalConfKey = "hbase.regionserver.kerberos.principal";
+
+ static int GetPluginPath(void *context, const char **path);
+ static void *MutexNew(void);
+ static int MutexLock(void *m);
+ static int MutexUnlock(void *m);
+ static void MutexDispose(void *m);
+ static std::once_flag library_inited_;
};
-}
-}
diff --git a/core/BUCK b/core/BUCK
index e9fc716..9cea1f6 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -92,7 +92,9 @@ cxx_library(
"//third-party:zookeeper_mt",
],
compiler_flags=['-Weffc++', '-ggdb'],
- visibility=['PUBLIC',],)
+ visibility=[
+ 'PUBLIC',
+ ],)
cxx_library(
name="conf",
exported_headers=[
@@ -105,10 +107,14 @@ cxx_library(
],
deps=["//third-party:folly"],
compiler_flags=['-Weffc++', '-ggdb'],
- visibility=['PUBLIC',],)
+ visibility=[
+ 'PUBLIC',
+ ],)
cxx_test(
name="location-cache-test",
- srcs=["location-cache-test.cc",],
+ srcs=[
+ "location-cache-test.cc",
+ ],
deps=[
":core",
"//test-util:test-util",
@@ -116,12 +122,18 @@ cxx_test(
run_test_separately=True,)
cxx_test(
name="cell-test",
- srcs=["cell-test.cc",],
- deps=[":core",],
+ srcs=[
+ "cell-test.cc",
+ ],
+ deps=[
+ ":core",
+ ],
run_test_separately=True,)
cxx_test(
name="filter-test",
- srcs=["filter-test.cc",],
+ srcs=[
+ "filter-test.cc",
+ ],
deps=[
":core",
"//if:if",
@@ -131,17 +143,27 @@ cxx_test(
run_test_separately=True,)
cxx_test(
name="get-test",
- srcs=["get-test.cc",],
- deps=[":core",],
+ srcs=[
+ "get-test.cc",
+ ],
+ deps=[
+ ":core",
+ ],
run_test_separately=True,)
cxx_test(
name="put-test",
- srcs=["put-test.cc",],
- deps=[":core",],
+ srcs=[
+ "put-test.cc",
+ ],
+ deps=[
+ ":core",
+ ],
run_test_separately=True,)
cxx_test(
name="retry-test",
- srcs=["async-rpc-retrying-test.cc",],
+ srcs=[
+ "async-rpc-retrying-test.cc",
+ ],
deps=[
":core",
"//test-util:test-util",
@@ -150,32 +172,54 @@ cxx_test(
run_test_separately=True,)
cxx_test(
name="time-range-test",
- srcs=["time-range-test.cc",],
- deps=[":core",],
+ srcs=[
+ "time-range-test.cc",
+ ],
+ deps=[
+ ":core",
+ ],
run_test_separately=True,)
cxx_test(
name="configuration-test",
- srcs=["configuration-test.cc",],
- deps=[":core",],
+ srcs=[
+ "configuration-test.cc",
+ ],
+ deps=[
+ ":core",
+ ],
run_test_separately=True,)
cxx_test(
name="hbase-configuration-test",
- srcs=["hbase-configuration-test.cc",],
- deps=[":core",],
+ srcs=[
+ "hbase-configuration-test.cc",
+ ],
+ deps=[
+ ":core",
+ ],
run_test_separately=True,)
cxx_test(
name="scan-test",
- srcs=["scan-test.cc",],
- deps=[":core",],
+ srcs=[
+ "scan-test.cc",
+ ],
+ deps=[
+ ":core",
+ ],
run_test_separately=True,)
cxx_test(
name="result-test",
- srcs=["result-test.cc",],
- deps=[":core",],
+ srcs=[
+ "result-test.cc",
+ ],
+ deps=[
+ ":core",
+ ],
run_test_separately=True,)
cxx_test(
name="request-converter-test",
- srcs=["request-converter-test.cc",],
+ srcs=[
+ "request-converter-test.cc",
+ ],
deps=[
":core",
"//connection:connection",
@@ -184,7 +228,9 @@ cxx_test(
run_test_separately=True,)
cxx_test(
name="client-test",
- srcs=["client-test.cc",],
+ srcs=[
+ "client-test.cc",
+ ],
deps=[
":core",
"//if:if",
@@ -194,10 +240,16 @@ cxx_test(
run_test_separately=True,)
cxx_test(
name="zk-util-test",
- srcs=["zk-util-test.cc",],
- deps=[":core",],
+ srcs=[
+ "zk-util-test.cc",
+ ],
+ deps=[
+ ":core",
+ ],
run_test_separately=True,)
cxx_binary(
name="simple-client",
- srcs=["simple-client.cc",],
+ srcs=[
+ "simple-client.cc",
+ ],
deps=[":core", "//connection:connection"],)
diff --git a/core/async-connection.cc b/core/async-connection.cc
index 4642c61..ef945fb 100644
--- a/core/async-connection.cc
+++ b/core/async-connection.cc
@@ -44,8 +44,8 @@ void AsyncConnectionImpl::Init() {
} else {
LOG(WARNING) << "Not using RPC Cell Codec";
}
- rpc_client_ =
- std::make_shared<hbase::RpcClient>(io_executor_, codec, connection_conf_->connect_timeout());
+ rpc_client_ = std::make_shared<hbase::RpcClient>(io_executor_, codec, conf_,
+ connection_conf_->connect_timeout());
location_cache_ =
std::make_shared<hbase::LocationCache>(conf_, cpu_executor_, rpc_client_->connection_pool());
caller_factory_ =
diff --git a/core/async-rpc-retrying-test.cc b/core/async-rpc-retrying-test.cc
index 487c34c..11750eb 100644
--- a/core/async-rpc-retrying-test.cc
+++ b/core/async-rpc-retrying-test.cc
@@ -317,7 +317,8 @@ void runTest(std::shared_ptr<AsyncRegionLocatorBase> region_locator, std::string
auto io_executor_ = client.async_connection()->io_executor();
auto retry_executor_ = std::make_shared<wangle::IOThreadPoolExecutor>(1);
auto codec = std::make_shared<hbase::KeyValueCodec>();
- auto rpc_client = std::make_shared<RpcClient>(io_executor_, codec);
+ auto rpc_client =
+ std::make_shared<RpcClient>(io_executor_, codec, AsyncRpcRetryTest::test_util->conf());
// auto retry_event_base_ = std::make_shared<folly::ScopedEventBaseThread>(true);
std::shared_ptr<folly::HHWheelTimer> retry_timer =
folly::HHWheelTimer::newTimer(retry_executor_->getEventBase());
diff --git a/core/location-cache-test.cc b/core/location-cache-test.cc
index 8d1ac5f..3253c56 100644
--- a/core/location-cache-test.cc
+++ b/core/location-cache-test.cc
@@ -52,7 +52,7 @@ TEST_F(LocationCacheTest, TestGetMetaNodeContents) {
auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
auto codec = std::make_shared<KeyValueCodec>();
- auto cp = std::make_shared<ConnectionPool>(io, codec);
+ auto cp = std::make_shared<ConnectionPool>(io, codec, LocationCacheTest::test_util_->conf());
LocationCache cache{LocationCacheTest::test_util_->conf(), cpu, cp};
auto f = cache.LocateMeta();
auto result = f.get();
@@ -68,7 +68,7 @@ TEST_F(LocationCacheTest, TestGetRegionLocation) {
auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
auto codec = std::make_shared<KeyValueCodec>();
- auto cp = std::make_shared<ConnectionPool>(io, codec);
+ auto cp = std::make_shared<ConnectionPool>(io, codec, LocationCacheTest::test_util_->conf());
LocationCache cache{LocationCacheTest::test_util_->conf(), cpu, cp};
// If there is no table this should throw an exception
@@ -87,7 +87,7 @@ TEST_F(LocationCacheTest, TestCaching) {
auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
auto codec = std::make_shared<KeyValueCodec>();
- auto cp = std::make_shared<ConnectionPool>(io, codec);
+ auto cp = std::make_shared<ConnectionPool>(io, codec, LocationCacheTest::test_util_->conf());
LocationCache cache{LocationCacheTest::test_util_->conf(), cpu, cp};
auto tn_1 = folly::to<hbase::pb::TableName>("t1");
diff --git a/docker-files/Dockerfile b/docker-files/Dockerfile
new file mode 100644
index 0000000..c5f47ad
--- /dev/null
+++ b/docker-files/Dockerfile
@@ -0,0 +1,88 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM pjameson/buck-folly-watchman:20160511
+
+ARG CC=/usr/bin/gcc-5
+ARG CXX=/usr/bin/g++-5
+ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -pthread"
+ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -pthread"
+
+ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
+
+RUN wget ftp://ftp.cyrusimap.org/cyrus-sasl/cyrus-sasl-2.1.26.tar.gz ; \
+ tar zxf cyrus-sasl-2.1.26.tar.gz ; \
+ cd cyrus-sasl-2.1.26 ; \
+ ./configure ; \
+ make ; \
+ make install ;\
+ cp /usr/local/lib/sasl2/* /usr/lib/sasl2/
+
+RUN apt-get install -y vim maven inetutils-ping python-pip doxygen graphviz clang-format && \
+ pip install yapf && \
+ apt-get -qq clean && \
+ apt-get -y -qq autoremove && \
+ rm -rf /var/lib/{apt,dpkg,cache,log}/ && \
+ rm -rf /tmp/*
+
+RUN apt-get update && \
+ apt-get install -y debconf-utils debconf-set-selections && \
+ echo "krb5-config krb5-config/kerberos_servers string localhost" | debconf-set-selections ; \
+ echo "krb5-config krb5-config/admin_server string localhost" | debconf-set-selections ; \
+ echo "krb5-config krb5-config/add_servers_realm string EXAMPLE.COM" | debconf-set-selections ; \
+ echo "krb5-config krb5-config/default_realm string EXAMPLE.COM" | debconf-set-selections ; \
+ apt-get install -y krb5-kdc krb5-admin-server ; \
+ echo "admin" > /tmp/krb-realm.pass ; \
+ echo "admin" >> /tmp/krb-realm.pass ; \
+ krb5_newrealm < /tmp/krb-realm.pass ; \
+ echo "addprinc hbase" > /tmp/krb-princ.pass ; \
+ echo "admin" >> /tmp/krb-princ.pass ; \
+ echo "admin" >> /tmp/krb-princ.pass ; \
+ kadmin.local < /tmp/krb-princ.pass ; \
+ echo 'addprinc hbase/securecluster' > /tmp/krb-princ.pass; echo 'admin' >> /tmp/krb-princ.pass ; \
+ rm hbase-host.keytab ; echo 'admin' >> /tmp/krb-princ.pass ; \
+ echo 'xst -k hbase-host.keytab hbase/securecluster@EXAMPLE.COM' >> /tmp/krb-princ.pass ; \
+ kadmin.local < /tmp/krb-princ.pass ;
+COPY docker-files/krb5.conf /etc
+
+RUN git clone https://github.com/google/protobuf.git /usr/src/protobuf && \
+ cd /usr/src/protobuf/ && \
+ git checkout 2.7.0 && \
+ mkdir gmock && \
+ ldconfig && \
+ ./autogen.sh && \
+ ./configure && \
+ make && \
+ make install && \
+ make clean && \
+ rm -rf .git && \
+ cd /usr/src && \
+ wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && \
+ tar zxf zookeeper-3.4.8.tar.gz && \
+ rm -rf zookeeper-3.4.8.tar.gz && \
+ cd zookeeper-3.4.8 && \
+ cd src/c && \
+ ldconfig && \
+ ./configure && \
+ make && \
+ make install && \
+ make clean && \
+ ldconfig
+
+ENTRYPOINT /usr/sbin/krb5kdc -P /var/run/krb5kdc.pid && /bin/bash
+
+WORKDIR /usr/src/hbase/hbase-native-client
diff --git a/docker-files/krb5.conf b/docker-files/krb5.conf
new file mode 100644
index 0000000..d449928
--- /dev/null
+++ b/docker-files/krb5.conf
@@ -0,0 +1,20 @@
+[logging]
+ default = FILE:/var/log/krb5libs.log
+ kdc = FILE:/var/log/krb5kdc.log
+ admin_server = FILE:/var/log/kadmind.log
+
+[libdefaults]
+ default_realm = EXAMPLE.COM
+ dns_lookup_realm = false
+ dns_lookup_kdc = false
+ ticket_lifetime = 24h
+ renew_lifetime = 7d
+ forwardable = true
+ rdns = false
+
+[realms]
+EXAMPLE.COM = {
+kdc = localhost
+admin_server = localhost
+}
+[domain_realm]
diff --git a/security/BUCK b/security/BUCK
index 7383028..f8a5695 100644
--- a/security/BUCK
+++ b/security/BUCK
@@ -19,8 +19,12 @@
# to a single server.
cxx_library(
name="security",
- exported_headers=["user.h",],
+ exported_headers=[
+ "user.h",
+ ],
srcs=[],
deps=["//core:conf"],
compiler_flags=['-Weffc++'],
- visibility=['//core/...', '//connection/...'],)
+ visibility=[
+ 'PUBLIC',
+ ],)
diff --git a/security/user.h b/security/user.h
index 035af31..307fc61 100644
--- a/security/user.h
+++ b/security/user.h
@@ -18,6 +18,7 @@
*/
#pragma once
+#include <glog/logging.h>
#include <mutex>
#include <string>
#include "core/configuration.h"
diff --git a/serde/BUCK b/serde/BUCK
index 38e7b4d..18e949c 100644
--- a/serde/BUCK
+++ b/serde/BUCK
@@ -31,7 +31,9 @@ cxx_library(
"rpc.cc",
"zk.cc",
],
- deps=["//if:if", "//third-party:folly", "//utils:utils"],
+ deps=[
+ "//if:if", "//third-party:folly", "//utils:utils", "//security:security"
+ ],
tests=[
":client-deserializer-test",
":client-serializer-test",
@@ -41,28 +43,54 @@ cxx_library(
":region-info-deserializer-test",
],
compiler_flags=['-Weffc++'],
- visibility=['PUBLIC',],)
+ visibility=[
+ 'PUBLIC',
+ ],)
cxx_test(
name="table-name-test",
- srcs=["table-name-test.cc",],
- deps=[":serde",],)
+ srcs=[
+ "table-name-test.cc",
+ ],
+ deps=[
+ ":serde",
+ ],)
cxx_test(
name="server-name-test",
- srcs=["server-name-test.cc",],
- deps=[":serde",],)
+ srcs=[
+ "server-name-test.cc",
+ ],
+ deps=[
+ ":serde",
+ ],)
cxx_test(
name="client-serializer-test",
- srcs=["client-serializer-test.cc",],
- deps=[":serde",],)
+ srcs=[
+ "client-serializer-test.cc",
+ ],
+ deps=[
+ ":serde",
+ ],)
cxx_test(
name="client-deserializer-test",
- srcs=["client-deserializer-test.cc",],
- deps=[":serde",],)
+ srcs=[
+ "client-deserializer-test.cc",
+ ],
+ deps=[
+ ":serde",
+ ],)
cxx_test(
name="zk-deserializer-test",
- srcs=["zk-deserializer-test.cc",],
- deps=[":serde",],)
+ srcs=[
+ "zk-deserializer-test.cc",
+ ],
+ deps=[
+ ":serde",
+ ],)
cxx_test(
name="region-info-deserializer-test",
- srcs=["region-info-deserializer-test.cc",],
- deps=[":serde",],)
+ srcs=[
+ "region-info-deserializer-test.cc",
+ ],
+ deps=[
+ ":serde",
+ ],)
diff --git a/serde/client-serializer-test.cc b/serde/client-serializer-test.cc
index 33c48f3..7d8b29c 100644
--- a/serde/client-serializer-test.cc
+++ b/serde/client-serializer-test.cc
@@ -33,7 +33,7 @@ using namespace folly::io;
TEST(RpcSerdeTest, PreambleIncludesHBas) {
RpcSerde ser{nullptr};
- auto buf = ser.Preamble();
+ auto buf = ser.Preamble(false);
const char *p = reinterpret_cast<const char *>(buf->data());
// Take the first for chars and make sure they are the
// magic string
@@ -44,7 +44,7 @@ TEST(RpcSerdeTest, PreambleIncludesHBas) {
TEST(RpcSerdeTest, PreambleIncludesVersion) {
RpcSerde ser{nullptr};
- auto buf = ser.Preamble();
+ auto buf = ser.Preamble(false);
EXPECT_EQ(0, static_cast<const uint8_t *>(buf->data())[4]);
EXPECT_EQ(80, static_cast<const uint8_t *>(buf->data())[5]);
}
diff --git a/serde/rpc.cc b/serde/rpc.cc
index e657a64..968cd5b 100644
--- a/serde/rpc.cc
+++ b/serde/rpc.cc
@@ -50,6 +50,7 @@ static const std::string PREAMBLE = "HBas";
static const std::string INTERFACE = "ClientService";
static const uint8_t RPC_VERSION = 0;
static const uint8_t DEFAULT_AUTH_TYPE = 80;
+static const uint8_t KERBEROS_AUTH_TYPE = 81;
int RpcSerde::ParseDelimited(const IOBuf *buf, Message *msg) {
if (buf == nullptr || msg == nullptr) {
@@ -85,17 +86,21 @@ int RpcSerde::ParseDelimited(const IOBuf *buf, Message *msg) {
return coded_stream.CurrentPosition();
}
-RpcSerde::RpcSerde(std::shared_ptr<Codec> codec) : auth_type_(DEFAULT_AUTH_TYPE), codec_(codec) {}
+RpcSerde::RpcSerde(std::shared_ptr<Codec> codec) : codec_(codec) {}
-unique_ptr<IOBuf> RpcSerde::Preamble() {
+std::unique_ptr<IOBuf> RpcSerde::Preamble(bool secure) {
auto magic = IOBuf::copyBuffer(PREAMBLE, 0, 2);
magic->append(2);
RWPrivateCursor c(magic.get());
c.skip(4);
// Version
c.write(RPC_VERSION);
- // Standard security aka Please don't lie to me.
- c.write(auth_type_);
+ if (secure) {
+ // for now support only KERBEROS (DIGEST is not supported)
+ c.write(KERBEROS_AUTH_TYPE);
+ } else {
+ c.write(DEFAULT_AUTH_TYPE);
+ }
return magic;
}
diff --git a/serde/rpc.h b/serde/rpc.h
index abebe94..15aa1ee 100644
--- a/serde/rpc.h
+++ b/serde/rpc.h
@@ -68,7 +68,7 @@ class RpcSerde {
/**
* Create a new connection preamble in a new IOBuf.
*/
- std::unique_ptr<folly::IOBuf> Preamble();
+ static std::unique_ptr<folly::IOBuf> Preamble(bool secure);
/**
* Create the header protobuf object and serialize it to a new IOBuf.
@@ -119,7 +119,6 @@ class RpcSerde {
private:
/* data */
- uint8_t auth_type_;
std::shared_ptr<Codec> codec_;
std::unique_ptr<pb::VersionInfo> CreateVersionInfo();
};
diff --git a/third-party/BUCK b/third-party/BUCK
index f37eb4e..418323b 100644
--- a/third-party/BUCK
+++ b/third-party/BUCK
@@ -91,7 +91,8 @@ wangle = add_system_libs(
genrule(
name="gen_zk",
out="gen_zk",
- bash="mkdir -p $OUT && wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && tar zxf zookeeper-3.4.8.tar.gz && rm -rf zookeeper-3.4.8.tar.gz && cd zookeeper-3.4.8 && cd src/c && ./configure --prefix=$OUT && make && make install && cd $OUT && rm -rf zookeeper-3.4.8*"
+ bash=
+ "mkdir -p $OUT && wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && tar zxf zookeeper-3.4.8.tar.gz && rm -rf zookeeper-3.4.8.tar.gz && cd zookeeper-3.4.8 && cd src/c && ./configure --prefix=$OUT && make && make install && cd $OUT && rm -rf zookeeper-3.4.8*"
)
cxx_library(
name='google-test',
@@ -112,4 +113,6 @@ cxx_library(
('googletest/googlemock', 'src/*.cc'),
]),
exported_deps=dynamic_rules,
- visibility=['PUBLIC',],)
+ visibility=[
+ 'PUBLIC',
+ ],)
diff --git a/utils/BUCK b/utils/BUCK
index 04e2b67..ed8e114 100644
--- a/utils/BUCK
+++ b/utils/BUCK
@@ -26,15 +26,29 @@ cxx_library(
"version.h",
],
srcs=["bytes-util.cc", "connection-util.cc", "user-util.cc"],
- deps=['//third-party:folly',],
+ deps=[
+ '//third-party:folly',
+ ],
tests=[":user-util-test"],
- visibility=['PUBLIC',],
+ linker_flags=['-L/usr/local/lib', '-lkrb5'],
+ exported_linker_flags=['-L/usr/local/lib', '-lkrb5'],
+ visibility=[
+ 'PUBLIC',
+ ],
compiler_flags=['-Weffc++'],)
cxx_test(
name="user-util-test",
- srcs=["user-util-test.cc",],
- deps=[":utils",],)
+ srcs=[
+ "user-util-test.cc",
+ ],
+ deps=[
+ ":utils",
+ ],)
cxx_test(
name="bytes-util-test",
- srcs=["bytes-util-test.cc",],
- deps=[":utils",],)
+ srcs=[
+ "bytes-util-test.cc",
+ ],
+ deps=[
+ ":utils",
+ ],)
diff --git a/utils/user-util-test.cc b/utils/user-util-test.cc
index 7c11d8c..aa3fa45 100644
--- a/utils/user-util-test.cc
+++ b/utils/user-util-test.cc
@@ -28,7 +28,7 @@ using namespace hbase;
TEST(TestUserUtil, TestGetSomething) {
UserUtil u_util;
- string name = u_util.user_name();
+ string name = u_util.user_name(false);
// TODO shell out to whoami to check this.
ASSERT_GT(name.length(), 0);
diff --git a/utils/user-util.cc b/utils/user-util.cc
index 9e170e0..71f0012 100644
--- a/utils/user-util.cc
+++ b/utils/user-util.cc
@@ -20,6 +20,7 @@
#include "utils/user-util.h"
#include <folly/Logging.h>
+#include <krb5/krb5.h>
#include <pwd.h>
#include <sys/types.h>
#include <unistd.h>
@@ -27,14 +28,14 @@
using namespace hbase;
using namespace std;
-UserUtil::UserUtil() : once_flag_{}, user_name_{"drwho"} {}
+UserUtil::UserUtil() : once_flag_{} {}
-string UserUtil::user_name() {
- std::call_once(once_flag_, [this]() { compute_user_name(); });
+string UserUtil::user_name(bool secure) {
+ std::call_once(once_flag_, [this, secure]() { compute_user_name(secure); });
return user_name_;
}
-void UserUtil::compute_user_name() {
+void UserUtil::compute_user_name(bool secure) {
// According to the man page of getpwuid
// this should never be free'd
//
@@ -45,4 +46,32 @@ void UserUtil::compute_user_name() {
if (passwd && passwd->pw_name) {
user_name_ = string{passwd->pw_name};
}
+ if (!secure) return;
+ krb5_context ctx;
+ krb5_error_code ret = krb5_init_context(&ctx);
+ if (ret != 0) {
+ throw std::runtime_error("cannot init krb ctx " + std::to_string(ret));
+ }
+ krb5_ccache ccache;
+ ret = krb5_cc_default(ctx, &ccache);
+ if (ret != 0) {
+ throw std::runtime_error("cannot get default cache " + std::to_string(ret));
+ }
+ // Here is sample principal: hbase/23a03935850c@EXAMPLE.COM
+ // There may be one (user) or two (user/host) components before the @ sign
+ krb5_principal princ;
+ ret = krb5_cc_get_principal(ctx, ccache, &princ);
+ if (ret != 0) {
+ throw std::runtime_error("cannot get default principal " + std::to_string(ret));
+ }
+ user_name_ = princ->data->data;
+ if (krb5_princ_size(ctx, princ) >= 2) {
+ user_name_ += "/";
+ user_name_ += static_cast<char *>(princ->data[1].data);
+ }
+ user_name_ += "@";
+ user_name_ += princ->realm.data;
+ VLOG(1) << "user " << user_name_;
+ krb5_free_principal(ctx, princ);
+ krb5_free_context(ctx);
}
diff --git a/utils/user-util.h b/utils/user-util.h
index 6f8fce1..6258c85 100644
--- a/utils/user-util.h
+++ b/utils/user-util.h
@@ -41,13 +41,13 @@ class UserUtil {
* Get the username of the user owning this process. This is thread safe and
* lockless for every invocation other than the first one.
*/
- std::string user_name();
+ std::string user_name(bool secure = false);
private:
/**
* Compute the username. This will block.
*/
- void compute_user_name();
+ void compute_user_name(bool secure);
std::once_flag once_flag_;
std::string user_name_;
};
[hbase] 127/137: HBASE-18777 Create one table for subtests in
multi-retry-test
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 37517773ade8bfa418d423a01f491e37ebe9271c
Author: tedyu <yu...@gmail.com>
AuthorDate: Thu Sep 7 19:24:34 2017 -0700
HBASE-18777 Create one table for subtests in multi-retry-test
---
core/async-batch-rpc-retrying-test.cc | 33 ++++++++-------------------------
1 file changed, 8 insertions(+), 25 deletions(-)
diff --git a/core/async-batch-rpc-retrying-test.cc b/core/async-batch-rpc-retrying-test.cc
index b8a0b81..00cf2b8 100644
--- a/core/async-batch-rpc-retrying-test.cc
+++ b/core/async-batch-rpc-retrying-test.cc
@@ -68,14 +68,20 @@ using folly::exception_wrapper;
class AsyncBatchRpcRetryTest : public ::testing::Test {
public:
static std::unique_ptr<hbase::TestUtil> test_util;
+ static std::string tableName;
static void SetUpTestCase() {
google::InstallFailureSignalHandler();
test_util = std::make_unique<hbase::TestUtil>();
test_util->StartMiniCluster(2);
+ std::vector<std::string> keys{"test0", "test100", "test200", "test300", "test400",
+ "test500", "test600", "test700", "test800", "test900"};
+ tableName = "split-table1";
+ test_util->CreateTable(tableName, "d", keys);
}
};
std::unique_ptr<hbase::TestUtil> AsyncBatchRpcRetryTest::test_util = nullptr;
+std::string AsyncBatchRpcRetryTest::tableName;
class AsyncRegionLocatorBase : public AsyncRegionLocator {
public:
@@ -283,18 +289,6 @@ class MockRawAsyncTableImpl {
std::shared_ptr<hbase::pb::TableName> tn_;
};
-std::string createTestTable(bool split_regions, const std::string &table_name) {
- std::vector<std::string> keys{"test0", "test100", "test200", "test300", "test400",
- "test500", "test600", "test700", "test800", "test900"};
- std::string tableName = (split_regions) ? ("split-" + table_name) : table_name;
- if (split_regions) {
- AsyncBatchRpcRetryTest::test_util->CreateTable(tableName, "d", keys);
- } else {
- AsyncBatchRpcRetryTest::test_util->CreateTable(tableName, "d");
- }
- return tableName;
-}
-
std::shared_ptr<MockAsyncConnection> getAsyncConnection(
Client &client, uint32_t operation_timeout_millis, uint32_t tries,
std::shared_ptr<AsyncRegionLocatorBase> region_locator) {
@@ -367,9 +361,8 @@ std::map<std::string, std::shared_ptr<RegionLocation>> getRegionLocationsAndActi
void runMultiGets(std::shared_ptr<AsyncRegionLocatorBase> region_locator,
const std::string &table_name, bool split_regions, uint32_t tries = 3,
uint32_t operation_timeout_millis = 600000, uint64_t num_rows = 1000) {
- auto tableName = createTestTable(split_regions, table_name);
// Create TableName and Row to be fetched from HBase
- auto tn = folly::to<hbase::pb::TableName>(tableName);
+ auto tn = folly::to<hbase::pb::TableName>(AsyncBatchRpcRetryTest::tableName);
// Create a client
Client client(*AsyncBatchRpcRetryTest::test_util->conf());
@@ -419,10 +412,8 @@ void runMultiGets(std::shared_ptr<AsyncRegionLocatorBase> region_locator,
void runMultiPuts(std::shared_ptr<AsyncRegionLocatorBase> region_locator,
const std::string &table_name, bool split_regions, uint32_t tries = 3,
uint32_t operation_timeout_millis = 600000, uint32_t num_rows = 1000) {
- auto tableName = createTestTable(split_regions, table_name);
-
// Create TableName and Row to be fetched from HBase
- auto tn = folly::to<hbase::pb::TableName>(tableName);
+ auto tn = folly::to<hbase::pb::TableName>(AsyncBatchRpcRetryTest::tableName);
// Create a client
Client client(*AsyncBatchRpcRetryTest::test_util->conf());
@@ -543,13 +534,6 @@ TEST_F(AsyncBatchRpcRetryTest, PutsFailWithOperationTimeout) {
EXPECT_ANY_THROW(runMultiPuts(region_locator, "table6", false, 5, 100, 1000));
}
-//////////////////////
-/*
- TODO: Below tests are failing with frequently with segfaults coming from
- JNI internals indicating that we are doing something wrong in the JNI boundary.
- However, we were not able to debug furhter yet. Disable the tests for now, and
- come back later to fix the issue.
-
// Test successful case
TEST_F(AsyncBatchRpcRetryTest, MultiGetsSplitRegions) {
std::shared_ptr<AsyncRegionLocatorBase> region_locator(
@@ -591,4 +575,3 @@ TEST_F(AsyncBatchRpcRetryTest, PutsFailWithOperationTimeout) {
std::make_shared<MockFailingAsyncRegionLocator>(6));
EXPECT_ANY_THROW(runMultiGets(region_locator, "table12", true, 5, 100, 1000));
}
- */
[hbase] 59/137: HBASE-17219 [C++] Reformat the code according to
the style guidelines - ADDENDUM after HBASE-17395
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit cc8c8f9cb2117c2a0f2b14aee627f7a1c56e7dea
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Fri Jan 13 17:42:10 2017 -0800
HBASE-17219 [C++] Reformat the code according to the style guidelines - ADDENDUM after HBASE-17395
---
connection/client-dispatcher.cc | 4 +--
connection/client-dispatcher.h | 3 +-
connection/client-handler.cc | 10 +++---
connection/client-handler.h | 12 +++----
connection/connection-factory.cc | 13 +++-----
connection/connection-factory.h | 6 ++--
connection/connection-id.h | 15 ++++-----
connection/connection-pool-test.cc | 28 +++++-----------
connection/connection-pool.cc | 9 ++---
connection/connection-pool.h | 23 +++++--------
connection/pipeline.h | 6 ++--
connection/request.cc | 14 +++-----
connection/response.h | 4 +--
connection/rpc-client.cc | 67 ++++++++++++++------------------------
connection/rpc-client.h | 29 +++++++----------
connection/rpc-connection.h | 8 ++---
core/cell-test.cc | 21 ++++--------
core/cell.cc | 14 +++-----
core/cell.h | 5 ++-
core/client-test.cc | 6 ++--
core/client.h | 4 +--
core/configuration.cc | 33 +++++++------------
core/configuration.h | 9 ++---
core/get-test.cc | 9 ++---
core/get.cc | 15 +++------
core/hbase_configuration-test.cc | 45 +++++++++----------------
core/hbase_configuration_loader.cc | 27 ++++++---------
core/hbase_configuration_loader.h | 8 ++---
core/location-cache.cc | 36 ++++++++------------
core/location-cache.h | 4 +--
core/meta-utils.cc | 10 +++---
core/meta-utils.h | 6 ++--
core/region-location.h | 3 +-
core/response_converter.h | 2 +-
core/result-test.cc | 54 ++++++++++++++----------------
core/result.cc | 28 +++++++---------
core/result.h | 20 +++++-------
core/scan-test.cc | 6 ++--
core/scan.cc | 21 ++++--------
core/simple-client.cc | 28 +++++++---------
core/time_range.cc | 7 ++--
security/BUCK | 12 +++----
security/user.h | 13 ++++----
serde/client-deserializer-test.cc | 3 +-
serde/rpc.cc | 13 +++-----
serde/rpc.h | 12 +++----
serde/server-name-test.cc | 4 +--
serde/zk.cc | 3 +-
48 files changed, 270 insertions(+), 462 deletions(-)
diff --git a/connection/client-dispatcher.cc b/connection/client-dispatcher.cc
index 0a03396..1ace99c 100644
--- a/connection/client-dispatcher.cc
+++ b/connection/client-dispatcher.cc
@@ -57,6 +57,4 @@ Future<Response> ClientDispatcher::operator()(std::unique_ptr<Request> arg) {
Future<Unit> ClientDispatcher::close() { return ClientDispatcherBase::close(); }
-Future<Unit> ClientDispatcher::close(Context *ctx) {
- return ClientDispatcherBase::close(ctx);
-}
+Future<Unit> ClientDispatcher::close(Context *ctx) { return ClientDispatcherBase::close(ctx); }
diff --git a/connection/client-dispatcher.h b/connection/client-dispatcher.h
index 67e604e..0489717 100644
--- a/connection/client-dispatcher.h
+++ b/connection/client-dispatcher.h
@@ -36,8 +36,7 @@ namespace hbase {
* future.
*/
class ClientDispatcher
- : public wangle::ClientDispatcherBase<SerializePipeline,
- std::unique_ptr<Request>, Response> {
+ : public wangle::ClientDispatcherBase<SerializePipeline, std::unique_ptr<Request>, Response> {
public:
/** Create a new ClientDispatcher */
ClientDispatcher();
diff --git a/connection/client-handler.cc b/connection/client-handler.cc
index e445444..5a6dce2 100644
--- a/connection/client-handler.cc
+++ b/connection/client-handler.cc
@@ -41,8 +41,8 @@ ClientHandler::ClientHandler(std::string user_name)
serde_(),
once_flag_(std::make_unique<std::once_flag>()),
resp_msgs_(
- make_unique<folly::AtomicHashMap<
- uint32_t, std::shared_ptr<google::protobuf::Message>>>(5000)) {}
+ make_unique<folly::AtomicHashMap<uint32_t, std::shared_ptr<google::protobuf::Message>>>(
+ 5000)) {}
void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
if (LIKELY(buf != nullptr)) {
@@ -51,8 +51,7 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
ResponseHeader header;
int used_bytes = serde_.ParseDelimited(buf.get(), &header);
- LOG(INFO) << "Read ResponseHeader size=" << used_bytes
- << " call_id=" << header.call_id()
+ LOG(INFO) << "Read ResponseHeader size=" << used_bytes << " call_id=" << header.call_id()
<< " has_exception=" << header.has_exception();
// Get the response protobuf from the map
@@ -96,6 +95,5 @@ Future<Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
// Now store the call id to response.
resp_msgs_->insert(r->call_id(), r->resp_msg());
// Send the data down the pipeline.
- return ctx->fireWrite(
- serde_.Request(r->call_id(), r->method(), r->req_msg().get()));
+ return ctx->fireWrite(serde_.Request(r->call_id(), r->method(), r->req_msg().get()));
}
diff --git a/connection/client-handler.h b/connection/client-handler.h
index 7306c9a..d860cc1 100644
--- a/connection/client-handler.h
+++ b/connection/client-handler.h
@@ -50,9 +50,9 @@ namespace hbase {
* This class deals with sending the connection header and preamble
* on first request.
*/
-class ClientHandler : public wangle::Handler<std::unique_ptr<folly::IOBuf>,
- Response, std::unique_ptr<Request>,
- std::unique_ptr<folly::IOBuf>> {
+class ClientHandler
+ : public wangle::Handler<std::unique_ptr<folly::IOBuf>, Response, std::unique_ptr<Request>,
+ std::unique_ptr<folly::IOBuf>> {
public:
/**
* Create the handler
@@ -70,8 +70,7 @@ class ClientHandler : public wangle::Handler<std::unique_ptr<folly::IOBuf>,
/**
* Write the data down the wire.
*/
- folly::Future<folly::Unit> write(Context *ctx,
- std::unique_ptr<Request> r) override;
+ folly::Future<folly::Unit> write(Context *ctx, std::unique_ptr<Request> r) override;
private:
std::unique_ptr<std::once_flag> once_flag_;
@@ -79,8 +78,7 @@ class ClientHandler : public wangle::Handler<std::unique_ptr<folly::IOBuf>,
RpcSerde serde_;
// in flight requests
- std::unique_ptr<folly::AtomicHashMap<
- uint32_t, std::shared_ptr<google::protobuf::Message>>>
+ std::unique_ptr<folly::AtomicHashMap<uint32_t, std::shared_ptr<google::protobuf::Message>>>
resp_msgs_;
};
} // namespace hbase
diff --git a/connection/connection-factory.cc b/connection/connection-factory.cc
index beec6d56..ff83212 100644
--- a/connection/connection-factory.cc
+++ b/connection/connection-factory.cc
@@ -26,13 +26,10 @@
using namespace folly;
using namespace hbase;
-ConnectionFactory::ConnectionFactory(
- std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool)
- : io_pool_(io_pool),
- pipeline_factory_(std::make_shared<RpcPipelineFactory>()) {}
+ConnectionFactory::ConnectionFactory(std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool)
+ : io_pool_(io_pool), pipeline_factory_(std::make_shared<RpcPipelineFactory>()) {}
-std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>
-ConnectionFactory::MakeBootstrap() {
+std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> ConnectionFactory::MakeBootstrap() {
auto client = std::make_shared<wangle::ClientBootstrap<SerializePipeline>>();
client->group(io_pool_);
client->pipelineFactory(pipeline_factory_);
@@ -40,8 +37,8 @@ ConnectionFactory::MakeBootstrap() {
return client;
}
std::shared_ptr<HBaseService> ConnectionFactory::Connect(
- std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> client,
- const std::string &hostname, int port) {
+ std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> client, const std::string &hostname,
+ int port) {
// Yes this will block however it makes dealing with connection pool soooooo
// much nicer.
// TODO see about using shared promise for this.
diff --git a/connection/connection-factory.h b/connection/connection-factory.h
index 17d9941..da44c35 100644
--- a/connection/connection-factory.h
+++ b/connection/connection-factory.h
@@ -40,16 +40,14 @@ class ConnectionFactory {
* Constructor.
* There should only be one ConnectionFactory per client.
*/
- explicit ConnectionFactory(
- std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool);
+ explicit ConnectionFactory(std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool);
/** Default Desctructor */
virtual ~ConnectionFactory() = default;
/**
* Create a BootStrap from which a connection can be made.
*/
- virtual std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>
- MakeBootstrap();
+ virtual std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> MakeBootstrap();
/**
* Connect a ClientBootstrap to a server and return the pipeline.
diff --git a/connection/connection-id.h b/connection/connection-id.h
index 62fe222..78b9780 100644
--- a/connection/connection-id.h
+++ b/connection/connection-id.h
@@ -21,9 +21,9 @@
#include "if/HBase.pb.h"
#include "security/user.h"
+#include <boost/functional/hash.hpp>
#include <memory>
#include <utility>
-#include <boost/functional/hash.hpp>
using hbase::pb::ServerName;
using hbase::security::User;
@@ -34,12 +34,11 @@ class ConnectionId {
ConnectionId(const std::string &host, uint16_t port)
: ConnectionId(host, port, User::defaultUser(), "") {}
- ConnectionId(const std::string &host, uint16_t port,
- std::shared_ptr<User> user)
+ ConnectionId(const std::string &host, uint16_t port, std::shared_ptr<User> user)
: ConnectionId(host, port, user, "") {}
- ConnectionId(const std::string &host, uint16_t port,
- std::shared_ptr<User> user, const std::string &service_name)
+ ConnectionId(const std::string &host, uint16_t port, std::shared_ptr<User> user,
+ const std::string &service_name)
: user_(user), service_name_(service_name), host_(host), port_(port) {}
virtual ~ConnectionId() = default;
@@ -66,11 +65,9 @@ struct ConnectionIdEquals {
}
private:
- bool userEquals(const std::shared_ptr<User> &lhs,
- const std::shared_ptr<User> &rhs) const {
+ bool userEquals(const std::shared_ptr<User> &lhs, const std::shared_ptr<User> &rhs) const {
return lhs == nullptr ? rhs == nullptr
- : (rhs == nullptr ? false : lhs->user_name() ==
- rhs->user_name());
+ : (rhs == nullptr ? false : lhs->user_name() == rhs->user_name());
}
};
diff --git a/connection/connection-pool-test.cc b/connection/connection-pool-test.cc
index 4547b30..0930095 100644
--- a/connection/connection-pool-test.cc
+++ b/connection/connection-pool-test.cc
@@ -17,9 +17,9 @@
*
*/
-#include "connection/connection-id.h"
#include "connection/connection-pool.h"
#include "connection/connection-factory.h"
+#include "connection/connection-id.h"
#include "if/HBase.pb.h"
#include "serde/server-name.h"
@@ -37,12 +37,10 @@ using hbase::ConnectionId;
class MockConnectionFactory : public ConnectionFactory {
public:
MockConnectionFactory() : ConnectionFactory(nullptr) {}
- MOCK_METHOD0(MakeBootstrap,
- std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>());
- MOCK_METHOD3(Connect,
- std::shared_ptr<HBaseService>(
- std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>,
- const std::string &hostname, int port));
+ MOCK_METHOD0(MakeBootstrap, std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>());
+ MOCK_METHOD3(Connect, std::shared_ptr<HBaseService>(
+ std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>,
+ const std::string &hostname, int port));
};
class MockBootstrap : public wangle::ClientBootstrap<SerializePipeline> {};
@@ -69,12 +67,8 @@ TEST(TestConnectionPool, TestOnlyCreateOnce) {
auto mock_cf = std::make_shared<MockConnectionFactory>();
uint32_t port{999};
- EXPECT_CALL((*mock_cf), Connect(_, _, _))
- .Times(1)
- .WillRepeatedly(Return(mock_service));
- EXPECT_CALL((*mock_cf), MakeBootstrap())
- .Times(1)
- .WillRepeatedly(Return(mock_boot));
+ EXPECT_CALL((*mock_cf), Connect(_, _, _)).Times(1).WillRepeatedly(Return(mock_service));
+ EXPECT_CALL((*mock_cf), MakeBootstrap()).Times(1).WillRepeatedly(Return(mock_boot));
ConnectionPool cp{mock_cf};
auto remote_id = std::make_shared<ConnectionId>(hostname, port);
@@ -92,12 +86,8 @@ TEST(TestConnectionPool, TestOnlyCreateMultipleDispose) {
auto mock_service = std::make_shared<MockService>();
auto mock_cf = std::make_shared<MockConnectionFactory>();
- EXPECT_CALL((*mock_cf), Connect(_, _, _))
- .Times(2)
- .WillRepeatedly(Return(mock_service));
- EXPECT_CALL((*mock_cf), MakeBootstrap())
- .Times(2)
- .WillRepeatedly(Return(mock_boot));
+ EXPECT_CALL((*mock_cf), Connect(_, _, _)).Times(2).WillRepeatedly(Return(mock_service));
+ EXPECT_CALL((*mock_cf), MakeBootstrap()).Times(2).WillRepeatedly(Return(mock_boot));
ConnectionPool cp{mock_cf};
{
diff --git a/connection/connection-pool.cc b/connection/connection-pool.cc
index ee14c9d..07518c5 100644
--- a/connection/connection-pool.cc
+++ b/connection/connection-pool.cc
@@ -33,8 +33,7 @@ using hbase::HBaseService;
using folly::SharedMutexWritePriority;
using folly::SocketAddress;
-ConnectionPool::ConnectionPool(
- std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor)
+ConnectionPool::ConnectionPool(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor)
: cf_(std::make_shared<ConnectionFactory>(io_executor)),
clients_(),
connections_(),
@@ -95,8 +94,7 @@ std::shared_ptr<RpcConnection> ConnectionPool::GetNewConnection(
/* create new connection */
auto clientBootstrap = cf_->MakeBootstrap();
- auto dispatcher =
- cf_->Connect(clientBootstrap, remote_id->host(), remote_id->port());
+ auto dispatcher = cf_->Connect(clientBootstrap, remote_id->host(), remote_id->port());
auto conneciton = std::make_shared<RpcConnection>(remote_id, dispatcher);
@@ -118,5 +116,4 @@ void ConnectionPool::Close(std::shared_ptr<ConnectionId> remote_id) {
connections_.erase(found);
}
-void ConnectionPool::Close() {
-}
+void ConnectionPool::Close() {}
diff --git a/connection/connection-pool.h b/connection/connection-pool.h
index 5101c68..1f2a182 100644
--- a/connection/connection-pool.h
+++ b/connection/connection-pool.h
@@ -30,7 +30,6 @@
#include "connection/service.h"
#include "if/HBase.pb.h"
-
using hbase::ConnectionId;
using hbase::ConnectionIdEquals;
using hbase::ConnectionIdHash;
@@ -47,8 +46,7 @@ namespace hbase {
class ConnectionPool {
public:
/** Create connection pool wit default connection factory */
- explicit ConnectionPool(
- std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor);
+ explicit ConnectionPool(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor);
/**
* Desctructor.
@@ -67,8 +65,7 @@ class ConnectionPool {
* Get a connection to the server name. Start time is ignored.
* This can be a blocking operation for a short time.
*/
- std::shared_ptr<RpcConnection> GetConnection(
- std::shared_ptr<ConnectionId> remote_id);
+ std::shared_ptr<RpcConnection> GetConnection(std::shared_ptr<ConnectionId> remote_id);
/**
* Close/remove a connection.
@@ -81,18 +78,14 @@ class ConnectionPool {
void Close();
private:
- std::shared_ptr<RpcConnection> GetCachedConnection(
- std::shared_ptr<ConnectionId> remote_id);
- std::shared_ptr<RpcConnection> GetNewConnection(
- std::shared_ptr<ConnectionId> remote_id);
+ std::shared_ptr<RpcConnection> GetCachedConnection(std::shared_ptr<ConnectionId> remote_id);
+ std::shared_ptr<RpcConnection> GetNewConnection(std::shared_ptr<ConnectionId> remote_id);
+ std::unordered_map<std::shared_ptr<ConnectionId>, std::shared_ptr<RpcConnection>,
+ ConnectionIdHash, ConnectionIdEquals>
+ connections_;
std::unordered_map<std::shared_ptr<ConnectionId>,
- std::shared_ptr<RpcConnection>, ConnectionIdHash,
+ std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>, ConnectionIdHash,
ConnectionIdEquals>
- connections_;
- std::unordered_map<
- std::shared_ptr<ConnectionId>,
- std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>,
- ConnectionIdHash, ConnectionIdEquals>
clients_;
folly::SharedMutexWritePriority map_mutex_;
std::shared_ptr<ConnectionFactory> cf_;
diff --git a/connection/pipeline.h b/connection/pipeline.h
index c4f3bbb..343219d 100644
--- a/connection/pipeline.h
+++ b/connection/pipeline.h
@@ -30,8 +30,7 @@
namespace hbase {
/** Pipeline to turn IOBuf into requests */
-using SerializePipeline =
- wangle::Pipeline<folly::IOBufQueue &, std::unique_ptr<Request>>;
+using SerializePipeline = wangle::Pipeline<folly::IOBufQueue &, std::unique_ptr<Request>>;
/**
* Factory to create new pipelines for HBase RPC's.
@@ -52,8 +51,7 @@ class RpcPipelineFactory : public wangle::PipelineFactory<SerializePipeline> {
* - Length Field Based Frame Decoder
* - Client Handler
*/
- SerializePipeline::Ptr newPipeline(
- std::shared_ptr<folly::AsyncTransportWrapper> sock) override;
+ SerializePipeline::Ptr newPipeline(std::shared_ptr<folly::AsyncTransportWrapper> sock) override;
private:
UserUtil user_util_;
diff --git a/connection/request.cc b/connection/request.cc
index d4a4603..189130e 100644
--- a/connection/request.cc
+++ b/connection/request.cc
@@ -24,22 +24,18 @@
using namespace hbase;
Request::Request(std::shared_ptr<google::protobuf::Message> req,
- std::shared_ptr<google::protobuf::Message> resp,
- std::string method)
+ std::shared_ptr<google::protobuf::Message> resp, std::string method)
: req_msg_(req), resp_msg_(resp), method_(method), call_id_(0) {}
std::unique_ptr<Request> Request::get() {
return std::make_unique<Request>(std::make_shared<hbase::pb::GetRequest>(),
- std::make_shared<hbase::pb::GetResponse>(),
- "Get");
+ std::make_shared<hbase::pb::GetResponse>(), "Get");
}
std::unique_ptr<Request> Request::mutate() {
- return std::make_unique<Request>(
- std::make_shared<hbase::pb::MutateRequest>(),
- std::make_shared<hbase::pb::MutateResponse>(), "Mutate");
+ return std::make_unique<Request>(std::make_shared<hbase::pb::MutateRequest>(),
+ std::make_shared<hbase::pb::MutateResponse>(), "Mutate");
}
std::unique_ptr<Request> Request::scan() {
return std::make_unique<Request>(std::make_shared<hbase::pb::ScanRequest>(),
- std::make_shared<hbase::pb::ScanResponse>(),
- "Scan");
+ std::make_shared<hbase::pb::ScanResponse>(), "Scan");
}
diff --git a/connection/response.h b/connection/response.h
index f032d09..560387c 100644
--- a/connection/response.h
+++ b/connection/response.h
@@ -55,9 +55,7 @@ class Response {
* The caller is reponsible for knowing the type. In practice the call id is
* used to figure out the type.
*/
- std::shared_ptr<google::protobuf::Message> resp_msg() const {
- return resp_msg_;
- }
+ std::shared_ptr<google::protobuf::Message> resp_msg() const { return resp_msg_; }
/** Set the response message. */
void set_resp_msg(std::shared_ptr<google::protobuf::Message> response) {
diff --git a/connection/rpc-client.cc b/connection/rpc-client.cc
index 3f0cfaf..9cfefb8 100644
--- a/connection/rpc-client.cc
+++ b/connection/rpc-client.cc
@@ -28,41 +28,32 @@ namespace hbase {
class RpcChannelImplementation : public AbstractRpcChannel {
public:
- RpcChannelImplementation(std::shared_ptr<RpcClient> rpc_client,
- const std::string& host, uint16_t port,
- std::shared_ptr<User> ticket, int rpc_timeout)
+ RpcChannelImplementation(std::shared_ptr<RpcClient> rpc_client, const std::string& host,
+ uint16_t port, std::shared_ptr<User> ticket, int rpc_timeout)
: AbstractRpcChannel(rpc_client, host, port, ticket, rpc_timeout) {}
- void CallMethod(const MethodDescriptor* method, RpcController* controller,
- const Message* request, Message* response,
- Closure* done) override {
- rpc_client_->CallMethod(method, controller, request, response, done, host_,
- port_, ticket_);
+ void CallMethod(const MethodDescriptor* method, RpcController* controller, const Message* request,
+ Message* response, Closure* done) override {
+ rpc_client_->CallMethod(method, controller, request, response, done, host_, port_, ticket_);
}
};
} // namespace hbase
RpcClient::RpcClient() {
- io_executor_ = std::make_shared<wangle::IOThreadPoolExecutor>(
- sysconf(_SC_NPROCESSORS_ONLN));
+ io_executor_ = std::make_shared<wangle::IOThreadPoolExecutor>(sysconf(_SC_NPROCESSORS_ONLN));
cp_ = std::make_shared<ConnectionPool>(io_executor_);
}
-void RpcClient::Close() {
- io_executor_->stop();
-}
+void RpcClient::Close() { io_executor_->stop(); }
-std::shared_ptr<Response> RpcClient::SyncCall(const std::string& host,
- uint16_t port,
+std::shared_ptr<Response> RpcClient::SyncCall(const std::string& host, uint16_t port,
std::unique_ptr<Request> req,
std::shared_ptr<User> ticket) {
- return std::make_shared<Response>(
- AsyncCall(host, port, std::move(req), ticket).get());
+ return std::make_shared<Response>(AsyncCall(host, port, std::move(req), ticket).get());
}
-std::shared_ptr<Response> RpcClient::SyncCall(const std::string& host,
- uint16_t port,
+std::shared_ptr<Response> RpcClient::SyncCall(const std::string& host, uint16_t port,
std::unique_ptr<Request> req,
std::shared_ptr<User> ticket,
const std::string& service_name) {
@@ -70,54 +61,44 @@ std::shared_ptr<Response> RpcClient::SyncCall(const std::string& host,
AsyncCall(host, port, std::move(req), ticket, service_name).get());
}
-folly::Future<Response> RpcClient::AsyncCall(const std::string& host,
- uint16_t port,
+folly::Future<Response> RpcClient::AsyncCall(const std::string& host, uint16_t port,
std::unique_ptr<Request> req,
std::shared_ptr<User> ticket) {
auto remote_id = std::make_shared<ConnectionId>(host, port, ticket);
return GetConnection(remote_id)->SendRequest(std::move(req));
}
-folly::Future<Response> RpcClient::AsyncCall(const std::string& host,
- uint16_t port,
+folly::Future<Response> RpcClient::AsyncCall(const std::string& host, uint16_t port,
std::unique_ptr<Request> req,
std::shared_ptr<User> ticket,
const std::string& service_name) {
- auto remote_id =
- std::make_shared<ConnectionId>(host, port, ticket, service_name);
+ auto remote_id = std::make_shared<ConnectionId>(host, port, ticket, service_name);
return GetConnection(remote_id)->SendRequest(std::move(req));
}
-std::shared_ptr<RpcConnection> RpcClient::GetConnection(
- std::shared_ptr<ConnectionId> remote_id) {
+std::shared_ptr<RpcConnection> RpcClient::GetConnection(std::shared_ptr<ConnectionId> remote_id) {
return cp_->GetConnection(remote_id);
}
-std::shared_ptr<RpcChannel> RpcClient::CreateRpcChannel(
- const std::string& host, uint16_t port, std::shared_ptr<User> ticket,
- int rpc_timeout) {
- std::shared_ptr<RpcChannelImplementation> channel =
- std::make_shared<RpcChannelImplementation>(shared_from_this(), host, port,
- ticket, rpc_timeout);
+std::shared_ptr<RpcChannel> RpcClient::CreateRpcChannel(const std::string& host, uint16_t port,
+ std::shared_ptr<User> ticket,
+ int rpc_timeout) {
+ std::shared_ptr<RpcChannelImplementation> channel = std::make_shared<RpcChannelImplementation>(
+ shared_from_this(), host, port, ticket, rpc_timeout);
/* static_pointer_cast is safe since RpcChannelImplementation derives
* from RpcChannel, otherwise, dynamic_pointer_cast should be used. */
return std::static_pointer_cast<RpcChannel>(channel);
}
-void RpcClient::CallMethod(const MethodDescriptor* method,
- RpcController* controller, const Message* req_msg,
- Message* resp_msg, Closure* done,
- const std::string& host, uint16_t port,
- std::shared_ptr<User> ticket) {
+void RpcClient::CallMethod(const MethodDescriptor* method, RpcController* controller,
+ const Message* req_msg, Message* resp_msg, Closure* done,
+ const std::string& host, uint16_t port, std::shared_ptr<User> ticket) {
std::shared_ptr<Message> shared_req(const_cast<Message*>(req_msg));
std::shared_ptr<Message> shared_resp(resp_msg);
- std::unique_ptr<Request> req =
- std::make_unique<Request>(shared_req, shared_resp, method->name());
+ std::unique_ptr<Request> req = std::make_unique<Request>(shared_req, shared_resp, method->name());
AsyncCall(host, port, std::move(req), ticket, method->service()->name())
- .then([done, this](Response resp) {
- done->Run();
- });
+ .then([done, this](Response resp) { done->Run(); });
}
diff --git a/connection/rpc-client.h b/connection/rpc-client.h
index dbf857d..407d588 100644
--- a/connection/rpc-client.h
+++ b/connection/rpc-client.h
@@ -53,41 +53,35 @@ class RpcClient : public std::enable_shared_from_this<RpcClient> {
virtual ~RpcClient() { Close(); }
- virtual std::shared_ptr<Response> SyncCall(const std::string &host,
- uint16_t port,
+ virtual std::shared_ptr<Response> SyncCall(const std::string &host, uint16_t port,
std::unique_ptr<Request> req,
std::shared_ptr<User> ticket);
- virtual std::shared_ptr<Response> SyncCall(const std::string &host,
- uint16_t port,
+ virtual std::shared_ptr<Response> SyncCall(const std::string &host, uint16_t port,
std::unique_ptr<Request> req,
std::shared_ptr<User> ticket,
const std::string &service_name);
- virtual folly::Future<Response> AsyncCall(const std::string &host,
- uint16_t port,
+ virtual folly::Future<Response> AsyncCall(const std::string &host, uint16_t port,
std::unique_ptr<Request> req,
std::shared_ptr<User> ticket);
- virtual folly::Future<Response> AsyncCall(const std::string &host,
- uint16_t port,
+ virtual folly::Future<Response> AsyncCall(const std::string &host, uint16_t port,
std::unique_ptr<Request> req,
std::shared_ptr<User> ticket,
const std::string &service_name);
virtual void Close();
- virtual std::shared_ptr<RpcChannel> CreateRpcChannel(
- const std::string &host, uint16_t port, std::shared_ptr<User> ticket,
- int rpc_timeout);
+ virtual std::shared_ptr<RpcChannel> CreateRpcChannel(const std::string &host, uint16_t port,
+ std::shared_ptr<User> ticket,
+ int rpc_timeout);
private:
- void CallMethod(const MethodDescriptor *method, RpcController *controller,
- const Message *req_msg, Message *resp_msg, Closure *done,
- const std::string &host, uint16_t port,
+ void CallMethod(const MethodDescriptor *method, RpcController *controller, const Message *req_msg,
+ Message *resp_msg, Closure *done, const std::string &host, uint16_t port,
std::shared_ptr<User> ticket);
- std::shared_ptr<RpcConnection> GetConnection(
- std::shared_ptr<ConnectionId> remote_id);
+ std::shared_ptr<RpcConnection> GetConnection(std::shared_ptr<ConnectionId> remote_id);
private:
std::shared_ptr<ConnectionPool> cp_;
@@ -96,8 +90,7 @@ class RpcClient : public std::enable_shared_from_this<RpcClient> {
class AbstractRpcChannel : public RpcChannel {
public:
- AbstractRpcChannel(std::shared_ptr<RpcClient> rpc_client,
- const std::string &host, uint16_t port,
+ AbstractRpcChannel(std::shared_ptr<RpcClient> rpc_client, const std::string &host, uint16_t port,
std::shared_ptr<User> ticket, int rpc_timeout)
: rpc_client_(rpc_client),
host_(host),
diff --git a/connection/rpc-connection.h b/connection/rpc-connection.h
index 2e06ec3..e2500b2 100644
--- a/connection/rpc-connection.h
+++ b/connection/rpc-connection.h
@@ -37,13 +37,9 @@ class RpcConnection {
virtual ~RpcConnection() { Close(); }
- virtual std::shared_ptr<ConnectionId> remote_id() const {
- return connection_id_;
- }
+ virtual std::shared_ptr<ConnectionId> remote_id() const { return connection_id_; }
- virtual std::shared_ptr<HBaseService> get_service() const {
- return hbase_service_;
- }
+ virtual std::shared_ptr<HBaseService> get_service() const { return hbase_service_; }
virtual folly::Future<Response> SendRequest(std::unique_ptr<Request> req) {
return (*hbase_service_)(std::move(req));
diff --git a/core/cell-test.cc b/core/cell-test.cc
index 53574e8..fb4d033 100644
--- a/core/cell-test.cc
+++ b/core/cell-test.cc
@@ -32,8 +32,7 @@ TEST(CellTest, CellFailureTest) {
std::string value = "value";
int64_t timestamp = std::numeric_limits<int64_t>::max();
std::string tags = "";
- std::unique_ptr<Cell> cell(
- new Cell(row, family, column, timestamp, value, cell_type));
+ std::unique_ptr<Cell> cell(new Cell(row, family, column, timestamp, value, cell_type));
if (cell.get()) {
EXPECT_NE("row-value", cell.get()->Row());
EXPECT_NE("family-value", cell.get()->Family());
@@ -51,8 +50,7 @@ TEST(CellTest, CellSuceessTest) {
std::string value = "value-value";
int64_t timestamp = std::numeric_limits<int64_t>::max();
CellType cell_type = CellType::PUT;
- const std::unique_ptr<Cell> cell(
- new Cell(row, family, column, timestamp, value, cell_type));
+ const std::unique_ptr<Cell> cell(new Cell(row, family, column, timestamp, value, cell_type));
if (cell.get()) {
EXPECT_EQ(row, cell.get()->Row());
EXPECT_EQ(family, cell.get()->Family());
@@ -74,8 +72,7 @@ TEST(CellTest, MultipleCellsTest) {
row += std::to_string(i);
value += std::to_string(i);
CellType cell_type = CellType::PUT;
- const Cell *cell =
- new Cell(row, family, column, timestamp, value, cell_type);
+ const Cell *cell = new Cell(row, family, column, timestamp, value, cell_type);
cells.push_back(cell);
}
int i = 0;
@@ -105,8 +102,7 @@ TEST(CellTest, CellRowTest) {
std::string value = "";
int64_t timestamp = std::numeric_limits<int64_t>::max();
CellType cell_type = CellType::PUT;
- std::unique_ptr<Cell> cell(
- new Cell(row, family, column, timestamp, value, cell_type));
+ std::unique_ptr<Cell> cell(new Cell(row, family, column, timestamp, value, cell_type));
if (cell.get()) {
EXPECT_EQ(row, cell.get()->Row());
EXPECT_EQ(family, cell.get()->Family());
@@ -124,8 +120,7 @@ TEST(CellTest, CellRowFamilyTest) {
std::string value = "";
int64_t timestamp = std::numeric_limits<int64_t>::max();
CellType cell_type = CellType::PUT;
- const std::unique_ptr<Cell> cell(
- new Cell(row, family, column, timestamp, value, cell_type));
+ const std::unique_ptr<Cell> cell(new Cell(row, family, column, timestamp, value, cell_type));
if (cell.get()) {
EXPECT_EQ(row, cell.get()->Row());
EXPECT_EQ(family, cell.get()->Family());
@@ -143,8 +138,7 @@ TEST(CellTest, CellRowFamilyValueTest) {
std::string value = "only-value";
int64_t timestamp = std::numeric_limits<int64_t>::max();
CellType cell_type = CellType::PUT;
- const std::unique_ptr<Cell> cell(
- new Cell(row, family, column, timestamp, value, cell_type));
+ const std::unique_ptr<Cell> cell(new Cell(row, family, column, timestamp, value, cell_type));
if (cell.get()) {
EXPECT_EQ(row, cell.get()->Row());
EXPECT_EQ(family, cell.get()->Family());
@@ -162,8 +156,7 @@ TEST(CellTest, CellRowFamilyColumnValueTest) {
std::string value = "only-value";
int64_t timestamp = std::numeric_limits<int64_t>::max();
CellType cell_type = CellType::PUT;
- std::unique_ptr<Cell> cell(
- new Cell(row, family, column, timestamp, value, cell_type));
+ std::unique_ptr<Cell> cell(new Cell(row, family, column, timestamp, value, cell_type));
if (cell.get()) {
EXPECT_EQ(row, cell.get()->Row());
EXPECT_EQ(family, cell.get()->Family());
diff --git a/core/cell.cc b/core/cell.cc
index 3102e53..4982406 100644
--- a/core/cell.cc
+++ b/core/cell.cc
@@ -22,9 +22,8 @@
namespace hbase {
-Cell::Cell(const std::string &row, const std::string &family,
- const std::string &qualifier, const int64_t timestamp,
- const std::string &value, const hbase::CellType &cell_type)
+Cell::Cell(const std::string &row, const std::string &family, const std::string &qualifier,
+ const int64_t timestamp, const std::string &value, const hbase::CellType &cell_type)
: row_(row),
family_(family),
qualifier_(qualifier),
@@ -32,14 +31,11 @@ Cell::Cell(const std::string &row, const std::string &family,
cell_type_(cell_type),
value_(value),
sequence_id_(0) {
- if (0 == row.size())
- throw std::runtime_error("Row size should be greater than 0");
+ if (0 == row.size()) throw std::runtime_error("Row size should be greater than 0");
- if (0 == family.size())
- throw std::runtime_error("Column family size should be greater than 0");
+ if (0 == family.size()) throw std::runtime_error("Column family size should be greater than 0");
- if (0 >= timestamp)
- throw std::runtime_error("Timestamp should be greater than 0");
+ if (0 >= timestamp) throw std::runtime_error("Timestamp should be greater than 0");
}
Cell::~Cell() {}
diff --git a/core/cell.h b/core/cell.h
index f55655c..5a39dc3 100644
--- a/core/cell.h
+++ b/core/cell.h
@@ -36,9 +36,8 @@ enum CellType {
class Cell {
public:
- Cell(const std::string &row, const std::string &family,
- const std::string &qualifier, const int64_t timestamp,
- const std::string &value, const hbase::CellType &cell_type);
+ Cell(const std::string &row, const std::string &family, const std::string &qualifier,
+ const int64_t timestamp, const std::string &value, const hbase::CellType &cell_type);
virtual ~Cell();
const std::string &Row() const;
const std::string &Family() const;
diff --git a/core/client-test.cc b/core/client-test.cc
index 0fe0225..28eec6f 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -17,8 +17,8 @@
*
*/
-#include <gtest/gtest.h>
#include "core/client.h"
+#include <gtest/gtest.h>
#include "core/configuration.h"
#include "core/get.h"
#include "core/hbase_configuration_loader.h"
@@ -84,9 +84,7 @@ const std::string ClientTest::kHBaseXmlData(
"the License.\n "
"*/\n-->\n<configuration>\n\n</configuration>");
-TEST(Client, EmptyConfigurationPassedToClient) {
- ASSERT_ANY_THROW(hbase::Client client);
-}
+TEST(Client, EmptyConfigurationPassedToClient) { ASSERT_ANY_THROW(hbase::Client client); }
TEST(Client, ConfigurationPassedToClient) {
// Remove already configured env if present.
diff --git a/core/client.h b/core/client.h
index 2bb506b..da71624 100644
--- a/core/client.h
+++ b/core/client.h
@@ -27,13 +27,13 @@
#include <memory>
#include <string>
+#include "connection/rpc-client.h"
#include "core/configuration.h"
#include "core/hbase_configuration_loader.h"
#include "core/location-cache.h"
-#include "connection/rpc-client.h"
#include "core/table.h"
-#include "serde/table-name.h"
#include "if/Cell.pb.h"
+#include "serde/table-name.h"
using hbase::pb::TableName;
diff --git a/core/configuration.cc b/core/configuration.cc
index 134f05f..bd582d8 100644
--- a/core/configuration.cc
+++ b/core/configuration.cc
@@ -28,13 +28,11 @@
namespace hbase {
-Configuration::Configuration(ConfigMap &config_map)
- : hb_property_(std::move(config_map)) {}
+Configuration::Configuration(ConfigMap &config_map) : hb_property_(std::move(config_map)) {}
Configuration::~Configuration() {}
-size_t Configuration::IsSubVariable(const std::string &expr,
- std::string &sub_variable) const {
+size_t Configuration::IsSubVariable(const std::string &expr, std::string &sub_variable) const {
size_t start_pos = expr.find("${");
if (std::string::npos != start_pos) {
size_t pos_next = expr.find("}", start_pos + 1);
@@ -57,8 +55,7 @@ std::string Configuration::SubstituteVars(const std::string &expr) const {
if (start_pos != std::string::npos) {
// We are blindly checking for environment property at first.
// If we don't get any value from GetEnv, check in hbase-site.xml.
- value_to_be_replaced =
- GetEnv(var).value_or(GetProperty(var).value_or(""));
+ value_to_be_replaced = GetEnv(var).value_or(GetProperty(var).value_or(""));
// we haven't found any value yet so we are returning eval
if (0 == value_to_be_replaced.size()) {
@@ -148,8 +145,7 @@ optional<std::string> Configuration::Get(const std::string &key) const {
}
}
-std::string Configuration::Get(const std::string &key,
- const std::string &default_value) const {
+std::string Configuration::Get(const std::string &key, const std::string &default_value) const {
return Get(key).value_or(default_value);
}
@@ -157,8 +153,7 @@ optional<int32_t> Configuration::GetInt(const std::string &key) const {
optional<std::string> raw = Get(key);
if (raw) {
try {
- return std::experimental::make_optional(
- boost::lexical_cast<int32_t>(*raw));
+ return std::experimental::make_optional(boost::lexical_cast<int32_t>(*raw));
} catch (const boost::bad_lexical_cast &blex) {
throw std::runtime_error(blex.what());
}
@@ -166,8 +161,7 @@ optional<int32_t> Configuration::GetInt(const std::string &key) const {
return optional<int32_t>();
}
-int32_t Configuration::GetInt(const std::string &key,
- int32_t default_value) const {
+int32_t Configuration::GetInt(const std::string &key, int32_t default_value) const {
return GetInt(key).value_or(default_value);
}
@@ -175,8 +169,7 @@ optional<int64_t> Configuration::GetLong(const std::string &key) const {
optional<std::string> raw = Get(key);
if (raw) {
try {
- return std::experimental::make_optional(
- boost::lexical_cast<int64_t>(*raw));
+ return std::experimental::make_optional(boost::lexical_cast<int64_t>(*raw));
} catch (const boost::bad_lexical_cast &blex) {
throw std::runtime_error(blex.what());
}
@@ -184,8 +177,7 @@ optional<int64_t> Configuration::GetLong(const std::string &key) const {
return optional<int64_t>();
}
-int64_t Configuration::GetLong(const std::string &key,
- int64_t default_value) const {
+int64_t Configuration::GetLong(const std::string &key, int64_t default_value) const {
return GetLong(key).value_or(default_value);
}
@@ -193,8 +185,7 @@ optional<double> Configuration::GetDouble(const std::string &key) const {
optional<std::string> raw = Get(key);
if (raw) {
try {
- return std::experimental::make_optional(
- boost::lexical_cast<double>(*raw));
+ return std::experimental::make_optional(boost::lexical_cast<double>(*raw));
} catch (const boost::bad_lexical_cast &blex) {
throw std::runtime_error(blex.what());
}
@@ -202,8 +193,7 @@ optional<double> Configuration::GetDouble(const std::string &key) const {
return optional<double>();
}
-double Configuration::GetDouble(const std::string &key,
- double default_value) const {
+double Configuration::GetDouble(const std::string &key, double default_value) const {
return GetDouble(key).value_or(default_value);
}
@@ -215,8 +205,7 @@ optional<bool> Configuration::GetBool(const std::string &key) const {
} else if (!strcasecmp((*raw).c_str(), "false")) {
return std::experimental::make_optional(false);
} else {
- throw std::runtime_error(
- "Unexpected value found while conversion to bool.");
+ throw std::runtime_error("Unexpected value found while conversion to bool.");
}
}
return optional<bool>();
diff --git a/core/configuration.h b/core/configuration.h
index f09bbac..69bef75 100644
--- a/core/configuration.h
+++ b/core/configuration.h
@@ -41,8 +41,7 @@ class Configuration {
* @param key Property whose value is to be fetched. SubstituteVars will be
* called for any variable expansion.
*/
- std::string Get(const std::string &key,
- const std::string &default_value) const;
+ std::string Get(const std::string &key, const std::string &default_value) const;
/**
* @brief Returns int32_t identified by key in ConfigMap else default value if
@@ -89,8 +88,7 @@ class Configuration {
std::string value;
bool final;
ConfigData() : final(false) {}
- explicit ConfigData(const std::string &value)
- : value(value), final(false) {}
+ explicit ConfigData(const std::string &value) : value(value), final(false) {}
void operator=(const std::string &new_value) {
value = new_value;
final = false;
@@ -174,8 +172,7 @@ class Configuration {
* @param sub_variable Extracted variable from expr which will be checked
* against environment value or ConfigMap values.
*/
- size_t IsSubVariable(const std::string &expr,
- std::string &sub_variable) const;
+ size_t IsSubVariable(const std::string &expr, std::string &sub_variable) const;
/**
* @brief This method will fetch value for key from environment if present.
diff --git a/core/get-test.cc b/core/get-test.cc
index cef99b9..07d0003 100644
--- a/core/get-test.cc
+++ b/core/get-test.cc
@@ -121,8 +121,7 @@ void GetMethods(Get &get, const std::string &row) {
// Test initial values
EXPECT_EQ(0, get.Timerange().MinTimeStamp());
- EXPECT_EQ(std::numeric_limits<int64_t>::max(),
- get.Timerange().MaxTimeStamp());
+ EXPECT_EQ(std::numeric_limits<int64_t>::max(), get.Timerange().MaxTimeStamp());
// Set & Test new values using TimeRange and TimeStamp
get.SetTimeRange(1000, 2000);
@@ -136,13 +135,11 @@ void GetMethods(Get &get, const std::string &row) {
ASSERT_THROW(get.SetTimeRange(-1000, 2000), std::runtime_error);
ASSERT_THROW(get.SetTimeRange(1000, -2000), std::runtime_error);
ASSERT_THROW(get.SetTimeRange(1000, 200), std::runtime_error);
- ASSERT_THROW(get.SetTimeStamp(std::numeric_limits<int64_t>::max()),
- std::runtime_error);
+ ASSERT_THROW(get.SetTimeStamp(std::numeric_limits<int64_t>::max()), std::runtime_error);
// Test some exceptions
ASSERT_THROW(get.SetMaxVersions(0), std::runtime_error);
- ASSERT_THROW(get.SetMaxVersions(std::numeric_limits<uint32_t>::max() + 1),
- std::runtime_error);
+ ASSERT_THROW(get.SetMaxVersions(std::numeric_limits<uint32_t>::max() + 1), std::runtime_error);
}
TEST(Get, SingleGet) {
diff --git a/core/get.cc b/core/get.cc
index 6ba4e86..5c5f446 100644
--- a/core/get.cc
+++ b/core/get.cc
@@ -34,8 +34,7 @@ Get::Get(const Get &get) {
cache_blocks_ = get.cache_blocks_;
check_existence_only_ = get.check_existence_only_;
consistency_ = get.consistency_;
- tr_.reset(new TimeRange(get.Timerange().MinTimeStamp(),
- get.Timerange().MaxTimeStamp()));
+ tr_.reset(new TimeRange(get.Timerange().MinTimeStamp(), get.Timerange().MaxTimeStamp()));
family_map_.insert(get.family_map_.begin(), get.family_map_.end());
}
@@ -45,8 +44,7 @@ Get &Get::operator=(const Get &get) {
cache_blocks_ = get.cache_blocks_;
check_existence_only_ = get.check_existence_only_;
consistency_ = get.consistency_;
- tr_.reset(new TimeRange(get.Timerange().MinTimeStamp(),
- get.Timerange().MaxTimeStamp()));
+ tr_.reset(new TimeRange(get.Timerange().MinTimeStamp(), get.Timerange().MaxTimeStamp()));
family_map_.insert(get.family_map_.begin(), get.family_map_.end());
return *this;
}
@@ -68,8 +66,7 @@ Get &Get::AddFamily(const std::string &family) {
}
Get &Get::AddColumn(const std::string &family, const std::string &qualifier) {
- const auto &it = std::find(family_map_[family].begin(),
- family_map_[family].end(), qualifier);
+ const auto &it = std::find(family_map_[family].begin(), family_map_[family].end(), qualifier);
/**
* Check if any qualifiers are already present or not.
@@ -97,8 +94,7 @@ const FamilyMap &Get::Family() const { return family_map_; }
int Get::MaxVersions() const { return max_versions_; }
Get &Get::SetMaxVersions(int32_t max_versions) {
- if (0 == max_versions)
- throw std::runtime_error("max_versions must be positive");
+ if (0 == max_versions) throw std::runtime_error("max_versions must be positive");
max_versions_ = max_versions;
return *this;
@@ -130,8 +126,7 @@ void Get::CheckRow(const std::string &row) {
throw std::runtime_error("Row length can't be 0");
}
if (row_length > kMaxRowLength) {
- throw std::runtime_error("Length of " + row +
- " is greater than max row size: " +
+ throw std::runtime_error("Length of " + row + " is greater than max row size: " +
std::to_string(kMaxRowLength));
}
}
diff --git a/core/hbase_configuration-test.cc b/core/hbase_configuration-test.cc
index eec500e..07a6358 100644
--- a/core/hbase_configuration-test.cc
+++ b/core/hbase_configuration-test.cc
@@ -28,10 +28,8 @@
using namespace hbase;
-const std::string kDefHBaseConfPath(
- "./build/test-data/hbase-configuration-test/conf/");
-const std::string kHBaseConfPath(
- "./build/test-data/hbase-configuration-test/custom-conf/");
+const std::string kDefHBaseConfPath("./build/test-data/hbase-configuration-test/conf/");
+const std::string kHBaseConfPath("./build/test-data/hbase-configuration-test/custom-conf/");
const std::string kHBaseDefaultXml("hbase-default.xml");
const std::string kHBaseSiteXml("hbase-site.xml");
@@ -121,8 +119,7 @@ void WriteDataToFile(const std::string &file, const std::string &xml_data) {
hbase_conf.close();
}
-void CreateHBaseConf(const std::string &dir, const std::string &file,
- const std::string xml_data) {
+void CreateHBaseConf(const std::string &dir, const std::string &file, const std::string xml_data) {
// Directory will be created if not present
if (!boost::filesystem::exists(dir)) {
boost::filesystem::create_directories(dir);
@@ -153,10 +150,8 @@ TEST(Configuration, LoadConfFromDefaultLocation) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
ASSERT_TRUE(conf) << "No configuration object present.";
- EXPECT_STREQ((*conf).Get("custom-prop", "Set this value").c_str(),
- "custom-value");
- EXPECT_STREQ((*conf).Get("default-prop", "Set this value").c_str(),
- "default-value");
+ EXPECT_STREQ((*conf).Get("custom-prop", "Set this value").c_str(), "custom-value");
+ EXPECT_STREQ((*conf).Get("default-prop", "Set this value").c_str(), "default-value");
}
/*
@@ -170,8 +165,7 @@ TEST(Configuration, LoadConfFromCustomLocation) {
HBaseConfigurationLoader loader;
std::vector<std::string> resources{kHBaseSiteXml};
- hbase::optional<Configuration> conf =
- loader.LoadResources(kHBaseConfPath, resources);
+ hbase::optional<Configuration> conf = loader.LoadResources(kHBaseConfPath, resources);
ASSERT_TRUE(conf) << "No configuration object present.";
EXPECT_STREQ((*conf).Get("custom-prop", "").c_str(), "custom-value");
EXPECT_STRNE((*conf).Get("custom-prop", "").c_str(), "some-value");
@@ -192,11 +186,9 @@ TEST(Configuration, LoadConfFromMultipleLocatons) {
HBaseConfigurationLoader loader;
std::string conf_paths = kDefHBaseConfPath + ":" + kHBaseConfPath;
std::vector<std::string> resources{kHBaseDefaultXml, kHBaseSiteXml};
- hbase::optional<Configuration> conf =
- loader.LoadResources(conf_paths, resources);
+ hbase::optional<Configuration> conf = loader.LoadResources(conf_paths, resources);
ASSERT_TRUE(conf) << "No configuration object present.";
- EXPECT_STREQ((*conf).Get("default-prop", "From hbase-default.xml").c_str(),
- "default-value");
+ EXPECT_STREQ((*conf).Get("default-prop", "From hbase-default.xml").c_str(), "default-value");
EXPECT_STREQ((*conf).Get("custom-prop", "").c_str(), "custom-value");
EXPECT_STRNE((*conf).Get("custom-prop", "").c_str(), "some-value");
}
@@ -215,10 +207,8 @@ TEST(Configuration, DefaultValues) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
ASSERT_TRUE(conf) << "No configuration object present.";
- EXPECT_STREQ((*conf).Get("default-prop", "Set this value.").c_str(),
- "default-value");
- EXPECT_STREQ((*conf).Get("custom-prop", "Set this value.").c_str(),
- "custom-value");
+ EXPECT_STREQ((*conf).Get("default-prop", "Set this value.").c_str(), "default-value");
+ EXPECT_STREQ((*conf).Get("custom-prop", "Set this value.").c_str(), "custom-value");
}
TEST(Configuration, FinalValues) {
@@ -229,12 +219,10 @@ TEST(Configuration, FinalValues) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
ASSERT_TRUE(conf) << "No configuration object present.";
- EXPECT_STREQ((*conf).Get("hbase.rootdir", "").c_str(),
- "/root/hbase-docker/apps/hbase/data");
+ EXPECT_STREQ((*conf).Get("hbase.rootdir", "").c_str(), "/root/hbase-docker/apps/hbase/data");
EXPECT_STREQ((*conf).Get("hbase.zookeeper.property.datadir", "").c_str(),
"/root/hbase-docker/zookeeper");
- EXPECT_STRNE((*conf).Get("hbase.rootdir", "").c_str(),
- "This value will not be be overwritten");
+ EXPECT_STRNE((*conf).Get("hbase.rootdir", "").c_str(), "This value will not be be overwritten");
EXPECT_STRNE((*conf).Get("hbase.zookeeper.property.datadir", "").c_str(),
"This value will be overwritten");
}
@@ -253,10 +241,8 @@ TEST(Configuration, EnvVars) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
ASSERT_TRUE(conf) << "No configuration object present.";
- EXPECT_STREQ((*conf).Get("hbase-client.user.name", "").c_str(),
- "${user.name}");
- EXPECT_STRNE((*conf).Get("hbase-client.user.name", "root").c_str(),
- "test-user");
+ EXPECT_STREQ((*conf).Get("hbase-client.user.name", "").c_str(), "${user.name}");
+ EXPECT_STRNE((*conf).Get("hbase-client.user.name", "root").c_str(), "test-user");
}
TEST(Configuration, SelfRef) {
@@ -292,8 +278,7 @@ TEST(Configuration, VarExpansionException) {
HBaseConfigurationLoader loader;
hbase::optional<Configuration> conf = loader.LoadDefaultResources();
ASSERT_TRUE(conf) << "No configuration object present.";
- ASSERT_THROW((*conf).Get("foo.substs.exception", "foo-value").c_str(),
- std::runtime_error);
+ ASSERT_THROW((*conf).Get("foo.substs.exception", "foo-value").c_str(), std::runtime_error);
}
TEST(Configuration, GetInt) {
diff --git a/core/hbase_configuration_loader.cc b/core/hbase_configuration_loader.cc
index 20d0449..38a617b 100644
--- a/core/hbase_configuration_loader.cc
+++ b/core/hbase_configuration_loader.cc
@@ -94,8 +94,7 @@ void HBaseConfigurationLoader::AddToSearchPath(const std::string &search_path) {
if (search_path.back() != kFileSeparator) {
path_to_add += kFileSeparator;
}
- if (std::find(search_paths_.begin(), search_paths_.end(), path_to_add) ==
- search_paths_.end())
+ if (std::find(search_paths_.begin(), search_paths_.end(), path_to_add) == search_paths_.end())
search_paths_.push_back(path_to_add);
}
@@ -105,8 +104,7 @@ void HBaseConfigurationLoader::AddDefaultResources() {
}
void HBaseConfigurationLoader::AddResources(const std::string &filename) {
- if (std::find(resources_.begin(), resources_.end(), filename) ==
- resources_.end())
+ if (std::find(resources_.begin(), resources_.end(), filename) == resources_.end())
resources_.push_back(filename);
}
@@ -127,8 +125,7 @@ optional<Configuration> HBaseConfigurationLoader::LoadDefaultResources() {
}
}
if (success) {
- return std::experimental::make_optional<Configuration>(
- Configuration(conf_property));
+ return std::experimental::make_optional<Configuration>(Configuration(conf_property));
} else {
return optional<Configuration>();
}
@@ -152,15 +149,13 @@ optional<Configuration> HBaseConfigurationLoader::LoadResources(
}
}
if (success) {
- return std::experimental::make_optional<Configuration>(
- Configuration(conf_property));
+ return std::experimental::make_optional<Configuration>(Configuration(conf_property));
} else {
return optional<Configuration>();
}
}
-bool HBaseConfigurationLoader::LoadProperties(const std::string &file,
- ConfigMap &property_map) {
+bool HBaseConfigurationLoader::LoadProperties(const std::string &file, ConfigMap &property_map) {
// Create empty property tree object
using boost::property_tree::ptree;
ptree pt;
@@ -179,23 +174,21 @@ bool HBaseConfigurationLoader::LoadProperties(const std::string &file,
std::string name_node = v.second.get<std::string>("name");
std::string value_node = v.second.get<std::string>("value");
if ((name_node.size() > 0) && (value_node.size() > 0)) {
- boost::optional<std::string> final_node =
- v.second.get_optional<std::string>("final");
+ boost::optional<std::string> final_node = v.second.get_optional<std::string>("final");
UpdateMapWithValue(property_map, name_node, value_node, final_node);
}
}
}
} catch (std::exception &ex) {
- DLOG(WARNING) << "Exception in parsing file [" << file << "]:[" << ex.what()
- << "]";
+ DLOG(WARNING) << "Exception in parsing file [" << file << "]:[" << ex.what() << "]";
return false;
}
return true;
}
-bool HBaseConfigurationLoader::UpdateMapWithValue(
- ConfigMap &map, const std::string &key, const std::string &value,
- boost::optional<std::string> final_text) {
+bool HBaseConfigurationLoader::UpdateMapWithValue(ConfigMap &map, const std::string &key,
+ const std::string &value,
+ boost::optional<std::string> final_text) {
auto map_value = map.find(key);
if (map_value != map.end() && map_value->second.final) {
return false;
diff --git a/core/hbase_configuration_loader.h b/core/hbase_configuration_loader.h
index 49dc2b9..a1c1d3f 100644
--- a/core/hbase_configuration_loader.h
+++ b/core/hbase_configuration_loader.h
@@ -58,9 +58,8 @@ class HBaseConfigurationLoader {
* @param search_path - ':' search paths to load resources.
* @param resources - list of resources used to load configuration properties.
*/
- optional<Configuration> LoadResources(
- const std::string &search_path,
- const std::vector<std::string> &resources);
+ optional<Configuration> LoadResources(const std::string &search_path,
+ const std::vector<std::string> &resources);
private:
using ConfigMap = Configuration::ConfigMap;
@@ -137,8 +136,7 @@ class HBaseConfigurationLoader {
* @param value value of value node.
* @param final_text value of final node true or false if present
*/
- bool UpdateMapWithValue(ConfigMap &map, const std::string &key,
- const std::string &value,
+ bool UpdateMapWithValue(ConfigMap &map, const std::string &key, const std::string &value,
boost::optional<std::string> final_text);
};
diff --git a/core/location-cache.cc b/core/location-cache.cc
index 4c29a61..6c2a790 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -53,10 +53,9 @@ using hbase::pb::RegionInfo;
// TODO(eclark): make this configurable on client creation
static const char META_ZNODE_NAME[] = "/hbase/meta-region-server";
-LocationCache::LocationCache(
- std::string quorum_spec,
- std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
- std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor)
+LocationCache::LocationCache(std::string quorum_spec,
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor)
: quorum_spec_(quorum_spec),
cpu_executor_(cpu_executor),
meta_promise_(nullptr),
@@ -91,9 +90,7 @@ void LocationCache::InvalidateMeta() {
/// MUST hold the meta_lock_
void LocationCache::RefreshMetaLocation() {
meta_promise_ = make_unique<SharedPromise<ServerName>>();
- cpu_executor_->add([&] {
- meta_promise_->setWith([&] { return this->ReadMetaLocation(); });
- });
+ cpu_executor_->add([&] { meta_promise_->setWith([&] { return this->ReadMetaLocation(); }); });
}
ServerName LocationCache::ReadMetaLocation() {
@@ -103,9 +100,8 @@ ServerName LocationCache::ReadMetaLocation() {
// This needs to be int rather than size_t as that's what ZK expects.
int len = buf->capacity();
// TODO(elliott): handle disconnects/reconntion as needed.
- int zk_result =
- zoo_get(this->zk_, META_ZNODE_NAME, 0,
- reinterpret_cast<char *>(buf->writableData()), &len, nullptr);
+ int zk_result = zoo_get(this->zk_, META_ZNODE_NAME, 0,
+ reinterpret_cast<char *>(buf->writableData()), &len, nullptr);
if (zk_result != ZOK || len < 9) {
LOG(ERROR) << "Error getting meta location.";
throw runtime_error("Error getting meta location");
@@ -119,18 +115,16 @@ ServerName LocationCache::ReadMetaLocation() {
return mrs.server();
}
-Future<std::shared_ptr<RegionLocation>> LocationCache::LocateFromMeta(
- const TableName &tn, const string &row) {
+Future<std::shared_ptr<RegionLocation>> LocationCache::LocateFromMeta(const TableName &tn,
+ const string &row) {
return this->LocateMeta()
.via(cpu_executor_.get())
.then([this](ServerName sn) {
- auto remote_id =
- std::make_shared<ConnectionId>(sn.host_name(), sn.port());
+ auto remote_id = std::make_shared<ConnectionId>(sn.host_name(), sn.port());
return this->cp_.GetConnection(remote_id);
})
.then([tn, row, this](std::shared_ptr<RpcConnection> rpc_connection) {
- return (*rpc_connection->get_service())(
- std::move(meta_util_.MetaRequest(tn, row)));
+ return (*rpc_connection->get_service())(std::move(meta_util_.MetaRequest(tn, row)));
})
.then([this](Response resp) {
// take the protobuf response and make it into
@@ -146,16 +140,15 @@ Future<std::shared_ptr<RegionLocation>> LocationCache::LocateFromMeta(
return rl;
})
.then([this](std::shared_ptr<RegionLocation> rl) {
- auto remote_id = std::make_shared<ConnectionId>(
- rl->server_name().host_name(), rl->server_name().port());
+ auto remote_id =
+ std::make_shared<ConnectionId>(rl->server_name().host_name(), rl->server_name().port());
// Now fill out the connection.
rl->set_service(cp_.GetConnection(remote_id)->get_service());
return rl;
});
}
-std::shared_ptr<RegionLocation> LocationCache::CreateLocation(
- const Response &resp) {
+std::shared_ptr<RegionLocation> LocationCache::CreateLocation(const Response &resp) {
auto resp_msg = static_pointer_cast<ScanResponse>(resp.resp_msg());
auto &results = resp_msg->results().Get(0);
auto &cells = results.cell();
@@ -168,6 +161,5 @@ std::shared_ptr<RegionLocation> LocationCache::CreateLocation(
auto region_info = folly::to<RegionInfo>(cell_zero);
auto server_name = folly::to<ServerName>(cell_one);
- return std::make_shared<RegionLocation>(row, std::move(region_info),
- server_name, nullptr);
+ return std::make_shared<RegionLocation>(row, std::move(region_info), server_name, nullptr);
}
diff --git a/core/location-cache.h b/core/location-cache.h
index 53e41ab..b290a1f 100644
--- a/core/location-cache.h
+++ b/core/location-cache.h
@@ -79,8 +79,8 @@ class LocationCache {
* @param row of the table to look up. This object must live until after the
* future is returned
*/
- folly::Future<std::shared_ptr<RegionLocation>> LocateFromMeta(
- const hbase::pb::TableName &tn, const std::string &row);
+ folly::Future<std::shared_ptr<RegionLocation>> LocateFromMeta(const hbase::pb::TableName &tn,
+ const std::string &row);
/**
* Remove the cached location of meta.
diff --git a/core/meta-utils.cc b/core/meta-utils.cc
index e1946d6..f92300c 100644
--- a/core/meta-utils.cc
+++ b/core/meta-utils.cc
@@ -37,13 +37,11 @@ using hbase::pb::RegionSpecifier_RegionSpecifierType;
static const std::string META_REGION = "1588230740";
-std::string MetaUtil::RegionLookupRowkey(const TableName &tn,
- const std::string &row) const {
+std::string MetaUtil::RegionLookupRowkey(const TableName &tn, const std::string &row) const {
return folly::to<std::string>(tn, ",", row, ",", "999999999999999999");
}
-std::unique_ptr<Request> MetaUtil::MetaRequest(const TableName tn,
- const std::string &row) const {
+std::unique_ptr<Request> MetaUtil::MetaRequest(const TableName tn, const std::string &row) const {
auto request = Request::scan();
auto msg = std::static_pointer_cast<ScanRequest>(request->req_msg());
@@ -53,8 +51,8 @@ std::unique_ptr<Request> MetaUtil::MetaRequest(const TableName tn,
// Set the region this scan goes to
auto region = msg->mutable_region();
region->set_value(META_REGION);
- region->set_type(RegionSpecifier_RegionSpecifierType::
- RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
+ region->set_type(
+ RegionSpecifier_RegionSpecifierType::RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
auto scan = msg->mutable_scan();
// We don't care about before, just now.
diff --git a/core/meta-utils.h b/core/meta-utils.h
index 33da1a8..075215e 100644
--- a/core/meta-utils.h
+++ b/core/meta-utils.h
@@ -36,14 +36,12 @@ class MetaUtil {
* Given a table and a row give the row key from which to start a scan to find
* region locations.
*/
- std::string RegionLookupRowkey(const hbase::pb::TableName &tn,
- const std::string &row) const;
+ std::string RegionLookupRowkey(const hbase::pb::TableName &tn, const std::string &row) const;
/**
* Given a row we're trying to access create a request to look up the
* location.
*/
- std::unique_ptr<Request> MetaRequest(const hbase::pb::TableName tn,
- const std::string &row) const;
+ std::unique_ptr<Request> MetaRequest(const hbase::pb::TableName tn, const std::string &row) const;
};
} // namespace hbase
diff --git a/core/region-location.h b/core/region-location.h
index f0295b1..e7b76d3 100644
--- a/core/region-location.h
+++ b/core/region-location.h
@@ -42,8 +42,7 @@ class RegionLocation {
* this region.
* @param service the connected service to the regionserver.
*/
- RegionLocation(std::string region_name, hbase::pb::RegionInfo ri,
- hbase::pb::ServerName sn,
+ RegionLocation(std::string region_name, hbase::pb::RegionInfo ri, hbase::pb::ServerName sn,
std::shared_ptr<HBaseService> service)
: region_name_(region_name), ri_(ri), sn_(sn), service_(service) {}
diff --git a/core/response_converter.h b/core/response_converter.h
index 7cbde81..86fb632 100644
--- a/core/response_converter.h
+++ b/core/response_converter.h
@@ -20,8 +20,8 @@
#pragma once
#include <memory>
-#include "core/result.h"
#include "connection/response.h"
+#include "core/result.h"
namespace hbase {
diff --git a/core/result-test.cc b/core/result-test.cc
index 134e80f..09d4e15 100644
--- a/core/result-test.cc
+++ b/core/result-test.cc
@@ -19,11 +19,11 @@
#include "core/result.h"
+#include <gtest/gtest.h>
#include <limits>
#include <memory>
-#include <vector>
#include <string>
-#include <gtest/gtest.h>
+#include <vector>
#include "core/cell.h"
using namespace hbase;
@@ -41,36 +41,35 @@ void PopulateCells(std::vector<std::shared_ptr<Cell> > &cells) {
switch (i) {
case 5: {
- cells.push_back(std::make_shared<Cell>(
- row, family, column, 1482113040506, "value-5", CellType::PUT));
- cells.push_back(std::make_shared<Cell>(
- row, family, column, 1482111803856, "value-X", CellType::PUT));
+ cells.push_back(
+ std::make_shared<Cell>(row, family, column, 1482113040506, "value-5", CellType::PUT));
+ cells.push_back(
+ std::make_shared<Cell>(row, family, column, 1482111803856, "value-X", CellType::PUT));
break;
}
case 8: {
- cells.push_back(std::make_shared<Cell>(
- row, family, column, 1482113040506, "value-8", CellType::PUT));
- cells.push_back(std::make_shared<Cell>(
- row, family, column, 1482111803856, "value-X", CellType::PUT));
- cells.push_back(std::make_shared<Cell>(
- row, family, column, 1482110969958, "value-Y", CellType::PUT));
+ cells.push_back(
+ std::make_shared<Cell>(row, family, column, 1482113040506, "value-8", CellType::PUT));
+ cells.push_back(
+ std::make_shared<Cell>(row, family, column, 1482111803856, "value-X", CellType::PUT));
+ cells.push_back(
+ std::make_shared<Cell>(row, family, column, 1482110969958, "value-Y", CellType::PUT));
break;
}
case 9: {
- cells.push_back(std::make_shared<Cell>(
- row, family, column, 1482113040506, "value-9", CellType::PUT));
- cells.push_back(std::make_shared<Cell>(
- row, family, column, 1482111803856, "value-X", CellType::PUT));
- cells.push_back(std::make_shared<Cell>(
- row, family, column, 1482110969958, "value-Y", CellType::PUT));
- cells.push_back(std::make_shared<Cell>(
- row, family, column, 1482110876075, "value-Z", CellType::PUT));
+ cells.push_back(
+ std::make_shared<Cell>(row, family, column, 1482113040506, "value-9", CellType::PUT));
+ cells.push_back(
+ std::make_shared<Cell>(row, family, column, 1482111803856, "value-X", CellType::PUT));
+ cells.push_back(
+ std::make_shared<Cell>(row, family, column, 1482110969958, "value-Y", CellType::PUT));
+ cells.push_back(
+ std::make_shared<Cell>(row, family, column, 1482110876075, "value-Z", CellType::PUT));
break;
}
default: {
- cells.push_back(std::make_shared<Cell>(row, family, column,
- std::numeric_limits<long>::max(),
- value, CellType::PUT));
+ cells.push_back(std::make_shared<Cell>(
+ row, family, column, std::numeric_limits<long>::max(), value, CellType::PUT));
}
}
}
@@ -111,12 +110,9 @@ TEST(Result, FilledResult) {
// Value will be present as family and qualifier is present
ASSERT_TRUE(result.Value("family-4", "column-4"));
// Value should be present and match.
- EXPECT_EQ(latest_cell->Value(),
- (*result.ColumnLatestCell("family-4", "column-4")).Value());
- EXPECT_EQ("value-5",
- (*result.ColumnLatestCell("family-5", "column-5")).Value());
- EXPECT_EQ("value-8",
- (*result.ColumnLatestCell("family-8", "column-8")).Value());
+ EXPECT_EQ(latest_cell->Value(), (*result.ColumnLatestCell("family-4", "column-4")).Value());
+ EXPECT_EQ("value-5", (*result.ColumnLatestCell("family-5", "column-5")).Value());
+ EXPECT_EQ("value-8", (*result.ColumnLatestCell("family-8", "column-8")).Value());
EXPECT_EQ("value-7", *result.Value("family-7", "column-7"));
// Get cells for the given family and qualifier
diff --git a/core/result.cc b/core/result.cc
index 9842915..d73a5b2 100644
--- a/core/result.cc
+++ b/core/result.cc
@@ -23,15 +23,14 @@ namespace hbase {
Result::~Result() {}
-Result::Result(const std::vector<std::shared_ptr<Cell> > &cells, bool exists,
- bool stale, bool partial)
+Result::Result(const std::vector<std::shared_ptr<Cell> > &cells, bool exists, bool stale,
+ bool partial)
: exists_(exists), stale_(stale), partial_(partial) {
for (const auto &cell : cells) {
cells_.push_back(cell);
// We create the map when cells are added. unlike java where map is created
// when result.getMap() is called
- result_map_[cell->Family()][cell->Qualifier()][cell->Timestamp()] =
- cell->Value();
+ result_map_[cell->Family()][cell->Qualifier()][cell->Timestamp()] = cell->Value();
}
row_ = (cells_.size() == 0 ? "" : cells_[0]->Row());
}
@@ -44,17 +43,14 @@ Result::Result(const Result &result) {
if (!result.cells_.empty()) {
for (const auto &cell : result.cells_) {
cells_.push_back(cell);
- result_map_[cell->Family()][cell->Qualifier()][cell->Timestamp()] =
- cell->Value();
+ result_map_[cell->Family()][cell->Qualifier()][cell->Timestamp()] = cell->Value();
}
}
}
-const std::vector<std::shared_ptr<Cell> > &Result::Cells() const {
- return cells_;
-}
+const std::vector<std::shared_ptr<Cell> > &Result::Cells() const { return cells_; }
-std::vector<std::shared_ptr<Cell> > Result::ColumnCells(
- const std::string &family, const std::string &qualifier) const {
+std::vector<std::shared_ptr<Cell> > Result::ColumnCells(const std::string &family,
+ const std::string &qualifier) const {
std::vector<std::shared_ptr<Cell> > column_cells;
// TODO implement a BinarySearch here ?
for (const auto &cell : cells_) {
@@ -65,8 +61,8 @@ std::vector<std::shared_ptr<Cell> > Result::ColumnCells(
return column_cells;
}
-const std::shared_ptr<Cell> Result::ColumnLatestCell(
- const std::string &family, const std::string &qualifier) const {
+const std::shared_ptr<Cell> Result::ColumnLatestCell(const std::string &family,
+ const std::string &qualifier) const {
// TODO implement a BinarySearch here ?
for (const auto &cell : cells_) {
// We find the latest(first) occurrence of the Cell for a given column and
@@ -100,10 +96,8 @@ const ResultFamilyMap Result::FamilyMap(const std::string &family) const {
if (!IsEmpty()) {
for (auto itr = result_map_.begin(); itr != result_map_.end(); ++itr) {
if (family == itr->first) {
- for (auto qitr = itr->second.begin(); qitr != itr->second.end();
- ++qitr) {
- for (auto vitr = qitr->second.begin(); vitr != qitr->second.end();
- ++vitr) {
+ for (auto qitr = itr->second.begin(); qitr != itr->second.end(); ++qitr) {
+ for (auto vitr = qitr->second.begin(); vitr != qitr->second.end(); ++vitr) {
// We break after inserting the first value. Result.java takes only
// the first value
family_map[qitr->first] = vitr->second;
diff --git a/core/result.h b/core/result.h
index ecfdd55..cd41cf0 100644
--- a/core/result.h
+++ b/core/result.h
@@ -34,9 +34,9 @@ namespace hbase {
* We need to have a reverse ordered map, when storing TS -> value, so that the
* most recent value is stored first
*/
-using ResultMap = std::map<
- std::string, std::map<std::string, std::map<int64_t, std::string,
- std::greater<int64_t> > > >;
+using ResultMap =
+ std::map<std::string,
+ std::map<std::string, std::map<int64_t, std::string, std::greater<int64_t> > > >;
/**
* @brief Map of qualifiers to values.
@@ -48,8 +48,7 @@ class Result {
/**
* Constructors
*/
- Result(const std::vector<std::shared_ptr<Cell> > &cells, bool exists,
- bool stale, bool partial);
+ Result(const std::vector<std::shared_ptr<Cell> > &cells, bool exists, bool stale, bool partial);
Result(const Result &result);
~Result();
@@ -67,8 +66,8 @@ class Result {
* @param family - column family
* @param qualifier - column qualifier
*/
- std::vector<std::shared_ptr<Cell> > ColumnCells(
- const std::string &family, const std::string &qualifier) const;
+ std::vector<std::shared_ptr<Cell> > ColumnCells(const std::string &family,
+ const std::string &qualifier) const;
/**
* @brief Returns the Cell for the most recent timestamp for a given family
@@ -77,16 +76,15 @@ class Result {
* @param family - column family.
* @param qualifier - column qualifier
*/
- const std::shared_ptr<Cell> ColumnLatestCell(
- const std::string &family, const std::string &qualifier) const;
+ const std::shared_ptr<Cell> ColumnLatestCell(const std::string &family,
+ const std::string &qualifier) const;
/**
* @brief Get the latest version of the specified family and qualifier.
* @param family - column family
* @param qualifier - column qualifier
*/
- std::shared_ptr<std::string> Value(const std::string &family,
- const std::string &qualifier) const;
+ std::shared_ptr<std::string> Value(const std::string &family, const std::string &qualifier) const;
/**
* @brief Returns if the underlying Cell vector is empty or not
diff --git a/core/scan-test.cc b/core/scan-test.cc
index 0a4001a..9a26407 100644
--- a/core/scan-test.cc
+++ b/core/scan-test.cc
@@ -154,8 +154,7 @@ void ScanMethods(Scan &scan) {
// Test initial values
EXPECT_EQ(0, scan.Timerange().MinTimeStamp());
- EXPECT_EQ(std::numeric_limits<int64_t>::max(),
- scan.Timerange().MaxTimeStamp());
+ EXPECT_EQ(std::numeric_limits<int64_t>::max(), scan.Timerange().MaxTimeStamp());
// Set & Test new values using TimeRange and TimeStamp
scan.SetTimeRange(1000, 2000);
@@ -169,8 +168,7 @@ void ScanMethods(Scan &scan) {
ASSERT_THROW(scan.SetTimeRange(-1000, 2000), std::runtime_error);
ASSERT_THROW(scan.SetTimeRange(1000, -2000), std::runtime_error);
ASSERT_THROW(scan.SetTimeRange(1000, 200), std::runtime_error);
- ASSERT_THROW(scan.SetTimeStamp(std::numeric_limits<int64_t>::max()),
- std::runtime_error);
+ ASSERT_THROW(scan.SetTimeStamp(std::numeric_limits<int64_t>::max()), std::runtime_error);
}
TEST(Scan, Object) {
diff --git a/core/scan.cc b/core/scan.cc
index 036acfb..5335abd 100644
--- a/core/scan.cc
+++ b/core/scan.cc
@@ -30,9 +30,7 @@ Scan::Scan() {}
Scan::~Scan() {}
-Scan::Scan(const std::string &start_row) : start_row_(start_row) {
- CheckRow(start_row_);
-}
+Scan::Scan(const std::string &start_row) : start_row_(start_row) { CheckRow(start_row_); }
Scan::Scan(const std::string &start_row, const std::string &stop_row)
: start_row_(start_row), stop_row_(stop_row) {
@@ -76,8 +74,7 @@ Scan &Scan::operator=(const Scan &scan) {
Scan::Scan(const Get &get) {
cache_blocks_ = get.CacheBlocks();
max_versions_ = get.MaxVersions();
- tr_.reset(new TimeRange(get.Timerange().MinTimeStamp(),
- get.Timerange().MaxTimeStamp()));
+ tr_.reset(new TimeRange(get.Timerange().MinTimeStamp(), get.Timerange().MaxTimeStamp()));
family_map_.insert(get.Family().begin(), get.Family().end());
}
@@ -97,8 +94,7 @@ Scan &Scan::AddFamily(const std::string &family) {
}
Scan &Scan::AddColumn(const std::string &family, const std::string &qualifier) {
- const auto &it = std::find(family_map_[family].begin(),
- family_map_[family].end(), qualifier);
+ const auto &it = std::find(family_map_[family].begin(), family_map_[family].end(), qualifier);
/**
* Check if any qualifiers are already present or not.
* Add only if qualifiers for a given family are not present
@@ -156,9 +152,7 @@ void Scan::SetLoadColumnFamiliesOnDemand(bool load_column_families_on_demand) {
load_column_families_on_demand_ = load_column_families_on_demand;
}
-bool Scan::LoadColumnFamiliesOnDemand() const {
- return load_column_families_on_demand_;
-}
+bool Scan::LoadColumnFamiliesOnDemand() const { return load_column_families_on_demand_; }
Scan &Scan::SetMaxVersions(uint32_t max_versions) {
max_versions_ = max_versions;
@@ -167,9 +161,7 @@ Scan &Scan::SetMaxVersions(uint32_t max_versions) {
int Scan::MaxVersions() const { return max_versions_; }
-void Scan::SetMaxResultSize(int64_t max_result_size) {
- max_result_size_ = max_result_size;
-}
+void Scan::SetMaxResultSize(int64_t max_result_size) { max_result_size_ = max_result_size; }
int64_t Scan::MaxResultSize() const { return max_result_size_; }
@@ -192,8 +184,7 @@ void Scan::CheckRow(const std::string &row) {
throw std::runtime_error("Row length can't be 0");
}
if (row_length > kMaxRowLength) {
- throw std::runtime_error("Length of " + row +
- " is greater than max row size: " +
+ throw std::runtime_error("Length of " + row + " is greater than max row size: " +
std::to_string(kMaxRowLength));
}
}
diff --git a/core/simple-client.cc b/core/simple-client.cc
index f3f6c42..90e7cd4 100644
--- a/core/simple-client.cc
+++ b/core/simple-client.cc
@@ -65,12 +65,11 @@ std::unique_ptr<Request> MakeRequest(uint64_t col, std::string region_name) {
auto suf = folly::to<std::string>(col);
region->set_value(region_name);
- region->set_type(RegionSpecifier_RegionSpecifierType::
- RegionSpecifier_RegionSpecifierType_REGION_NAME);
+ region->set_type(
+ RegionSpecifier_RegionSpecifierType::RegionSpecifier_RegionSpecifierType_REGION_NAME);
auto mutation = msg->mutable_mutation();
mutation->set_row(FLAGS_row + suf);
- mutation->set_mutate_type(
- MutationProto_MutationType::MutationProto_MutationType_PUT);
+ mutation->set_mutate_type(MutationProto_MutationType::MutationProto_MutationType_PUT);
auto column = mutation->add_column_value();
column->set_family("d");
auto qual = column->add_qualifier_value();
@@ -81,14 +80,12 @@ std::unique_ptr<Request> MakeRequest(uint64_t col, std::string region_name) {
}
int main(int argc, char *argv[]) {
- google::SetUsageMessage(
- "Simple client to get a single row from HBase on the comamnd line");
+ google::SetUsageMessage("Simple client to get a single row from HBase on the comamnd line");
google::ParseCommandLineFlags(&argc, &argv, true);
google::InitGoogleLogging(argv[0]);
// Set up thread pools.
- auto cpu_pool =
- std::make_shared<wangle::CPUThreadPoolExecutor>(FLAGS_threads);
+ auto cpu_pool = std::make_shared<wangle::CPUThreadPoolExecutor>(FLAGS_threads);
auto io_pool = std::make_shared<wangle::IOThreadPoolExecutor>(5);
// Create the cache.
@@ -105,14 +102,13 @@ int main(int argc, char *argv[]) {
auto results = std::vector<Future<Response>>{};
auto col = uint64_t{0};
for (; col < num_puts; col++) {
- results.push_back(folly::makeFuture(col)
- .via(cpu_pool.get())
- .then([loc](uint64_t col) {
- return MakeRequest(col, loc->region_name());
- })
- .then([connection](std::unique_ptr<Request> req) {
- return (*connection)(std::move(req));
- }));
+ results.push_back(
+ folly::makeFuture(col)
+ .via(cpu_pool.get())
+ .then([loc](uint64_t col) { return MakeRequest(col, loc->region_name()); })
+ .then([connection](std::unique_ptr<Request> req) {
+ return (*connection)(std::move(req));
+ }));
}
auto allf = folly::collect(results).get();
diff --git a/core/time_range.cc b/core/time_range.cc
index 8c37b29..04b70ec 100644
--- a/core/time_range.cc
+++ b/core/time_range.cc
@@ -25,9 +25,7 @@
namespace hbase {
TimeRange::TimeRange()
- : min_timestamp_(0L),
- max_timestamp_(std::numeric_limits<int64_t>::max()),
- all_time_(true) {}
+ : min_timestamp_(0L), max_timestamp_(std::numeric_limits<int64_t>::max()), all_time_(true) {}
TimeRange::TimeRange(const TimeRange &tr) {
this->all_time_ = tr.all_time_;
@@ -53,8 +51,7 @@ TimeRange::TimeRange(int64_t min_timestamp) {
TimeRange::TimeRange(int64_t min_timestamp, int64_t max_timestamp) {
if (min_timestamp < 0 || max_timestamp < 0) {
throw std::runtime_error("Timestamp cannot be negative. min_timestamp: " +
- std::to_string(min_timestamp) +
- ", max_timestamp:" +
+ std::to_string(min_timestamp) + ", max_timestamp:" +
std::to_string(max_timestamp));
}
if (max_timestamp < min_timestamp) {
diff --git a/security/BUCK b/security/BUCK
index 5b935d3..e176c90 100644
--- a/security/BUCK
+++ b/security/BUCK
@@ -19,12 +19,8 @@
# to a single server.
cxx_library(
name="security",
- exported_headers=[
- "user.h",
- ],
- srcs=[
- ],
- deps=[
- ],
+ exported_headers=["user.h",],
+ srcs=[],
+ deps=[],
compiler_flags=['-Weffc++'],
- visibility=['//core/...','//connection/...'],)
+ visibility=['//core/...', '//connection/...'],)
diff --git a/security/user.h b/security/user.h
index 795f5ac..8a15891 100644
--- a/security/user.h
+++ b/security/user.h
@@ -23,16 +23,15 @@
namespace hbase {
namespace security {
class User {
-public:
+ public:
explicit User(const std::string& user_name) : user_name_(user_name) {}
- virtual ~User() = default;
+ virtual ~User() = default;
- std::string user_name() {return user_name_;}
+ std::string user_name() { return user_name_; }
- static std::shared_ptr<User> defaultUser() {
- return std::make_shared<User>("__drwho");
- }
-private:
+ static std::shared_ptr<User> defaultUser() { return std::make_shared<User>("__drwho"); }
+
+ private:
std::string user_name_;
};
}
diff --git a/serde/client-deserializer-test.cc b/serde/client-deserializer-test.cc
index 8c571b1..a30f904 100644
--- a/serde/client-deserializer-test.cc
+++ b/serde/client-deserializer-test.cc
@@ -50,8 +50,7 @@ TEST(TestRpcSerde, TestGoodGetRequestFullRoundTrip) {
// fill up the GetRequest.
in.mutable_region()->set_value("test_region_id");
in.mutable_region()->set_type(
- RegionSpecifier_RegionSpecifierType::
- RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
+ RegionSpecifier_RegionSpecifierType::RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
in.mutable_get()->set_row("test_row");
// Create the buffer
diff --git a/serde/rpc.cc b/serde/rpc.cc
index e4ade22..3bdb489 100644
--- a/serde/rpc.cc
+++ b/serde/rpc.cc
@@ -70,15 +70,13 @@ int RpcSerde::ParseDelimited(const IOBuf *buf, Message *msg) {
coded_stream.PushLimit(msg_size);
// Parse the message.
if (msg->MergeFromCodedStream(&coded_stream) == false) {
- FB_LOG_EVERY_MS(ERROR, 1000)
- << "Unable to read a protobuf message from data.";
+ FB_LOG_EVERY_MS(ERROR, 1000) << "Unable to read a protobuf message from data.";
return -4;
}
// Make sure all the data was consumed.
if (coded_stream.ConsumedEntireMessage() == false) {
- FB_LOG_EVERY_MS(ERROR, 1000)
- << "Orphaned data left after reading protobuf message";
+ FB_LOG_EVERY_MS(ERROR, 1000) << "Orphaned data left after reading protobuf message";
return -5;
}
@@ -115,8 +113,8 @@ unique_ptr<IOBuf> RpcSerde::Header(const string &user) {
return PrependLength(SerializeMessage(h));
}
-unique_ptr<IOBuf> RpcSerde::Request(const uint32_t call_id,
- const string &method, const Message *msg) {
+unique_ptr<IOBuf> RpcSerde::Request(const uint32_t call_id, const string &method,
+ const Message *msg) {
pb::RequestHeader rq;
rq.set_method_name(method);
rq.set_call_id(call_id);
@@ -169,8 +167,7 @@ unique_ptr<IOBuf> RpcSerde::SerializeDelimited(const Message &msg) {
// Now write the rest out.
// We're using the protobuf output streams here to keep track
// of where in the output array we are rather than IOBuf.
- msg.SerializeWithCachedSizesToArray(
- cos.GetDirectBufferForNBytesAndAdvance(msg_size));
+ msg.SerializeWithCachedSizesToArray(cos.GetDirectBufferForNBytesAndAdvance(msg_size));
// Return the buffer.
return buf;
diff --git a/serde/rpc.h b/serde/rpc.h
index 05ceee9..7d060c7 100644
--- a/serde/rpc.h
+++ b/serde/rpc.h
@@ -83,8 +83,7 @@ class RpcSerde {
* - RequestHeader object
* - The passed in Message object
*/
- std::unique_ptr<folly::IOBuf> Request(const uint32_t call_id,
- const std::string &method,
+ std::unique_ptr<folly::IOBuf> Request(const uint32_t call_id, const std::string &method,
const google::protobuf::Message *msg);
/**
@@ -94,21 +93,18 @@ class RpcSerde {
* - A protobuf var int32
* - The message object seriailized after that.
*/
- std::unique_ptr<folly::IOBuf> SerializeDelimited(
- const google::protobuf::Message &msg);
+ std::unique_ptr<folly::IOBuf> SerializeDelimited(const google::protobuf::Message &msg);
/**
* Serilalize a message. This does not add any length prepend.
*/
- std::unique_ptr<folly::IOBuf> SerializeMessage(
- const google::protobuf::Message &msg);
+ std::unique_ptr<folly::IOBuf> SerializeMessage(const google::protobuf::Message &msg);
/**
* Prepend a length IOBuf to the given IOBuf chain.
* This involves no copies or moves of the passed in data.
*/
- std::unique_ptr<folly::IOBuf> PrependLength(
- std::unique_ptr<folly::IOBuf> msg);
+ std::unique_ptr<folly::IOBuf> PrependLength(std::unique_ptr<folly::IOBuf> msg);
private:
/* data */
diff --git a/serde/server-name-test.cc b/serde/server-name-test.cc
index 73a68d6..87c493a 100644
--- a/serde/server-name-test.cc
+++ b/serde/server-name-test.cc
@@ -37,9 +37,7 @@ TEST(TestServerName, TestIps) {
ASSERT_EQ(999, sn.port());
}
-TEST(TestServerName, TestThrow) {
- ASSERT_ANY_THROW(folly::to<ServerName>("Ther's no colon here"));
-}
+TEST(TestServerName, TestThrow) { ASSERT_ANY_THROW(folly::to<ServerName>("Ther's no colon here")); }
TEST(TestServerName, TestIPV6) {
auto sn = folly::to<ServerName>("[::::1]:123");
diff --git a/serde/zk.cc b/serde/zk.cc
index 69b55fa..bf68400 100644
--- a/serde/zk.cc
+++ b/serde/zk.cc
@@ -46,8 +46,7 @@ bool ZkDeserializer::Parse(IOBuf *buf, Message *out) {
// There should be a magic number for recoverable zk
uint8_t magic_num = c.read<uint8_t>();
if (magic_num != 255) {
- LOG(ERROR) << "Magic number not in ZK znode data expected 255 got ="
- << unsigned(magic_num);
+ LOG(ERROR) << "Magic number not in ZK znode data expected 255 got =" << unsigned(magic_num);
throw runtime_error("Magic number not in znode data");
}
// How long is the id?
[hbase] 134/137: HBASE-18901 [C++] Provide CMAKE infrastructure
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 69841292024ca888d3b33ea36cae7910a0cbf471
Author: Marc Parisi <ph...@apache.org>
AuthorDate: Thu Sep 21 14:40:40 2017 -0400
HBASE-18901 [C++] Provide CMAKE infrastructure
* Provided cmake files for packages in which a default module
did not exist.
* Moved tests to a location where we could automatically build
the test suite.
* Resolved minor issues with tests
* Tested across OSX, RHEL7, and Ubuntu16
* Fix PROTOBUF_LIBRARY by removing static override in FindZookeeper
* Get the Dockerfile working again
Signed-off-by: Josh Elser <el...@apache.org>
---
.gitignore | 16 ++
BUILDING.md | 28 +-
CMakeLists.txt | 194 +++++++++++++
Makefile | 176 ------------
Makefile.protos | 80 ------
bin/cpplint.sh | 9 +-
cmake/BuildTests.cmake | 80 ++++++
cmake/FindFolly.cmake | 48 ++++
cmake/FindGMock.cmake | 23 ++
cmake/FindGflags.cmake | 23 ++
cmake/FindGlog.cmake | 23 ++
cmake/FindKrb5.cmake | 43 +++
cmake/FindSasl2.cmake | 29 ++
cmake/FindWangle.cmake | 43 +++
cmake/FindZookeeper.cmake | 51 ++++
cmake/ProtobufGen.cmake | 62 +++++
docker-files/Dockerfile | 49 ++--
include/hbase/client/BUCK | 93 -------
include/hbase/connection/BUCK | 57 ----
include/hbase/exceptions/BUCK | 28 --
include/hbase/security/BUCK | 30 --
include/hbase/serde/BUCK | 37 ---
{src => include}/hbase/test-util/mini-cluster.h | 3 +-
{src => include}/hbase/test-util/test-util.h | 0
include/hbase/utils/BUCK | 40 ---
src/hbase/client/BUCK | 301 ---------------------
src/hbase/connection/BUCK | 68 -----
src/hbase/{client => examples}/load-client.cc | 0
src/hbase/{client => examples}/simple-client.cc | 0
src/hbase/exceptions/BUCK | 37 ---
src/hbase/if/BUCK | 49 ----
src/hbase/security/BUCK | 27 --
src/hbase/serde/BUCK | 86 ------
src/hbase/test-util/BUCK | 53 ----
src/hbase/test-util/mini-cluster.cc | 3 +-
src/hbase/utils/BUCK | 57 ----
src/hbase/utils/bytes-util.cc | 1 -
src/{hbase/client => test}/append-test.cc | 0
.../async-batch-rpc-retrying-test.cc | 0
.../client => test}/async-rpc-retrying-test.cc | 1 -
src/{hbase/utils => test}/bytes-util-test.cc | 0
src/{hbase/client => test}/cell-test.cc | 0
.../serde => test}/client-deserializer-test.cc | 0
.../serde => test}/client-serializer-test.cc | 0
src/{hbase/client => test}/client-test.cc | 1 +
src/{hbase/utils => test}/concurrent-map-test.cc | 0
src/{hbase/client => test}/configuration-test.cc | 0
.../connection => test}/connection-pool-test.cc | 0
src/{hbase/client => test}/delete-test.cc | 0
src/{hbase/exceptions => test}/exception-test.cc | 0
src/{hbase/client => test}/filter-test.cc | 0
src/{hbase/client => test}/get-test.cc | 0
.../client => test}/hbase-configuration-test.cc | 0
src/{hbase/client => test}/increment-test.cc | 0
.../client => test}/location-cache-retry-test.cc | 0
src/{hbase/client => test}/location-cache-test.cc | 0
src/{hbase/client => test}/put-test.cc | 0
.../region-info-deserializer-test.cc | 0
.../client => test}/request-converter-test.cc | 0
src/{hbase/client => test}/result-test.cc | 0
src/{hbase/connection => test}/rpc-test.cc | 0
.../client => test}/scan-result-cache-test.cc | 0
src/{hbase/client => test}/scan-test.cc | 0
src/{hbase/client => test}/scanner-test.cc | 0
src/{hbase/serde => test}/server-name-test.cc | 0
src/{hbase/serde => test}/table-name-test.cc | 0
src/{hbase/client => test}/time-range-test.cc | 0
src/{hbase/utils => test}/user-util-test.cc | 0
src/{hbase/serde => test}/zk-deserializer-test.cc | 0
src/{hbase/client => test}/zk-util-test.cc | 0
third-party/BUCK | 118 --------
71 files changed, 676 insertions(+), 1391 deletions(-)
diff --git a/.gitignore b/.gitignore
index e8dd42d..5f3e387 100644
--- a/.gitignore
+++ b/.gitignore
@@ -27,3 +27,19 @@ third-party/*
# Generated files
src/hbase/utils/version.h
+
+# Tests
+*-test
+
+# Executables
+load-client
+simple-client
+
+# CMake temporary files
+CMakeCache.txt
+CMakeFiles
+*.cmake
+Makefile
+
+# Copied from hbase-common at build-time
+include/hbase/utils/version.h
diff --git a/BUILDING.md b/BUILDING.md
index 4c06776..53eb6d9 100644
--- a/BUILDING.md
+++ b/BUILDING.md
@@ -47,31 +47,11 @@ Then go into the hbase-native-client directory and run `./bin/start-docker.sh`
that will build the docker development environment and when complete will
drop you into a shell on a linux vm with all the tools needed installed.
-# Buck
-From then on we can use [buck](https://buckbuild.com/) to build everything.
-For example:
-```
-buck build //core:core
-buck test --all
-buck build //core:simple-client
-```
-
-That will build the library, then build and test everything, then build
-the simple-client binary. Buck will find all modules used, and compile
-them in parallel, caching the results. Output from buck is in the buck-out
-foulder. Generated binaries are in buck-out/gen logs are in buck-out/logs
-
-
-# Make
-
-If learning buck isn't your thing there is a Makefile wrapper for your
-convenience.
+# CMake
```
-make help
-make check
-make clean
-make all
-make build
+cmake .
+make
+make install
```
diff --git a/CMakeLists.txt b/CMakeLists.txt
new file mode 100644
index 0000000..3afe4d7
--- /dev/null
+++ b/CMakeLists.txt
@@ -0,0 +1,194 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+#
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+cmake_minimum_required(VERSION 3.0)
+PROJECT(hbaseclient CXX)
+set(PROJECT_NAME "hbaseclient")
+set(PROJECT_VERSION_MAJOR 0)
+set(PROJECT_VERSION_MINOR 1 )
+set(PROJECT_VERSION_PATCH 0)
+set(BUILD_SHARED_LIBS ON)
+## set our cmake module path
+list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+## include the Protobuf generation code
+include(ProtobufGen)
+if(COMPILER_SUPPORTS_CXX14)
+ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++14")
+else()
+ message(STATUS "The compiler ${CMAKE_CXX_COMPILER} has no C++14 support. Please use a different C++ compiler.")
+endif()
+set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(HBASE_SRC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src/hbase")
+set(HBASE_PROTO_GEN_SRC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src/hbase/if")
+set(HBASE_PROTO_GEN_INC_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include/hbase/if")
+# Set the right openssl root path
+if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+ set(OPENSSL_ROOT_DIR "/usr/local/opt/openssl/")
+else()
+ set(OPENSSL_ROOT_DIR "/usr/lib/x86_64-linux-gnu")
+endif()
+# Include OpenSSL
+find_package (OpenSSL REQUIRED)
+if (OPENSSL_FOUND)
+ include_directories(${OPENSSL_INCLUDE_DIR})
+else ()
+ message( FATAL_ERROR "OpenSSL was not found. Please install OpenSSL" )
+endif (OPENSSL_FOUND)
+# ensure we have required dependencies
+find_package(Threads)
+find_package(Boost REQUIRED COMPONENTS thread system filesystem)
+find_package(Gflags REQUIRED)
+find_package(Folly REQUIRED)
+find_package(Krb5 REQUIRED)
+find_package(Sasl2 REQUIRED)
+find_package(Wangle REQUIRED)
+find_package(GTest)
+find_package(Glog)
+find_package(Java REQUIRED)
+find_package(JNI REQUIRED)
+find_package(Zookeeper REQUIRED)
+find_package(Protobuf REQUIRED)
+if (NOT FOLLY_FOUND)
+ message(FATAL_ERROR "-- Folly not found")
+endif()
+if (NOT WANGLE_FOUND)
+ message(FATAL_ERROR "-- Wangle not found")
+endif()
+if (NOT PROTOBUF_FOUND)
+ message(FATAL_ERROR "-- Protocol buffer include directory not found ${PROTOBUF_INCLUDE_DIRS}")
+endif()
+if (NOT JNI_FOUND)
+ message(FATAL_ERROR "-- JAVA include directory not found")
+endif()
+### provide the include directories, starting at the base
+### and including those from our
+include_directories(include)
+include_directories(${PROTOBUF_INCLUDE_DIRS})
+include_directories(${Zookeeper_INCLUDE_DIRS})
+include_directories(${Boost_INCLUDE_DIR})
+include_directories(${KRB5_INCLUDE_DIRS})
+include_directories(${JAVA_INCLUDE_DIRS})
+include_directories(${FOLLY_INCLUDE_DIRS})
+### create a directory for the hbase protobuf headers.
+### this is helpful so that when we include it, later, we can generate
+### the protocol buffer source/headers without polluting our tree.
+set(CMAKE_BINARY_DIR_GEN "${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_FILES_DIRECTORY}/hbase/if/")
+file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR_GEN})
+## build a recursive list of sources
+file(GLOB_RECURSE CONNECTION_SRC "${HBASE_SRC_DIR}/connection/*.cc" )
+file(GLOB_RECURSE EXCEPTION_SRC "${HBASE_SRC_DIR}/exceptions/*.cc" )
+file(GLOB_RECURSE PROTO_SRC "${HBASE_SRC_DIR}/if/*.cc" )
+file(GLOB_RECURSE UTILS_SRC "${HBASE_SRC_DIR}/utils/*.cc" )
+file(GLOB_RECURSE CLIENT_SRC "${HBASE_SRC_DIR}/client/*.cc" )
+file(GLOB_RECURSE SECURITY_SRC "${HBASE_SRC_DIR}/security/*.cc" )
+file(GLOB_RECURSE SRDE_SRC "${HBASE_SRC_DIR}/serde/*.cc" )
+file(GLOB_RECURSE TEST_UTIL "${HBASE_SRC_DIR}/test-util/*.cc" )
+include_directories(${CMAKE_CURRENT_BINARY_DIR}/${CMAKE_FILES_DIRECTORY})
+include_directories(${SASL_INCLUDE_DIRS})
+include_directories(${GFLAGS_INCLUDE_DIR})
+file(GLOB_RECURSE PROTO_FILES "${HBASE_SRC_DIR}/if/*.proto" )
+### generate the protocol buffers.
+generate_protobuf_src(PROTO_SOURCES PROTO_HEADERS ${PROTO_FILES})
+
+add_library(hbaseclient-static STATIC ${PROTO_SOURCES} ${CLIENT_SRC} ${CONNECTION_SRC} ${EXCEPTION_SRC} ${PROTO_SRC} ${SECURITY_SRC} ${SRDE_SRC} ${UTILS_SRC})
+set_target_properties(hbaseclient-static PROPERTIES LINKER_LANGUAGE CXX)
+SET_TARGET_PROPERTIES(hbaseclient-static PROPERTIES OUTPUT_NAME hbaseclient CLEAN_DIRECT_OUTPUT 1)
+target_link_libraries(hbaseclient-static ${PROTOBUF_LIBRARY})
+target_link_libraries(hbaseclient-static ${FOLLY_LIBRARIES})
+target_link_libraries(hbaseclient-static ${Boost_LIBRARIES})
+target_link_libraries(hbaseclient-static ${SASL_LIBS})
+target_link_libraries(hbaseclient-static ${GLOG_SHARED_LIB})
+target_link_libraries(hbaseclient-static ${GFLAGS_SHARED_LIB})
+target_link_libraries(hbaseclient-static ${KRB5_LIBRARIES})
+target_link_libraries(hbaseclient-static ${WANGLE_LIBRARIES})
+target_link_libraries(hbaseclient-static ${Zookeeper_LIBRARIES})
+target_link_libraries(hbaseclient-static ${OPENSSL_LIBRARIES})
+add_library(hbaseclient-shared SHARED ${PROTO_SOURCES} ${CLIENT_SRC} ${CONNECTION_SRC} ${EXCEPTION_SRC} ${PROTO_SRC} ${SECURITY_SRC} ${SRDE_SRC} ${UTILS_SRC})
+set_target_properties(hbaseclient-shared PROPERTIES LINKER_LANGUAGE CXX)
+SET_TARGET_PROPERTIES(hbaseclient-shared PROPERTIES COMPILE_FLAGS " -fPIC")
+SET_TARGET_PROPERTIES(hbaseclient-shared PROPERTIES OUTPUT_NAME hbaseclient CLEAN_DIRECT_OUTPUT 1)
+target_link_libraries(hbaseclient-shared ${PROTOBUF_LIBRARY})
+target_link_libraries(hbaseclient-shared ${FOLLY_LIBRARIES})
+target_link_libraries(hbaseclient-shared ${Boost_LIBRARIES})
+target_link_libraries(hbaseclient-shared ${SASL_LIBS})
+target_link_libraries(hbaseclient-shared ${GLOG_SHARED_LIB})
+target_link_libraries(hbaseclient-shared ${GFLAGS_SHARED_LIB})
+target_link_libraries(hbaseclient-shared ${KRB5_LIBRARIES})
+target_link_libraries(hbaseclient-shared ${WANGLE_LIBRARIES})
+target_link_libraries(hbaseclient-shared ${Zookeeper_LIBRARIES})
+add_executable(simple-client "${HBASE_SRC_DIR}/examples/simple-client.cc")
+SET_TARGET_PROPERTIES(simple-client PROPERTIES COMPILE_FLAGS " ")
+target_link_libraries(simple-client ${PROTOBUF_LIBRARY})
+target_link_libraries(simple-client ${Boost_LIBRARIES})
+target_link_libraries(simple-client ${SASL_LIBS})
+target_link_libraries(simple-client ${GFLAGS_SHARED_LIB})
+target_link_libraries(simple-client ${KRB5_LIBRARIES})
+target_link_libraries(simple-client ${Zookeeper_LIBRARIES})
+target_link_libraries(simple-client hbaseclient-static ${CMAKE_THREAD_LIBS_INIT})
+add_executable(load-client "${HBASE_SRC_DIR}/examples/load-client.cc")
+SET_TARGET_PROPERTIES(load-client PROPERTIES COMPILE_FLAGS " ")
+target_link_libraries(load-client ${PROTOBUF_LIBRARY})
+target_link_libraries(load-client ${Boost_LIBRARIES})
+target_link_libraries(load-client ${SASL_LIBS})
+target_link_libraries(load-client ${GFLAGS_SHARED_LIB})
+target_link_libraries(load-client ${KRB5_LIBRARIES})
+target_link_libraries(load-client ${Zookeeper_LIBRARIES})
+target_link_libraries(load-client hbaseclient-static ${CMAKE_THREAD_LIBS_INIT})
+if (JNI_FOUND)
+ message (STATUS "JNI_INCLUDE_DIRS=${JNI_INCLUDE_DIRS}")
+ message (STATUS "JNI_LIBRARIES=${JNI_LIBRARIES}")
+endif()
+if (NOT SKIP_TESTS)
+ include(BuildTests)
+endif()
+## Create a custom target for our linter
+add_custom_target(
+ linter
+ COMMAND ${CMAKE_SOURCE_DIR}/bin/cpplint.sh)
+# Copy the version.h file in before doing anything
+add_custom_target (
+ copy_version_h
+ COMMAND "${CMAKE_CURRENT_SOURCE_DIR}/bin/copy-version.sh"
+)
+add_dependencies(hbaseclient-static copy_version_h)
+add_dependencies(hbaseclient-shared copy_version_h)
+
+# Install library headers
+include(GNUInstallDirs)
+file(GLOB RECURSE HEADERS include/*.h)
+set_target_properties(hbaseclient-static PROPERTIES PUBLIC_HEADER "${HEADERS}")
+install(TARGETS hbaseclient-static
+ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+ PUBLIC_HEADER DESTINATION include/
+ COMPONENT LIBRARY )
+install(TARGETS hbaseclient-shared
+ ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}
+ LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}
+ PUBLIC_HEADER DESTINATION include/
+ COMPONENT LIBRARY )
+INSTALL (
+ DIRECTORY ${CMAKE_SOURCE_DIR}/include/
+ DESTINATION include/
+ FILES_MATCHING PATTERN "*.h*")
+# Install pb-generated headers too
+INSTALL (
+ DIRECTORY "${CMAKE_BINARY_DIR_GEN}"
+ DESTINATION include/hbase/if
+ FILES_MATCHING PATTERN "hbase/if/*.h")
diff --git a/Makefile b/Makefile
deleted file mode 100644
index cd818ca..0000000
--- a/Makefile
+++ /dev/null
@@ -1,176 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#use "g++" to compile source files
-CC := g++
-LD := g++
-
-SRC_HBASE := src/hbase
-INCLUDE_HBASE := include/
-BUILD_PATH := build
-DEBUG_PATH := $(BUILD_PATH)/debug
-RELEASE_PATH := $(BUILD_PATH)/release
-PROTO_SRC_DIR := $(SRC_HBASE)/if
-PROTO_CXX_DIR := $(BUILD_PATH)/$(PROTO_SRC_DIR)
-MODULES := connection client exceptions security serde utils
-TEST_MODULES := test-util # These modules contain test code, not included in the build for the lib
-SRC_DIR := $(addprefix $(SRC_HBASE)/,$(MODULES))
-DEBUG_BUILD_DIR := $(addprefix $(DEBUG_PATH)/hbase/,$(MODULES))
-RELEASE_BUILD_DIR := $(addprefix $(RELEASE_PATH)/hbase/,$(MODULES))
-
-INCLUDE_DIR := . src $(BUILD_PATH)/src $(INCLUDE_HBASE)
-TEST_BUILD_INCLUDE_DIR := $(INLCUDE_DIR) $(JAVA_HOME)/include/ $(JAVA_HOME)/include/linux
-
-#flags to pass to the CPP compiler & linker
-CPPFLAGS_DEBUG := -D_GLIBCXX_USE_CXX11_ABI=0 -g -Wall -std=c++14 -pedantic -fPIC -MMD -MP
-CPPFLAGS_RELEASE := -D_GLIBCXX_USE_CXX11_ABI=0 -DNDEBUG -O2 -Wall -std=c++14 -pedantic -fPIC -MMD -MP
-LDFLAGS := -lprotobuf -lzookeeper_mt -lsasl2 -lfolly -lwangle
-TEST_BUILD_LDFLAGS := $(LDFLAGS) -L $(JAVA_HOME)/jre/lib/amd64/server -ljvm
-LINKFLAG := -shared
-
-#define list of source files and object files
-ALLSRC := $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc))
-EXCLUDE_SRC := $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*-test.cc)) \
- $(SRC_HBASE)/client/simple-client.cc $(SRC_HBASE)/client/load-client.cc
-SRC := $(filter-out $(EXCLUDE_SRC), $(ALLSRC))
-PROTOSRC := $(patsubst $(PROTO_SRC_DIR)/%.proto, $(addprefix $(PROTO_CXX_DIR)/,%.pb.cc),$(wildcard $(PROTO_SRC_DIR)/*.proto))
-PROTOHDR := $(patsubst $(PROTO_SRC_DIR)/%.proto, $(addprefix $(PROTO_CXX_DIR)/,%.pb.h),$(wildcard $(PROTO_SRC_DIR)/*.proto))
-DEBUG_OBJ := $(patsubst $(SRC_HBASE)/%.cc,$(DEBUG_PATH)/hbase/%.o,$(SRC))
-DEBUG_OBJ += $(patsubst $(PROTO_CXX_DIR)/%.cc,$(DEBUG_PATH)/hbase/if/%.o,$(PROTOSRC))
-RELEASE_OBJ := $(patsubst $(SRC_HBASE)/%.cc,$(RELEASE_PATH)/hbase/%.o,$(SRC))
-RELEASE_OBJ += $(patsubst $(PROTO_CXX_DIR)/%.cc,$(RELEASE_PATH)/hbase/if/%.o,$(PROTOSRC))
-INCLUDES := $(addprefix -I,$(INCLUDE_DIR))
-
-LIB_DIR := /usr/local
-LIB_LIBDIR := $(LIB_DIR)/lib
-LIB_INCDIR := $(LIB_DIR)/include
-LIB_RELEASE := $(RELEASE_PATH)/libHBaseClient.so
-ARC_RELEASE := $(RELEASE_PATH)/libHBaseClient.a
-LIB_DEBUG := $(DEBUG_PATH)/libHBaseClient_d.so
-ARC_DEBUG := $(DEBUG_PATH)/libHBaseClient_d.a
-LOCAL_INCLUDE_DIR := /usr/local/include/
-
-build: checkdirs protos copyfiles $(LIB_DEBUG) $(LIB_RELEASE) $(ARC_DEBUG) $(ARC_RELEASE)
-
-vpath %.cc $(SRC_DIR)
-
-$(LIB_DEBUG):
-define make-goal-dbg
-DEPS := $(DEBUG_OBJ:.o=.d)
--include $(DEPS)
-$1/%.o: %.cc
- $(CC) -c $$< -o $$@ -MF$$(@:%.o=%.d) -MT$$@ $(CPPFLAGS_DEBUG) $(INCLUDES)
-endef
-
-$(LIB_RELEASE):
-define make-goal-rel
-DEPS := $(RELEASE_OBJ:.o=.d)
--include $(DEPS)
-$1/%.o: %.cc
- $(CC) -c $$< -o $$@ -MF$$(@:%.o=%.d) -MT$$@ $(CPPFLAGS_RELEASE) $(INCLUDES)
-endef
-
-.PHONY: all clean install copyfiles
-
-checkdirs: $(DEBUG_BUILD_DIR) $(RELEASE_BUILD_DIR) $(PROTO_CXX_DIR)
-
-copyfiles:
- @bin/copy-protobuf.sh
- @bin/copy-version.sh
-
-# .proto files are in src/hbase/if. These are compiled into C++ code by the
-# protoc compiler, and turned into .cc and .h files under build/src/hbase/if
-$(PROTO_CXX_DIR)/%.pb.cc $(PROTO_CXX_DIR)/%.pb.h: $(PROTO_SRC_DIR)/%.proto
- @protoc --proto_path=$(PROTO_SRC_DIR) --cpp_out=$(PROTO_CXX_DIR) $<
-
-# protos target compiles the .cc and .h files into .o files for the protobuf
-# generated source files
-protos: $(PROTO_CXX_DIR) $(PROTOSRC) $(PROTOHDR)
- @make -j8 all -f Makefile.protos
-
-install_headers:
- cp -r $(INCLUDE_HBASE)/hbase $(LOCAL_INCLUDE_DIR)
- mkdir -p $(LOCAL_INCLUDE_DIR)/hbase/if
- cp -r $(PROTO_CXX_DIR)/*.h $(LOCAL_INCLUDE_DIR)/hbase/if
-
-uninstall_headers:
- rm -rf $(LOCAL_INCLUDE_DIR)/hbase
-
-install: install_headers
- cp $(LIB_RELEASE) $(LIB_LIBDIR)/libHBaseClient.so
- cp $(ARC_RELEASE) $(LIB_LIBDIR)/libHBaseClient.a
- cp $(LIB_DEBUG) $(LIB_LIBDIR)/libHBaseClient_d.so
- cp $(ARC_DEBUG) $(LIB_LIBDIR)/libHBaseClient_d.a
- ldconfig
-
-uninstall: uninstall_headers
- rm -f $(LIB_LIBDIR)/libHBaseClient.so $(LIB_LIBDIR)/libHBaseClient.a $(LIB_LIBDIR)/libHBaseClient_d.so $(LIB_LIBDIR)/libHBaseClient_d.a
- ldconfig
-
-$(PROTO_CXX_DIR):
- @mkdir -p $@
-
-$(DEBUG_BUILD_DIR):
- @mkdir -p $@
-
-$(RELEASE_BUILD_DIR):
- @mkdir -p $@
-
-$(ARC_DEBUG): $(DEBUG_OBJ)
- ar rcs $@ $^
-
-$(ARC_RELEASE): $(RELEASE_OBJ)
- ar rcs $@ $^
-
-$(LIB_RELEASE): $(RELEASE_OBJ)
- $(LD) $(LINKFLAG) -o $@ $(LDFLAGS) $(RELEASE_OBJ)
-
-$(LIB_DEBUG): $(DEBUG_OBJ)
- $(LD) $(LINKFLAG) -o $@ $(LDFLAGS) $(DEBUG_OBJ)
-
-clean:
- @rm -rf docs buck-out $(BUILD_PATH)
-
-$(foreach bdir,$(DEBUG_BUILD_DIR), $(eval $(call make-goal-dbg,$(bdir))))
-
-$(foreach bdir,$(RELEASE_BUILD_DIR),$(eval $(call make-goal-rel,$(bdir))))
-
-check:
- $(shell buck test --all --no-results-cache)
-
-lint:
- bin/cpplint.sh
-
-doc:
- $(shell doxygen hbase.doxygen > /dev/null)
-
-help:
- @echo "Available targets:"
- @echo ""
- @echo " all : builds everything, creates doc and runs tests."
- @echo " build : will build/rebuild everything."
- @echo " check : will test everything."
- @echo " clean : removes docs folder, object files and local libraries from build/ directory."
- @echo " copyfiles : copies native version.h from mvn build and proto locally to hbase-native-client."
- @echo " doc : generates documentation."
- @echo " install : will copy the libs to $(LIB_LIBDIR). super user priviliege would be required."
- @echo " protos : will create PB CPP sec and headers from if/*.proto and build them."
- @echo " uninstall : removes the libs from $(LIB_LIBDIR)."
- @echo " lint : will ensure that code conforms to Google coding style."
- @echo "If no target is specified 'build' will be executed"
-
-all: copyfiles build doc check
diff --git a/Makefile.protos b/Makefile.protos
deleted file mode 100644
index 8712029..0000000
--- a/Makefile.protos
+++ /dev/null
@@ -1,80 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-#use "g++" to compile source files
-CC := g++
-
-BUILD_PATH := build
-DEBUG_PATH := $(BUILD_PATH)/debug
-RELEASE_PATH := $(BUILD_PATH)/release
-MODULES := if
-SRC_DIR := $(BUILD_PATH)/src/hbase/$(MODULES)
-DEBUG_BUILD_DIR := $(addprefix $(DEBUG_PATH)/hbase/,$(MODULES))
-RELEASE_BUILD_DIR := $(addprefix $(RELEASE_PATH)/hbase/,$(MODULES))
-INCLUDE_DIR := . $(BUILD_PATH)/src/hbase/if
-
-#flags to pass to the CPP compiler & linker
-CPPFLAGS_DEBUG := -D_GLIBCXX_USE_CXX11_ABI=0 -g -Wall -std=c++14 -pedantic -fPIC -MMD -MP
-CPPFLAGS_RELEASE := -D_GLIBCXX_USE_CXX11_ABI=0 -DNDEBUG -O2 -Wall -std=c++14 -pedantic -fPIC -MMD -MP
-
-#define list of source files and object files
-SRC := $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc))
-DEBUG_OBJ := $(patsubst $(SRC_DIR)/%.cc,$(DEBUG_PATH)/hbase/if/%.o,$(SRC))
-RELEASE_OBJ := $(patsubst $(SRC_DIR)/%.cc,$(RELEASE_PATH)/hbase/if/%.o,$(SRC))
-INCLUDES := $(addprefix -I,$(INCLUDE_DIR))
-
-all: $(DEBUG_BUILD_DIR) $(RELEASE_BUILD_DIR) $(DEBUG_OBJ) $(RELEASE_OBJ)
-
-vpath %.cc $(SRC_DIR)
-
-$(DEBUG_OBJ):
-define make-goal-dbg
-DEPS := $(DEBUG_OBJ:.o=.d)
--include $(DEPS)
-$1/%.o: %.cc
- $(CC) -c $$< -o $$@ -MF$$(@:%.o=%.d) -MT$$@ $(CPPFLAGS_DEBUG) $(INCLUDES)
-endef
-
-$(RELEASE_OBJ):
-define make-goal-rel
-DEPS := $(RELEASE_OBJ:.o=.d)
--include $(DEPS)
-$1/%.o: %.cc
- $(CC) -c $$< -o $$@ -MF$$(@:%.o=%.d) -MT$$@ $(CPPFLAGS_RELEASE) $(INCLUDES)
-endef
-
-.PHONY: all clean
-
-$(DEBUG_BUILD_DIR):
- @mkdir -p $@
-
-$(RELEASE_BUILD_DIR):
- @mkdir -p $@
-
-clean:
- @rm -rf $(DEBUG_BUILD_DIR) $(RELEASE_BUILD_DIR)
-
-$(foreach bdir,$(DEBUG_BUILD_DIR),$(eval $(call make-goal-dbg,$(bdir))))
-
-$(foreach bdir,$(RELEASE_BUILD_DIR),$(eval $(call make-goal-rel,$(bdir))))
-
-help:
- @echo "This Makefile invocation will only work when protobuf sources and headers have been generated by running 'make protos'"
- @echo "Available targets:"
- @echo ""
- @echo " all : creates objects for the generated PB src."
- @echo " clean : removes PB objects."
diff --git a/bin/cpplint.sh b/bin/cpplint.sh
index 7a27945..497ab24 100755
--- a/bin/cpplint.sh
+++ b/bin/cpplint.sh
@@ -16,9 +16,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
IFS=$'\n\t'
-
+OUTPUT_DIR=$1
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
CPPLINT_LOC=https://raw.githubusercontent.com/google/styleguide/gh-pages/cpplint/cpplint.py
-OUTPUT=build/cpplint.py
+OUTPUT=$OUTPUT_DIR/cpplint.py
declare -a MODULES=( client connection exceptions security serde utils test-util )
@@ -31,9 +32,9 @@ wget -nc $CPPLINT_LOC -O $OUTPUT
# build/c++11 (We are building with c++14)
for m in ${MODULES[@]}; do
if [ $m != "security" ]; then #These are empty
- exec find src/hbase/$m -name "*.h" -or -name "*.cc" | xargs -P8 python $OUTPUT --filter=-build/header_guard,-readability/todo,-build/c++11 --linelength=100
+ exec find ${SCRIPT_DIR}/../src/hbase/$m -name "*.h" -or -name "*.cc" | xargs -P8 python $OUTPUT --filter=-build/header_guard,-readability/todo,-build/c++11 --linelength=100
fi
if [ $m != "test-util" ]; then
- exec find include/hbase/$m -name "*.h" -or -name "*.cc" | xargs -P8 python $OUTPUT --filter=-build/header_guard,-readability/todo,-build/c++11 --linelength=100
+ exec find ${SCRIPT_DIR}/../include/hbase/$m -name "*.h" -or -name "*.cc" | xargs -P8 python $OUTPUT --filter=-build/header_guard,-readability/todo,-build/c++11 --linelength=100
fi
done
diff --git a/cmake/BuildTests.cmake b/cmake/BuildTests.cmake
new file mode 100644
index 0000000..7ab2102
--- /dev/null
+++ b/cmake/BuildTests.cmake
@@ -0,0 +1,80 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+### test functions
+MACRO(GETSOURCEFILES result curdir)
+ FILE(GLOB children RELATIVE ${curdir} ${curdir}/*)
+ SET(dirlist "")
+ FOREACH(child ${children})
+ IF( "${child}" MATCHES ^[^.].*\\.cc)
+
+ LIST(APPEND dirlist ${child})
+ ENDIF()
+ ENDFOREACH()
+ SET(${result} ${dirlist})
+ENDMACRO()
+find_package(GMock REQUIRED)
+add_library(testutil STATIC ${TEST_UTIL})
+target_include_directories(testutil PRIVATE BEFORE "include")
+target_include_directories(testutil PRIVATE BEFORE "${Java_INCLUDE_DIRS}")
+target_include_directories(testutil PRIVATE BEFORE "${JNI_INCLUDE_DIRS}")
+target_include_directories(testutil PRIVATE BEFORE "${Boost_INCLUDE_DIR}")
+target_include_directories(testutil PRIVATE BEFORE "${GTEST_INCLUDE_DIRS}")
+target_include_directories(testutil PRIVATE BEFORE${PROTOBUF_INCLUDE_DIRS})
+target_include_directories(testutil PRIVATE BEFORE${Zookeeper_INCLUDE_DIRS})
+target_include_directories(testutil PRIVATE BEFORE${KRB5_INCLUDE_DIRS})
+target_include_directories(testutil PRIVATE BEFORE${Java_INCLUDE_DIRS})
+target_include_directories(testutil PRIVATE BEFORE${FOLLY_INCLUDE_DIRS})
+target_link_libraries(testutil hbaseclient-static ${CMAKE_THREAD_LIBS_INIT} ${Java_LIBRARIES} ${JNI_LIBRARIES} ${PROTOBUF_LIBRARY} ${Boost_LIBRARIES} ${GFLAGS_SHARED_LIB} ${GMOCK_SHARED_LIB} ${GTEST_BOTH_LIBRARIES} ${SASL_LIBS} ${GFLAGS_SHARED_LIB} ${KRB5_LIBRARIES} ${OPENSSL_LIBRARIES} ${Zookeeper_LIBRARIES})
+function(createTests testName)
+ message ("-- Including Test: ${testName}")
+ target_include_directories(${testName} PRIVATE BEFORE "include")
+ target_include_directories(${testName} PRIVATE BEFORE "${Java_INCLUDE_DIRS}")
+ target_include_directories(${testName} PRIVATE BEFORE "${JNI_INCLUDE_DIRS}")
+ target_include_directories(${testName} PRIVATE BEFORE "${Boost_INCLUDE_DIR}")
+ target_include_directories(${testName} PRIVATE BEFORE "${GTEST_INCLUDE_DIRS}")
+ target_include_directories(${testName} PRIVATE BEFORE "${OPENSSL_INCLUDE_DIR}")
+
+ target_link_libraries(hbaseclient-static ${PROTOBUF_LIBRARY})
+ target_link_libraries(hbaseclient-static ${FOLLY_LIBRARIES})
+
+ target_link_libraries(${testName} hbaseclient-static testutil ${CMAKE_THREAD_LIBS_INIT}
+ ${Java_LIBRARIES}
+ ${JNI_LIBRARIES}
+ ${PROTOBUF_LIBRARY}
+ ${Boost_LIBRARIES}
+ ${GFLAGS_SHARED_LIB}
+ ${GTEST_BOTH_LIBRARIES}
+ ${SASL_LIBS}
+ ${GFLAGS_SHARED_LIB}
+ ${KRB5_LIBRARIES}
+ ${Zookeeper_LIBRARIES} ${OPENSSL_LIBRARIES}
+ ${WANGLE_LIBRARIES}
+ ${FOLLY_LIBRARIES}
+ ${GLOG_SHARED_LIB})
+endfunction()
+enable_testing(test)
+SET(TEST_DIR ${CMAKE_SOURCE_DIR}/src/test)
+GETSOURCEFILES(UNIT_TESTS "${TEST_DIR}")
+SET(UNIT_TEST_COUNT 0)
+FOREACH(testfile ${UNIT_TESTS})
+ get_filename_component(testfilename "${testfile}" NAME_WE)
+ add_executable("${testfilename}" "${TEST_DIR}/${testfile}")
+ createTests("${testfilename}")
+ MATH(EXPR UNIT_TEST_COUNT "${UNIT_TEST_COUNT}+1")
+ add_test(NAME "${testfilename}" COMMAND "${testfilename}" WORKING_DIRECTORY ${TEST_DIR})
+ENDFOREACH()
+message("-- Finished building ${UNIT_TEST_COUNT} unit test file(s)...")
diff --git a/cmake/FindFolly.cmake b/cmake/FindFolly.cmake
new file mode 100644
index 0000000..e7854a0
--- /dev/null
+++ b/cmake/FindFolly.cmake
@@ -0,0 +1,48 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+#
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+find_path(FOLLY_ROOT_DIR
+ NAMES include/folly/AtomicHashMap.h
+)
+find_library(FOLLY_LIBRARIES
+ NAMES folly
+ HINTS ${FOLLY_ROOT_DIR}/lib /usr/lib/ /usr/local/lib/ /usr/lib/x86_64-linux-gnu/
+)
+find_library(FOLLY_BENCHMARK_LIBRARIES
+ NAMES follybenchmark
+ HINTS ${FOLLY_ROOT_DIR}/lib
+)
+find_path(FOLLY_INCLUDE_DIR
+ NAMES folly/AtomicHashMap.h
+ HINTS ${FOLLY_ROOT_DIR}/include
+)
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(Folly DEFAULT_MSG
+ FOLLY_LIBRARIES
+ FOLLY_INCLUDE_DIR
+)
+mark_as_advanced(
+ FOLLY_ROOT_DIR
+ FOLLY_LIBRARIES
+ FOLLY_BENCHMARK_LIBRARIES
+ FOLLY_INCLUDE_DIR
+)
+if (FOLLY_LIBRARIES)
+ set(FOLLY_FOUND "true")
+ message("-- Folly found, ${FOLLY_LIBRARIES}")
+endif(FOLLY_LIBRARIES)
diff --git a/cmake/FindGMock.cmake b/cmake/FindGMock.cmake
new file mode 100644
index 0000000..998fa6d
--- /dev/null
+++ b/cmake/FindGMock.cmake
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+find_path(GMOCK_INCLUDE_DIR gmock/gmock.h)
+ # make sure we don't accidentally pick up a different version
+find_library(GMOCK_SHARED_LIB gmock)
+find_library(GMOCK_STATIC_LIB libgmock.a)
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(GMOCK REQUIRED_VARS
+ GMOCK_SHARED_LIB GMOCK_STATIC_LIB GMOCK_INCLUDE_DIR)
diff --git a/cmake/FindGflags.cmake b/cmake/FindGflags.cmake
new file mode 100644
index 0000000..59c65df
--- /dev/null
+++ b/cmake/FindGflags.cmake
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+find_path(GFLAGS_INCLUDE_DIR gflags/gflags.h)
+ # make sure we don't accidentally pick up a different version
+find_library(GFLAGS_SHARED_LIB gflags)
+find_library(GFLAGS_STATIC_LIB libgflags.a)
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(GFLAGS REQUIRED_VARS
+ GFLAGS_SHARED_LIB GFLAGS_STATIC_LIB GFLAGS_INCLUDE_DIR)
diff --git a/cmake/FindGlog.cmake b/cmake/FindGlog.cmake
new file mode 100644
index 0000000..a2c541d
--- /dev/null
+++ b/cmake/FindGlog.cmake
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+find_path(GLOG_INCLUDE_DIR glog/logging.h)
+ # make sure we don't accidentally pick up a different version
+find_library(GLOG_SHARED_LIB glog)
+find_library(GLOG_STATIC_LIB libglog.a)
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(GLOG REQUIRED_VARS
+ GLOG_SHARED_LIB GLOG_STATIC_LIB GLOG_INCLUDE_DIR)
diff --git a/cmake/FindKrb5.cmake b/cmake/FindKrb5.cmake
new file mode 100644
index 0000000..e4c6fb8
--- /dev/null
+++ b/cmake/FindKrb5.cmake
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+#
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+find_path(KRB5_ROOT_DIR
+ NAMES include/krb5/krb5.h
+)
+find_library(KRB5_LIBRARIES
+ NAMES krb5
+ HINTS ${KRB5_ROOT_DIR}/lib
+)
+find_path(KRB5_INCLUDE_DIR
+ NAMES krb5/krb5.h
+ HINTS ${KRB5_ROOT_DIR}/include
+)
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(krb5 DEFAULT_MSG
+ KRB5_LIBRARIES
+ KRB5_INCLUDE_DIR
+)
+if (KRB5_LIBRARIES)
+ set(KRB5_FOUND "true")
+ message("-- KRB5 Libs Found, ${KRB5_LIBRARIES}")
+endif()
+mark_as_advanced(
+ KRB5_ROOT_DIR
+ KRB5_LIBRARIES
+ KRB5_INCLUDE_DIR
+)
diff --git a/cmake/FindSasl2.cmake b/cmake/FindSasl2.cmake
new file mode 100644
index 0000000..871385b
--- /dev/null
+++ b/cmake/FindSasl2.cmake
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+#
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+include(CheckSymbolExists)
+find_path (
+ SASL_INCLUDE_DIRS NAMES sasl/sasl.h
+ PATHS /include /usr/include /usr/local/include /usr/share/include)
+find_library(
+ SASL_LIBS NAMES sasl2
+ PATHS /usr/lib /lib /usr/local/lib /usr/lib/x86_64-linux-gnu/)
+if (SASL_INCLUDE_DIRS AND SASL_LIBS)
+ set (SASL_FOUND 1)
+ message("-- LibSASL found, ${SASL_LIBS}")
+endif ()
diff --git a/cmake/FindWangle.cmake b/cmake/FindWangle.cmake
new file mode 100644
index 0000000..6e69faa
--- /dev/null
+++ b/cmake/FindWangle.cmake
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+#
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+find_path(WANGLE_ROOT_DIR
+ NAMES include/wangle/acceptor/Acceptor.h
+)
+find_library(WANGLE_LIBRARIES
+ NAMES wangle
+ HINTS ${WANGLE_ROOT_DIR}/lib /usr/lib/ /usr/local/lib/
+)
+find_path(WANGLE_INCLUDE_DIR
+ NAMES wangle/acceptor/Acceptor.h
+ HINTS ${WANGLE_ROOT_DIR}/include /usr/local/include/
+)
+include(FindPackageHandleStandardArgs)
+find_package_handle_standard_args(WANGLE DEFAULT_MSG
+ WANGLE_LIBRARIES
+ WANGLE_INCLUDE_DIR
+)
+mark_as_advanced(
+ WANGLE_ROOT_DIR
+ WANGLE_LIBRARIES
+ WANGLE_INCLUDE_DIR
+)
+if (WANGLE_LIBRARIES)
+ set(WANGLE_FOUND "true")
+ message("-- Wangle found, ${WANGLE_LIBRARIES}")
+endif(WANGLE_LIBRARIES)
diff --git a/cmake/FindZookeeper.cmake b/cmake/FindZookeeper.cmake
new file mode 100644
index 0000000..a9e9d3d
--- /dev/null
+++ b/cmake/FindZookeeper.cmake
@@ -0,0 +1,51 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+if (MSVC)
+ if(${CMAKE_BUILD_TYPE} MATCHES "Debug")
+ set(ZK_BuildOutputDir "Debug")
+ else()
+ set(ZK_BuildOutputDir "Release")
+ endif()
+ if("${ZOOKEEPER_HOME}_" MATCHES "^_$")
+ message(" ")
+ message("- Please set the cache variable ZOOKEEPER_HOME to point to the directory with the zookeeper source.")
+ message("- CMAKE will look for zookeeper include files in $ZOOKEEPER_HOME/src/c/include.")
+ message("- CMAKE will look for zookeeper library files in $ZOOKEEPER_HOME/src/c/Debug or $ZOOKEEPER_HOME/src/c/Release.")
+ else()
+ FILE(TO_CMAKE_PATH ${ZOOKEEPER_HOME} Zookeeper_HomePath)
+ set(Zookeeper_LIB_PATHS ${Zookeeper_HomePath}/src/c/${ZK_BuildOutputDir})
+ find_path(ZK_INCLUDE_DIR zookeeper.h ${Zookeeper_HomePath}/src/c/include)
+ find_path(ZK_INCLUDE_DIR_GEN zookeeper.jute.h ${Zookeeper_HomePath}/src/c/generated)
+ set(Zookeeper_INCLUDE_DIR zookeeper.h ${ZK_INCLUDE_DIR} ${ZK_INCLUDE_DIR_GEN} )
+ find_library(Zookeeper_LIBRARY NAMES zookeeper PATHS ${Zookeeper_LIB_PATHS})
+ endif()
+else()
+ set(Zookeeper_LIB_PATHS /usr/local/lib /usr/lib/ /usr/lib/x86_64-linux-gnu/)
+ find_path(Zookeeper_INCLUDE_DIR zookeeper/zookeeper.h /usr/local/include)
+ find_library(Zookeeper_LIBRARY NAMES libzookeeper_mt.a PATHS ${Zookeeper_LIB_PATHS})
+endif()
+set(Zookeeper_LIBRARIES ${Zookeeper_LIBRARY} )
+set(Zookeeper_INCLUDE_DIRS ${Zookeeper_INCLUDE_DIR} )
+include(FindPackageHandleStandardArgs)
+# handle the QUIETLY and REQUIRED arguments and set Zookeeper_FOUND to TRUE
+# if all listed variables are TRUE
+find_package_handle_standard_args(Zookeeper DEFAULT_MSG
+ Zookeeper_LIBRARY Zookeeper_INCLUDE_DIR)
+if (Zookeeper_LIBRARY)
+ message("-- Zookeeper found, ${Zookeeper_LIBRARY}")
+endif()
+mark_as_advanced(Zookeeper_INCLUDE_DIR Zookeeper_LIBRARY )
diff --git a/cmake/ProtobufGen.cmake b/cmake/ProtobufGen.cmake
new file mode 100644
index 0000000..0cc2c2f
--- /dev/null
+++ b/cmake/ProtobufGen.cmake
@@ -0,0 +1,62 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+# copied in most part from protobuf cmake
+# there are similar protobuf gen changes online, all of which do
+# a similar job of customizing their generation.
+function(generate_protobuf_src SRCS HDRS)
+ if(NOT ARGN)
+ message(SEND_ERROR "Error: generate_protobuf_src() called without any proto files")
+ return()
+ endif()
+
+ set(_protobuf_include_path -I .)
+ if(DEFINED PROTOBUF_INCLUDE_DIRS)
+ foreach(DIR ${PROTOBUF_INCLUDE_DIRS})
+ file(RELATIVE_PATH REL_PATH ${CMAKE_SOURCE_DIR} ${DIR})
+ list(FIND _protobuf_include_path ${REL_PATH} _contains_already)
+ if(${_contains_already} EQUAL -1)
+ list(APPEND _protobuf_include_path -I ${REL_PATH})
+ endif()
+ endforeach()
+ endif()
+ set(${SRCS})
+ set(${HDRS})
+ foreach(FIL ${ARGN})
+ get_filename_component(ABS_FIL ${FIL} ABSOLUTE)
+ get_filename_component(FIL_WE ${FIL} NAME_WE)
+ ## get the directory where our protobufs are stored
+ file(RELATIVE_PATH REL_FIL ${CMAKE_SOURCE_DIR} ${ABS_FIL})
+ get_filename_component(REL_DIR ${REL_FIL} DIRECTORY)
+ list(APPEND ${SRCS} "${CMAKE_BINARY_DIR_GEN}/${FIL_WE}.pb.cc")
+ list(APPEND ${HDRS} "${CMAKE_BINARY_DIR_GEN}/${FIL_WE}.pb.h")
+ add_custom_command(
+ OUTPUT "${CMAKE_BINARY_DIR_GEN}/${FIL_WE}.pb.cc"
+ "${CMAKE_BINARY_DIR_GEN}/${FIL_WE}.pb.h"
+ COMMAND ${PROTOBUF_PROTOC_EXECUTABLE}
+ ARGS --cpp_out=${CMAKE_BINARY_DIR_GEN}
+ --proto_path=${REL_DIR}
+ ${_protobuf_include_path}
+ ${REL_FIL}
+ DEPENDS ${ABS_FIL} ${PROTOBUF_PROTOC_EXECUTABLE}
+ WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}
+ COMMENT "Generating ${FIL}"
+ VERBATIM)
+ endforeach()
+ set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+ set(${SRCS} ${${SRCS}} PARENT_SCOPE)
+ set(${HDRS} ${${HDRS}} PARENT_SCOPE)
+endfunction()
diff --git a/docker-files/Dockerfile b/docker-files/Dockerfile
index c8844ac..045f4f5 100644
--- a/docker-files/Dockerfile
+++ b/docker-files/Dockerfile
@@ -15,31 +15,33 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM pjameson/buck-folly-watchman:20160511
+FROM ubuntu:16.04
ARG CC=/usr/bin/gcc-5
ARG CXX=/usr/bin/g++-5
-ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -pthread"
-ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -pthread"
+ARG CFLAGS="-fPIC -g -fno-omit-frame-pointer -O2 -pthread"
+ARG CXXFLAGS="-fPIC -g -fno-omit-frame-pointer -O2 -pthread"
ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
+RUN apt-get update && \
+ apt-get install -y vim maven inetutils-ping python-pip doxygen graphviz clang-format valgrind \
+ wget libgflags-dev libgoogle-glog-dev dh-autoreconf pkg-config libssl-dev build-essential \
+ libboost-all-dev libevent-dev libdouble-conversion-dev cmake libkrb5-dev git openjdk-8-jdk && \
+ pip install yapf && \
+ apt-get -qq clean && \
+ apt-get -y -qq autoremove && \
+ rm -rf /var/lib/{apt,dpkg,cache,log}/ && \
+ rm -rf /tmp/*
+
RUN wget ftp://ftp.cyrusimap.org/cyrus-sasl/cyrus-sasl-2.1.26.tar.gz ; \
tar zxf cyrus-sasl-2.1.26.tar.gz ; \
cd cyrus-sasl-2.1.26 ; \
./configure ; \
- make ; \
+ make -j4; \
make install ;\
cp /usr/local/lib/sasl2/* /usr/lib/sasl2/
-RUN apt-get install -y vim maven inetutils-ping python-pip doxygen graphviz clang-format \
- valgrind && \
- pip install yapf && \
- apt-get -qq clean && \
- apt-get -y -qq autoremove && \
- rm -rf /var/lib/{apt,dpkg,cache,log}/ && \
- rm -rf /tmp/*
-
RUN apt-get update && \
apt-get install -y debconf-utils debconf-set-selections && \
echo "krb5-config krb5-config/kerberos_servers string localhost" | debconf-set-selections ; \
@@ -67,7 +69,7 @@ RUN git clone https://github.com/google/protobuf.git /usr/src/protobuf && \
ldconfig && \
./autogen.sh && \
./configure && \
- make && \
+ make -j4 && \
make install && \
make clean && \
rm -rf .git && \
@@ -79,12 +81,12 @@ RUN git clone https://github.com/google/protobuf.git /usr/src/protobuf && \
cd src/c && \
ldconfig && \
./configure && \
- make && \
+ make -j4 && \
make install && \
make clean && \
ldconfig
-# Update folly
+# Update folly. `make check` removed due to a failing test, explicitly installing gtest ourselves
RUN cd /usr/src/ && \
ver=2017.09.04.00 && \
wget https://github.com/facebook/folly/archive/v$ver.tar.gz && \
@@ -96,13 +98,17 @@ RUN cd /usr/src/ && \
tar zxf release-1.8.0.tar.gz && \
rm -f release-1.8.0.tar.gz && \
mv googletest-release-1.8.0 gtest && \
- cd ../ && \
+ cd gtest && \
+ cmake . && \
+ make -j4 && \
+ make install && \
+ cd ../.. && \
ldconfig && \
autoreconf -ivf && \
./configure && \
- make && \
- make check && \
- make install
+ make -j4 && \
+ make install && \
+ make clean
# Update wangle
RUN cd /usr/src/ && \
@@ -113,9 +119,10 @@ RUN cd /usr/src/ && \
cd wangle-$ver/wangle && \
ldconfig && \
cmake . -DBUILD_TESTS=OFF && \
- make && \
+ make -j4 && \
ctest && \
- make install
+ make install && \
+ make clean
RUN echo "enabled=1" >> /etc/default/apport
diff --git a/include/hbase/client/BUCK b/include/hbase/client/BUCK
deleted file mode 100644
index 66d6896..0000000
--- a/include/hbase/client/BUCK
+++ /dev/null
@@ -1,93 +0,0 @@
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This is the main library.
-cxx_library(
- name="client",
- header_namespace="hbase/client",
- exported_headers=[
- "async-client-scanner.h",
- "async-connection.h",
- "async-region-locator.h",
- "async-rpc-retrying-caller-factory.h",
- "async-rpc-retrying-caller.h",
- "async-table-result-scanner.h",
- "client.h",
- "cell.h",
- "filter.h",
- "query.h",
- "keyvalue-codec.h",
- "region-location.h",
- "location-cache.h",
- "connection-configuration.h",
- # TODO: move this out of exported
- # Once meta lookup works
- "meta-utils.h",
- "get.h",
- "increment.h",
- "mutation.h",
- "put.h",
- "delete.h",
- "scan.h",
- "append.h",
- "result.h",
- "result-scanner.h",
- "request-converter.h",
- "response-converter.h",
- "table.h",
- "async-scan-rpc-retrying-caller.h",
- "raw-async-table.h",
- "raw-scan-result-consumer.h",
- "scan-result-cache.h",
- "hbase-rpc-controller.h",
- "time-range.h",
- "zk-util.h",
- "action.h",
- "multi-response.h",
- "region-request.h",
- "region-result.h",
- "row.h",
- "server-request.h",
- "async-batch-rpc-retrying-caller.h",
- ],
- deps=[
- "//include/hbase/exceptions:exceptions",
- "//include/hbase/utils:utils",
- "//include/hbase/connection:connection",
- "//include/hbase/client:conf",
- "//src/hbase/if:if",
- "//include/hbase/serde:serde",
- "//third-party:folly",
- "//third-party:wangle",
- "//third-party:zookeeper_mt",
- ],
- compiler_flags=['-Weffc++', '-ggdb'],
- visibility=[
- 'PUBLIC',
- ],)
-cxx_library(
- name="conf",
- header_namespace="hbase/client",
- exported_headers=[
- "configuration.h",
- "hbase-configuration-loader.h",
- ],
- deps=["//src/hbase/utils:utils", "//third-party:folly"],
- compiler_flags=['-Weffc++', '-ggdb'],
- visibility=[
- 'PUBLIC',
- ],)
diff --git a/include/hbase/connection/BUCK b/include/hbase/connection/BUCK
deleted file mode 100644
index 9a0b0cf..0000000
--- a/include/hbase/connection/BUCK
+++ /dev/null
@@ -1,57 +0,0 @@
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This is the library dealing with a single connection
-# to a single server.
-cxx_library(
- name="connection",
- header_namespace="hbase/connection",
- exported_headers=[
- "client-dispatcher.h",
- "client-handler.h",
- "sasl-handler.h",
- "connection-factory.h",
- "connection-pool.h",
- "connection-id.h",
- "pipeline.h",
- "request.h",
- "rpc-connection.h",
- "response.h",
- "service.h",
- "rpc-client.h",
- "sasl-util.h",
- "rpc-test-server.h",
- "rpc-test-server-handler.h",
- "rpc-fault-injector.h",
- "rpc-fault-injector-inl.h",
- ],
- deps=[
- "//src/hbase/if:if",
- "//include/hbase/utils:utils",
- "//include/hbase/serde:serde",
- "//include/hbase/security:security",
- "//third-party:folly",
- "//third-party:wangle",
- "//include/hbase/exceptions:exceptions",
- ],
- compiler_flags=['-Weffc++'],
- linker_flags=['-L/usr/local/lib', '-lsasl2', '-lkrb5'],
- exported_linker_flags=['-L/usr/local/lib', '-lsasl2', '-lkrb5'],
- visibility=[
- '//include/hbase/client/...',
- '//src/hbase/connection/...',
- ],)
diff --git a/include/hbase/exceptions/BUCK b/include/hbase/exceptions/BUCK
deleted file mode 100644
index ee296ff..0000000
--- a/include/hbase/exceptions/BUCK
+++ /dev/null
@@ -1,28 +0,0 @@
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-cxx_library(
- name="exceptions",
- header_namespace="hbase/exceptions",
- exported_headers=[
- "exception.h",
- ],
- deps=[
- "//third-party:folly",
- ],
- compiler_flags=['-Weffc++'],
- visibility=['//src/hbase/exceptions/...', '//include/hbase/client/...', '//include/hbase/connection/...'],)
diff --git a/include/hbase/security/BUCK b/include/hbase/security/BUCK
deleted file mode 100644
index 2e7530b..0000000
--- a/include/hbase/security/BUCK
+++ /dev/null
@@ -1,30 +0,0 @@
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This is the library dealing with a single connection
-# to a single server.
-cxx_library(
- name="security",
- header_namespace="hbase/security",
- exported_headers=[
- "user.h",
- ],
- deps=["//src/hbase/client:conf"],
- compiler_flags=['-Weffc++'],
- visibility=[
- 'PUBLIC',
- ],)
diff --git a/include/hbase/serde/BUCK b/include/hbase/serde/BUCK
deleted file mode 100644
index ee8ef82..0000000
--- a/include/hbase/serde/BUCK
+++ /dev/null
@@ -1,37 +0,0 @@
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-cxx_library(
- name="serde",
- header_namespace="hbase/serde",
- exported_headers=[
- "cell-scanner.h",
- "cell-outputstream.h",
- "codec.h",
- "region-info.h",
- "rpc-serde.h",
- "server-name.h",
- "table-name.h",
- "zk.h",
- ],
- deps=[
- "//src/hbase/if:if", "//third-party:folly", "//src/hbase/utils:utils", "//src/hbase/security:security"
- ],
- compiler_flags=['-Weffc++'],
- visibility=[
- 'PUBLIC',
- ],)
diff --git a/src/hbase/test-util/mini-cluster.h b/include/hbase/test-util/mini-cluster.h
similarity index 98%
rename from src/hbase/test-util/mini-cluster.h
rename to include/hbase/test-util/mini-cluster.h
index 6b4547c..978ded8 100644
--- a/src/hbase/test-util/mini-cluster.h
+++ b/include/hbase/test-util/mini-cluster.h
@@ -21,6 +21,7 @@
#include <jni.h>
#include <string>
#include <vector>
+#include <mutex>
namespace hbase {
@@ -69,7 +70,7 @@ class MiniCluster {
jmethodID str_ctor_mid_;
jobject htu_;
jobject cluster_;
- pthread_mutex_t count_mutex_;
+ std::mutex count_mutex_;
JavaVM *jvm_;
JNIEnv *CreateVM(JavaVM **jvm);
void Setup();
diff --git a/src/hbase/test-util/test-util.h b/include/hbase/test-util/test-util.h
similarity index 100%
rename from src/hbase/test-util/test-util.h
rename to include/hbase/test-util/test-util.h
diff --git a/include/hbase/utils/BUCK b/include/hbase/utils/BUCK
deleted file mode 100644
index d44e908..0000000
--- a/include/hbase/utils/BUCK
+++ /dev/null
@@ -1,40 +0,0 @@
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-cxx_library(
- name="utils",
- header_namespace="hbase/utils",
- exported_headers=[
- "bytes-util.h",
- "connection-util.h",
- "concurrent-map.h",
- "optional.h",
- "sys-util.h",
- "time-util.h",
- "user-util.h",
- "version.h",
- ],
- deps=[
- '//third-party:folly',
- ],
- tests=[":user-util-test"],
- linker_flags=['-L/usr/local/lib', '-lkrb5'],
- exported_linker_flags=['-L/usr/local/lib', '-lkrb5'],
- visibility=[
- 'PUBLIC',
- ],
- compiler_flags=['-Weffc++'],)
diff --git a/src/hbase/client/BUCK b/src/hbase/client/BUCK
deleted file mode 100644
index 1a8f434..0000000
--- a/src/hbase/client/BUCK
+++ /dev/null
@@ -1,301 +0,0 @@
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This is the main library.
-cxx_library(
- name="client",
- srcs=[
- "async-client-scanner.cc",
- "async-connection.cc",
- "async-rpc-retrying-caller-factory.cc",
- "async-rpc-retrying-caller.cc",
- "async-scan-rpc-retrying-caller.cc",
- "async-table-result-scanner.cc",
- "cell.cc",
- "client.cc",
- "hbase-rpc-controller.cc",
- "keyvalue-codec.cc",
- "location-cache.cc",
- "meta-utils.cc",
- "increment.cc",
- "get.cc",
- "mutation.cc",
- "put.cc",
- "delete.cc",
- "scan.cc",
- "append.cc",
- "scan-result-cache.cc",
- "raw-async-table.cc",
- "result.cc",
- "request-converter.cc",
- "response-converter.cc",
- "table.cc",
- "time-range.cc",
- "zk-util.cc",
- "multi-response.cc",
- "region-result.cc",
- "async-batch-rpc-retrying-caller.cc",
- ],
- deps=[
- "//include/hbase/client:client",
- "//src/hbase/exceptions:exceptions",
- "//src/hbase/utils:utils",
- "//src/hbase/connection:connection",
- "//src/hbase/client:conf",
- "//src/hbase/if:if",
- "//src/hbase/serde:serde",
- "//third-party:folly",
- "//third-party:wangle",
- "//third-party:zookeeper_mt",
- ],
- compiler_flags=['-Weffc++', '-ggdb'],
- visibility=[
- 'PUBLIC',
- ],)
-cxx_library(
- name="conf",
- exported_headers=[
- ],
- srcs=[
- "configuration.cc",
- "hbase-configuration-loader.cc",
- ],
- deps=["//include/hbase/client:conf", "//src/hbase/utils:utils", "//third-party:folly"],
- compiler_flags=['-Weffc++', '-ggdb'],
- visibility=[
- 'PUBLIC',
- ],)
-cxx_test(
- name="location-cache-test",
- srcs=[
- "location-cache-test.cc",
- ],
- deps=[
- ":client",
- "//src/hbase/test-util:test-util",
- ],
- run_test_separately=True,)
-cxx_test(
- name="location-cache-retry-test",
- srcs=[
- "location-cache-retry-test.cc",
- ],
- deps=[
- ":client",
- "//src/hbase/if:if",
- "//src/hbase/serde:serde",
- "//src/hbase/test-util:test-util",
- ],
- run_test_separately=True,)
-cxx_test(
- name="cell-test",
- srcs=[
- "cell-test.cc",
- ],
- deps=[
- ":client",
- ],
- run_test_separately=True,)
-cxx_test(
- name="filter-test",
- srcs=[
- "filter-test.cc",
- ],
- deps=[
- ":client",
- "//src/hbase/if:if",
- "//src/hbase/serde:serde",
- "//src/hbase/test-util:test-util",
- ],
- run_test_separately=True,)
-cxx_test(
- name="get-test",
- srcs=[
- "get-test.cc",
- ],
- deps=[
- ":client",
- ],
- run_test_separately=True,)
-cxx_test(
- name="delete-test",
- srcs=[
- "delete-test.cc",
- ],
- deps=[
- ":client",
- ],
- run_test_separately=True,)
-cxx_test(
- name="increment-test",
- srcs=[
- "increment-test.cc",
- ],
- deps=[
- ":client",
- ],
- run_test_separately=True,)
-cxx_test(
- name="put-test",
- srcs=[
- "put-test.cc",
- ],
- deps=[
- ":client",
- ],
- run_test_separately=True,)
-cxx_test(
- name="append-test",
- srcs=[
- "append-test.cc",
- ],
- deps=[
- ":client",
- ],
- run_test_separately=True,)
-cxx_test(
- name="retry-test",
- srcs=[
- "async-rpc-retrying-test.cc",
- ],
- deps=[
- ":client",
- "//src/hbase/test-util:test-util",
- "//src/hbase/exceptions:exceptions",
- ],
- run_test_separately=True,)
-cxx_test(
- name="time-range-test",
- srcs=[
- "time-range-test.cc",
- ],
- deps=[
- ":client",
- ],
- run_test_separately=True,)
-cxx_test(
- name="configuration-test",
- srcs=[
- "configuration-test.cc",
- ],
- deps=[
- ":client",
- ],
- run_test_separately=True,)
-cxx_test(
- name="hbase-configuration-test",
- srcs=[
- "hbase-configuration-test.cc",
- ],
- deps=[
- ":client",
- ],
- run_test_separately=True,)
-cxx_test(
- name="scan-test",
- srcs=[
- "scan-test.cc",
- ],
- deps=[
- ":client",
- ],
- run_test_separately=True,)
-cxx_test(
- name="result-test",
- srcs=[
- "result-test.cc",
- ],
- deps=[
- ":client",
- ],
- run_test_separately=True,)
-cxx_test(
- name="request-converter-test",
- srcs=[
- "request-converter-test.cc",
- ],
- deps=[
- ":client",
- "//src/hbase/connection:connection",
- "//src/hbase/if:if",
- ],
- run_test_separately=True,)
-cxx_test(
- name="client-test",
- srcs=[
- "client-test.cc",
- ],
- deps=[
- ":client",
- "//src/hbase/if:if",
- "//src/hbase/serde:serde",
- "//src/hbase/test-util:test-util",
- ],
- run_test_separately=True,)
-cxx_test(
- name="scan-result-cache-test",
- srcs=[
- "scan-result-cache-test.cc",
- ],
- deps=[
- ":client",
- ],
- run_test_separately=True,)
-cxx_test(
- name="scanner-test",
- srcs=[
- "scanner-test.cc",
- ],
- deps=[
- ":client",
- "//src/hbase/if:if",
- "//src/hbase/serde:serde",
- "//src/hbase/test-util:test-util",
- ],
- run_test_separately=True,)
-cxx_test(
- name="zk-util-test",
- srcs=[
- "zk-util-test.cc",
- ],
- deps=[
- ":client",
- ],
- run_test_separately=True,)
-cxx_test(
- name="multi-retry-test",
- srcs=[
- "async-batch-rpc-retrying-test.cc",
- ],
- deps=[
- ":client",
- "//src/hbase/test-util:test-util",
- "//src/hbase/exceptions:exceptions",
- ],
- run_test_separately=True,)
-cxx_binary(
- name="simple-client",
- srcs=[
- "simple-client.cc",
- ],
- deps=[":client", "//src/hbase/connection:connection"],)
-cxx_binary(
- name="load-client",
- srcs=[
- "load-client.cc",
- ],
- deps=[":client", "//src/hbase/connection:connection"],)
diff --git a/src/hbase/connection/BUCK b/src/hbase/connection/BUCK
deleted file mode 100644
index 1a856fb..0000000
--- a/src/hbase/connection/BUCK
+++ /dev/null
@@ -1,68 +0,0 @@
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This is the library dealing with a single connection
-# to a single server.
-cxx_library(
- name="connection",
- srcs=[
- "client-dispatcher.cc",
- "client-handler.cc",
- "connection-factory.cc",
- "connection-pool.cc",
- "pipeline.cc",
- "request.cc",
- "rpc-client.cc",
- "sasl-handler.cc",
- "sasl-util.cc",
- "rpc-test-server.cc",
- "rpc-test-server-handler.cc",
- "rpc-fault-injector.cc",
- ],
- deps=[
- "//include/hbase/connection:connection",
- "//src/hbase/if:if",
- "//src/hbase/utils:utils",
- "//src/hbase/serde:serde",
- "//src/hbase/security:security",
- "//third-party:folly",
- "//third-party:wangle",
- "//src/hbase/exceptions:exceptions",
- ],
- compiler_flags=['-Weffc++'],
- linker_flags=['-L/usr/local/lib', '-lsasl2', '-lkrb5'],
- exported_linker_flags=['-L/usr/local/lib', '-lsasl2', '-lkrb5'],
- visibility=[
- '//src/hbase/client/...',
- ],)
-cxx_test(
- name="connection-pool-test",
- srcs=[
- "connection-pool-test.cc",
- ],
- deps=[
- ":connection",
- ],)
-cxx_test(
- name="rpc-test",
- srcs=[
- "rpc-test.cc",
- ],
- deps=[
- ":connection",
- ],
- run_test_separately=True,)
diff --git a/src/hbase/client/load-client.cc b/src/hbase/examples/load-client.cc
similarity index 100%
rename from src/hbase/client/load-client.cc
rename to src/hbase/examples/load-client.cc
diff --git a/src/hbase/client/simple-client.cc b/src/hbase/examples/simple-client.cc
similarity index 100%
rename from src/hbase/client/simple-client.cc
rename to src/hbase/examples/simple-client.cc
diff --git a/src/hbase/exceptions/BUCK b/src/hbase/exceptions/BUCK
deleted file mode 100644
index 00ed344..0000000
--- a/src/hbase/exceptions/BUCK
+++ /dev/null
@@ -1,37 +0,0 @@
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-cxx_library(
- name="exceptions",
- srcs=[
- "exception.cc",
- ],
- deps=[
- "//include/hbase/exceptions:exceptions",
- "//third-party:folly",
- ],
- compiler_flags=['-Weffc++'],
- visibility=['//src/hbase/client/...', '//src/hbase/connection/...'],)
-cxx_test(
- name="exception-test",
- srcs=[
- "exception-test.cc",
- ],
- deps=[
- ":exceptions",
- ],
- run_test_separately=True,)
diff --git a/src/hbase/if/BUCK b/src/hbase/if/BUCK
deleted file mode 100644
index c8d51f2..0000000
--- a/src/hbase/if/BUCK
+++ /dev/null
@@ -1,49 +0,0 @@
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-PROTO_SRCS = glob(['*.proto'])
-HEADER_FILENAMES = [x.replace('.proto', '.pb.h') for x in PROTO_SRCS]
-CC_FILENAMES = [x.replace('.proto', '.pb.cc') for x in PROTO_SRCS]
-
-genrule(
- name='generate-proto-sources',
- srcs=PROTO_SRCS,
- cmd='mkdir -p $OUT && pwd && protoc --proto_path=. --cpp_out=$OUT *.proto',
- out='output', )
-
-for header_filename in HEADER_FILENAMES:
- genrule(name=header_filename,
- cmd='mkdir -p `dirname $OUT` '
- ' && cp $(location :generate-proto-sources)/{} $OUT'.format(
- header_filename),
- out=header_filename, )
-for cc_filename in CC_FILENAMES:
- genrule(
- name=cc_filename,
- cmd='mkdir -p `dirname $OUT` '
- ' && cp $(location :generate-proto-sources)/*.cc `dirname $OUT` '
- ' && cp $(location :generate-proto-sources)/*.h `dirname $OUT`'.format(
- cc_filename),
- out=cc_filename, )
-
-cxx_library(name='if',
- header_namespace="hbase/if",
- exported_headers=[':' + x for x in HEADER_FILENAMES],
- srcs=[':' + x for x in CC_FILENAMES],
- deps=['//third-party:protobuf'],
- visibility=['PUBLIC', ],
- exported_deps=['//third-party:protobuf'])
diff --git a/src/hbase/security/BUCK b/src/hbase/security/BUCK
deleted file mode 100644
index c329f30..0000000
--- a/src/hbase/security/BUCK
+++ /dev/null
@@ -1,27 +0,0 @@
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This is the library dealing with a single connection
-# to a single server.
-cxx_library(
- name="security",
- srcs=[],
- deps=["//include/hbase/security:security", "//src/hbase/client:conf"],
- compiler_flags=['-Weffc++'],
- visibility=[
- 'PUBLIC',
- ],)
diff --git a/src/hbase/serde/BUCK b/src/hbase/serde/BUCK
deleted file mode 100644
index 6b39e0b..0000000
--- a/src/hbase/serde/BUCK
+++ /dev/null
@@ -1,86 +0,0 @@
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-cxx_library(
- name="serde",
- srcs=[
- "rpc-serde.cc",
- "zk.cc",
- ],
- deps=[
- "//include/hbase/serde:serde", "//src/hbase/if:if", "//third-party:folly", "//src/hbase/utils:utils", "//src/hbase/security:security"
- ],
- tests=[
- ":client-deserializer-test",
- ":client-serializer-test",
- ":server-name-test",
- ":table-name-test",
- ":zk-deserializer-test",
- ":region-info-deserializer-test",
- ],
- compiler_flags=['-Weffc++'],
- visibility=[
- 'PUBLIC',
- ],)
-cxx_test(
- name="table-name-test",
- srcs=[
- "table-name-test.cc",
- ],
- deps=[
- ":serde",
- ],)
-cxx_test(
- name="server-name-test",
- srcs=[
- "server-name-test.cc",
- ],
- deps=[
- ":serde",
- ],)
-cxx_test(
- name="client-serializer-test",
- srcs=[
- "client-serializer-test.cc",
- ],
- deps=[
- ":serde",
- ],)
-cxx_test(
- name="client-deserializer-test",
- srcs=[
- "client-deserializer-test.cc",
- ],
- deps=[
- ":serde",
- ],)
-cxx_test(
- name="zk-deserializer-test",
- srcs=[
- "zk-deserializer-test.cc",
- ],
- deps=[
- ":serde",
- ],)
-cxx_test(
- name="region-info-deserializer-test",
- srcs=[
- "region-info-deserializer-test.cc",
- ],
- deps=[
- ":serde",
- ],)
diff --git a/src/hbase/test-util/BUCK b/src/hbase/test-util/BUCK
deleted file mode 100644
index f1aedab..0000000
--- a/src/hbase/test-util/BUCK
+++ /dev/null
@@ -1,53 +0,0 @@
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import os
-
-cxx_library(
- name="test-util",
- header_namespace="hbase/test-util",
- exported_headers=["test-util.h", "mini-cluster.h"],
- srcs=["test-util.cc", "mini-cluster.cc"],
- deps=[
- "//third-party:folly",
- "//src/hbase/client:client",
- ],
- preprocessor_flags=[
- '-I' + os.environ['JAVA_HOME'] + '/include',
- '-I' + os.environ['JAVA_HOME'] + '/include/darwin',
- '-I' + os.environ['JAVA_HOME'] + '/include/linux'
- ],
- exported_preprocessor_flags=[
- '-I' + os.environ['JAVA_HOME'] + '/include',
- '-I' + os.environ['JAVA_HOME'] + '/include/darwin',
- '-I' + os.environ['JAVA_HOME'] + '/include/linux'
- ],
- compiler_flags=[
- '-I' + os.environ['JAVA_HOME'] + '/include',
- '-I' + os.environ['JAVA_HOME'] + '/include/darwin',
- '-I' + os.environ['JAVA_HOME'] + '/include/linux', '-ggdb'
- ],
- linker_flags=[
- '-ljvm', '-L' + os.environ['JAVA_HOME'] + '/jre/lib/amd64/server',
- '-ggdb'
- ],
- exported_linker_flags=[
- '-ljvm', '-L' + os.environ['JAVA_HOME'] + '/jre/lib/amd64/server',
- '-Wl,-rpath=' + os.environ['JAVA_HOME'] + '/jre/lib/amd64/server'
- ],
- visibility=[
- 'PUBLIC',
- ],)
diff --git a/src/hbase/test-util/mini-cluster.cc b/src/hbase/test-util/mini-cluster.cc
index 1e491a2..98a4e7e 100644
--- a/src/hbase/test-util/mini-cluster.cc
+++ b/src/hbase/test-util/mini-cluster.cc
@@ -80,7 +80,7 @@ MiniCluster::~MiniCluster() {
void MiniCluster::Setup() {
jmethodID constructor;
- pthread_mutex_lock(&count_mutex_);
+ std::lock_guard<std::mutex> lock(count_mutex_);
if (env_ == NULL) {
env_ = CreateVM(&jvm_);
if (env_ == NULL) {
@@ -176,7 +176,6 @@ void MiniCluster::Setup() {
exit(-1);
}
}
- pthread_mutex_unlock(&count_mutex_);
}
jobject MiniCluster::htu() {
diff --git a/src/hbase/utils/BUCK b/src/hbase/utils/BUCK
deleted file mode 100644
index ab55d8f..0000000
--- a/src/hbase/utils/BUCK
+++ /dev/null
@@ -1,57 +0,0 @@
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-cxx_library(
- name="utils",
- exported_headers=[
- ],
- srcs=["bytes-util.cc", "connection-util.cc", "user-util.cc"],
- deps=[
- '//include/hbase/utils:utils',
- '//third-party:folly',
- ],
- tests=[":user-util-test"],
- linker_flags=['-L/usr/local/lib', '-lkrb5'],
- exported_linker_flags=['-L/usr/local/lib', '-lkrb5'],
- visibility=[
- 'PUBLIC',
- ],
- compiler_flags=['-Weffc++'],)
-cxx_test(
- name="bytes-util-test",
- srcs=[
- "bytes-util-test.cc",
- ],
- deps=[
- ":utils",
- ],)
-cxx_test(
- name="concurrent-map-test",
- srcs=[
- "concurrent-map-test.cc",
- ],
- deps=[
- ":utils",
- ],)
-cxx_test(
- name="user-util-test",
- srcs=[
- "user-util-test.cc",
- ],
- deps=[
- ":utils",
- ],)
diff --git a/src/hbase/utils/bytes-util.cc b/src/hbase/utils/bytes-util.cc
index 144b866..5b34aa0 100644
--- a/src/hbase/utils/bytes-util.cc
+++ b/src/hbase/utils/bytes-util.cc
@@ -19,7 +19,6 @@
#include "hbase/utils/bytes-util.h"
-#include <bits/stdc++.h>
#include <boost/predef.h>
#include <glog/logging.h>
diff --git a/src/hbase/client/append-test.cc b/src/test/append-test.cc
similarity index 100%
rename from src/hbase/client/append-test.cc
rename to src/test/append-test.cc
diff --git a/src/hbase/client/async-batch-rpc-retrying-test.cc b/src/test/async-batch-rpc-retrying-test.cc
similarity index 100%
rename from src/hbase/client/async-batch-rpc-retrying-test.cc
rename to src/test/async-batch-rpc-retrying-test.cc
diff --git a/src/hbase/client/async-rpc-retrying-test.cc b/src/test/async-rpc-retrying-test.cc
similarity index 99%
rename from src/hbase/client/async-rpc-retrying-test.cc
rename to src/test/async-rpc-retrying-test.cc
index 6782d05..b590b43 100644
--- a/src/hbase/client/async-rpc-retrying-test.cc
+++ b/src/test/async-rpc-retrying-test.cc
@@ -23,7 +23,6 @@
#include <folly/io/async/EventBase.h>
#include <folly/io/async/ScopedEventBaseThread.h>
#include <gmock/gmock.h>
-#include <google/protobuf/stubs/callback.h>
#include <wangle/concurrent/IOThreadPoolExecutor.h>
#include <chrono>
diff --git a/src/hbase/utils/bytes-util-test.cc b/src/test/bytes-util-test.cc
similarity index 100%
rename from src/hbase/utils/bytes-util-test.cc
rename to src/test/bytes-util-test.cc
diff --git a/src/hbase/client/cell-test.cc b/src/test/cell-test.cc
similarity index 100%
rename from src/hbase/client/cell-test.cc
rename to src/test/cell-test.cc
diff --git a/src/hbase/serde/client-deserializer-test.cc b/src/test/client-deserializer-test.cc
similarity index 100%
rename from src/hbase/serde/client-deserializer-test.cc
rename to src/test/client-deserializer-test.cc
diff --git a/src/hbase/serde/client-serializer-test.cc b/src/test/client-serializer-test.cc
similarity index 100%
rename from src/hbase/serde/client-serializer-test.cc
rename to src/test/client-serializer-test.cc
diff --git a/src/hbase/client/client-test.cc b/src/test/client-test.cc
similarity index 99%
rename from src/hbase/client/client-test.cc
rename to src/test/client-test.cc
index 0773d3d..b1d5180 100644
--- a/src/hbase/client/client-test.cc
+++ b/src/test/client-test.cc
@@ -19,6 +19,7 @@
#include <gtest/gtest.h>
+#include <fstream>
#include "hbase/client/append.h"
#include "hbase/client/cell.h"
#include "hbase/client/client.h"
diff --git a/src/hbase/utils/concurrent-map-test.cc b/src/test/concurrent-map-test.cc
similarity index 100%
rename from src/hbase/utils/concurrent-map-test.cc
rename to src/test/concurrent-map-test.cc
diff --git a/src/hbase/client/configuration-test.cc b/src/test/configuration-test.cc
similarity index 100%
rename from src/hbase/client/configuration-test.cc
rename to src/test/configuration-test.cc
diff --git a/src/hbase/connection/connection-pool-test.cc b/src/test/connection-pool-test.cc
similarity index 100%
rename from src/hbase/connection/connection-pool-test.cc
rename to src/test/connection-pool-test.cc
diff --git a/src/hbase/client/delete-test.cc b/src/test/delete-test.cc
similarity index 100%
rename from src/hbase/client/delete-test.cc
rename to src/test/delete-test.cc
diff --git a/src/hbase/exceptions/exception-test.cc b/src/test/exception-test.cc
similarity index 100%
rename from src/hbase/exceptions/exception-test.cc
rename to src/test/exception-test.cc
diff --git a/src/hbase/client/filter-test.cc b/src/test/filter-test.cc
similarity index 100%
rename from src/hbase/client/filter-test.cc
rename to src/test/filter-test.cc
diff --git a/src/hbase/client/get-test.cc b/src/test/get-test.cc
similarity index 100%
rename from src/hbase/client/get-test.cc
rename to src/test/get-test.cc
diff --git a/src/hbase/client/hbase-configuration-test.cc b/src/test/hbase-configuration-test.cc
similarity index 100%
rename from src/hbase/client/hbase-configuration-test.cc
rename to src/test/hbase-configuration-test.cc
diff --git a/src/hbase/client/increment-test.cc b/src/test/increment-test.cc
similarity index 100%
rename from src/hbase/client/increment-test.cc
rename to src/test/increment-test.cc
diff --git a/src/hbase/client/location-cache-retry-test.cc b/src/test/location-cache-retry-test.cc
similarity index 100%
rename from src/hbase/client/location-cache-retry-test.cc
rename to src/test/location-cache-retry-test.cc
diff --git a/src/hbase/client/location-cache-test.cc b/src/test/location-cache-test.cc
similarity index 100%
rename from src/hbase/client/location-cache-test.cc
rename to src/test/location-cache-test.cc
diff --git a/src/hbase/client/put-test.cc b/src/test/put-test.cc
similarity index 100%
rename from src/hbase/client/put-test.cc
rename to src/test/put-test.cc
diff --git a/src/hbase/serde/region-info-deserializer-test.cc b/src/test/region-info-deserializer-test.cc
similarity index 100%
rename from src/hbase/serde/region-info-deserializer-test.cc
rename to src/test/region-info-deserializer-test.cc
diff --git a/src/hbase/client/request-converter-test.cc b/src/test/request-converter-test.cc
similarity index 100%
rename from src/hbase/client/request-converter-test.cc
rename to src/test/request-converter-test.cc
diff --git a/src/hbase/client/result-test.cc b/src/test/result-test.cc
similarity index 100%
rename from src/hbase/client/result-test.cc
rename to src/test/result-test.cc
diff --git a/src/hbase/connection/rpc-test.cc b/src/test/rpc-test.cc
similarity index 100%
rename from src/hbase/connection/rpc-test.cc
rename to src/test/rpc-test.cc
diff --git a/src/hbase/client/scan-result-cache-test.cc b/src/test/scan-result-cache-test.cc
similarity index 100%
rename from src/hbase/client/scan-result-cache-test.cc
rename to src/test/scan-result-cache-test.cc
diff --git a/src/hbase/client/scan-test.cc b/src/test/scan-test.cc
similarity index 100%
rename from src/hbase/client/scan-test.cc
rename to src/test/scan-test.cc
diff --git a/src/hbase/client/scanner-test.cc b/src/test/scanner-test.cc
similarity index 100%
rename from src/hbase/client/scanner-test.cc
rename to src/test/scanner-test.cc
diff --git a/src/hbase/serde/server-name-test.cc b/src/test/server-name-test.cc
similarity index 100%
rename from src/hbase/serde/server-name-test.cc
rename to src/test/server-name-test.cc
diff --git a/src/hbase/serde/table-name-test.cc b/src/test/table-name-test.cc
similarity index 100%
rename from src/hbase/serde/table-name-test.cc
rename to src/test/table-name-test.cc
diff --git a/src/hbase/client/time-range-test.cc b/src/test/time-range-test.cc
similarity index 100%
rename from src/hbase/client/time-range-test.cc
rename to src/test/time-range-test.cc
diff --git a/src/hbase/utils/user-util-test.cc b/src/test/user-util-test.cc
similarity index 100%
rename from src/hbase/utils/user-util-test.cc
rename to src/test/user-util-test.cc
diff --git a/src/hbase/serde/zk-deserializer-test.cc b/src/test/zk-deserializer-test.cc
similarity index 100%
rename from src/hbase/serde/zk-deserializer-test.cc
rename to src/test/zk-deserializer-test.cc
diff --git a/src/hbase/client/zk-util-test.cc b/src/test/zk-util-test.cc
similarity index 100%
rename from src/hbase/client/zk-util-test.cc
rename to src/test/zk-util-test.cc
diff --git a/third-party/BUCK b/third-party/BUCK
deleted file mode 100644
index 418323b..0000000
--- a/third-party/BUCK
+++ /dev/null
@@ -1,118 +0,0 @@
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-def add_system_libs(names=[],
- lib_dir="/usr/lib/x86_64-linux-gnu",
- deps=[],
- exported_deps=[],
- exported_linker_flags=[]):
- rules = []
- for name in names:
- rule_visibility = ['PUBLIC']
- gen_rule_name = "gen_lib{}".format(name)
- genrule(
- name=gen_rule_name,
- out=gen_rule_name,
- bash="mkdir -p $OUT && cp {}/lib{}.a $OUT".format(lib_dir, name),)
- prebuilt_cxx_library(
- name=name,
- lib_name=name,
- lib_dir='$(location :{})'.format(gen_rule_name),
- deps=deps,
- force_static=True,
- exported_deps=exported_deps,
- visibility=rule_visibility,)
- rules.append(":" + name)
- return rules
-
-
-def add_dynamic_libs(names=[]):
- rules = []
- for name in names:
- prebuilt_cxx_library(
- name=name,
- header_only=True,
- exported_linker_flags=["-l" + name],
- visibility=["PUBLIC"],)
- rules.append(":" + name)
- return rules
-
-
-system_libs = [
- "lzma",
- "event",
-]
-local_libs = [
- "double-conversion",
- "boost_regex",
- "boost_context",
- "boost_thread",
- "boost_system",
- "boost_filesystem",
- "boost_program_options",
- "boost_chrono",
- "gflags",
- "glog",
- "protobuf",
-]
-dynamic_libs = ["stdc++", "pthread", "ssl", "crypto", "dl", "atomic", "unwind"]
-dynamic_rules = add_dynamic_libs(dynamic_libs)
-tp_dep_rules = add_system_libs(system_libs,) \
- + add_system_libs(local_libs, lib_dir = "/usr/local/lib") \
- + dynamic_rules
-
-zookeeper = add_system_libs(["zookeeper_mt"], lib_dir="/usr/local/lib")
-folly = add_system_libs(
- ['folly'],
- lib_dir='/usr/local/lib',
- exported_deps=tp_dep_rules,)
-folly_bench = add_system_libs(
- ['follybenchmark'],
- lib_dir='/usr/local/lib',
- exported_deps=folly + tp_dep_rules,)
-wangle = add_system_libs(
- ['wangle'], lib_dir='/usr/local/lib', exported_deps=folly + tp_dep_rules)
-
-genrule(
- name="gen_zk",
- out="gen_zk",
- bash=
- "mkdir -p $OUT && wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && tar zxf zookeeper-3.4.8.tar.gz && rm -rf zookeeper-3.4.8.tar.gz && cd zookeeper-3.4.8 && cd src/c && ./configure --prefix=$OUT && make && make install && cd $OUT && rm -rf zookeeper-3.4.8*"
-)
-cxx_library(
- name='google-test',
- srcs=[
- 'googletest/googletest/src/gtest-all.cc',
- 'googletest/googlemock/src/gmock-all.cc',
- 'googletest/googlemock/src/gmock_main.cc',
- ],
- header_namespace='',
- exported_headers=subdir_glob([
- ('googletest/googletest/include', '**/*.h'),
- ('googletest/googlemock/include', '**/*.h'),
- ]),
- headers=subdir_glob([
- ('googletest/googletest', 'src/*.h'),
- ('googletest/googletest', 'src/*.cc'),
- ('googletest/googlemock', 'src/*.h'),
- ('googletest/googlemock', 'src/*.cc'),
- ]),
- exported_deps=dynamic_rules,
- visibility=[
- 'PUBLIC',
- ],)
[hbase] 26/137: HBASE-15766 Show working puts
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit f8cb53bbeeeea9bc09409404c3ade30b65caa78d
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Wed May 4 01:54:21 2016 -0700
HBASE-15766 Show working puts
Summary:
Add on showing how a set of puts to a single connection will work.
This still needs retries and looking up what region each request is going to
Test Plan:
./buck-out/gen/core/simple-client -columns 100
../bin/hbase shell
count 't'
100 row(s) in 0.2470 seconds
Differential Revision: https://reviews.facebook.net/D57603
---
connection/client-dispatcher.cc | 9 ++--
connection/client-dispatcher.h | 11 ++++-
connection/client-handler.cc | 13 +++---
connection/client-handler.h | 4 +-
connection/connection-factory.cc | 46 ++++++++++----------
connection/connection-factory.h | 11 +++--
connection/connection-pool-test.cc | 54 ++++++++++++++++++++----
connection/connection-pool.cc | 33 +++++++++++----
connection/connection-pool.h | 10 ++++-
core/client.cc | 6 +--
core/location-cache.cc | 25 +++++------
core/location-cache.h | 6 +--
core/meta-utils.cc | 6 +--
core/meta-utils.h | 4 +-
core/region-location.h | 8 +++-
core/simple-client.cc | 76 ++++++++++++++++++++++++++++++----
serde/region-info-deserializer-test.cc | 1 -
serde/region-info.h | 6 +--
18 files changed, 231 insertions(+), 98 deletions(-)
diff --git a/connection/client-dispatcher.cc b/connection/client-dispatcher.cc
index 6e2dc54..655d765 100644
--- a/connection/client-dispatcher.cc
+++ b/connection/client-dispatcher.cc
@@ -22,10 +22,11 @@ using namespace folly;
using namespace hbase;
using namespace wangle;
-ClientDispatcher::ClientDispatcher() : requests_(), current_call_id_(9) {}
+ClientDispatcher::ClientDispatcher() : requests_(5000), current_call_id_(9) {}
void ClientDispatcher::read(Context *ctx, Response in) {
auto call_id = in.call_id();
+
auto search = requests_.find(call_id);
CHECK(search != requests_.end());
auto p = std::move(search->second);
@@ -38,10 +39,10 @@ void ClientDispatcher::read(Context *ctx, Response in) {
}
Future<Response> ClientDispatcher::operator()(std::unique_ptr<Request> arg) {
- auto call_id = ++current_call_id_;
-
+ auto call_id = current_call_id_++;
arg->set_call_id(call_id);
- auto &p = requests_[call_id];
+ requests_.insert(call_id, Promise<Response>{});
+ auto &p = requests_.find(call_id)->second;
auto f = p.getFuture();
p.setInterruptHandler([call_id, this](const folly::exception_wrapper &e) {
LOG(ERROR) << "e = " << call_id;
diff --git a/connection/client-dispatcher.h b/connection/client-dispatcher.h
index 826fc6a..4435a1b 100644
--- a/connection/client-dispatcher.h
+++ b/connection/client-dispatcher.h
@@ -19,8 +19,12 @@
#pragma once
+#include <folly/AtomicHashMap.h>
+#include <folly/Logging.h>
#include <wangle/service/ClientDispatcher.h>
+#include <atomic>
+
#include "connection/pipeline.h"
#include "connection/request.h"
#include "connection/response.h"
@@ -31,13 +35,16 @@ class ClientDispatcher
std::unique_ptr<Request>, Response> {
public:
ClientDispatcher();
+ ~ClientDispatcher() {
+ LOG(ERROR) << "Killing ClientDispatcher call_id = " << current_call_id_;
+ }
void read(Context *ctx, Response in) override;
folly::Future<Response> operator()(std::unique_ptr<Request> arg) override;
folly::Future<folly::Unit> close(Context *ctx) override;
folly::Future<folly::Unit> close() override;
private:
- std::unordered_map<uint32_t, folly::Promise<Response>> requests_;
+ folly::AtomicHashMap<uint32_t, folly::Promise<Response>> requests_;
// Start at some number way above what could
// be there for un-initialized call id counters.
//
@@ -46,6 +53,6 @@ private:
//
// uint32_t has a max of 4Billion so 10 more or less is
// not a big deal.
- uint32_t current_call_id_;
+ std::atomic<uint32_t> current_call_id_;
};
} // namespace hbase
diff --git a/connection/client-handler.cc b/connection/client-handler.cc
index 496e4f2..b92ad89 100644
--- a/connection/client-handler.cc
+++ b/connection/client-handler.cc
@@ -37,7 +37,10 @@ using hbase::pb::GetResponse;
using google::protobuf::Message;
ClientHandler::ClientHandler(std::string user_name)
- : user_name_(user_name), need_send_header_(true), serde_(), resp_msgs_() {}
+ : user_name_(user_name), need_send_header_(true), serde_(),
+ resp_msgs_(
+ make_unique<folly::AtomicHashMap<
+ uint32_t, std::shared_ptr<google::protobuf::Message>>>(5000)) {}
void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
if (LIKELY(buf != nullptr)) {
@@ -51,14 +54,14 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
<< " has_exception=" << header.has_exception();
// Get the response protobuf from the map
- auto search = resp_msgs_.find(header.call_id());
+ auto search = resp_msgs_->find(header.call_id());
// It's an error if it's not there.
- CHECK(search != resp_msgs_.end());
+ CHECK(search != resp_msgs_->end());
auto resp_msg = search->second;
CHECK(resp_msg != nullptr);
// Make sure we don't leak the protobuf
- resp_msgs_.erase(search);
+ resp_msgs_->erase(header.call_id());
// set the call_id.
// This will be used to by the dispatcher to match up
@@ -96,7 +99,7 @@ Future<Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
ctx->fireWrite(std::move(pre));
}
- resp_msgs_[r->call_id()] = r->resp_msg();
+ resp_msgs_->insert(r->call_id(), r->resp_msg());
return ctx->fireWrite(
serde_.Request(r->call_id(), r->method(), r->req_msg().get()));
}
diff --git a/connection/client-handler.h b/connection/client-handler.h
index ce99c9e..be5143c 100644
--- a/connection/client-handler.h
+++ b/connection/client-handler.h
@@ -18,6 +18,7 @@
*/
#pragma once
+#include <folly/AtomicHashMap.h>
#include <wangle/channel/Handler.h>
#include <string>
@@ -51,7 +52,8 @@ private:
RpcSerde serde_;
// in flight requests
- std::unordered_map<uint32_t, std::shared_ptr<google::protobuf::Message>>
+ std::unique_ptr<folly::AtomicHashMap<
+ uint32_t, std::shared_ptr<google::protobuf::Message>>>
resp_msgs_;
};
} // namespace hbase
diff --git a/connection/connection-factory.cc b/connection/connection-factory.cc
index 9102d60..635d12d 100644
--- a/connection/connection-factory.cc
+++ b/connection/connection-factory.cc
@@ -19,40 +19,36 @@
#include "connection/connection-factory.h"
-#include <folly/futures/Future.h>
-#include <wangle/bootstrap/ClientBootstrap.h>
-#include <wangle/channel/AsyncSocketHandler.h>
-#include <wangle/channel/EventBaseHandler.h>
-#include <wangle/channel/OutputBufferingHandler.h>
-#include <wangle/service/ClientDispatcher.h>
-#include <wangle/service/CloseOnReleaseFilter.h>
-#include <wangle/service/ExpiringFilter.h>
-
-#include <string>
+#include <wangle/concurrent/GlobalExecutor.h>
#include "connection/client-dispatcher.h"
#include "connection/pipeline.h"
-#include "connection/request.h"
-#include "connection/response.h"
#include "connection/service.h"
using namespace folly;
using namespace hbase;
-using namespace wangle;
-ConnectionFactory::ConnectionFactory() : bootstrap_() {
- bootstrap_.group(std::make_shared<wangle::IOThreadPoolExecutor>(1));
- bootstrap_.pipelineFactory(std::make_shared<RpcPipelineFactory>());
-}
+ConnectionFactory::ConnectionFactory()
+ : io_pool_(std::static_pointer_cast<wangle::IOThreadPoolExecutor>(
+ wangle::getIOExecutor())),
+ pipeline_factory_(std::make_shared<RpcPipelineFactory>()) {}
-std::shared_ptr<HBaseService>
-ConnectionFactory::make_connection(const std::string &host, int port) {
- // Connect to a given server
- // Then when connected create a ClientDispactcher.
- auto pipeline = bootstrap_.connect(SocketAddress(host, port, true)).get();
+std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>
+ConnectionFactory::MakeBootstrap() {
+ auto client = std::make_shared<wangle::ClientBootstrap<SerializePipeline>>();
+ client->group(io_pool_);
+ client->pipelineFactory(pipeline_factory_);
+
+ return client;
+}
+std::shared_ptr<HBaseService> ConnectionFactory::Connect(
+ std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> client,
+ const std::string &hostname, int port) {
+ // Yes this will block however it makes dealing with connection pool soooooo
+ // much nicer.
+ // TODO see about using shared promise for this.
+ auto pipeline = client->connect(SocketAddress(hostname, port, true)).get();
auto dispatcher = std::make_shared<ClientDispatcher>();
dispatcher->setPipeline(pipeline);
- auto service = std::make_shared<
- CloseOnReleaseFilter<std::unique_ptr<Request>, Response>>(dispatcher);
- return service;
+ return dispatcher;
}
diff --git a/connection/connection-factory.h b/connection/connection-factory.h
index fc4e161..2284a7c 100644
--- a/connection/connection-factory.h
+++ b/connection/connection-factory.h
@@ -33,10 +33,15 @@ public:
ConnectionFactory();
virtual ~ConnectionFactory() = default;
- virtual std::shared_ptr<HBaseService> make_connection(const std::string &host,
- int port);
+ virtual std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>
+ MakeBootstrap();
+
+ virtual std::shared_ptr<HBaseService>
+ Connect(std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> client,
+ const std::string &hostname, int port);
private:
- wangle::ClientBootstrap<SerializePipeline> bootstrap_;
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool_;
+ std::shared_ptr<RpcPipelineFactory> pipeline_factory_;
};
} // namespace hbase
diff --git a/connection/connection-pool-test.cc b/connection/connection-pool-test.cc
index 975bc5e..b1a0ba0 100644
--- a/connection/connection-pool-test.cc
+++ b/connection/connection-pool-test.cc
@@ -24,6 +24,7 @@
#include "connection/connection-factory.h"
#include "if/HBase.pb.h"
+#include "serde/server-name.h"
using namespace hbase;
@@ -33,11 +34,16 @@ using ::testing::_;
class MockConnectionFactory : public ConnectionFactory {
public:
- MOCK_METHOD2(make_connection,
- std::shared_ptr<HBaseService>(const std::string &hostname,
- int port));
+ MOCK_METHOD0(MakeBootstrap,
+ std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>());
+ MOCK_METHOD3(Connect,
+ std::shared_ptr<HBaseService>(
+ std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>,
+ const std::string &hostname, int port));
};
+class MockBootstrap : public wangle::ClientBootstrap<SerializePipeline> {};
+
class MockServiceBase : public HBaseService {
public:
folly::Future<Response> operator()(std::unique_ptr<Request> req) override {
@@ -54,19 +60,20 @@ public:
};
TEST(TestConnectionPool, TestOnlyCreateOnce) {
- std::string hostname{"hostname"};
+ auto hostname = std::string{"hostname"};
+ auto mock_boot = std::make_shared<MockBootstrap>();
auto mock_service = std::make_shared<MockService>();
+ auto mock_cf = std::make_shared<MockConnectionFactory>();
uint32_t port{999};
- LOG(ERROR) << "About to make a MockConnectionFactory";
- auto mock_cf = std::make_shared<MockConnectionFactory>();
- EXPECT_CALL((*mock_cf), make_connection(_, _))
+ EXPECT_CALL((*mock_cf), Connect(_, _, _))
.Times(1)
.WillRepeatedly(Return(mock_service));
+ EXPECT_CALL((*mock_cf), MakeBootstrap())
+ .Times(1)
+ .WillRepeatedly(Return(mock_boot));
ConnectionPool cp{mock_cf};
- LOG(ERROR) << "Created ConnectionPool";
-
ServerName sn;
sn.set_host_name(hostname);
sn.set_port(port);
@@ -75,3 +82,32 @@ TEST(TestConnectionPool, TestOnlyCreateOnce) {
ASSERT_TRUE(result != nullptr);
result = cp.get(sn);
}
+
+TEST(TestConnectionPool, TestOnlyCreateMultipleDispose) {
+ std::string hostname_one{"hostname"};
+ std::string hostname_two{"hostname_two"};
+ uint32_t port{999};
+
+ auto mock_boot = std::make_shared<MockBootstrap>();
+ auto mock_service = std::make_shared<MockService>();
+ auto mock_cf = std::make_shared<MockConnectionFactory>();
+
+ EXPECT_CALL((*mock_cf), Connect(_, _, _))
+ .Times(2)
+ .WillRepeatedly(Return(mock_service));
+ EXPECT_CALL((*mock_cf), MakeBootstrap())
+ .Times(2)
+ .WillRepeatedly(Return(mock_boot));
+ ConnectionPool cp{mock_cf};
+
+ {
+ auto result_one = cp.get(folly::to<ServerName>(
+ hostname_one + ":" + folly::to<std::string>(port)));
+ auto result_two = cp.get(folly::to<ServerName>(
+ hostname_two + ":" + folly::to<std::string>(port)));
+ }
+ auto result_one = cp.get(
+ folly::to<ServerName>(hostname_one + ":" + folly::to<std::string>(port)));
+ auto result_two = cp.get(
+ folly::to<ServerName>(hostname_two + ":" + folly::to<std::string>(port)));
+}
diff --git a/connection/connection-pool.cc b/connection/connection-pool.cc
index eafe60a..6ed5ad9 100644
--- a/connection/connection-pool.cc
+++ b/connection/connection-pool.cc
@@ -19,6 +19,7 @@
#include "connection/connection-pool.h"
+#include <folly/SocketAddress.h>
#include <wangle/service/Service.h>
using std::mutex;
@@ -26,28 +27,46 @@ using std::unique_ptr;
using std::shared_ptr;
using hbase::pb::ServerName;
using folly::SharedMutexWritePriority;
+using folly::SocketAddress;
namespace hbase {
ConnectionPool::ConnectionPool()
- : cf_(std::make_shared<ConnectionFactory>()), connections_(), map_mutex_() {
-}
+ : cf_(std::make_shared<ConnectionFactory>()), clients_(), connections_(),
+ map_mutex_() {}
ConnectionPool::ConnectionPool(std::shared_ptr<ConnectionFactory> cf)
- : cf_(cf), connections_(), map_mutex_() {}
+ : cf_(cf), clients_(), connections_(), map_mutex_() {}
+
+ConnectionPool::~ConnectionPool() {
+ SharedMutexWritePriority::WriteHolder holder(map_mutex_);
+ for (auto &item : connections_) {
+ auto &con = item.second;
+ con->close();
+ }
+ connections_.clear();
+ clients_.clear();
+}
std::shared_ptr<HBaseService> ConnectionPool::get(const ServerName &sn) {
+ // Create a read lock.
SharedMutexWritePriority::UpgradeHolder holder(map_mutex_);
+
auto found = connections_.find(sn);
if (found == connections_.end() || found->second == nullptr) {
+ // Move the upgradable lock into the write lock if the connection
+ // hasn't been found.
SharedMutexWritePriority::WriteHolder holder(std::move(holder));
- auto new_con = cf_->make_connection(sn.host_name(), sn.port());
- connections_[sn] = new_con;
- return new_con;
+ auto client = cf_->MakeBootstrap();
+ auto dispatcher = cf_->Connect(client, sn.host_name(), sn.port());
+ clients_.insert(std::make_pair(sn, client));
+ connections_.insert(std::make_pair(sn, dispatcher));
+ return dispatcher;
}
return found->second;
}
+
void ConnectionPool::close(const ServerName &sn) {
- SharedMutexWritePriority::WriteHolder holder(map_mutex_);
+ SharedMutexWritePriority::WriteHolder holder{map_mutex_};
auto found = connections_.find(sn);
if (found == connections_.end() || found->second == nullptr) {
diff --git a/connection/connection-pool.h b/connection/connection-pool.h
index b8330e3..907afdb 100644
--- a/connection/connection-pool.h
+++ b/connection/connection-pool.h
@@ -37,23 +37,29 @@ struct ServerNameHash {
std::size_t operator()(hbase::pb::ServerName const &s) const {
std::size_t h1 = std::hash<std::string>()(s.host_name());
std::size_t h2 = std::hash<uint32_t>()(s.port());
- return h1 ^ (h2 << 1);
+ return h1 ^ (h2 << 2);
}
};
class ConnectionPool {
public:
ConnectionPool();
+ ~ConnectionPool();
explicit ConnectionPool(std::shared_ptr<ConnectionFactory> cf);
std::shared_ptr<HBaseService> get(const hbase::pb::ServerName &sn);
void close(const hbase::pb::ServerName &sn);
private:
- std::shared_ptr<ConnectionFactory> cf_;
std::unordered_map<hbase::pb::ServerName, std::shared_ptr<HBaseService>,
ServerNameHash, ServerNameEquals>
connections_;
+ std::unordered_map<
+ hbase::pb::ServerName,
+ std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>,
+ ServerNameHash, ServerNameEquals>
+ clients_;
folly::SharedMutexWritePriority map_mutex_;
+ std::shared_ptr<ConnectionFactory> cf_;
};
} // namespace hbase
diff --git a/core/client.cc b/core/client.cc
index 266c239..4b9f844 100644
--- a/core/client.cc
+++ b/core/client.cc
@@ -33,8 +33,4 @@ using namespace folly;
using namespace std;
using namespace hbase::pb;
-namespace hbase {
-
-Client::Client(string quorum_spec)
- : location_cache_(quorum_spec, wangle::getCPUExecutor()) {}
-}
+namespace hbase {}
diff --git a/core/location-cache.cc b/core/location-cache.cc
index 2667f11..e2a6251 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -25,8 +25,8 @@
#include "connection/response.h"
#include "if/Client.pb.h"
#include "if/ZooKeeper.pb.h"
-#include "serde/server-name.h"
#include "serde/region-info.h"
+#include "serde/server-name.h"
#include "serde/zk.h"
using namespace std;
@@ -109,17 +109,22 @@ ServerName LocationCache::ReadMetaLocation() {
Future<std::shared_ptr<RegionLocation>>
LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
- auto exc = wangle::getIOExecutor();
+ auto exec = wangle::getCPUExecutor();
return this->LocateMeta()
- .then([&](ServerName sn) { return this->cp_.get(sn); })
- .via(exc.get()) // Need to handle all rpc's on the IOExecutor.
+ .via(exec.get())
+ .then([ exec = exec, this ](ServerName sn) { return this->cp_.get(sn); })
.then([&](std::shared_ptr<HBaseService> service) {
return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
})
- .then([&](Response resp) {
+ .then([this](Response resp) {
// take the protobuf response and make it into
// a region location.
return this->CreateLocation(std::move(resp));
+ })
+ .then([ exec = exec, this ](std::shared_ptr<RegionLocation> rl) {
+ // Now fill out the connection.
+ rl->set_service(cp_.get(rl->server_name()));
+ return rl;
});
}
@@ -162,16 +167,12 @@ private:
};
std::shared_ptr<RegionLocation>
-LocationCache::CreateLocation(const Response &resp){
+LocationCache::CreateLocation(const Response &resp) {
auto resp_msg = static_pointer_cast<ScanResponse>(resp.response());
auto &results = resp_msg->results().Get(0);
auto &cells = results.cell();
- LOG(ERROR) << "resp_msg = " << resp_msg->DebugString();
auto ri = folly::to<RegionInfo>(cells.Get(0).value());
auto sn = folly::to<ServerName>(cells.Get(1).value());
-
- LOG(ERROR) << "RegionInfo = " << ri.DebugString();
- LOG(ERROR) << "ServerName = " << sn.DebugString();
- auto wrapped = make_shared<RemoveServiceFilter>(cp_.get(sn), sn, this->cp_);
- return std::make_shared<RegionLocation>(std::move(ri), std::move(sn), wrapped);
+ return std::make_shared<RegionLocation>(cells.Get(0).row(), std::move(ri), sn,
+ nullptr);
}
diff --git a/core/location-cache.h b/core/location-cache.h
index 99b5e5e..7f76428 100644
--- a/core/location-cache.h
+++ b/core/location-cache.h
@@ -48,9 +48,10 @@ public:
// Meta Related Methods.
// These are only public until testing is complete
folly::Future<hbase::pb::ServerName> LocateMeta();
- folly::Future<std::shared_ptr<RegionLocation>> LocateFromMeta(const hbase::pb::TableName &tn,
- const std::string &row);
+ folly::Future<std::shared_ptr<RegionLocation>>
+ LocateFromMeta(const hbase::pb::TableName &tn, const std::string &row);
void InvalidateMeta();
+ ConnectionPool cp_;
private:
void RefreshMetaLocation();
@@ -61,7 +62,6 @@ private:
std::shared_ptr<folly::Executor> executor_;
std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> meta_promise_;
std::mutex meta_lock_;
- ConnectionPool cp_;
MetaUtil meta_util_;
// TODO: migrate this to a smart pointer with a deleter.
diff --git a/core/meta-utils.cc b/core/meta-utils.cc
index 1325d83..23d2041 100644
--- a/core/meta-utils.cc
+++ b/core/meta-utils.cc
@@ -37,12 +37,12 @@ using hbase::pb::RegionSpecifier_RegionSpecifierType;
static const std::string META_REGION = "1588230740";
std::string MetaUtil::RegionLookupRowkey(const TableName &tn,
- const std::string &row) const {
+ const std::string &row) const {
return folly::to<std::string>(tn, ",", row, ",", "999999999999999999");
}
-std::unique_ptr<Request>
-MetaUtil::MetaRequest(const TableName tn, const std::string &row) const {
+std::unique_ptr<Request> MetaUtil::MetaRequest(const TableName tn,
+ const std::string &row) const {
auto request = Request::scan();
auto msg = std::static_pointer_cast<ScanRequest>(request->req_msg());
diff --git a/core/meta-utils.h b/core/meta-utils.h
index 5a659f3..dfef065 100644
--- a/core/meta-utils.h
+++ b/core/meta-utils.h
@@ -29,8 +29,8 @@ namespace hbase {
class MetaUtil {
public:
std::string RegionLookupRowkey(const hbase::pb::TableName &tn,
- const std::string &row) const;
+ const std::string &row) const;
std::unique_ptr<Request> MetaRequest(const hbase::pb::TableName tn,
- const std::string &row) const;
+ const std::string &row) const;
};
} // namespace hbase
diff --git a/core/region-location.h b/core/region-location.h
index 7922c95..7887526 100644
--- a/core/region-location.h
+++ b/core/region-location.h
@@ -27,15 +27,19 @@ namespace hbase {
class RegionLocation {
public:
- RegionLocation(hbase::pb::RegionInfo ri, hbase::pb::ServerName sn,
+ RegionLocation(std::string region_name, hbase::pb::RegionInfo ri,
+ hbase::pb::ServerName sn,
std::shared_ptr<HBaseService> service)
- : ri_(ri), sn_(sn), service_(service) {}
+ : region_name_(region_name), ri_(ri), sn_(sn), service_(service) {}
const hbase::pb::RegionInfo ®ion_info() { return ri_; }
const hbase::pb::ServerName &server_name() { return sn_; }
+ const std::string ®ion_name() { return region_name_; }
std::shared_ptr<HBaseService> service() { return service_; }
+ void set_service(std::shared_ptr<HBaseService> s) { service_ = s; }
private:
+ std::string region_name_;
hbase::pb::RegionInfo ri_;
hbase::pb::ServerName sn_;
std::shared_ptr<HBaseService> service_;
diff --git a/core/simple-client.cc b/core/simple-client.cc
index 00e3369..39c82c3 100644
--- a/core/simple-client.cc
+++ b/core/simple-client.cc
@@ -19,16 +19,21 @@
#include <folly/Logging.h>
#include <folly/Random.h>
+#include <folly/futures/Future.h>
#include <gflags/gflags.h>
+#include <wangle/concurrent/CPUThreadPoolExecutor.h>
#include <wangle/concurrent/GlobalExecutor.h>
+#include <atomic>
#include <chrono>
#include <iostream>
+#include <thread>
#include "connection/connection-pool.h"
#include "core/client.h"
#include "if/Client.pb.h"
#include "if/ZooKeeper.pb.h"
+#include "serde/server-name.h"
#include "serde/table-name.h"
using namespace folly;
@@ -39,16 +44,41 @@ using hbase::Request;
using hbase::HBaseService;
using hbase::LocationCache;
using hbase::ConnectionPool;
+using hbase::ConnectionFactory;
using hbase::pb::TableName;
using hbase::pb::ServerName;
using hbase::pb::RegionSpecifier_RegionSpecifierType;
-using hbase::pb::GetRequest;
-using hbase::pb::GetResponse;
+using hbase::pb::MutateRequest;
+using hbase::pb::MutationProto_MutationType;
// TODO(eclark): remove the need for this.
DEFINE_string(table, "t", "What region to send a get");
DEFINE_string(row, "test", "What row to get");
DEFINE_string(zookeeper, "localhost:2181", "What zk quorum to talk to");
+DEFINE_uint64(columns, 10000, "How many columns to write");
+DEFINE_int32(threads, 6, "How many cpu threads");
+
+std::unique_ptr<Request> MakeRequest(uint64_t col, std::string region_name) {
+ auto req = Request::mutate();
+ auto msg = std::static_pointer_cast<MutateRequest>(req->req_msg());
+ auto region = msg->mutable_region();
+ auto suf = folly::to<std::string>(col);
+
+ region->set_value(region_name);
+ region->set_type(RegionSpecifier_RegionSpecifierType::
+ RegionSpecifier_RegionSpecifierType_REGION_NAME);
+ auto mutation = msg->mutable_mutation();
+ mutation->set_row(FLAGS_row + suf);
+ mutation->set_mutate_type(
+ MutationProto_MutationType::MutationProto_MutationType_PUT);
+ auto column = mutation->add_column_value();
+ column->set_family("d");
+ auto qual = column->add_qualifier_value();
+ qual->set_qualifier(suf);
+ qual->set_value(".");
+
+ return std::move(req);
+}
int main(int argc, char *argv[]) {
google::SetUsageMessage(
@@ -56,13 +86,41 @@ int main(int argc, char *argv[]) {
google::ParseCommandLineFlags(&argc, &argv, true);
google::InitGoogleLogging(argv[0]);
- // Create a connection factory
- ConnectionPool cp;
- auto cpu_ex = wangle::getCPUExecutor();
- LocationCache cache{FLAGS_zookeeper, cpu_ex};
- auto result =
- cache.LocateFromMeta(folly::to<TableName>(FLAGS_table), FLAGS_row)
- .get(milliseconds(5000));
+ // Set up thread pools.
+ auto cpu_pool =
+ std::make_shared<wangle::CPUThreadPoolExecutor>(FLAGS_threads);
+ wangle::setCPUExecutor(cpu_pool);
+ auto io_pool = std::make_shared<wangle::IOThreadPoolExecutor>(5);
+ wangle::setIOExecutor(io_pool);
+
+ // Create the cache.
+ LocationCache cache{FLAGS_zookeeper, cpu_pool};
+
+ auto row = FLAGS_row;
+ auto tn = folly::to<TableName>(FLAGS_table);
+
+ auto loc = cache.LocateFromMeta(tn, row).get(milliseconds(5000));
+ auto connection = loc->service();
+
+ auto num_puts = FLAGS_columns;
+
+ auto results = std::vector<Future<Response>>{};
+ uint64_t col{0};
+ for (; col < num_puts; col++) {
+ results.push_back(folly::makeFuture(col)
+ .via(cpu_pool.get())
+ .then([loc](uint64_t col) {
+ return MakeRequest(col, loc->region_name());
+ })
+ .then([connection](std::unique_ptr<Request> req) {
+ return (*connection)(std::move(req));
+ }));
+ }
+ auto allf = folly::collect(results).get();
+
+ LOG(ERROR) << "Successfully sent " << allf.size() << " requests.";
+
+ io_pool->stop();
return 0;
}
diff --git a/serde/region-info-deserializer-test.cc b/serde/region-info-deserializer-test.cc
index ce8dedf..5cb8482 100644
--- a/serde/region-info-deserializer-test.cc
+++ b/serde/region-info-deserializer-test.cc
@@ -44,7 +44,6 @@ TEST(TestRegionInfoDesializer, TestDeserialize) {
ri_out.set_start_key(start_row);
ri_out.set_end_key(stop_row);
-
string header{"PBUF"};
string ser = header + ri_out.SerializeAsString();
diff --git a/serde/region-info.h b/serde/region-info.h
index 6af351c..e2ecfc9 100644
--- a/serde/region-info.h
+++ b/serde/region-info.h
@@ -21,16 +21,16 @@
#include "if/HBase.pb.h"
-#include <folly/Conv.h>
#include <boost/algorithm/string/predicate.hpp>
+#include <folly/Conv.h>
namespace hbase {
namespace pb {
-template <class String> void parseTo(String in, RegionInfo& out) {
+template <class String> void parseTo(String in, RegionInfo &out) {
// TODO(eclark): there has to be something better.
std::string s = folly::to<std::string>(in);
- if (!boost::starts_with(s, "PBUF") ) {
+ if (!boost::starts_with(s, "PBUF")) {
throw std::runtime_error("Region Info field doesn't contain preamble");
}
if (!out.ParseFromArray(s.data() + 4, s.size() - 4)) {
[hbase] 114/137: HBASE-18078. [C++] Harden RPC by handling various
communication abnormalities
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 0368621908b07b5b84642dc9b12caddc51d23140
Author: Xiaobing Zhou <xz...@hortonworks.com>
AuthorDate: Mon Aug 7 17:44:59 2017 -0700
HBASE-18078. [C++] Harden RPC by handling various communication abnormalities
Signed-off-by: Enis Soztutar <en...@apache.org>
---
connection/client-dispatcher.cc | 16 ++++++-
connection/client-handler.cc | 14 ++++--
connection/connection-factory.cc | 29 +++++++-----
connection/rpc-client.cc | 49 +++++++++++++++++++-
connection/rpc-client.h | 8 ++++
connection/rpc-test-server-handler.cc | 3 ++
connection/rpc-test-server.cc | 3 ++
connection/rpc-test.cc | 84 ++++++++++++++++++++++++++++-------
exceptions/exception.h | 16 ++++++-
if/test_rpc_service.proto | 1 +
10 files changed, 186 insertions(+), 37 deletions(-)
diff --git a/connection/client-dispatcher.cc b/connection/client-dispatcher.cc
index b9b2c34..d5d7f5f 100644
--- a/connection/client-dispatcher.cc
+++ b/connection/client-dispatcher.cc
@@ -18,8 +18,10 @@
*/
#include "connection/client-dispatcher.h"
#include <folly/ExceptionWrapper.h>
-
+#include <folly/Format.h>
+#include <folly/io/async/AsyncSocketException.h>
#include <utility>
+#include "exceptions/exception.h"
using std::unique_ptr;
@@ -31,6 +33,9 @@ void ClientDispatcher::read(Context *ctx, unique_ptr<Response> in) {
auto call_id = in->call_id();
auto p = requests_.find_and_erase(call_id);
+ VLOG(3) << folly::sformat("Read hbase::Response, call_id: {}, hasException: {}, what: {}",
+ in->call_id(), bool(in->exception()), in->exception().what());
+
if (in->exception()) {
p.setException(in->exception());
} else {
@@ -51,7 +56,14 @@ folly::Future<unique_ptr<Response>> ClientDispatcher::operator()(unique_ptr<Requ
LOG(ERROR) << "e = " << call_id;
this->requests_.erase(call_id);
});
- this->pipeline_->write(std::move(arg));
+
+ try {
+ this->pipeline_->write(std::move(arg));
+ } catch (const folly::AsyncSocketException &e) {
+ p.setException(folly::exception_wrapper{ConnectionException{folly::exception_wrapper{e}}});
+ /* clear folly::Promise to avoid overflow. */
+ requests_.erase(call_id);
+ }
return f;
}
diff --git a/connection/client-handler.cc b/connection/client-handler.cc
index 39227d3..983a68c 100644
--- a/connection/client-handler.cc
+++ b/connection/client-handler.cc
@@ -21,8 +21,8 @@
#include <folly/ExceptionWrapper.h>
#include <folly/Likely.h>
+#include <folly/io/async/AsyncSocketException.h>
#include <glog/logging.h>
-
#include <string>
#include "connection/request.h"
@@ -95,7 +95,7 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<folly::IOBuf> buf) {
: "";
std::string stack_trace =
exceptionResponse.has_stack_trace() ? exceptionResponse.stack_trace() : "";
- what.append(exception_class_name).append(stack_trace);
+ what.append(stack_trace);
auto remote_exception = std::make_unique<RemoteException>(what);
remote_exception->set_exception_class_name(exception_class_name)
@@ -133,7 +133,13 @@ folly::Future<folly::Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Re
// Now store the call id to response.
resp_msgs_->insert(std::make_pair(r->call_id(), r->resp_msg()));
- // Send the data down the pipeline.
- return ctx->fireWrite(serde_.Request(r->call_id(), r->method(), r->req_msg().get()));
+ try {
+ // Send the data down the pipeline.
+ return ctx->fireWrite(serde_.Request(r->call_id(), r->method(), r->req_msg().get()));
+ } catch (const folly::AsyncSocketException &e) {
+ /* clear protobuf::Message to avoid overflow. */
+ resp_msgs_->erase(r->call_id());
+ throw e;
+ }
}
} // namespace hbase
diff --git a/connection/connection-factory.cc b/connection/connection-factory.cc
index a0c7f96..e763c03 100644
--- a/connection/connection-factory.cc
+++ b/connection/connection-factory.cc
@@ -22,11 +22,16 @@
#include <chrono>
+#include <folly/ExceptionWrapper.h>
+#include <folly/SocketAddress.h>
+#include <folly/io/async/AsyncSocketException.h>
+
#include "connection/client-dispatcher.h"
#include "connection/connection-factory.h"
#include "connection/pipeline.h"
#include "connection/sasl-handler.h"
#include "connection/service.h"
+#include "exceptions/exception.h"
using std::chrono::milliseconds;
using std::chrono::nanoseconds;
@@ -56,15 +61,19 @@ std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> ConnectionFactory::M
std::shared_ptr<HBaseService> ConnectionFactory::Connect(
std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> client, const std::string &hostname,
uint16_t port) {
- // Yes this will block however it makes dealing with connection pool soooooo
- // much nicer.
- // TODO see about using shared promise for this.
- auto pipeline = client
- ->connect(folly::SocketAddress(hostname, port, true),
- std::chrono::duration_cast<milliseconds>(connect_timeout_))
- .get();
- auto dispatcher = std::make_shared<ClientDispatcher>();
- dispatcher->setPipeline(pipeline);
- return dispatcher;
+ try {
+ // Yes this will block however it makes dealing with connection pool soooooo
+ // much nicer.
+ // TODO see about using shared promise for this.
+ auto pipeline = client
+ ->connect(folly::SocketAddress(hostname, port, true),
+ std::chrono::duration_cast<milliseconds>(connect_timeout_))
+ .get();
+ auto dispatcher = std::make_shared<ClientDispatcher>();
+ dispatcher->setPipeline(pipeline);
+ return dispatcher;
+ } catch (const folly::AsyncSocketException &e) {
+ throw ConnectionException(folly::exception_wrapper{e});
+ }
}
} // namespace hbase
diff --git a/connection/rpc-client.cc b/connection/rpc-client.cc
index 10faa7a..a16dca6 100644
--- a/connection/rpc-client.cc
+++ b/connection/rpc-client.cc
@@ -19,9 +19,12 @@
#include "connection/rpc-client.h"
+#include <folly/Format.h>
#include <folly/Logging.h>
+#include <folly/futures/Future.h>
#include <unistd.h>
#include <wangle/concurrent/IOThreadPoolExecutor.h>
+#include "exceptions/exception.h"
using hbase::security::User;
using std::chrono::nanoseconds;
@@ -55,7 +58,7 @@ folly::Future<std::unique_ptr<Response>> RpcClient::AsyncCall(const std::string&
std::unique_ptr<Request> req,
std::shared_ptr<User> ticket) {
auto remote_id = std::make_shared<ConnectionId>(host, port, ticket);
- return GetConnection(remote_id)->SendRequest(std::move(req));
+ return SendRequest(remote_id, std::move(req));
}
folly::Future<std::unique_ptr<Response>> RpcClient::AsyncCall(const std::string& host,
@@ -64,7 +67,49 @@ folly::Future<std::unique_ptr<Response>> RpcClient::AsyncCall(const std::string&
std::shared_ptr<User> ticket,
const std::string& service_name) {
auto remote_id = std::make_shared<ConnectionId>(host, port, ticket, service_name);
- return GetConnection(remote_id)->SendRequest(std::move(req));
+ return SendRequest(remote_id, std::move(req));
+}
+
+/**
+ * There are two cases for ConnectionException:
+ * 1. The first time connection
+ * establishment, i.e. GetConnection(remote_id), AsyncSocketException being a cause.
+ * 2. Writing request down the pipeline, i.e. RpcConnection::SendRequest, AsyncSocketException being
+ * a cause as well.
+ */
+folly::Future<std::unique_ptr<Response>> RpcClient::SendRequest(
+ std::shared_ptr<ConnectionId> remote_id, std::unique_ptr<Request> req) {
+ try {
+ return GetConnection(remote_id)
+ ->SendRequest(std::move(req))
+ .onError([&, this](const folly::exception_wrapper& ew) {
+ VLOG(3) << folly::sformat("RpcClient Exception: {}", ew.what());
+ ew.with_exception([&, this](const hbase::ConnectionException& re) {
+ /* bad connection, remove it from pool. */
+ cp_->Close(remote_id);
+ });
+ return GetFutureWithException(ew);
+ });
+ } catch (const ConnectionException& e) {
+ CHECK(e.cause().get_exception() != nullptr);
+ VLOG(3) << folly::sformat("RpcClient Exception: {}", e.cause().what());
+ /* bad connection, remove it from pool. */
+ cp_->Close(remote_id);
+ return GetFutureWithException(e);
+ }
+}
+
+template <typename EXCEPTION>
+folly::Future<std::unique_ptr<Response>> RpcClient::GetFutureWithException(const EXCEPTION& e) {
+ return GetFutureWithException(folly::exception_wrapper{e});
+}
+
+folly::Future<std::unique_ptr<Response>> RpcClient::GetFutureWithException(
+ const folly::exception_wrapper& ew) {
+ folly::Promise<std::unique_ptr<Response>> promise;
+ auto future = promise.getFuture();
+ promise.setException(ew);
+ return future;
}
std::shared_ptr<RpcConnection> RpcClient::GetConnection(std::shared_ptr<ConnectionId> remote_id) {
diff --git a/connection/rpc-client.h b/connection/rpc-client.h
index 0ecde5b..8145be4 100644
--- a/connection/rpc-client.h
+++ b/connection/rpc-client.h
@@ -20,6 +20,7 @@
#include <google/protobuf/service.h>
+#include <folly/ExceptionWrapper.h>
#include <chrono>
#include <memory>
#include <string>
@@ -65,6 +66,13 @@ class RpcClient {
private:
std::shared_ptr<RpcConnection> GetConnection(std::shared_ptr<ConnectionId> remote_id);
+ folly::Future<std::unique_ptr<Response>> SendRequest(std::shared_ptr<ConnectionId> remote_id,
+ std::unique_ptr<Request> req);
+ template <typename EXCEPTION>
+ folly::Future<std::unique_ptr<Response>> GetFutureWithException(const EXCEPTION &e);
+
+ folly::Future<std::unique_ptr<Response>> GetFutureWithException(
+ const folly::exception_wrapper &ew);
private:
std::shared_ptr<ConnectionPool> cp_;
diff --git a/connection/rpc-test-server-handler.cc b/connection/rpc-test-server-handler.cc
index 7f41b7e..8e405ef 100644
--- a/connection/rpc-test-server-handler.cc
+++ b/connection/rpc-test-server-handler.cc
@@ -72,6 +72,9 @@ std::unique_ptr<Request> RpcTestServerSerializeHandler::CreateReceivedRequest(
} else if (method_name == "addr") {
result = std::make_unique<Request>(std::make_shared<EmptyRequestProto>(),
std::make_shared<AddrResponseProto>(), method_name);
+ } else if (method_name == "socketNotOpen") {
+ result = std::make_unique<Request>(std::make_shared<EmptyRequestProto>(),
+ std::make_shared<EmptyResponseProto>(), method_name);
}
return result;
}
diff --git a/connection/rpc-test-server.cc b/connection/rpc-test-server.cc
index b9e1f13..f350d6a 100644
--- a/connection/rpc-test-server.cc
+++ b/connection/rpc-test-server.cc
@@ -88,6 +88,9 @@ Future<std::unique_ptr<Response>> RpcTestService::operator()(std::unique_ptr<Req
// TODO:
} else if (method_name == "addr") {
// TODO:
+ } else if (method_name == "socketNotOpen") {
+ auto pb_resp_msg = std::make_shared<EmptyResponseProto>();
+ response->set_resp_msg(pb_resp_msg);
}
return folly::makeFuture<std::unique_ptr<Response>>(std::move(response));
diff --git a/connection/rpc-test.cc b/connection/rpc-test.cc
index 2949fe9..e7f678d 100644
--- a/connection/rpc-test.cc
+++ b/connection/rpc-test.cc
@@ -20,10 +20,12 @@
#include <wangle/bootstrap/ClientBootstrap.h>
#include <wangle/channel/Handler.h>
+#include <folly/Format.h>
#include <folly/Logging.h>
#include <folly/SocketAddress.h>
#include <folly/String.h>
#include <folly/experimental/TestUtil.h>
+#include <folly/io/async/AsyncSocketException.h>
#include <gflags/gflags.h>
#include <glog/logging.h>
#include <gtest/gtest.h>
@@ -41,6 +43,9 @@ using namespace folly;
using namespace hbase;
DEFINE_int32(port, 0, "test server port");
+DEFINE_string(result_format, "RPC {} returned: {}.", "output format of RPC result");
+DEFINE_string(fail_format, "Shouldn't get here, exception is expected for RPC {}.",
+ "output format of enforcing fail");
typedef ServerBootstrap<RpcTestServerSerializePipeline> ServerTestBootstrap;
typedef std::shared_ptr<ServerTestBootstrap> ServerPtr;
@@ -91,9 +96,10 @@ TEST_F(RpcTest, Echo) {
auto server_addr = GetRpcServerAddress(server);
auto client = CreateRpcClient(conf);
- std::string greetings = "hello, hbase server!";
+ auto method = "echo";
+ auto greetings = "hello, hbase server!";
auto request = std::make_unique<Request>(std::make_shared<EchoRequestProto>(),
- std::make_shared<EchoResponseProto>(), "echo");
+ std::make_shared<EchoResponseProto>(), method);
auto pb_msg = std::static_pointer_cast<EchoRequestProto>(request->req_msg());
pb_msg->set_message(greetings);
@@ -101,14 +107,14 @@ TEST_F(RpcTest, Echo) {
client
->AsyncCall(server_addr->getAddressStr(), server_addr->getPort(), std::move(request),
hbase::security::User::defaultUser())
- .then([=](std::unique_ptr<Response> response) {
+ .then([&](std::unique_ptr<Response> response) {
auto pb_resp = std::static_pointer_cast<EchoResponseProto>(response->resp_msg());
EXPECT_TRUE(pb_resp != nullptr);
- VLOG(1) << "RPC echo returned: " + pb_resp->message();
+ VLOG(1) << folly::sformat(FLAGS_result_format, method, pb_resp->message());
EXPECT_EQ(greetings, pb_resp->message());
})
- .onError([](const folly::exception_wrapper& ew) {
- FAIL() << "Shouldn't get here, no exception is expected for RPC echo.";
+ .onError([&](const folly::exception_wrapper& ew) {
+ FAIL() << folly::sformat(FLAGS_fail_format, method);
});
server->stop();
@@ -118,23 +124,24 @@ TEST_F(RpcTest, Echo) {
/**
* test error
*/
-TEST_F(RpcTest, error) {
+TEST_F(RpcTest, Error) {
auto conf = CreateConf();
auto server = CreateRpcServer();
auto server_addr = GetRpcServerAddress(server);
auto client = CreateRpcClient(conf);
+ auto method = "error";
auto request = std::make_unique<Request>(std::make_shared<EmptyRequestProto>(),
- std::make_shared<EmptyResponseProto>(), "error");
+ std::make_shared<EmptyResponseProto>(), method);
/* sending out request */
client
->AsyncCall(server_addr->getAddressStr(), server_addr->getPort(), std::move(request),
hbase::security::User::defaultUser())
- .then([=](std::unique_ptr<Response> response) {
- FAIL() << "Shouldn't get here, exception is expected for RPC error.";
+ .then([&](std::unique_ptr<Response> response) {
+ FAIL() << folly::sformat(FLAGS_fail_format, method);
})
- .onError([](const folly::exception_wrapper& ew) {
- VLOG(1) << "RPC error returned with exception.";
+ .onError([&](const folly::exception_wrapper& ew) {
+ VLOG(1) << folly::sformat(FLAGS_result_format, method, ew.what());
std::string kRemoteException = demangle(typeid(hbase::RemoteException)).toStdString();
std::string kRpcTestException = demangle(typeid(hbase::RpcTestException)).toStdString();
@@ -142,14 +149,57 @@ TEST_F(RpcTest, error) {
EXPECT_TRUE(bool(ew));
EXPECT_EQ(kRemoteException, ew.class_name());
- /* verify RemoteException */
- EXPECT_TRUE(ew.with_exception([&](const hbase::RemoteException& re) {
- /* verify DoNotRetryIOException*/
- EXPECT_EQ(kRpcTestException, re.exception_class_name());
- EXPECT_EQ(kRpcTestException + ": server error!", re.stack_trace());
+ /* verify exception */
+ EXPECT_TRUE(ew.with_exception([&](const hbase::RemoteException& e) {
+ EXPECT_EQ(kRpcTestException, e.exception_class_name());
+ EXPECT_EQ(kRpcTestException + ": server error!", e.stack_trace());
}));
});
server->stop();
server->join();
}
+
+TEST_F(RpcTest, SocketNotOpen) {
+ auto conf = CreateConf();
+ auto server = CreateRpcServer();
+ auto server_addr = GetRpcServerAddress(server);
+ auto client = CreateRpcClient(conf);
+
+ auto method = "socketNotOpen";
+ auto request = std::make_unique<Request>(std::make_shared<EmptyRequestProto>(),
+ std::make_shared<EmptyResponseProto>(), method);
+
+ server->stop();
+ server->join();
+
+ /* sending out request */
+ client
+ ->AsyncCall(server_addr->getAddressStr(), server_addr->getPort(), std::move(request),
+ hbase::security::User::defaultUser())
+ .then([&](std::unique_ptr<Response> response) {
+ FAIL() << folly::sformat(FLAGS_fail_format, method);
+ })
+ .onError([&](const folly::exception_wrapper& ew) {
+ VLOG(1) << folly::sformat(FLAGS_result_format, method, ew.what());
+ std::string kConnectionException =
+ demangle(typeid(hbase::ConnectionException)).toStdString();
+ std::string kAsyncSocketException =
+ demangle(typeid(folly::AsyncSocketException)).toStdString();
+
+ /* verify exception_wrapper */
+ EXPECT_TRUE(bool(ew));
+ EXPECT_EQ(kConnectionException, ew.class_name());
+
+ /* verify exception */
+ EXPECT_TRUE(ew.with_exception([&](const hbase::ConnectionException& e) {
+ EXPECT_TRUE(bool(e.cause()));
+ EXPECT_EQ(kAsyncSocketException, e.cause().class_name());
+ VLOG(1) << folly::sformat(FLAGS_result_format, method, e.cause().what());
+ e.cause().with_exception([&](const folly::AsyncSocketException& ase) {
+ EXPECT_EQ(AsyncSocketException::AsyncSocketExceptionType::NOT_OPEN, ase.getType());
+ EXPECT_EQ(111 /*ECONNREFUSED*/, ase.getErrno());
+ });
+ }));
+ });
+}
diff --git a/exceptions/exception.h b/exceptions/exception.h
index bdedff4..bc3b291 100644
--- a/exceptions/exception.h
+++ b/exceptions/exception.h
@@ -59,7 +59,7 @@ class IOException : public std::logic_error {
IOException(const std::string& what, bool do_not_retry)
: logic_error(what), do_not_retry_(do_not_retry) {}
- IOException(const std::string& what, folly::exception_wrapper cause)
+ IOException(const std::string& what, const folly::exception_wrapper& cause)
: logic_error(what), cause_(cause), do_not_retry_(false) {}
IOException(const std::string& what, folly::exception_wrapper cause, bool do_not_retry)
@@ -67,7 +67,7 @@ class IOException : public std::logic_error {
virtual ~IOException() = default;
- virtual folly::exception_wrapper cause() { return cause_; }
+ virtual folly::exception_wrapper cause() const { return cause_; }
bool do_not_retry() const { return do_not_retry_; }
@@ -115,6 +115,18 @@ class RetriesExhaustedException : public IOException {
int32_t num_retries_;
};
+class ConnectionException : public IOException {
+ public:
+ ConnectionException() {}
+
+ ConnectionException(const std::string& what) : IOException(what) {}
+
+ ConnectionException(const folly::exception_wrapper& cause) : IOException("", cause) {}
+
+ ConnectionException(const std::string& what, const folly::exception_wrapper& cause)
+ : IOException(what, cause) {}
+};
+
class RemoteException : public IOException {
public:
RemoteException() : IOException(), port_(0) {}
diff --git a/if/test_rpc_service.proto b/if/test_rpc_service.proto
index 5f91dc4..2730403 100644
--- a/if/test_rpc_service.proto
+++ b/if/test_rpc_service.proto
@@ -32,4 +32,5 @@ service TestProtobufRpcProto {
rpc error(EmptyRequestProto) returns (EmptyResponseProto);
rpc pause(PauseRequestProto) returns (EmptyResponseProto);
rpc addr(EmptyRequestProto) returns (AddrResponseProto);
+ rpc socketNotOpen(EmptyRequestProto) returns (EmptyResponseProto);
}
[hbase] 79/137: HBASE-15894 [C++] Put and Mutation objects
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 59bfb2142809a72d1322264261a2259009b67604
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Wed Mar 29 16:34:20 2017 -0700
HBASE-15894 [C++] Put and Mutation objects
---
core/BUCK | 8 +++-
core/cell-test.cc | 18 ++++++++-
core/cell.cc | 39 ++++++++++++++++++-
core/cell.h | 6 ++-
core/get-test.cc | 36 ++++++++---------
core/get.cc | 4 +-
core/get.h | 14 ++-----
core/mutation.cc | 69 +++++++++++++++++++++++++++++++++
core/mutation.h | 99 +++++++++++++++++++++++++++++++++++++++++++++++
core/put.cc | 67 ++++++++++++++++++++++++++++++++
core/put.h | 68 ++++++++++++++++++++++++++++++++
core/request-converter.cc | 5 ++-
core/result.cc | 4 +-
core/result.h | 7 +---
core/scan-test.cc | 28 +++++++-------
core/scan.cc | 6 ++-
core/scan.h | 14 ++-----
core/table.h | 1 +
utils/BUCK | 14 +++++--
utils/bytes-util-test.cc | 49 +++++++++++++++++++++++
utils/bytes-util.cc | 53 +++++++++++++++++++++++++
utils/bytes-util.h | 45 +++++++++++++++++++++
22 files changed, 580 insertions(+), 74 deletions(-)
diff --git a/core/BUCK b/core/BUCK
index 7483980..03fc0fc 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -34,7 +34,8 @@ cxx_library(
# Once meta lookup works
"meta-utils.h",
"get.h",
- "time-range.h",
+ "mutation.h",
+ "put.h",
"scan.h",
"result.h",
"request-converter.h",
@@ -44,6 +45,7 @@ cxx_library(
"async-rpc-retrying-caller-factory.h",
"async-rpc-retrying-caller.h",
"hbase-rpc-controller.h",
+ "time-range.h",
"zk-util.h",
"action.h",
"multi-response.h",
@@ -60,13 +62,15 @@ cxx_library(
"location-cache.cc",
"meta-utils.cc",
"get.cc",
- "time-range.cc",
+ "mutation.cc",
+ "put.cc",
"scan.cc",
"raw-async-table.cc",
"result.cc",
"request-converter.cc",
"response-converter.cc",
"table.cc",
+ "time-range.cc",
"zk-util.cc",
"multi-response.cc",
"region-result.cc",
diff --git a/core/cell-test.cc b/core/cell-test.cc
index fb4d033..2ec983b 100644
--- a/core/cell-test.cc
+++ b/core/cell-test.cc
@@ -31,7 +31,6 @@ TEST(CellTest, CellFailureTest) {
std::string column = "column";
std::string value = "value";
int64_t timestamp = std::numeric_limits<int64_t>::max();
- std::string tags = "";
std::unique_ptr<Cell> cell(new Cell(row, family, column, timestamp, value, cell_type));
if (cell.get()) {
EXPECT_NE("row-value", cell.get()->Row());
@@ -166,3 +165,20 @@ TEST(CellTest, CellRowFamilyColumnValueTest) {
EXPECT_EQ(cell_type, cell.get()->Type());
}
}
+
+TEST(CellTest, CellDebugString) {
+ CellType cell_type = CellType::PUT;
+ std::string row = "row";
+ std::string family = "family";
+ std::string column = "column";
+ std::string value = "value";
+ int64_t timestamp = std::numeric_limits<int64_t>::max();
+
+ Cell cell{row, family, column, timestamp, value, cell_type};
+ LOG(INFO) << cell.DebugString();
+ EXPECT_EQ("row/family:column/LATEST_TIMESTAMP/PUT/vlen=5/seqid=0", cell.DebugString());
+
+ Cell cell2{row, "", column, 42, value, CellType::DELETE};
+ LOG(INFO) << cell2.DebugString();
+ EXPECT_EQ("row/column/42/DELETE/vlen=5/seqid=0", cell2.DebugString());
+}
diff --git a/core/cell.cc b/core/cell.cc
index 4982406..14a8014 100644
--- a/core/cell.cc
+++ b/core/cell.cc
@@ -18,8 +18,12 @@
*/
#include "core/cell.h"
+#include <climits>
#include <stdexcept>
+#include "folly/Conv.h"
+#include "utils/bytes-util.h"
+
namespace hbase {
Cell::Cell(const std::string &row, const std::string &family, const std::string &qualifier,
@@ -33,8 +37,6 @@ Cell::Cell(const std::string &row, const std::string &family, const std::string
sequence_id_(0) {
if (0 == row.size()) throw std::runtime_error("Row size should be greater than 0");
- if (0 == family.size()) throw std::runtime_error("Column family size should be greater than 0");
-
if (0 >= timestamp) throw std::runtime_error("Timestamp should be greater than 0");
}
@@ -54,4 +56,37 @@ hbase::CellType Cell::Type() const { return cell_type_; }
int64_t Cell::SequenceId() const { return sequence_id_; }
+std::string Cell::DebugString() const {
+ std::string timestamp_str;
+ if (timestamp_ == std::numeric_limits<int64_t>::max()) {
+ timestamp_str = "LATEST_TIMESTAMP";
+ } else {
+ timestamp_str = folly::to<std::string>(timestamp_);
+ }
+
+ return BytesUtil::ToStringBinary(row_) + "/" + BytesUtil::ToStringBinary(family_) +
+ (family_.empty() ? "" : ":") + BytesUtil::ToStringBinary(qualifier_) + "/" +
+ timestamp_str + "/" + TypeToString(cell_type_) + "/vlen=" +
+ folly::to<std::string>(value_.size()) + "/seqid=" + folly::to<std::string>(sequence_id_);
+}
+
+const char *Cell::TypeToString(CellType type) {
+ switch (type) {
+ case MINIMUM:
+ return "MINIMUM";
+ case PUT:
+ return "PUT";
+ case DELETE:
+ return "DELETE";
+ case DELETE_COLUMN:
+ return "DELETE_COLUMN";
+ case DELETE_FAMILY:
+ return "DELETE_FAMILY";
+ case MAXIMUM:
+ return "MAXIMUM";
+ default:
+ return "UNKNOWN";
+ }
+}
+
} /* namespace hbase */
diff --git a/core/cell.h b/core/cell.h
index 5a39dc3..c062e45 100644
--- a/core/cell.h
+++ b/core/cell.h
@@ -28,7 +28,7 @@ enum CellType {
MINIMUM = 0,
PUT = 4,
DELETE = 8,
- DELETEFAMILYVERSION = 10,
+ DELETE_FAMILY_VERSION = 10,
DELETE_COLUMN = 12,
DELETE_FAMILY = 14,
MAXIMUM = 255
@@ -46,6 +46,7 @@ class Cell {
const std::string &Value() const;
CellType Type() const;
int64_t SequenceId() const;
+ std::string DebugString() const;
private:
std::string row_;
@@ -58,6 +59,9 @@ class Cell {
hbase::CellType cell_type_;
std::string value_;
int64_t sequence_id_;
+
+ private:
+ static const char *TypeToString(CellType type);
};
} // namespace hbase
diff --git a/core/get-test.cc b/core/get-test.cc
index 6ee2715..fafff39 100644
--- a/core/get-test.cc
+++ b/core/get-test.cc
@@ -29,20 +29,20 @@ void CheckFamilies(Get &get) {
EXPECT_EQ(false, get.HasFamilies());
get.AddFamily("family-1");
EXPECT_EQ(true, get.HasFamilies());
- EXPECT_EQ(1, get.Family().size());
- for (const auto &family : get.Family()) {
+ EXPECT_EQ(1, get.FamilyMap().size());
+ for (const auto &family : get.FamilyMap()) {
EXPECT_STREQ("family-1", family.first.c_str());
EXPECT_EQ(0, family.second.size());
}
// Not allowed to add the same CF.
get.AddFamily("family-1");
- EXPECT_EQ(1, get.Family().size());
+ EXPECT_EQ(1, get.FamilyMap().size());
get.AddFamily("family-2");
- EXPECT_EQ(2, get.Family().size());
+ EXPECT_EQ(2, get.FamilyMap().size());
get.AddFamily("family-3");
- EXPECT_EQ(3, get.Family().size());
+ EXPECT_EQ(3, get.FamilyMap().size());
int i = 1;
- for (const auto &family : get.Family()) {
+ for (const auto &family : get.FamilyMap()) {
std::string family_name = "family-" + std::to_string(i);
EXPECT_STREQ(family_name.c_str(), family.first.c_str());
EXPECT_EQ(0, family.second.size());
@@ -55,8 +55,8 @@ void CheckFamilies(Get &get) {
get.AddColumn("family-1", "column-3");
get.AddColumn("family-2", "column-X");
- EXPECT_EQ(3, get.Family().size());
- auto it = get.Family().begin();
+ EXPECT_EQ(3, get.FamilyMap().size());
+ auto it = get.FamilyMap().begin();
EXPECT_STREQ("family-1", it->first.c_str());
EXPECT_EQ(4, it->second.size());
EXPECT_STREQ("column-1", it->second[0].c_str());
@@ -71,20 +71,20 @@ void CheckFamilies(Get &get) {
EXPECT_STREQ("family-3", it->first.c_str());
EXPECT_EQ(0, it->second.size());
++it;
- EXPECT_EQ(it, get.Family().end());
+ EXPECT_EQ(it, get.FamilyMap().end());
}
void CheckFamiliesAfterCopy(Get &get) {
EXPECT_EQ(true, get.HasFamilies());
- EXPECT_EQ(3, get.Family().size());
+ EXPECT_EQ(3, get.FamilyMap().size());
int i = 1;
- for (const auto &family : get.Family()) {
+ for (const auto &family : get.FamilyMap()) {
std::string family_name = "family-" + std::to_string(i);
EXPECT_STREQ(family_name.c_str(), family.first.c_str());
i += 1;
}
// Check if the alreaday added CF's and CQ's are as expected
- auto it = get.Family().begin();
+ auto it = get.FamilyMap().begin();
EXPECT_STREQ("family-1", it->first.c_str());
EXPECT_EQ(4, it->second.size());
EXPECT_STREQ("column-1", it->second[0].c_str());
@@ -99,7 +99,7 @@ void CheckFamiliesAfterCopy(Get &get) {
EXPECT_STREQ("family-3", it->first.c_str());
EXPECT_EQ(0, it->second.size());
++it;
- EXPECT_EQ(it, get.Family().end());
+ EXPECT_EQ(it, get.FamilyMap().end());
}
void GetMethods(Get &get, const std::string &row) {
@@ -159,19 +159,19 @@ TEST(Get, SingleGet) {
// Adding the below tests as there were some concerns raised that the same
// vector of qualifiers in FamilyMap is being shared between copied objects
// Verify the source object's family map size before using it to copy.
- EXPECT_EQ(3, get.Family().size());
+ EXPECT_EQ(3, get.FamilyMap().size());
Get getcp_fam(get);
// address of family maps should be different.
- EXPECT_NE(&(get.Family()), &(getcp_fam.Family()));
+ EXPECT_NE(&(get.FamilyMap()), &(getcp_fam.FamilyMap()));
// Add family to the source object
get.AddColumn("family-4", "column-A");
get.AddColumn("family-4", "column-B");
// Verify the source object's family map size
- EXPECT_EQ(4, get.Family().size());
+ EXPECT_EQ(4, get.FamilyMap().size());
// Verify the source object's family elements
- auto it = get.Family().begin();
+ auto it = get.FamilyMap().begin();
EXPECT_STREQ("family-1", it->first.c_str());
EXPECT_EQ(4, it->second.size());
EXPECT_STREQ("column-1", it->second[0].c_str());
@@ -191,7 +191,7 @@ TEST(Get, SingleGet) {
EXPECT_STREQ("column-A", it->second[0].c_str());
EXPECT_STREQ("column-B", it->second[1].c_str());
++it;
- EXPECT_EQ(it, get.Family().end());
+ EXPECT_EQ(it, get.FamilyMap().end());
// Verifying the copied object's families. It will remain unchanged and below
// tests should pass
diff --git a/core/get.cc b/core/get.cc
index afeb429..52b2ec5 100644
--- a/core/get.cc
+++ b/core/get.cc
@@ -87,7 +87,9 @@ Get &Get::SetConsistency(hbase::pb::Consistency consistency) {
bool Get::HasFamilies() const { return !family_map_.empty(); }
-const FamilyMap &Get::Family() const { return family_map_; }
+const std::map<std::string, std::vector<std::string>> &Get::FamilyMap() const {
+ return family_map_;
+}
int Get::MaxVersions() const { return max_versions_; }
diff --git a/core/get.h b/core/get.h
index e0be4e7..098635d 100644
--- a/core/get.h
+++ b/core/get.h
@@ -32,12 +32,6 @@
using hbase::Row;
namespace hbase {
-/**
- * @brief Map consisting of column families and qualifiers to be used for Get
- * operation
- */
-using FamilyMap = std::map<std::string, std::vector<std::string>>;
-
class Get : public Row, public Query {
public:
/**
@@ -73,10 +67,10 @@ class Get : public Row, public Query {
Get& SetCacheBlocks(bool cache_blocks);
/**
- * @brief Returns the Get family map (FamilyMap) for this Get operation. Used
+ * @brief Returns the Get family map for this Get operation. Used
* for constructing Scan object with an already constructed Get
*/
- const FamilyMap& Family() const;
+ const std::map<std::string, std::vector<std::string>>& FamilyMap() const;
/**
* @brief Returns the timerange for this Get
@@ -112,7 +106,7 @@ class Get : public Row, public Query {
Get& AddColumn(const std::string& family, const std::string& qualifier);
/**
- * @brief Returns true if family map (FamilyMap) is non empty false otherwise
+ * @brief Returns true if family map is non empty false otherwise
*/
bool HasFamilies() const;
@@ -131,7 +125,7 @@ class Get : public Row, public Query {
int32_t max_versions_ = 1;
bool cache_blocks_ = true;
bool check_existence_only_ = false;
- FamilyMap family_map_;
+ std::map<std::string, std::vector<std::string>> family_map_;
hbase::pb::Consistency consistency_ = hbase::pb::Consistency::STRONG;
std::unique_ptr<TimeRange> tr_ = std::make_unique<TimeRange>();
};
diff --git a/core/mutation.cc b/core/mutation.cc
new file mode 100644
index 0000000..ab33105
--- /dev/null
+++ b/core/mutation.cc
@@ -0,0 +1,69 @@
+
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/mutation.h"
+#include <algorithm>
+#include <limits>
+#include <stdexcept>
+
+namespace hbase {
+
+Mutation::Mutation(const std::string &row) : Row(row) { }
+Mutation::Mutation(const std::string &row, int64_t timestamp) : Row(row), timestamp_(timestamp) { }
+
+Mutation::Mutation(const Mutation &mutation) {
+ row_ = mutation.row_;
+ durability_ = mutation.durability_;
+ timestamp_ = mutation.timestamp_;
+ for (auto const &e : mutation.family_map_) {
+ for (auto const &c : e.second) {
+ family_map_[e.first].push_back(std::make_unique<Cell>(*c));
+ }
+ }
+}
+
+Mutation &Mutation::operator=(const Mutation &mutation) {
+ row_ = mutation.row_;
+ durability_ = mutation.durability_;
+ timestamp_ = mutation.timestamp_;
+ for (auto const &e : mutation.family_map_) {
+ for (auto const &c : e.second) {
+ family_map_[e.first].push_back(std::make_unique<Cell>(*c));
+ }
+ }
+ return *this;
+}
+
+pb::MutationProto_Durability Mutation::Durability() const { return durability_; }
+
+Mutation &Mutation::SetDurability(pb::MutationProto_Durability durability) {
+ durability_ = durability;
+ return *this;
+}
+
+bool Mutation::HasFamilies() const { return !family_map_.empty(); }
+
+std::unique_ptr<Cell> Mutation::CreateCell(const std::string &family, const std::string &qualifier,
+ int64_t timestamp, const std::string &value) {
+ return std::make_unique<Cell>(row_, family, qualifier, timestamp, value, hbase::CellType::PUT);
+}
+
+} // namespace hbase
diff --git a/core/mutation.h b/core/mutation.h
new file mode 100644
index 0000000..83392e1
--- /dev/null
+++ b/core/mutation.h
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <limits>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+#include "core/cell.h"
+#include "core/row.h"
+#include "if/Client.pb.h"
+
+namespace hbase {
+
+class Mutation: public Row {
+ public:
+ /**
+ * Constructors
+ */
+ explicit Mutation(const std::string& row);
+ Mutation(const std::string& row, int64_t timestamp);
+ Mutation(const Mutation& cmutation);
+ Mutation& operator=(const Mutation& cmutation);
+
+ virtual ~Mutation() = default;
+
+ /**
+ * @brief Returns the Mutation family map for this operation.
+ */
+ const std::map<std::string, std::vector<std::unique_ptr<Cell>>>& FamilyMap() const {
+ return family_map_;
+ }
+
+ /**
+ * @brief Returns the timerange for this Get
+ */
+ int64_t TimeStamp() const { return timestamp_; }
+
+ /**
+ * @brief Get versions of columns with the specified timestamp.
+ * @param The timestamp to be set
+ */
+ Mutation& SetTimeStamp(int64_t timestamp) {
+ timestamp_ = timestamp;
+ return *this;
+ }
+
+ /**
+ * @brief Returns the row for this operation
+ */
+ const std::string& row() const;
+
+ /**
+ * @brief Returns true if family map is non empty false otherwise
+ */
+ bool HasFamilies() const;
+
+ /**
+ * @brief Returns the durability level for this Mutation operation
+ */
+ pb::MutationProto_Durability Durability() const;
+
+ /**
+ * @brief Sets the durability level for this Mutation operation
+ * @param durability the durability to be set
+ */
+ Mutation& SetDurability(pb::MutationProto_Durability durability);
+
+ protected:
+ static const constexpr int64_t kLatestTimestamp = std::numeric_limits<int64_t>::max();
+ std::map<std::string, std::vector<std::unique_ptr<Cell>>> family_map_;
+ pb::MutationProto_Durability durability_ =
+ hbase::pb::MutationProto_Durability::MutationProto_Durability_USE_DEFAULT;
+ int64_t timestamp_ = kLatestTimestamp;
+
+ std::unique_ptr<Cell> CreateCell(const std::string& family, const std::string& qualifier,
+ int64_t timestamp, const std::string& value);
+};
+
+} // namespace hbase
diff --git a/core/put.cc b/core/put.cc
new file mode 100644
index 0000000..b81051e
--- /dev/null
+++ b/core/put.cc
@@ -0,0 +1,67 @@
+
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/put.h"
+#include <folly/Conv.h>
+#include <algorithm>
+#include <limits>
+#include <stdexcept>
+
+namespace hbase {
+
+/**
+ * @brief Add the specified column and value to this Put operation.
+ * @param family family name
+ * @param qualifier column qualifier
+ * @param value column value
+ */
+Put& Put::AddColumn(const std::string& family, const std::string& qualifier,
+ const std::string& value) {
+ return AddColumn(family, qualifier, kLatestTimestamp, value);
+}
+
+/**
+ * @brief Add the specified column and value to this Put operation.
+ * @param family family name
+ * @param qualifier column qualifier
+ * @param timestamp version timestamp
+ * @param value column value
+ */
+Put& Put::AddColumn(const std::string& family, const std::string& qualifier, int64_t timestamp,
+ const std::string& value) {
+ if (timestamp < 0) {
+ throw std::runtime_error("Timestamp cannot be negative. ts=" +
+ folly::to<std::string>(timestamp));
+ }
+
+ return Add(CreateCell(family, qualifier, timestamp, value));
+}
+
+Put& Put::Add(std::unique_ptr<Cell> cell) {
+ if (cell->Row() != row_) {
+ throw std::runtime_error("The row in" + cell->DebugString() +
+ " doesn't match the original one " + row_);
+ }
+
+ family_map_[cell->Family()].push_back(std::move(cell));
+ return *this;
+}
+} // namespace hbase
diff --git a/core/put.h b/core/put.h
new file mode 100644
index 0000000..8a43dec
--- /dev/null
+++ b/core/put.h
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+#include "core/cell.h"
+#include "core/mutation.h"
+
+namespace hbase {
+
+class Put : public Mutation {
+ public:
+ /**
+ * Constructors
+ */
+ explicit Put(const std::string& row) : Mutation(row) {}
+ Put(const std::string& row, int64_t timestamp) : Mutation(row, timestamp) {}
+ Put(const Put& cput) : Mutation(cput) {}
+ Put& operator=(const Put& cput) {
+ Mutation::operator=(cput);
+ return *this;
+ }
+
+ ~Put() = default;
+
+ /**
+ * @brief Add the specified column and value to this Put operation.
+ * @param family family name
+ * @param qualifier column qualifier
+ * @param value column value
+ */
+ Put& AddColumn(const std::string& family, const std::string& qualifier, const std::string& value);
+
+ /**
+ * @brief Add the specified column and value to this Put operation.
+ * @param family family name
+ * @param qualifier column qualifier
+ * @param timestamp version timestamp
+ * @param value column value
+ */
+ Put& AddColumn(const std::string& family, const std::string& qualifier, int64_t timestamp,
+ const std::string& value);
+
+ Put& Add(std::unique_ptr<Cell> cell);
+};
+
+} // namespace hbase
diff --git a/core/request-converter.cc b/core/request-converter.cc
index ff92b5c..c40a56e 100644
--- a/core/request-converter.cc
+++ b/core/request-converter.cc
@@ -47,6 +47,7 @@ std::unique_ptr<Request> RequestConverter::ToGetRequest(const Get &get,
auto pb_msg = std::static_pointer_cast<GetRequest>(pb_req->req_msg());
RequestConverter::SetRegion(region_name, pb_msg->mutable_region());
pb_msg->set_allocated_get((RequestConverter::ToGet(get)).release());
+
return pb_req;
}
@@ -78,7 +79,7 @@ std::unique_ptr<Request> RequestConverter::ToScanRequest(const Scan &scan,
}
if (scan.HasFamilies()) {
- for (const auto &family : scan.Family()) {
+ for (const auto &family : scan.FamilyMap()) {
auto column = pb_scan->add_column();
column->set_family(family.first);
for (const auto &qualifier : family.second) {
@@ -137,7 +138,7 @@ std::unique_ptr<hbase::pb::Get> RequestConverter::ToGet(const Get &get) {
}
pb_get->set_row(get.row());
if (get.HasFamilies()) {
- for (const auto &family : get.Family()) {
+ for (const auto &family : get.FamilyMap()) {
auto column = pb_get->add_column();
column->set_family(family.first);
for (const auto &qualifier : family.second) {
diff --git a/core/result.cc b/core/result.cc
index 4db3fca..eb28b32 100644
--- a/core/result.cc
+++ b/core/result.cc
@@ -91,8 +91,8 @@ int Result::Size() const { return cells_.size(); }
const ResultMap &Result::Map() const { return result_map_; }
-const ResultFamilyMap Result::FamilyMap(const std::string &family) const {
- ResultFamilyMap family_map;
+const std::map<std::string, std::string> Result::FamilyMap(const std::string &family) const {
+ std::map<std::string, std::string> family_map;
if (!IsEmpty()) {
for (auto itr = result_map_.begin(); itr != result_map_.end(); ++itr) {
if (family == itr->first) {
diff --git a/core/result.h b/core/result.h
index 8ff4311..8064197 100644
--- a/core/result.h
+++ b/core/result.h
@@ -38,11 +38,6 @@ using ResultMap =
std::map<std::string,
std::map<std::string, std::map<int64_t, std::string, std::greater<int64_t> > > >;
-/**
- * @brief Map of qualifiers to values.
- */
-using ResultFamilyMap = std::map<std::string, std::string>;
-
class Result {
public:
/**
@@ -116,7 +111,7 @@ class Result {
* Returns a Map of the form: Map<qualifier,value>
* @param family - column family to get
*/
- const ResultFamilyMap FamilyMap(const std::string &family) const;
+ const std::map<std::string, std::string> FamilyMap(const std::string &family) const;
private:
bool exists_ = false;
diff --git a/core/scan-test.cc b/core/scan-test.cc
index 9a26407..73fb6df 100644
--- a/core/scan-test.cc
+++ b/core/scan-test.cc
@@ -29,20 +29,20 @@ void CheckFamilies(Scan &scan) {
EXPECT_EQ(false, scan.HasFamilies());
scan.AddFamily("family-1");
EXPECT_EQ(true, scan.HasFamilies());
- EXPECT_EQ(1, scan.Family().size());
- for (const auto &family : scan.Family()) {
+ EXPECT_EQ(1, scan.FamilyMap().size());
+ for (const auto &family : scan.FamilyMap()) {
EXPECT_STREQ("family-1", family.first.c_str());
EXPECT_EQ(0, family.second.size());
}
// Not allowed to add the same CF.
scan.AddFamily("family-1");
- EXPECT_EQ(1, scan.Family().size());
+ EXPECT_EQ(1, scan.FamilyMap().size());
scan.AddFamily("family-2");
- EXPECT_EQ(2, scan.Family().size());
+ EXPECT_EQ(2, scan.FamilyMap().size());
scan.AddFamily("family-3");
- EXPECT_EQ(3, scan.Family().size());
+ EXPECT_EQ(3, scan.FamilyMap().size());
int i = 1;
- for (const auto &family : scan.Family()) {
+ for (const auto &family : scan.FamilyMap()) {
std::string family_name = "family-" + std::to_string(i);
EXPECT_STREQ(family_name.c_str(), family.first.c_str());
EXPECT_EQ(0, family.second.size());
@@ -55,8 +55,8 @@ void CheckFamilies(Scan &scan) {
scan.AddColumn("family-1", "column-3");
scan.AddColumn("family-2", "column-X");
- EXPECT_EQ(3, scan.Family().size());
- auto it = scan.Family().begin();
+ EXPECT_EQ(3, scan.FamilyMap().size());
+ auto it = scan.FamilyMap().begin();
EXPECT_STREQ("family-1", it->first.c_str());
EXPECT_EQ(4, it->second.size());
EXPECT_STREQ("column-1", it->second[0].c_str());
@@ -71,20 +71,20 @@ void CheckFamilies(Scan &scan) {
EXPECT_STREQ("family-3", it->first.c_str());
EXPECT_EQ(0, it->second.size());
++it;
- EXPECT_EQ(it, scan.Family().end());
+ EXPECT_EQ(it, scan.FamilyMap().end());
}
void CheckFamiliesAfterCopy(Scan &scan) {
EXPECT_EQ(true, scan.HasFamilies());
- EXPECT_EQ(3, scan.Family().size());
+ EXPECT_EQ(3, scan.FamilyMap().size());
int i = 1;
- for (const auto &family : scan.Family()) {
+ for (const auto &family : scan.FamilyMap()) {
std::string family_name = "family-" + std::to_string(i);
EXPECT_STREQ(family_name.c_str(), family.first.c_str());
i += 1;
}
// Check if the alreaday added CF's and CQ's are as expected
- auto it = scan.Family().begin();
+ auto it = scan.FamilyMap().begin();
EXPECT_STREQ("family-1", it->first.c_str());
EXPECT_EQ(4, it->second.size());
EXPECT_STREQ("column-1", it->second[0].c_str());
@@ -99,7 +99,7 @@ void CheckFamiliesAfterCopy(Scan &scan) {
EXPECT_STREQ("family-3", it->first.c_str());
EXPECT_EQ(0, it->second.size());
++it;
- EXPECT_EQ(it, scan.Family().end());
+ EXPECT_EQ(it, scan.FamilyMap().end());
}
void ScanMethods(Scan &scan) {
@@ -219,7 +219,7 @@ TEST(Scan, FromGet) {
get.AddColumn("family-1", "column-3");
get.AddColumn("family-2", "column-X");
- EXPECT_EQ(3, get.Family().size());
+ EXPECT_EQ(3, get.FamilyMap().size());
Scan scan(get);
ScanMethods(scan);
diff --git a/core/scan.cc b/core/scan.cc
index 5335abd..5f315ec 100644
--- a/core/scan.cc
+++ b/core/scan.cc
@@ -75,7 +75,7 @@ Scan::Scan(const Get &get) {
cache_blocks_ = get.CacheBlocks();
max_versions_ = get.MaxVersions();
tr_.reset(new TimeRange(get.Timerange().MinTimeStamp(), get.Timerange().MaxTimeStamp()));
- family_map_.insert(get.Family().begin(), get.Family().end());
+ family_map_.insert(get.FamilyMap().begin(), get.FamilyMap().end());
}
Scan &Scan::AddFamily(const std::string &family) {
@@ -191,5 +191,7 @@ void Scan::CheckRow(const std::string &row) {
bool Scan::HasFamilies() const { return !family_map_.empty(); }
-const FamilyMap &Scan::Family() const { return family_map_; }
+const std::map<std::string, std::vector<std::string>> &Scan::FamilyMap() const {
+ return family_map_;
+}
} // namespace hbase
diff --git a/core/scan.h b/core/scan.h
index cedbe0a..fb302b7 100644
--- a/core/scan.h
+++ b/core/scan.h
@@ -30,12 +30,6 @@
namespace hbase {
-/**
- * @brief Map consisting of column families and qualifiers to be used for Get
- * operation
- */
-using FamilyMap = std::map<std::string, std::vector<std::string>>;
-
class Scan : public Query {
public:
/**
@@ -251,14 +245,14 @@ class Scan : public Query {
const TimeRange &Timerange() const;
/**
- * @brief Returns true if family map (FamilyMap) is non empty false otherwise
+ * @brief Returns true if family map is non empty false otherwise
*/
bool HasFamilies() const;
/**
- * @brief Returns the Scan family map (FamilyMap) for this Scan operation.
+ * @brief Returns the Scan family map for this Scan operation.
*/
- const FamilyMap &Family() const;
+ const std::map<std::string, std::vector<std::string>> &FamilyMap() const;
private:
std::string start_row_ = "";
@@ -273,7 +267,7 @@ class Scan : public Query {
bool allow_partial_results_ = false;
hbase::pb::Consistency consistency_ = hbase::pb::Consistency::STRONG;
std::unique_ptr<TimeRange> tr_ = std::make_unique<TimeRange>();
- FamilyMap family_map_;
+ std::map<std::string, std::vector<std::string>> family_map_;
/**
* @brief Checks for row length validity, throws if length check fails,
diff --git a/core/table.h b/core/table.h
index 93ab91b..803befe 100644
--- a/core/table.h
+++ b/core/table.h
@@ -30,6 +30,7 @@
#include "core/configuration.h"
#include "core/get.h"
#include "core/location-cache.h"
+#include "core/put.h"
#include "core/raw-async-table.h"
#include "core/result.h"
#include "serde/table-name.h"
diff --git a/utils/BUCK b/utils/BUCK
index eae929e..04e2b67 100644
--- a/utils/BUCK
+++ b/utils/BUCK
@@ -18,10 +18,14 @@
cxx_library(
name="utils",
exported_headers=[
- "user-util.h", "version.h", "connection-util.h", "sys-util.h",
- "time-util.h"
+ "bytes-util.h",
+ "connection-util.h",
+ "sys-util.h",
+ "time-util.h",
+ "user-util.h",
+ "version.h",
],
- srcs=["user-util.cc", "connection-util.cc"],
+ srcs=["bytes-util.cc", "connection-util.cc", "user-util.cc"],
deps=['//third-party:folly',],
tests=[":user-util-test"],
visibility=['PUBLIC',],
@@ -30,3 +34,7 @@ cxx_test(
name="user-util-test",
srcs=["user-util-test.cc",],
deps=[":utils",],)
+cxx_test(
+ name="bytes-util-test",
+ srcs=["bytes-util-test.cc",],
+ deps=[":utils",],)
diff --git a/utils/bytes-util-test.cc b/utils/bytes-util-test.cc
new file mode 100644
index 0000000..d997ee2
--- /dev/null
+++ b/utils/bytes-util-test.cc
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <folly/Logging.h>
+#include <gtest/gtest.h>
+#include <string>
+
+#include "utils/bytes-util.h"
+
+using namespace std;
+using namespace hbase;
+
+TEST(TestBytesUtil, TestToStringBinary) {
+ std::string empty{""};
+ EXPECT_EQ(empty, BytesUtil::ToStringBinary(empty));
+
+ std::string foo_bar{"foo bar"};
+ EXPECT_EQ(foo_bar, BytesUtil::ToStringBinary(foo_bar));
+
+ std::string foo_bar2{"foo bar_/!@#$%^&*(){}[]|1234567890"};
+ EXPECT_EQ(foo_bar2, BytesUtil::ToStringBinary(foo_bar2));
+
+ char zero = 0;
+ EXPECT_EQ("\\x00", BytesUtil::ToStringBinary(std::string{zero}));
+
+ char max = 255;
+ EXPECT_EQ("\\xFF", BytesUtil::ToStringBinary(std::string{max}));
+
+ EXPECT_EQ("\\x00\\xFF", BytesUtil::ToStringBinary(std::string{zero} + std::string{max}));
+
+ EXPECT_EQ("foo_\\x00\\xFF_bar",
+ BytesUtil::ToStringBinary("foo_" + std::string{zero} + std::string{max} + "_bar"));
+}
diff --git a/utils/bytes-util.cc b/utils/bytes-util.cc
new file mode 100644
index 0000000..5e4d728
--- /dev/null
+++ b/utils/bytes-util.cc
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "utils/bytes-util.h"
+
+#include <memory>
+#include <string>
+
+#include <glog/logging.h>
+
+namespace hbase {
+
+constexpr char BytesUtil::kHexChars[];
+
+std::string BytesUtil::ToStringBinary(const std::string& b, size_t off, size_t len) {
+ std::string result;
+ // Just in case we are passed a 'len' that is > buffer length...
+ if (off >= b.size()) {
+ return result;
+ }
+ if (off + len > b.size()) {
+ len = b.size() - off;
+ }
+ for (size_t i = off; i < off + len; ++i) {
+ int32_t ch = b[i] & 0xFF;
+ if (ch >= ' ' && ch <= '~' && ch != '\\') {
+ result += ch;
+ } else {
+ result += "\\x";
+ result += kHexChars[ch / 0x10];
+ result += kHexChars[ch % 0x10];
+ }
+ }
+ return result;
+}
+
+} /* namespace hbase */
diff --git a/utils/bytes-util.h b/utils/bytes-util.h
new file mode 100644
index 0000000..541b2d7
--- /dev/null
+++ b/utils/bytes-util.h
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+
+namespace hbase {
+
+class BytesUtil {
+ private:
+ static const constexpr char kHexChars[] = "0123456789ABCDEF";
+
+ public:
+ static std::string ToStringBinary(const std::string& b) { return ToStringBinary(b, 0, b.size()); }
+ /**
+ * Write a printable representation of a byte array. Non-printable
+ * characters are hex escaped in the format \\x%02X, eg:
+ * \x00 \x05 etc
+ *
+ * @param b array to write out
+ * @param off offset to start at
+ * @param len length to write
+ * @return string output
+ */
+ static std::string ToStringBinary(const std::string& b, size_t off, size_t len);
+};
+} /* namespace hbase */
[hbase] 74/137: HBASE-17768 [C++] Makefile should recompile only
the changed sources (Sudeep Sunthankar)
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit a2caa164cf69aae6bed1d38a971b326456c31657
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Tue Mar 14 14:35:22 2017 -0700
HBASE-17768 [C++] Makefile should recompile only the changed sources (Sudeep Sunthankar)
---
Makefile | 132 +++++++++++++++++++++++++++++++-------------------------
Makefile.protos | 68 ++++++++++++++++-------------
2 files changed, 113 insertions(+), 87 deletions(-)
diff --git a/Makefile b/Makefile
index 166c6ab..143c00c 100644
--- a/Makefile
+++ b/Makefile
@@ -15,82 +15,94 @@
# specific language governing permissions and limitations
# under the License.
-#use "gcc" to compile source files
-CC:=g++
-LD:=g++
-
-DEBUG_PATH = build/debug
-RELEASE_PATH = build/release
-PROTO_SRC_DIR = build/if
-MODULES = connection core serde test-util utils security exceptions
-SRC_DIR = $(MODULES)
-DEBUG_BUILD_DIR = $(addprefix $(DEBUG_PATH)/,$(MODULES))
-RELEASE_BUILD_DIR = $(addprefix $(RELEASE_PATH)/,$(MODULES))
-INCLUDE_DIR = . build/ $(JAVA_HOME)/include/ $(JAVA_HOME)/include/linux
+#use "g++" to compile source files
+CC := g++
+LD := g++
+
+BUILD_PATH := build
+DEBUG_PATH := $(BUILD_PATH)/debug
+RELEASE_PATH := $(BUILD_PATH)/release
+PROTO_SRC_DIR := if
+PROTO_CXX_DIR := $(BUILD_PATH)/if
+MODULES := connection core exceptions security serde test-util utils
+SRC_DIR := $(MODULES)
+DEBUG_BUILD_DIR := $(addprefix $(DEBUG_PATH)/,$(MODULES))
+RELEASE_BUILD_DIR := $(addprefix $(RELEASE_PATH)/,$(MODULES))
+INCLUDE_DIR := . $(BUILD_PATH) $(JAVA_HOME)/include/ $(JAVA_HOME)/include/linux
#flags to pass to the CPP compiler & linker
-CPPFLAGS_DEBUG = -D_GLIBCXX_USE_CXX11_ABI=0 -g -Wall -std=c++14 -pedantic -fPIC
-CPPFLAGS_RELEASE = -D_GLIBCXX_USE_CXX11_ABI=0 -DNDEBUG -O2 -Wall -std=c++14 -pedantic -fPIC
-LDFLAGS = -lprotobuf -lzookeeper_mt -lsasl2 -lfolly -lwangle
-LINKFLAG = -shared -L$(JAVA_HOME)/jre/lib/amd64/server -ljvm
+CPPFLAGS_DEBUG := -D_GLIBCXX_USE_CXX11_ABI=0 -g -Wall -std=c++14 -pedantic -fPIC -MMD -MP
+CPPFLAGS_RELEASE := -D_GLIBCXX_USE_CXX11_ABI=0 -DNDEBUG -O2 -Wall -std=c++14 -pedantic -fPIC -MMD -MP
+LDFLAGS := -lprotobuf -lzookeeper_mt -lsasl2 -lfolly -lwangle -L $(JAVA_HOME)/jre/lib/amd64/server -ljvm
+LINKFLAG := -shared
#define list of source files and object files
-ALLSRC = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc))
-EXCLUDE_SRC = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*-test.cc)) core/simple-client.cc
-SRC = $(filter-out $(EXCLUDE_SRC), $(ALLSRC))
-PROTOSRC = $(patsubst %.proto, $(addprefix build/,%.pb.cc),$(wildcard if/*.proto))
-DEPS = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.h))
-PROTODEPS = $(patsubst %.proto, $(addprefix build/,%.pb.h),$(wildcard if/*.proto))
-DEBUG_OBJ = $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(SRC))
+ALLSRC := $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc))
+EXCLUDE_SRC := $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*-test.cc)) core/simple-client.cc
+SRC := $(filter-out $(EXCLUDE_SRC), $(ALLSRC))
+PROTOSRC := $(patsubst %.proto, $(addprefix $(BUILD_PATH)/,%.pb.cc),$(wildcard if/*.proto))
+PROTOHDR := $(patsubst %.proto, $(addprefix $(BUILD_PATH)/,%.pb.h),$(wildcard if/*.proto))
+DEBUG_OBJ := $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(SRC))
DEBUG_OBJ += $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(PROTOSRC))
-RELEASE_OBJ = $(patsubst %.cc,$(RELEASE_PATH)/%.o,$(SRC))
+RELEASE_OBJ := $(patsubst %.cc,$(RELEASE_PATH)/%.o,$(SRC))
RELEASE_OBJ += $(patsubst %.cc,$(RELEASE_PATH)/%.o,$(PROTOSRC))
-INCLUDES = $(addprefix -I,$(INCLUDE_DIR))
-
-LIB_DIR = /usr/local
-LIB_LIBDIR = $(LIB_DIR)/lib
-LIB_INCDIR = $(LIB_DIR)/include
-LIB_RELEASE=$(RELEASE_PATH)/libHbaseClient.so
-ARC_RELEASE=$(RELEASE_PATH)/libHbaseClient.a
-LIB_DEBUG=$(DEBUG_PATH)/libHbaseClient_d.so
-ARC_DEBUG=$(DEBUG_PATH)/libHbaseClient_d.a
+INCLUDES := $(addprefix -I,$(INCLUDE_DIR))
+
+LIB_DIR := /usr/local
+LIB_LIBDIR := $(LIB_DIR)/lib
+LIB_INCDIR := $(LIB_DIR)/include
+LIB_RELEASE := $(RELEASE_PATH)/libHbaseClient.so
+ARC_RELEASE := $(RELEASE_PATH)/libHbaseClient.a
+LIB_DEBUG := $(DEBUG_PATH)/libHbaseClient_d.so
+ARC_DEBUG := $(DEBUG_PATH)/libHbaseClient_d.a
+
+build: checkdirs protos $(LIB_DEBUG) $(LIB_RELEASE) $(ARC_DEBUG) $(ARC_RELEASE)
vpath %.cc $(SRC_DIR)
-build: checkdirs copyfiles protos $(LIB_DEBUG) $(LIB_RELEASE) $(ARC_DEBUG) $(ARC_RELEASE)
-$(LIB_DEBUG): $(DEBUG_BUILD_DIR)
+$(LIB_DEBUG):
define make-goal-dbg
-$1/%.o: %.cc $(DEPS) $(PROTODEPS) $(PROTOSRC)
- $(CC) -c $$< -o $$@ $(CPPFLAGS_DEBUG) $(INCLUDES)
+DEPS := $(DEBUG_OBJ:.o=.d)
+-include $(DEPS)
+$1/%.o: %.cc
+ $(CC) -c $$< -o $$@ -MF$$(@:%.o=%.d) -MT$$@ $(CPPFLAGS_DEBUG) $(INCLUDES)
endef
-$(LIB_RELEASE): $(RELEASE_BUILD_DIR)
+$(LIB_RELEASE):
define make-goal-rel
-$1/%.o: %.cc $(DEPS) $(PROTODEPS) $(PROTOSRC)
- $(CC) -c $$< -o $$@ $(CPPFLAGS_RELEASE) $(INCLUDES)
+DEPS := $(RELEASE_OBJ:.o=.d)
+-include $(DEPS)
+$1/%.o: %.cc
+ $(CC) -c $$< -o $$@ -MF$$(@:%.o=%.d) -MT$$@ $(CPPFLAGS_RELEASE) $(INCLUDES)
endef
.PHONY: all clean install
-checkdirs: $(DEBUG_BUILD_DIR) $(RELEASE_BUILD_DIR) $(PROTO_SRC_DIR)
+checkdirs: $(DEBUG_BUILD_DIR) $(RELEASE_BUILD_DIR) $(PROTO_CXX_DIR)
copyfiles:
- bin/copy-protobuf.sh
- bin/copy-version.sh
+ @bin/copy-protobuf.sh
+ @bin/copy-version.sh
-protos: createprotosrc
- @make all -f Makefile.protos
+$(PROTO_CXX_DIR)/%.pb.cc $(PROTO_CXX_DIR)/%.pb.h: $(PROTO_SRC_DIR)/%.proto
+ @protoc --proto_path=$(PROTO_SRC_DIR) --cpp_out=$(PROTO_CXX_DIR) $<
-createprotosrc: $(PROTO_SRC_DIR)
- @protoc --proto_path=if --cpp_out=$(PROTO_SRC_DIR) if/*.proto
+#Run parallel jobs to speed up compilation
+protos: $(PROTO_CXX_DIR) $(PROTOSRC) $(PROTOHDR)
+ @make -j8 all -f Makefile.protos
install:
cp $(LIB_RELEASE) $(LIB_LIBDIR)/libHbaseClient.so
cp $(ARC_RELEASE) $(LIB_LIBDIR)/libHbaseClient.a
cp $(LIB_DEBUG) $(LIB_LIBDIR)/libHbaseClient_d.so
cp $(ARC_DEBUG) $(LIB_LIBDIR)/libHbaseClient_d.a
+ ldconfig
-$(PROTO_SRC_DIR):
+uninstall:
+ rm -f $(LIB_LIBDIR)/libHbaseClient.so $(LIB_LIBDIR)/libHbaseClient.a $(LIB_LIBDIR)/libHbaseClient_d.so $(ARC_DEBUG) $(LIB_LIBDIR)/libHbaseClient_d.a
+ ldconfig
+
+$(PROTO_CXX_DIR):
@mkdir -p $@
$(DEBUG_BUILD_DIR):
@@ -111,9 +123,8 @@ $(LIB_RELEASE): $(RELEASE_OBJ)
$(LIB_DEBUG): $(DEBUG_OBJ)
$(LD) $(LINKFLAG) -o $@ $(LDFLAGS) $(DEBUG_OBJ)
-#to clean re-compilable files
clean:
- @rm -rf $(DEBUG_BUILD_DIR) $(RELEASE_BUILD_DIR) $(LIB_RELEASE) $(LIB_DEBUG) $(ARC_RELEASE) $(ARC_DEBUG) docs buck-out build
+ @rm -rf docs buck-out $(BUILD_PATH)
$(foreach bdir,$(DEBUG_BUILD_DIR), $(eval $(call make-goal-dbg,$(bdir))))
@@ -131,11 +142,16 @@ doc:
help:
@echo "Available targets:"
@echo ""
- @echo " build : will build everything."
- @echo " clean : will remove the docs folder, object files and local libraries"
- @echo " install : will copy the libs to $(LIB_LIBDIR). super user priviliege would be required."
- @echo " check : will test everything."
- @echo " protos : will build the corresponding sources for protobufs present in if/ directory."
- @echo " lint : will ensure that code conforms to Google coding style."
-
-all: build doc check
+ @echo " all : builds everything, creates doc and runs tests."
+ @echo " build : will build/rebuild everything."
+ @echo " check : will test everything."
+ @echo " clean : removes docs folder, object files and local libraries from build/ directory."
+ @echo " copyfiles : copies native version.h from mvn build and proto locally to hbase-native-client."
+ @echo " doc : generates documentation."
+ @echo " install : will copy the libs to $(LIB_LIBDIR). super user priviliege would be required."
+ @echo " protos : will create PB CPP sec and headers from if/*.proto and build them."
+ @echo " uninstall : removes the libs from $(LIB_LIBDIR)."
+ @echo " lint : will ensure that code conforms to Google coding style."
+ @echo "If no target is specified 'build' will be executed"
+
+all: copyfiles build doc check
diff --git a/Makefile.protos b/Makefile.protos
index 2c6316b..4cf8982 100644
--- a/Makefile.protos
+++ b/Makefile.protos
@@ -15,56 +15,66 @@
# specific language governing permissions and limitations
# under the License.
-#use "gcc" to compile source files
-CC:=g++
-LD:=g++
+#use "g++" to compile source files
+CC := g++
-DEBUG_PATH = build/debug
-RELEASE_PATH = build/release
-MODULES = build/if
-SRC_DIR = $(MODULES)
-DEBUG_BUILD_DIR = $(addprefix $(DEBUG_PATH)/,$(MODULES))
-RELEASE_BUILD_DIR = $(addprefix $(RELEASE_PATH)/,$(MODULES))
-INCLUDE_DIR = .
+BUILD_PATH := build
+DEBUG_PATH := $(BUILD_PATH)/debug
+RELEASE_PATH := $(BUILD_PATH)/release
+MODULES := $(BUILD_PATH)/if
+SRC_DIR := $(MODULES)
+DEBUG_BUILD_DIR := $(addprefix $(DEBUG_PATH)/,$(MODULES))
+RELEASE_BUILD_DIR := $(addprefix $(RELEASE_PATH)/,$(MODULES))
+INCLUDE_DIR := . $(BUILD_PATH)/if
#flags to pass to the CPP compiler & linker
-CPPFLAGS_DEBUG = -D_GLIBCXX_USE_CXX11_ABI=0 -g -Wall -std=c++14 -pedantic -fPIC
-CPPFLAGS_RELEASE = -D_GLIBCXX_USE_CXX11_ABI=0 -DNDEBUG -O2 -Wall -std=c++14 -pedantic -fPIC
+CPPFLAGS_DEBUG := -D_GLIBCXX_USE_CXX11_ABI=0 -g -Wall -std=c++14 -pedantic -fPIC -MMD -MP
+CPPFLAGS_RELEASE := -D_GLIBCXX_USE_CXX11_ABI=0 -DNDEBUG -O2 -Wall -std=c++14 -pedantic -fPIC -MMD -MP
#define list of source files and object files
-SRC = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc))
-DEPS = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.h))
-DEBUG_OBJ = $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(SRC))
-RELEASE_OBJ = $(patsubst %.cc,$(RELEASE_PATH)/%.o,$(SRC))
-INCLUDES = $(addprefix -I,$(INCLUDE_DIR))
+SRC := $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc))
+DEBUG_OBJ := $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(SRC))
+RELEASE_OBJ := $(patsubst %.cc,$(RELEASE_PATH)/%.o,$(SRC))
+INCLUDES := $(addprefix -I,$(INCLUDE_DIR))
+
+all: $(DEBUG_BUILD_DIR) $(RELEASE_BUILD_DIR) $(DEBUG_OBJ) $(RELEASE_OBJ)
vpath %.cc $(SRC_DIR)
-$(DEBUG_OBJ): $(DEBUG_BUILD_DIR)
+$(DEBUG_OBJ):
define make-goal-dbg
-$1/%.o: %.cc $(DEPS)
- $(CC) -c $$< -o $$@ $(CPPFLAGS_DEBUG) $(INCLUDES)
+DEPS := $(DEBUG_OBJ:.o=.d)
+-include $(DEPS)
+$1/%.o: %.cc
+ $(CC) -c $$< -o $$@ -MF$$(@:%.o=%.d) -MT$$@ $(CPPFLAGS_DEBUG) $(INCLUDES)
endef
-$(RELEASE_OBJ): $(RELEASE_BUILD_DIR)
+$(RELEASE_OBJ):
define make-goal-rel
-$1/%.o: %.cc $(DEPS)
- $(CC) -c $$< -o $$@ $(CPPFLAGS_RELEASE) $(INCLUDES)
+DEPS := $(RELEASE_OBJ:.o=.d)
+-include $(DEPS)
+$1/%.o: %.cc
+ $(CC) -c $$< -o $$@ -MF$$(@:%.o=%.d) -MT$$@ $(CPPFLAGS_RELEASE) $(INCLUDES)
endef
+.PHONY: all clean
+
$(DEBUG_BUILD_DIR):
@mkdir -p $@
$(RELEASE_BUILD_DIR):
@mkdir -p $@
-.PHONY: all clean
-
-all: $(DEBUG_OBJ) $(RELEASE_OBJ)
-
clean:
- @rm -rf $(DEBUG_OBJ) $(RELEASE_OBJ)
+ @rm -rf $(DEBUG_BUILD_DIR) $(RELEASE_BUILD_DIR)
-$(foreach bdir,$(DEBUG_BUILD_DIR), $(eval $(call make-goal-dbg,$(bdir))))
+$(foreach bdir,$(DEBUG_BUILD_DIR),$(eval $(call make-goal-dbg,$(bdir))))
$(foreach bdir,$(RELEASE_BUILD_DIR),$(eval $(call make-goal-rel,$(bdir))))
+
+help:
+ @echo "This Makefile invocation will only work when protobuf sources and headers have been generated by running 'make protos'"
+ @echo "Available targets:"
+ @echo ""
+ @echo " all : creates objects for the generated PB src."
+ @echo " clean : removes PB objects."
[hbase] 20/137: HBASE-15730 Add on script to format all .h, .cc,
and BUCK files.
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 5f84b288273d4c06be2ed6d1f44cd251deaf9256
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Wed Apr 27 15:52:27 2016 -0700
HBASE-15730 Add on script to format all .h,.cc, and BUCK files.
---
bin/format-code.sh | 7 +++++++
connection/request.cc | 14 +++++++-------
utils/BUCK | 8 ++++----
utils/user-util-test.cc | 2 +-
utils/user-util.cc | 4 ++--
utils/user-util.h | 2 +-
6 files changed, 22 insertions(+), 15 deletions(-)
diff --git a/bin/format-code.sh b/bin/format-code.sh
new file mode 100755
index 0000000..24d6515
--- /dev/null
+++ b/bin/format-code.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+
+
+find core connection serde utils -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i
+find core connection serde utils third-party -name "BUCK" | xargs -P8 yapf -i
diff --git a/connection/request.cc b/connection/request.cc
index 50ea029..d4a4603 100644
--- a/connection/request.cc
+++ b/connection/request.cc
@@ -30,16 +30,16 @@ Request::Request(std::shared_ptr<google::protobuf::Message> req,
std::unique_ptr<Request> Request::get() {
return std::make_unique<Request>(std::make_shared<hbase::pb::GetRequest>(),
- std::make_shared<hbase::pb::GetResponse>(),
- "Get");
+ std::make_shared<hbase::pb::GetResponse>(),
+ "Get");
}
std::unique_ptr<Request> Request::mutate() {
- return std::make_unique<Request>(std::make_shared<hbase::pb::MutateRequest>(),
- std::make_shared<hbase::pb::MutateResponse>(),
- "Mutate");
+ return std::make_unique<Request>(
+ std::make_shared<hbase::pb::MutateRequest>(),
+ std::make_shared<hbase::pb::MutateResponse>(), "Mutate");
}
std::unique_ptr<Request> Request::scan() {
return std::make_unique<Request>(std::make_shared<hbase::pb::ScanRequest>(),
- std::make_shared<hbase::pb::ScanResponse>(),
- "Scan");
+ std::make_shared<hbase::pb::ScanResponse>(),
+ "Scan");
}
diff --git a/utils/BUCK b/utils/BUCK
index 2b65b12..ad12f7d 100644
--- a/utils/BUCK
+++ b/utils/BUCK
@@ -23,18 +23,18 @@ cxx_library(name="utils",
"user-util.cc",
],
deps=[
- '//third-party:folly',
+ '//third-party:folly',
],
visibility=[
'PUBLIC',
],
tests=[
- ":user-util-test"
- ],)
+ ":user-util-test"
+ ], )
cxx_test(name="user-util-test",
srcs=[
"user-util-test.cc",
],
deps=[
":utils",
- ],)
+ ], )
diff --git a/utils/user-util-test.cc b/utils/user-util-test.cc
index 1e72bcc..7c11d8c 100644
--- a/utils/user-util-test.cc
+++ b/utils/user-util-test.cc
@@ -17,9 +17,9 @@
*
*/
+#include <folly/Logging.h>
#include <gtest/gtest.h>
#include <string>
-#include <folly/Logging.h>
#include "utils/user-util.h"
diff --git a/utils/user-util.cc b/utils/user-util.cc
index c4427e3..b801537 100644
--- a/utils/user-util.cc
+++ b/utils/user-util.cc
@@ -19,10 +19,10 @@
#include "utils/user-util.h"
+#include <folly/Logging.h>
#include <pwd.h>
-#include <unistd.h>
#include <sys/types.h>
-#include <folly/Logging.h>
+#include <unistd.h>
using namespace hbase;
using namespace std;
diff --git a/utils/user-util.h b/utils/user-util.h
index 4f74948..9ca28e3 100644
--- a/utils/user-util.h
+++ b/utils/user-util.h
@@ -19,8 +19,8 @@
#pragma once
-#include <memory>
#include <atomic>
+#include <memory>
#include <mutex>
namespace hbase {
[hbase] 102/137: HBASE-18214 Replace the folly::AtomicHashMap usage
in the RPC layer
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 448ec33fbc76d92adc34788be5cd3f715cc38139
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Mon Jun 26 14:30:49 2017 -0700
HBASE-18214 Replace the folly::AtomicHashMap usage in the RPC layer
---
connection/client-dispatcher.cc | 16 +++--
connection/client-dispatcher.h | 6 +-
connection/client-handler.cc | 17 ++----
connection/client-handler.h | 5 +-
connection/connection-factory.cc | 8 +--
utils/BUCK | 17 ++++--
utils/concurrent-map-test.cc | 36 +++++++++++
utils/concurrent-map.h | 125 +++++++++++++++++++++++++++++++++++++++
8 files changed, 196 insertions(+), 34 deletions(-)
diff --git a/connection/client-dispatcher.cc b/connection/client-dispatcher.cc
index 35a1f7d..b9b2c34 100644
--- a/connection/client-dispatcher.cc
+++ b/connection/client-dispatcher.cc
@@ -25,16 +25,11 @@ using std::unique_ptr;
namespace hbase {
-ClientDispatcher::ClientDispatcher() : requests_(5000), current_call_id_(9) {}
+ClientDispatcher::ClientDispatcher() : current_call_id_(9), requests_(5000) {}
void ClientDispatcher::read(Context *ctx, unique_ptr<Response> in) {
auto call_id = in->call_id();
-
- auto search = requests_.find(call_id);
- CHECK(search != requests_.end());
- auto p = std::move(search->second);
-
- requests_.erase(call_id);
+ auto p = requests_.find_and_erase(call_id);
if (in->exception()) {
p.setException(in->exception());
@@ -46,8 +41,11 @@ void ClientDispatcher::read(Context *ctx, unique_ptr<Response> in) {
folly::Future<unique_ptr<Response>> ClientDispatcher::operator()(unique_ptr<Request> arg) {
auto call_id = current_call_id_++;
arg->set_call_id(call_id);
- requests_.insert(call_id, folly::Promise<unique_ptr<Response>>{});
- auto &p = requests_.find(call_id)->second;
+
+ // TODO: if the map is full (or we have more than hbase.client.perserver.requests.threshold)
+ // then throw ServerTooBusyException so that upper layers will retry.
+ auto &p = requests_[call_id];
+
auto f = p.getFuture();
p.setInterruptHandler([call_id, this](const folly::exception_wrapper &e) {
LOG(ERROR) << "e = " << call_id;
diff --git a/connection/client-dispatcher.h b/connection/client-dispatcher.h
index 857042c..1f8e6b3 100644
--- a/connection/client-dispatcher.h
+++ b/connection/client-dispatcher.h
@@ -19,16 +19,18 @@
#pragma once
-#include <folly/AtomicHashMap.h>
#include <folly/Logging.h>
#include <wangle/service/ClientDispatcher.h>
#include <atomic>
+#include <map>
#include <memory>
+#include <mutex>
#include "connection/pipeline.h"
#include "connection/request.h"
#include "connection/response.h"
+#include "utils/concurrent-map.h"
namespace hbase {
/**
@@ -51,7 +53,7 @@ class ClientDispatcher
folly::Future<folly::Unit> close() override;
private:
- folly::AtomicHashMap<uint32_t, folly::Promise<std::unique_ptr<Response>>> requests_;
+ concurrent_map<uint32_t, folly::Promise<std::unique_ptr<Response>>> requests_;
// Start at some number way above what could
// be there for un-initialized call id counters.
//
diff --git a/connection/client-handler.cc b/connection/client-handler.cc
index 775df68..052c171 100644
--- a/connection/client-handler.cc
+++ b/connection/client-handler.cc
@@ -40,8 +40,9 @@ ClientHandler::ClientHandler(std::string user_name, std::shared_ptr<Codec> codec
serde_(codec),
server_(server),
once_flag_(std::make_unique<std::once_flag>()),
- resp_msgs_(std::make_unique<folly::AtomicHashMap<uint32_t, std::shared_ptr<Message>>>(5000)) {
-}
+ resp_msgs_(
+ std::make_unique<concurrent_map<uint32_t, std::shared_ptr<google::protobuf::Message>>>(
+ 5000)) {}
void ClientHandler::read(Context *ctx, std::unique_ptr<folly::IOBuf> buf) {
if (LIKELY(buf != nullptr)) {
@@ -53,15 +54,7 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<folly::IOBuf> buf) {
VLOG(3) << "Read RPC ResponseHeader size=" << used_bytes << " call_id=" << header.call_id()
<< " has_exception=" << header.has_exception() << ", server: " << server_;
- // Get the response protobuf from the map
- auto search = resp_msgs_->find(header.call_id());
- // It's an error if it's not there.
- CHECK(search != resp_msgs_->end());
- auto resp_msg = search->second;
- CHECK(resp_msg != nullptr);
-
- // Make sure we don't leak the protobuf
- resp_msgs_->erase(header.call_id());
+ auto resp_msg = resp_msgs_->find_and_erase(header.call_id());
// set the call_id.
// This will be used to by the dispatcher to match up
@@ -133,7 +126,7 @@ folly::Future<folly::Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Re
VLOG(3) << "Writing RPC Request:" << r->DebugString() << ", server: " << server_;
// Now store the call id to response.
- resp_msgs_->insert(r->call_id(), r->resp_msg());
+ resp_msgs_->insert(std::make_pair(r->call_id(), r->resp_msg()));
// Send the data down the pipeline.
return ctx->fireWrite(serde_.Request(r->call_id(), r->method(), r->req_msg().get()));
diff --git a/connection/client-handler.h b/connection/client-handler.h
index 4c106e0..8de3a8b 100644
--- a/connection/client-handler.h
+++ b/connection/client-handler.h
@@ -18,7 +18,6 @@
*/
#pragma once
-#include <folly/AtomicHashMap.h>
#include <wangle/channel/Handler.h>
#include <atomic>
@@ -30,6 +29,7 @@
#include "exceptions/exception.h"
#include "serde/codec.h"
#include "serde/rpc.h"
+#include "utils/concurrent-map.h"
// Forward decs.
namespace hbase {
@@ -81,7 +81,6 @@ class ClientHandler
std::string server_; // for logging
// in flight requests
- std::unique_ptr<folly::AtomicHashMap<uint32_t, std::shared_ptr<google::protobuf::Message>>>
- resp_msgs_;
+ std::unique_ptr<concurrent_map<uint32_t, std::shared_ptr<google::protobuf::Message>>> resp_msgs_;
};
} // namespace hbase
diff --git a/connection/connection-factory.cc b/connection/connection-factory.cc
index d1bfbce..a0c7f96 100644
--- a/connection/connection-factory.cc
+++ b/connection/connection-factory.cc
@@ -17,15 +17,15 @@
*
*/
-#include "connection/connection-factory.h"
-#include "connection/sasl-handler.h"
+#include <glog/logging.h>
+#include <wangle/channel/Handler.h>
#include <chrono>
-#include <glog/logging.h>
-#include <wangle/channel/Handler.h>
#include "connection/client-dispatcher.h"
+#include "connection/connection-factory.h"
#include "connection/pipeline.h"
+#include "connection/sasl-handler.h"
#include "connection/service.h"
using std::chrono::milliseconds;
diff --git a/utils/BUCK b/utils/BUCK
index 788056b..96f02b8 100644
--- a/utils/BUCK
+++ b/utils/BUCK
@@ -20,6 +20,7 @@ cxx_library(
exported_headers=[
"bytes-util.h",
"connection-util.h",
+ "concurrent-map.h",
"optional.h",
"sys-util.h",
"time-util.h",
@@ -38,18 +39,26 @@ cxx_library(
],
compiler_flags=['-Weffc++'],)
cxx_test(
- name="user-util-test",
+ name="bytes-util-test",
srcs=[
- "user-util-test.cc",
+ "bytes-util-test.cc",
],
deps=[
":utils",
],)
cxx_test(
- name="bytes-util-test",
+ name="concurrent-map-test",
srcs=[
- "bytes-util-test.cc",
+ "concurrent-map-test.cc",
],
deps=[
":utils",
],)
+cxx_test(
+ name="user-util-test",
+ srcs=[
+ "user-util-test.cc",
+ ],
+ deps=[
+ ":utils",
+ ],)
\ No newline at end of file
diff --git a/utils/concurrent-map-test.cc b/utils/concurrent-map-test.cc
new file mode 100644
index 0000000..588bd08
--- /dev/null
+++ b/utils/concurrent-map-test.cc
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <folly/Logging.h>
+#include <gtest/gtest.h>
+#include <string>
+
+#include "utils/concurrent-map.h"
+
+using hbase::concurrent_map;
+
+TEST(TestConcurrentMap, TestFindAndErase) {
+ concurrent_map<std::string, std::string> map{500};
+
+ map.insert(std::make_pair("foo", "bar"));
+ auto prev = map.find_and_erase("foo");
+ ASSERT_EQ("bar", prev);
+
+ ASSERT_EQ(map.end(), map.find("foo"));
+}
diff --git a/utils/concurrent-map.h b/utils/concurrent-map.h
new file mode 100644
index 0000000..d9703e1
--- /dev/null
+++ b/utils/concurrent-map.h
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <map>
+#include <memory>
+#include <shared_mutex>
+#include <unordered_map>
+#include <utility>
+
+namespace hbase {
+
+/**
+ * A concurrent version of std::unordered_map where we acquire a shared or exclusive
+ * lock for operations. This is NOT a highly-concurrent and scalable implementation
+ * since there is only one lock object.
+ * Replace this with tbb::concurrent_unordered_map or similar.
+ *
+ * Concurrency here is different than in Java. For example, the iterators returned from
+ * find() will not copy the key, value pairs.
+ */
+template <typename K, typename V>
+class concurrent_map {
+ public:
+ typedef K key_type;
+ typedef V mapped_type;
+ typedef std::pair<const key_type, mapped_type> value_type;
+ typedef typename std::unordered_map<K, V>::iterator iterator;
+ typedef typename std::unordered_map<K, V>::const_iterator const_iterator;
+
+ concurrent_map() : map_(), mutex_() {}
+ explicit concurrent_map(int32_t n) : map_(n), mutex_() {}
+
+ void insert(const value_type& value) {
+ std::unique_lock<std::shared_timed_mutex> lock(mutex_);
+ map_.insert(value);
+ }
+
+ /**
+ * Return the mapped object for this key. Be careful to not use the return reference
+ * to do assignment. I think it won't be thread safe
+ */
+ mapped_type& at(const key_type& key) {
+ std::shared_lock<std::shared_timed_mutex> lock(mutex_);
+ iterator where = map_.find(key);
+ if (where == end()) {
+ std::runtime_error("Key not found");
+ }
+ return where->second;
+ }
+
+ mapped_type& operator[](const key_type& key) {
+ std::unique_lock<std::shared_timed_mutex> lock(mutex_);
+ iterator where = map_.find(key);
+ if (where == end()) {
+ return map_[key];
+ }
+ return where->second;
+ }
+
+ /**
+ * Atomically finds the entry and removes it from the map, returning
+ * the previously associated value.
+ */
+ mapped_type find_and_erase(const K& key) {
+ std::unique_lock<std::shared_timed_mutex> lock(mutex_);
+ auto search = map_.find(key);
+ // It's an error if it's not there.
+ CHECK(search != end());
+ auto val = std::move(search->second);
+ map_.erase(key);
+ return val;
+ }
+
+ void erase(const K& key) {
+ std::unique_lock<std::shared_timed_mutex> lock(mutex_);
+ map_.erase(key);
+ }
+
+ iterator begin() { return map_.begin(); }
+
+ const_iterator begin() const { return map_.begin(); }
+
+ const_iterator cbegin() const { return map_.begin(); }
+
+ iterator end() { return map_.end(); }
+
+ const_iterator end() const { return map_.end(); }
+
+ const_iterator cend() const { return map_.end(); }
+
+ iterator find(const K& key) {
+ std::shared_lock<std::shared_timed_mutex> lock(mutex_);
+ return map_.find(key);
+ }
+
+ // TODO: find(), at() returning const_iterator
+
+ bool empty() const {
+ std::unique_lock<std::shared_timed_mutex> lock(mutex_);
+ return map_.empty();
+ }
+
+ private:
+ std::shared_timed_mutex mutex_;
+ std::unordered_map<K, V> map_;
+};
+} /* namespace hbase */
[hbase] 90/137: HBASE-18153 [C++] Add valgrind to the docker image
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 395ea2c9a646c9b35d376b0b644745f253618e84
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Fri Jun 2 17:19:57 2017 -0700
HBASE-18153 [C++] Add valgrind to the docker image
---
docker-files/Dockerfile | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/docker-files/Dockerfile b/docker-files/Dockerfile
index c5f47ad..d5a1eab 100644
--- a/docker-files/Dockerfile
+++ b/docker-files/Dockerfile
@@ -32,7 +32,8 @@ RUN wget ftp://ftp.cyrusimap.org/cyrus-sasl/cyrus-sasl-2.1.26.tar.gz ; \
make install ;\
cp /usr/local/lib/sasl2/* /usr/lib/sasl2/
-RUN apt-get install -y vim maven inetutils-ping python-pip doxygen graphviz clang-format && \
+RUN apt-get install -y vim maven inetutils-ping python-pip doxygen graphviz clang-format \
+ valgrind && \
pip install yapf && \
apt-get -qq clean && \
apt-get -y -qq autoremove && \
[hbase] 131/137: HBASE-18725 [C++] Install header files as well as
library
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 725bdedecfffa4716b23fc9ea24b31cb149acd1e
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Mon Sep 11 18:36:11 2017 -0700
HBASE-18725 [C++] Install header files as well as library
---
.gitignore | 2 +-
Makefile | 54 +++++---
Makefile.protos | 14 +-
bin/copy-protobuf.sh | 2 +-
bin/copy-version.sh | 2 +-
bin/cpplint.sh | 11 +-
bin/format-code.sh | 9 +-
include/hbase/client/BUCK | 93 +++++++++++++
{core => include/hbase/client}/action.h | 2 +-
{core => include/hbase/client}/append.h | 4 +-
.../client}/async-batch-rpc-retrying-caller.h | 40 +++---
.../hbase/client}/async-client-scanner.h | 36 ++---
{core => include/hbase/client}/async-connection.h | 20 +--
.../hbase/client}/async-region-locator.h | 10 +-
.../client}/async-rpc-retrying-caller-factory.h | 24 ++--
.../hbase/client}/async-rpc-retrying-caller.h | 10 +-
.../hbase/client}/async-scan-rpc-retrying-caller.h | 36 ++---
.../hbase/client}/async-table-result-scanner.h | 10 +-
{core => include/hbase/client}/cell.h | 0
{core => include/hbase/client}/client.h | 10 +-
{core => include/hbase/client}/configuration.h | 0
.../hbase/client}/connection-configuration.h | 2 +-
{core => include/hbase/client}/delete.h | 4 +-
{core => include/hbase/client}/filter.h | 6 +-
{core => include/hbase/client}/get.h | 8 +-
.../hbase/client}/hbase-configuration-loader.h | 4 +-
.../hbase/client}/hbase-rpc-controller.h | 0
{core => include/hbase/client}/increment.h | 4 +-
{core => include/hbase/client}/keyvalue-codec.h | 4 +-
{core => include/hbase/client}/location-cache.h | 14 +-
{core => include/hbase/client}/meta-utils.h | 10 +-
{core => include/hbase/client}/multi-response.h | 6 +-
{core => include/hbase/client}/mutation.h | 6 +-
{core => include/hbase/client}/put.h | 4 +-
{core => include/hbase/client}/query.h | 2 +-
{core => include/hbase/client}/raw-async-table.h | 23 ++--
.../hbase/client}/raw-scan-result-consumer.h | 6 +-
{core => include/hbase/client}/region-location.h | 2 +-
{core => include/hbase/client}/region-request.h | 4 +-
{core => include/hbase/client}/region-result.h | 4 +-
{core => include/hbase/client}/request-converter.h | 28 ++--
.../hbase/client}/response-converter.h | 14 +-
{core => include/hbase/client}/result-scanner.h | 4 +-
{core => include/hbase/client}/result.h | 4 +-
{core => include/hbase/client}/row.h | 0
{core => include/hbase/client}/scan-result-cache.h | 6 +-
{core => include/hbase/client}/scan.h | 6 +-
{core => include/hbase/client}/server-request.h | 6 +-
{core => include/hbase/client}/table.h | 22 +--
{core => include/hbase/client}/time-range.h | 0
{core => include/hbase/client}/zk-util.h | 2 +-
{connection => include/hbase/connection}/BUCK | 45 ++-----
.../hbase/connection}/client-dispatcher.h | 8 +-
.../hbase/connection}/client-handler.h | 10 +-
.../hbase/connection}/connection-factory.h | 10 +-
.../hbase/connection}/connection-id.h | 4 +-
.../hbase/connection}/connection-pool.h | 10 +-
.../hbase/connection}/pipeline.h | 10 +-
{connection => include/hbase/connection}/request.h | 0
.../hbase/connection}/response.h | 2 +-
.../hbase/connection}/rpc-client.h | 10 +-
.../hbase/connection}/rpc-connection.h | 10 +-
.../hbase/connection}/rpc-fault-injector-inl.h | 0
.../hbase/connection}/rpc-fault-injector.h | 4 +-
.../hbase/connection}/rpc-test-server-handler.h | 6 +-
.../hbase/connection}/rpc-test-server.h | 6 +-
.../hbase/connection}/sasl-handler.h | 8 +-
.../hbase/connection}/sasl-util.h | 2 +-
{connection => include/hbase/connection}/service.h | 4 +-
{exceptions => include/hbase/exceptions}/BUCK | 15 +--
.../hbase/exceptions}/exception.h | 0
{security => include/hbase/security}/BUCK | 4 +-
{security => include/hbase/security}/user.h | 2 +-
{security => include/hbase/serde}/BUCK | 19 ++-
{serde => include/hbase/serde}/cell-outputstream.h | 0
{serde => include/hbase/serde}/cell-scanner.h | 0
{serde => include/hbase/serde}/codec.h | 4 +-
{serde => include/hbase/serde}/region-info.h | 2 +-
{serde => include/hbase/serde}/rpc-serde.h | 6 +-
{serde => include/hbase/serde}/server-name.h | 2 +-
{serde => include/hbase/serde}/table-name.h | 2 +-
{serde => include/hbase/serde}/zk.h | 0
{utils => include/hbase/utils}/BUCK | 26 +---
{utils => include/hbase/utils}/bytes-util.h | 0
{utils => include/hbase/utils}/concurrent-map.h | 0
{utils => include/hbase/utils}/connection-util.h | 2 +-
{utils => include/hbase/utils}/optional.h | 0
{utils => include/hbase/utils}/sys-util.h | 0
{utils => include/hbase/utils}/time-util.h | 0
{utils => include/hbase/utils}/user-util.h | 0
{core => src/hbase/client}/BUCK | 149 +++++++--------------
{core => src/hbase/client}/append-test.cc | 6 +-
{core => src/hbase/client}/append.cc | 2 +-
.../client}/async-batch-rpc-retrying-caller.cc | 2 +-
.../hbase/client}/async-batch-rpc-retrying-test.cc | 24 ++--
{core => src/hbase/client}/async-client-scanner.cc | 2 +-
{core => src/hbase/client}/async-connection.cc | 4 +-
.../client}/async-rpc-retrying-caller-factory.cc | 2 +-
.../hbase/client}/async-rpc-retrying-caller.cc | 22 +--
.../hbase/client}/async-rpc-retrying-test.cc | 38 +++---
.../client}/async-scan-rpc-retrying-caller.cc | 2 +-
.../hbase/client}/async-table-result-scanner.cc | 2 +-
{core => src/hbase/client}/cell-test.cc | 2 +-
{core => src/hbase/client}/cell.cc | 4 +-
{core => src/hbase/client}/client-test.cc | 30 ++---
{core => src/hbase/client}/client.cc | 2 +-
{core => src/hbase/client}/configuration-test.cc | 2 +-
{core => src/hbase/client}/configuration.cc | 2 +-
{core => src/hbase/client}/delete-test.cc | 6 +-
{core => src/hbase/client}/delete.cc | 2 +-
{core => src/hbase/client}/filter-test.cc | 20 +--
{core => src/hbase/client}/get-test.cc | 4 +-
{core => src/hbase/client}/get.cc | 2 +-
.../hbase/client}/hbase-configuration-loader.cc | 2 +-
.../hbase/client}/hbase-configuration-test.cc | 4 +-
{core => src/hbase/client}/hbase-rpc-controller.cc | 2 +-
{core => src/hbase/client}/increment-test.cc | 8 +-
{core => src/hbase/client}/increment.cc | 4 +-
{core => src/hbase/client}/keyvalue-codec.cc | 2 +-
{core => src/hbase/client}/load-client.cc | 12 +-
.../hbase/client}/location-cache-retry-test.cc | 32 ++---
{core => src/hbase/client}/location-cache-test.cc | 10 +-
{core => src/hbase/client}/location-cache.cc | 20 +--
{core => src/hbase/client}/meta-utils.cc | 18 +--
{core => src/hbase/client}/multi-response.cc | 4 +-
{core => src/hbase/client}/mutation.cc | 2 +-
{core => src/hbase/client}/put-test.cc | 6 +-
{core => src/hbase/client}/put.cc | 2 +-
{core => src/hbase/client}/raw-async-table.cc | 7 +-
{core => src/hbase/client}/region-result.cc | 2 +-
.../hbase/client}/request-converter-test.cc | 8 +-
{core => src/hbase/client}/request-converter.cc | 4 +-
{core => src/hbase/client}/response-converter.cc | 8 +-
{core => src/hbase/client}/result-test.cc | 4 +-
{core => src/hbase/client}/result.cc | 2 +-
.../hbase/client}/scan-result-cache-test.cc | 6 +-
{core => src/hbase/client}/scan-result-cache.cc | 2 +-
{core => src/hbase/client}/scan-test.cc | 2 +-
{core => src/hbase/client}/scan.cc | 2 +-
{core => src/hbase/client}/scanner-test.cc | 34 ++---
{core => src/hbase/client}/simple-client.cc | 20 +--
{core => src/hbase/client}/table.cc | 18 +--
{core => src/hbase/client}/time-range-test.cc | 2 +-
{core => src/hbase/client}/time-range.cc | 2 +-
{core => src/hbase/client}/zk-util-test.cc | 2 +-
{core => src/hbase/client}/zk-util.cc | 2 +-
{connection => src/hbase/connection}/BUCK | 32 +----
.../hbase/connection}/client-dispatcher.cc | 6 +-
.../hbase/connection}/client-handler.cc | 10 +-
.../hbase/connection}/connection-factory.cc | 12 +-
.../hbase/connection}/connection-pool-test.cc | 10 +-
.../hbase/connection}/connection-pool.cc | 2 +-
{connection => src/hbase/connection}/pipeline.cc | 6 +-
{connection => src/hbase/connection}/request.cc | 4 +-
{connection => src/hbase/connection}/rpc-client.cc | 4 +-
.../hbase/connection}/rpc-fault-injector.cc | 2 +-
.../hbase/connection}/rpc-test-server-handler.cc | 6 +-
.../hbase/connection}/rpc-test-server.cc | 6 +-
{connection => src/hbase/connection}/rpc-test.cc | 12 +-
.../hbase/connection}/sasl-handler.cc | 6 +-
{connection => src/hbase/connection}/sasl-util.cc | 2 +-
{exceptions => src/hbase/exceptions}/BUCK | 6 +-
.../hbase/exceptions}/exception-test.cc | 2 +-
{exceptions => src/hbase/exceptions}/exception.cc | 2 +-
{if => src/hbase/if}/AccessControl.proto | 0
{if => src/hbase/if}/Admin.proto | 0
{if => src/hbase/if}/Aggregate.proto | 0
{if => src/hbase/if}/Authentication.proto | 0
{if => src/hbase/if}/BUCK | 1 +
{if => src/hbase/if}/Cell.proto | 0
{if => src/hbase/if}/Client.proto | 0
{if => src/hbase/if}/ClusterId.proto | 0
{if => src/hbase/if}/ClusterStatus.proto | 0
{if => src/hbase/if}/Comparator.proto | 0
{if => src/hbase/if}/Encryption.proto | 0
{if => src/hbase/if}/ErrorHandling.proto | 0
{if => src/hbase/if}/FS.proto | 0
{if => src/hbase/if}/Filter.proto | 0
{if => src/hbase/if}/HBase.proto | 0
{if => src/hbase/if}/HFile.proto | 0
{if => src/hbase/if}/LoadBalancer.proto | 0
{if => src/hbase/if}/MapReduce.proto | 0
{if => src/hbase/if}/Master.proto | 0
{if => src/hbase/if}/MasterProcedure.proto | 0
{if => src/hbase/if}/MultiRowMutation.proto | 0
{if => src/hbase/if}/Procedure.proto | 0
{if => src/hbase/if}/Quota.proto | 0
{if => src/hbase/if}/RPC.proto | 0
{if => src/hbase/if}/RSGroup.proto | 0
{if => src/hbase/if}/RSGroupAdmin.proto | 0
{if => src/hbase/if}/RegionNormalizer.proto | 0
{if => src/hbase/if}/RegionServerStatus.proto | 0
{if => src/hbase/if}/RowProcessor.proto | 0
{if => src/hbase/if}/SecureBulkLoad.proto | 0
{if => src/hbase/if}/Snapshot.proto | 0
{if => src/hbase/if}/Tracing.proto | 0
{if => src/hbase/if}/VisibilityLabels.proto | 0
{if => src/hbase/if}/WAL.proto | 0
{if => src/hbase/if}/ZooKeeper.proto | 0
{if => src/hbase/if}/test.proto | 0
{if => src/hbase/if}/test_rpc_service.proto | 0
{security => src/hbase/security}/BUCK | 5 +-
{serde => src/hbase/serde}/BUCK | 12 +-
.../hbase/serde}/client-deserializer-test.cc | 4 +-
.../hbase/serde}/client-serializer-test.cc | 6 +-
.../hbase/serde}/region-info-deserializer-test.cc | 6 +-
{serde => src/hbase/serde}/rpc-serde.cc | 6 +-
{serde => src/hbase/serde}/server-name-test.cc | 2 +-
{serde => src/hbase/serde}/table-name-test.cc | 2 +-
{serde => src/hbase/serde}/zk-deserializer-test.cc | 4 +-
{serde => src/hbase/serde}/zk.cc | 2 +-
{test-util => src/hbase/test-util}/BUCK | 3 +-
{test-util => src/hbase/test-util}/mini-cluster.cc | 2 +-
{test-util => src/hbase/test-util}/mini-cluster.h | 0
{test-util => src/hbase/test-util}/test-util.cc | 4 +-
{test-util => src/hbase/test-util}/test-util.h | 4 +-
{utils => src/hbase/utils}/BUCK | 11 +-
{utils => src/hbase/utils}/bytes-util-test.cc | 2 +-
{utils => src/hbase/utils}/bytes-util.cc | 2 +-
{utils => src/hbase/utils}/concurrent-map-test.cc | 2 +-
{utils => src/hbase/utils}/connection-util.cc | 2 +-
{utils => src/hbase/utils}/user-util-test.cc | 2 +-
{utils => src/hbase/utils}/user-util.cc | 2 +-
223 files changed, 804 insertions(+), 825 deletions(-)
diff --git a/.gitignore b/.gitignore
index c99ce59..e8dd42d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,4 +26,4 @@ third-party/*
/gcc-debug/
# Generated files
-utils/version.h
+src/hbase/utils/version.h
diff --git a/Makefile b/Makefile
index a291dfa..4d6430c 100644
--- a/Makefile
+++ b/Makefile
@@ -19,17 +19,20 @@
CC := g++
LD := g++
+SRC_HBASE := src/hbase
+INCLUDE_HBASE := include/
BUILD_PATH := build
DEBUG_PATH := $(BUILD_PATH)/debug
RELEASE_PATH := $(BUILD_PATH)/release
-PROTO_SRC_DIR := if
-PROTO_CXX_DIR := $(BUILD_PATH)/if
-MODULES := connection core exceptions security serde utils
+PROTO_SRC_DIR := $(SRC_HBASE)/if
+PROTO_CXX_DIR := $(BUILD_PATH)/$(PROTO_SRC_DIR)
+MODULES := connection client exceptions security serde utils
TEST_MODULES := test-util # These modules contain test code, not included in the build for the lib
-SRC_DIR := $(MODULES)
-DEBUG_BUILD_DIR := $(addprefix $(DEBUG_PATH)/,$(MODULES))
-RELEASE_BUILD_DIR := $(addprefix $(RELEASE_PATH)/,$(MODULES))
-INCLUDE_DIR := . $(BUILD_PATH)
+SRC_DIR := $(addprefix $(SRC_HBASE)/,$(MODULES))
+DEBUG_BUILD_DIR := $(addprefix $(DEBUG_PATH)/hbase/,$(MODULES))
+RELEASE_BUILD_DIR := $(addprefix $(RELEASE_PATH)/hbase/,$(MODULES))
+
+INCLUDE_DIR := . src $(BUILD_PATH)/src $(INCLUDE_HBASE)
TEST_BUILD_INCLUDE_DIR := $(INLCUDE_DIR) $(JAVA_HOME)/include/ $(JAVA_HOME)/include/linux
#flags to pass to the CPP compiler & linker
@@ -42,14 +45,14 @@ LINKFLAG := -shared
#define list of source files and object files
ALLSRC := $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc))
EXCLUDE_SRC := $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*-test.cc)) \
- core/simple-client.cc core/load-client.cc
+ $(SRC_HBASE)/client/simple-client.cc $(SRC_HBASE)/client/load-client.cc
SRC := $(filter-out $(EXCLUDE_SRC), $(ALLSRC))
-PROTOSRC := $(patsubst %.proto, $(addprefix $(BUILD_PATH)/,%.pb.cc),$(wildcard if/*.proto))
-PROTOHDR := $(patsubst %.proto, $(addprefix $(BUILD_PATH)/,%.pb.h),$(wildcard if/*.proto))
-DEBUG_OBJ := $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(SRC))
-DEBUG_OBJ += $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(PROTOSRC))
-RELEASE_OBJ := $(patsubst %.cc,$(RELEASE_PATH)/%.o,$(SRC))
-RELEASE_OBJ += $(patsubst %.cc,$(RELEASE_PATH)/%.o,$(PROTOSRC))
+PROTOSRC := $(patsubst $(PROTO_SRC_DIR)/%.proto, $(addprefix $(PROTO_CXX_DIR)/,%.pb.cc),$(wildcard $(PROTO_SRC_DIR)/*.proto))
+PROTOHDR := $(patsubst $(PROTO_SRC_DIR)/%.proto, $(addprefix $(PROTO_CXX_DIR)/,%.pb.h),$(wildcard $(PROTO_SRC_DIR)/*.proto))
+DEBUG_OBJ := $(patsubst $(SRC_HBASE)/%.cc,$(DEBUG_PATH)/hbase/%.o,$(SRC))
+DEBUG_OBJ += $(patsubst $(PROTO_CXX_DIR)/%.cc,$(DEBUG_PATH)/hbase/if/%.o,$(PROTOSRC))
+RELEASE_OBJ := $(patsubst $(SRC_HBASE)/%.cc,$(RELEASE_PATH)/hbase/%.o,$(SRC))
+RELEASE_OBJ += $(patsubst $(PROTO_CXX_DIR)/%.cc,$(RELEASE_PATH)/hbase/if/%.o,$(PROTOSRC))
INCLUDES := $(addprefix -I,$(INCLUDE_DIR))
LIB_DIR := /usr/local
@@ -59,8 +62,9 @@ LIB_RELEASE := $(RELEASE_PATH)/libHBaseClient.so
ARC_RELEASE := $(RELEASE_PATH)/libHBaseClient.a
LIB_DEBUG := $(DEBUG_PATH)/libHBaseClient_d.so
ARC_DEBUG := $(DEBUG_PATH)/libHBaseClient_d.a
+LOCAL_INCLUDE_DIR := /usr/local/include/
-build: checkdirs protos $(LIB_DEBUG) $(LIB_RELEASE) $(ARC_DEBUG) $(ARC_RELEASE)
+build: checkdirs protos copyfiles $(LIB_DEBUG) $(LIB_RELEASE) $(ARC_DEBUG) $(ARC_RELEASE)
vpath %.cc $(SRC_DIR)
@@ -80,7 +84,7 @@ $1/%.o: %.cc
$(CC) -c $$< -o $$@ -MF$$(@:%.o=%.d) -MT$$@ $(CPPFLAGS_RELEASE) $(INCLUDES)
endef
-.PHONY: all clean install
+.PHONY: all clean install copyfiles
checkdirs: $(DEBUG_BUILD_DIR) $(RELEASE_BUILD_DIR) $(PROTO_CXX_DIR)
@@ -88,22 +92,32 @@ copyfiles:
@bin/copy-protobuf.sh
@bin/copy-version.sh
+# .proto files are in src/hbase/if. These are compiled into C++ code by the
+# protoc compiler, and turned into .cc and .h files under build/src/hbase/if
$(PROTO_CXX_DIR)/%.pb.cc $(PROTO_CXX_DIR)/%.pb.h: $(PROTO_SRC_DIR)/%.proto
@protoc --proto_path=$(PROTO_SRC_DIR) --cpp_out=$(PROTO_CXX_DIR) $<
-#Run parallel jobs to speed up compilation
+# protos target compiles the .cc and .h files into .o files for the protobuf
+# generated source files
protos: $(PROTO_CXX_DIR) $(PROTOSRC) $(PROTOHDR)
@make -j8 all -f Makefile.protos
-install:
+install_headers:
+ cp -r $(INCLUDE_HBASE)/hbase $(LOCAL_INCLUDE_DIR)
+ cp -r $(PROTO_CXX_DIR) $(LOCAL_INCLUDE_DIR)/hbase/
+
+uninstall_headers:
+ rm -rf $(LOCAL_INCLUDE_DIR)/hbase
+
+install: install_headers
cp $(LIB_RELEASE) $(LIB_LIBDIR)/libHBaseClient.so
cp $(ARC_RELEASE) $(LIB_LIBDIR)/libHBaseClient.a
cp $(LIB_DEBUG) $(LIB_LIBDIR)/libHBaseClient_d.so
cp $(ARC_DEBUG) $(LIB_LIBDIR)/libHBaseClient_d.a
ldconfig
-uninstall:
- rm -f $(LIB_LIBDIR)/libHBaseClient.so $(LIB_LIBDIR)/libHBaseClient.a $(LIB_LIBDIR)/libHBaseClient_d.so $(ARC_DEBUG) $(LIB_LIBDIR)/libHBaseClient_d.a
+uninstall: uninstall_headers
+ rm -f $(LIB_LIBDIR)/libHBaseClient.so $(LIB_LIBDIR)/libHBaseClient.a $(LIB_LIBDIR)/libHBaseClient_d.so $(LIB_LIBDIR)/libHBaseClient_d.a
ldconfig
$(PROTO_CXX_DIR):
diff --git a/Makefile.protos b/Makefile.protos
index 4cf8982..8712029 100644
--- a/Makefile.protos
+++ b/Makefile.protos
@@ -21,11 +21,11 @@ CC := g++
BUILD_PATH := build
DEBUG_PATH := $(BUILD_PATH)/debug
RELEASE_PATH := $(BUILD_PATH)/release
-MODULES := $(BUILD_PATH)/if
-SRC_DIR := $(MODULES)
-DEBUG_BUILD_DIR := $(addprefix $(DEBUG_PATH)/,$(MODULES))
-RELEASE_BUILD_DIR := $(addprefix $(RELEASE_PATH)/,$(MODULES))
-INCLUDE_DIR := . $(BUILD_PATH)/if
+MODULES := if
+SRC_DIR := $(BUILD_PATH)/src/hbase/$(MODULES)
+DEBUG_BUILD_DIR := $(addprefix $(DEBUG_PATH)/hbase/,$(MODULES))
+RELEASE_BUILD_DIR := $(addprefix $(RELEASE_PATH)/hbase/,$(MODULES))
+INCLUDE_DIR := . $(BUILD_PATH)/src/hbase/if
#flags to pass to the CPP compiler & linker
CPPFLAGS_DEBUG := -D_GLIBCXX_USE_CXX11_ABI=0 -g -Wall -std=c++14 -pedantic -fPIC -MMD -MP
@@ -33,8 +33,8 @@ CPPFLAGS_RELEASE := -D_GLIBCXX_USE_CXX11_ABI=0 -DNDEBUG -O2 -Wall -std=c++14 -pe
#define list of source files and object files
SRC := $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc))
-DEBUG_OBJ := $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(SRC))
-RELEASE_OBJ := $(patsubst %.cc,$(RELEASE_PATH)/%.o,$(SRC))
+DEBUG_OBJ := $(patsubst $(SRC_DIR)/%.cc,$(DEBUG_PATH)/hbase/if/%.o,$(SRC))
+RELEASE_OBJ := $(patsubst $(SRC_DIR)/%.cc,$(RELEASE_PATH)/hbase/if/%.o,$(SRC))
INCLUDES := $(addprefix -I,$(INCLUDE_DIR))
all: $(DEBUG_BUILD_DIR) $(RELEASE_BUILD_DIR) $(DEBUG_OBJ) $(RELEASE_OBJ)
diff --git a/bin/copy-protobuf.sh b/bin/copy-protobuf.sh
index f727800..47544c8 100755
--- a/bin/copy-protobuf.sh
+++ b/bin/copy-protobuf.sh
@@ -20,5 +20,5 @@ IFS=$'\n\t'
BIN_DIR=$(dirname "$0")
PB_SOURCE_DIR="${BIN_DIR}/../../hbase-protocol/src/main/protobuf/"
-PB_DEST_DIR="${BIN_DIR}/../if/"
+PB_DEST_DIR="${BIN_DIR}/../src/hbase/if/"
rsync -r --exclude BUCK ${PB_SOURCE_DIR} ${PB_DEST_DIR}
diff --git a/bin/copy-version.sh b/bin/copy-version.sh
index b33db7a..e2b707f 100755
--- a/bin/copy-version.sh
+++ b/bin/copy-version.sh
@@ -21,5 +21,5 @@ IFS=$'\n\t'
# Copy the version.h generated from hbase-common/src/saveVersion.sh script via the mvn build
BIN_DIR=$(dirname "$0")
VERSION_SOURCE_DIR="${BIN_DIR}/../../hbase-common/target/generated-sources/native/utils/"
-VERSION_DEST_DIR="${BIN_DIR}/../utils/"
+VERSION_DEST_DIR="${BIN_DIR}/../include/hbase/utils/"
cp $VERSION_SOURCE_DIR/* $VERSION_DEST_DIR/
diff --git a/bin/cpplint.sh b/bin/cpplint.sh
index 3684c2e..7a27945 100755
--- a/bin/cpplint.sh
+++ b/bin/cpplint.sh
@@ -20,6 +20,8 @@ IFS=$'\n\t'
CPPLINT_LOC=https://raw.githubusercontent.com/google/styleguide/gh-pages/cpplint/cpplint.py
OUTPUT=build/cpplint.py
+declare -a MODULES=( client connection exceptions security serde utils test-util )
+
# Download if not already there
wget -nc $CPPLINT_LOC -O $OUTPUT
@@ -27,4 +29,11 @@ wget -nc $CPPLINT_LOC -O $OUTPUT
# Exclude the following rules: build/header_guard (We use #pragma once instead)
# readability/todo (TODOs are generic)
# build/c++11 (We are building with c++14)
-find core connection exceptions serde utils test-util security -name "*.h" -or -name "*.cc" | xargs -P8 python $OUTPUT --filter=-build/header_guard,-readability/todo,-build/c++11 --linelength=100
+for m in ${MODULES[@]}; do
+ if [ $m != "security" ]; then #These are empty
+ exec find src/hbase/$m -name "*.h" -or -name "*.cc" | xargs -P8 python $OUTPUT --filter=-build/header_guard,-readability/todo,-build/c++11 --linelength=100
+ fi
+ if [ $m != "test-util" ]; then
+ exec find include/hbase/$m -name "*.h" -or -name "*.cc" | xargs -P8 python $OUTPUT --filter=-build/header_guard,-readability/todo,-build/c++11 --linelength=100
+ fi
+done
diff --git a/bin/format-code.sh b/bin/format-code.sh
index fe236d8..301167e 100755
--- a/bin/format-code.sh
+++ b/bin/format-code.sh
@@ -18,6 +18,11 @@
set -euo pipefail
IFS=$'\n\t'
+declare -a MODULES=( client connection exceptions security serde utils test-util )
-find core connection exceptions serde utils test-util security -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i --style='{BasedOnStyle: Google, ColumnLimit: 100}'
-find core connection exceptions serde utils test-util third-party security -name "BUCK" | xargs -P8 yapf -i --style=google
+for m in ${MODULES[@]}; do
+ find src/hbase/$m -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i --style='{BasedOnStyle: Google, ColumnLimit: 100}'
+ find src/hbase/$m -name "BUCK" | xargs -P8 yapf -i --style=google
+done
+
+find third-party -name "BUCK" | xargs -P8 yapf -i --style=google
diff --git a/include/hbase/client/BUCK b/include/hbase/client/BUCK
new file mode 100644
index 0000000..66d6896
--- /dev/null
+++ b/include/hbase/client/BUCK
@@ -0,0 +1,93 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is the main library.
+cxx_library(
+ name="client",
+ header_namespace="hbase/client",
+ exported_headers=[
+ "async-client-scanner.h",
+ "async-connection.h",
+ "async-region-locator.h",
+ "async-rpc-retrying-caller-factory.h",
+ "async-rpc-retrying-caller.h",
+ "async-table-result-scanner.h",
+ "client.h",
+ "cell.h",
+ "filter.h",
+ "query.h",
+ "keyvalue-codec.h",
+ "region-location.h",
+ "location-cache.h",
+ "connection-configuration.h",
+ # TODO: move this out of exported
+ # Once meta lookup works
+ "meta-utils.h",
+ "get.h",
+ "increment.h",
+ "mutation.h",
+ "put.h",
+ "delete.h",
+ "scan.h",
+ "append.h",
+ "result.h",
+ "result-scanner.h",
+ "request-converter.h",
+ "response-converter.h",
+ "table.h",
+ "async-scan-rpc-retrying-caller.h",
+ "raw-async-table.h",
+ "raw-scan-result-consumer.h",
+ "scan-result-cache.h",
+ "hbase-rpc-controller.h",
+ "time-range.h",
+ "zk-util.h",
+ "action.h",
+ "multi-response.h",
+ "region-request.h",
+ "region-result.h",
+ "row.h",
+ "server-request.h",
+ "async-batch-rpc-retrying-caller.h",
+ ],
+ deps=[
+ "//include/hbase/exceptions:exceptions",
+ "//include/hbase/utils:utils",
+ "//include/hbase/connection:connection",
+ "//include/hbase/client:conf",
+ "//src/hbase/if:if",
+ "//include/hbase/serde:serde",
+ "//third-party:folly",
+ "//third-party:wangle",
+ "//third-party:zookeeper_mt",
+ ],
+ compiler_flags=['-Weffc++', '-ggdb'],
+ visibility=[
+ 'PUBLIC',
+ ],)
+cxx_library(
+ name="conf",
+ header_namespace="hbase/client",
+ exported_headers=[
+ "configuration.h",
+ "hbase-configuration-loader.h",
+ ],
+ deps=["//src/hbase/utils:utils", "//third-party:folly"],
+ compiler_flags=['-Weffc++', '-ggdb'],
+ visibility=[
+ 'PUBLIC',
+ ],)
diff --git a/core/action.h b/include/hbase/client/action.h
similarity index 97%
rename from core/action.h
rename to include/hbase/client/action.h
index a00f079..2288f12 100644
--- a/core/action.h
+++ b/include/hbase/client/action.h
@@ -20,7 +20,7 @@
#pragma once
#include <memory>
-#include "core/row.h"
+#include "hbase/client/row.h"
namespace hbase {
class Action {
diff --git a/core/append.h b/include/hbase/client/append.h
similarity index 96%
rename from core/append.h
rename to include/hbase/client/append.h
index cf9ac24..e7f9a6d 100644
--- a/core/append.h
+++ b/include/hbase/client/append.h
@@ -24,8 +24,8 @@
#include <memory>
#include <string>
#include <vector>
-#include "core/cell.h"
-#include "core/mutation.h"
+#include "hbase/client/cell.h"
+#include "hbase/client/mutation.h"
namespace hbase {
diff --git a/core/async-batch-rpc-retrying-caller.h b/include/hbase/client/async-batch-rpc-retrying-caller.h
similarity index 89%
rename from core/async-batch-rpc-retrying-caller.h
rename to include/hbase/client/async-batch-rpc-retrying-caller.h
index 9194b04..39b7aa9 100644
--- a/core/async-batch-rpc-retrying-caller.h
+++ b/include/hbase/client/async-batch-rpc-retrying-caller.h
@@ -42,26 +42,26 @@
#include <utility>
#include <vector>
-#include "connection/rpc-client.h"
-#include "core/action.h"
-#include "core/async-connection.h"
-#include "core/location-cache.h"
-#include "core/multi-response.h"
-#include "core/region-location.h"
-#include "core/region-request.h"
-#include "core/region-result.h"
-#include "core/request-converter.h"
-#include "core/response-converter.h"
-#include "core/result.h"
-#include "core/row.h"
-#include "core/server-request.h"
-#include "exceptions/exception.h"
-#include "if/Client.pb.h"
-#include "if/HBase.pb.h"
-#include "security/user.h"
-#include "utils/connection-util.h"
-#include "utils/sys-util.h"
-#include "utils/time-util.h"
+#include "hbase/connection/rpc-client.h"
+#include "hbase/client/action.h"
+#include "hbase/client/async-connection.h"
+#include "hbase/client/location-cache.h"
+#include "hbase/client/multi-response.h"
+#include "hbase/client/region-location.h"
+#include "hbase/client/region-request.h"
+#include "hbase/client/region-result.h"
+#include "hbase/client/request-converter.h"
+#include "hbase/client/response-converter.h"
+#include "hbase/client/result.h"
+#include "hbase/client/row.h"
+#include "hbase/client/server-request.h"
+#include "hbase/exceptions/exception.h"
+#include "hbase/if/Client.pb.h"
+#include "hbase/if/HBase.pb.h"
+#include "hbase/security/user.h"
+#include "hbase/utils/connection-util.h"
+#include "hbase/utils/sys-util.h"
+#include "hbase/utils/time-util.h"
namespace hbase {
/* Equals function for ServerName */
diff --git a/core/async-client-scanner.h b/include/hbase/client/async-client-scanner.h
similarity index 82%
rename from core/async-client-scanner.h
rename to include/hbase/client/async-client-scanner.h
index 8663468..cccf50b 100644
--- a/core/async-client-scanner.h
+++ b/include/hbase/client/async-client-scanner.h
@@ -33,24 +33,24 @@
#include <utility>
#include <vector>
-#include "connection/rpc-client.h"
-#include "core/async-connection.h"
-#include "core/async-rpc-retrying-caller-factory.h"
-#include "core/async-rpc-retrying-caller.h"
-#include "core/hbase-rpc-controller.h"
-#include "core/raw-scan-result-consumer.h"
-#include "core/region-location.h"
-#include "core/request-converter.h"
-#include "core/response-converter.h"
-#include "core/result.h"
-#include "core/scan-result-cache.h"
-#include "core/scan.h"
-#include "exceptions/exception.h"
-#include "if/Client.pb.h"
-#include "if/HBase.pb.h"
-#include "utils/connection-util.h"
-#include "utils/sys-util.h"
-#include "utils/time-util.h"
+#include "hbase/connection/rpc-client.h"
+#include "hbase/client/async-connection.h"
+#include "hbase/client/async-rpc-retrying-caller-factory.h"
+#include "hbase/client/async-rpc-retrying-caller.h"
+#include "hbase/client/hbase-rpc-controller.h"
+#include "hbase/client/raw-scan-result-consumer.h"
+#include "hbase/client/region-location.h"
+#include "hbase/client/request-converter.h"
+#include "hbase/client/response-converter.h"
+#include "hbase/client/result.h"
+#include "hbase/client/scan-result-cache.h"
+#include "hbase/client/scan.h"
+#include "hbase/exceptions/exception.h"
+#include "hbase/if/Client.pb.h"
+#include "hbase/if/HBase.pb.h"
+#include "hbase/utils/connection-util.h"
+#include "hbase/utils/sys-util.h"
+#include "hbase/utils/time-util.h"
using std::chrono::nanoseconds;
using std::chrono::milliseconds;
diff --git a/core/async-connection.h b/include/hbase/client/async-connection.h
similarity index 91%
rename from core/async-connection.h
rename to include/hbase/client/async-connection.h
index 7b260a5..547ac76 100644
--- a/core/async-connection.h
+++ b/include/hbase/client/async-connection.h
@@ -28,16 +28,16 @@
#include <string>
#include <utility>
-#include "connection/rpc-client.h"
-#include "core/async-region-locator.h"
-#include "core/configuration.h"
-#include "core/connection-configuration.h"
-#include "core/hbase-configuration-loader.h"
-#include "core/hbase-rpc-controller.h"
-#include "core/keyvalue-codec.h"
-#include "core/location-cache.h"
-#include "if/Cell.pb.h"
-#include "serde/table-name.h"
+#include "hbase/connection/rpc-client.h"
+#include "hbase/client/async-region-locator.h"
+#include "hbase/client/configuration.h"
+#include "hbase/client/connection-configuration.h"
+#include "hbase/client/hbase-configuration-loader.h"
+#include "hbase/client/hbase-rpc-controller.h"
+#include "hbase/client/keyvalue-codec.h"
+#include "hbase/client/location-cache.h"
+#include "hbase/if/Cell.pb.h"
+#include "hbase/serde/table-name.h"
namespace hbase {
diff --git a/core/async-region-locator.h b/include/hbase/client/async-region-locator.h
similarity index 92%
rename from core/async-region-locator.h
rename to include/hbase/client/async-region-locator.h
index f75cb7e..d42038e 100644
--- a/core/async-region-locator.h
+++ b/include/hbase/client/async-region-locator.h
@@ -24,11 +24,11 @@
#include <memory>
#include <string>
-#include "core/region-location.h"
-#include "if/Client.pb.h"
-#include "serde/region-info.h"
-#include "serde/server-name.h"
-#include "serde/table-name.h"
+#include "hbase/client/region-location.h"
+#include "hbase/if/Client.pb.h"
+#include "hbase/serde/region-info.h"
+#include "hbase/serde/server-name.h"
+#include "hbase/serde/table-name.h"
namespace hbase {
diff --git a/core/async-rpc-retrying-caller-factory.h b/include/hbase/client/async-rpc-retrying-caller-factory.h
similarity index 95%
rename from core/async-rpc-retrying-caller-factory.h
rename to include/hbase/client/async-rpc-retrying-caller-factory.h
index 188f469..a580896 100644
--- a/core/async-rpc-retrying-caller-factory.h
+++ b/include/hbase/client/async-rpc-retrying-caller-factory.h
@@ -25,18 +25,18 @@
#include <string>
#include <vector>
-#include "connection/rpc-client.h"
-#include "core/async-batch-rpc-retrying-caller.h"
-#include "core/async-rpc-retrying-caller.h"
-#include "core/async-scan-rpc-retrying-caller.h"
-#include "core/raw-scan-result-consumer.h"
-#include "core/region-location.h"
-#include "core/row.h"
-#include "core/scan-result-cache.h"
-#include "core/scan.h"
-
-#include "if/Client.pb.h"
-#include "if/HBase.pb.h"
+#include "hbase/connection/rpc-client.h"
+#include "hbase/client/async-batch-rpc-retrying-caller.h"
+#include "hbase/client/async-rpc-retrying-caller.h"
+#include "hbase/client/async-scan-rpc-retrying-caller.h"
+#include "hbase/client/raw-scan-result-consumer.h"
+#include "hbase/client/region-location.h"
+#include "hbase/client/row.h"
+#include "hbase/client/scan-result-cache.h"
+#include "hbase/client/scan.h"
+
+#include "hbase/if/Client.pb.h"
+#include "hbase/if/HBase.pb.h"
namespace hbase {
diff --git a/core/async-rpc-retrying-caller.h b/include/hbase/client/async-rpc-retrying-caller.h
similarity index 94%
rename from core/async-rpc-retrying-caller.h
rename to include/hbase/client/async-rpc-retrying-caller.h
index c7e28d0..b5f4ad2 100644
--- a/core/async-rpc-retrying-caller.h
+++ b/include/hbase/client/async-rpc-retrying-caller.h
@@ -31,11 +31,11 @@
#include <type_traits>
#include <utility>
#include <vector>
-#include "core/async-connection.h"
-#include "core/hbase-rpc-controller.h"
-#include "core/region-location.h"
-#include "exceptions/exception.h"
-#include "if/HBase.pb.h"
+#include "hbase/client/async-connection.h"
+#include "hbase/client/hbase-rpc-controller.h"
+#include "hbase/client/region-location.h"
+#include "hbase/exceptions/exception.h"
+#include "hbase/if/HBase.pb.h"
namespace hbase {
diff --git a/core/async-scan-rpc-retrying-caller.h b/include/hbase/client/async-scan-rpc-retrying-caller.h
similarity index 92%
rename from core/async-scan-rpc-retrying-caller.h
rename to include/hbase/client/async-scan-rpc-retrying-caller.h
index 9555e80..68382e6 100644
--- a/core/async-scan-rpc-retrying-caller.h
+++ b/include/hbase/client/async-scan-rpc-retrying-caller.h
@@ -35,24 +35,24 @@
#include <utility>
#include <vector>
-#include "connection/rpc-client.h"
-#include "core/async-connection.h"
-#include "core/hbase-rpc-controller.h"
-#include "core/raw-scan-result-consumer.h"
-#include "core/region-location.h"
-#include "core/request-converter.h"
-#include "core/response-converter.h"
-#include "core/result.h"
-#include "core/scan-result-cache.h"
-#include "core/scan.h"
-#include "exceptions/exception.h"
-#include "if/Client.pb.h"
-#include "if/HBase.pb.h"
-#include "utils/bytes-util.h"
-#include "utils/connection-util.h"
-#include "utils/optional.h"
-#include "utils/sys-util.h"
-#include "utils/time-util.h"
+#include "hbase/connection/rpc-client.h"
+#include "hbase/client/async-connection.h"
+#include "hbase/client/hbase-rpc-controller.h"
+#include "hbase/client/raw-scan-result-consumer.h"
+#include "hbase/client/region-location.h"
+#include "hbase/client/request-converter.h"
+#include "hbase/client/response-converter.h"
+#include "hbase/client/result.h"
+#include "hbase/client/scan-result-cache.h"
+#include "hbase/client/scan.h"
+#include "hbase/exceptions/exception.h"
+#include "hbase/if/Client.pb.h"
+#include "hbase/if/HBase.pb.h"
+#include "hbase/utils/bytes-util.h"
+#include "hbase/utils/connection-util.h"
+#include "hbase/utils/optional.h"
+#include "hbase/utils/sys-util.h"
+#include "hbase/utils/time-util.h"
using std::chrono::nanoseconds;
using std::chrono::milliseconds;
diff --git a/core/async-table-result-scanner.h b/include/hbase/client/async-table-result-scanner.h
similarity index 94%
rename from core/async-table-result-scanner.h
rename to include/hbase/client/async-table-result-scanner.h
index dcdf871..0e1d444 100644
--- a/core/async-table-result-scanner.h
+++ b/include/hbase/client/async-table-result-scanner.h
@@ -29,11 +29,11 @@
#include <string>
#include <vector>
-#include "core/raw-scan-result-consumer.h"
-#include "core/result-scanner.h"
-#include "core/result.h"
-#include "if/Client.pb.h"
-#include "if/HBase.pb.h"
+#include "hbase/client/raw-scan-result-consumer.h"
+#include "hbase/client/result-scanner.h"
+#include "hbase/client/result.h"
+#include "hbase/if/Client.pb.h"
+#include "hbase/if/HBase.pb.h"
namespace hbase {
diff --git a/core/cell.h b/include/hbase/client/cell.h
similarity index 100%
rename from core/cell.h
rename to include/hbase/client/cell.h
diff --git a/core/client.h b/include/hbase/client/client.h
similarity index 90%
rename from core/client.h
rename to include/hbase/client/client.h
index 5563a15..2e77df9 100644
--- a/core/client.h
+++ b/include/hbase/client/client.h
@@ -22,12 +22,12 @@
#include <memory>
#include <string>
-#include "connection/rpc-client.h"
-#include "core/async-connection.h"
-#include "core/configuration.h"
+#include "hbase/connection/rpc-client.h"
+#include "hbase/client/async-connection.h"
+#include "hbase/client/configuration.h"
-#include "core/table.h"
-#include "serde/table-name.h"
+#include "hbase/client/table.h"
+#include "hbase/serde/table-name.h"
namespace hbase {
diff --git a/core/configuration.h b/include/hbase/client/configuration.h
similarity index 100%
rename from core/configuration.h
rename to include/hbase/client/configuration.h
diff --git a/core/connection-configuration.h b/include/hbase/client/connection-configuration.h
similarity index 99%
rename from core/connection-configuration.h
rename to include/hbase/client/connection-configuration.h
index 995798e..308e1ba 100644
--- a/core/connection-configuration.h
+++ b/include/hbase/client/connection-configuration.h
@@ -23,7 +23,7 @@
#include <climits>
#include <string>
-#include "core/configuration.h"
+#include "hbase/client/configuration.h"
namespace hbase {
diff --git a/core/delete.h b/include/hbase/client/delete.h
similarity index 98%
rename from core/delete.h
rename to include/hbase/client/delete.h
index 9ebb5a6..5fdbcc1 100644
--- a/core/delete.h
+++ b/include/hbase/client/delete.h
@@ -24,8 +24,8 @@
#include <memory>
#include <string>
#include <vector>
-#include "core/cell.h"
-#include "core/mutation.h"
+#include "hbase/client/cell.h"
+#include "hbase/client/mutation.h"
namespace hbase {
diff --git a/core/filter.h b/include/hbase/client/filter.h
similarity index 99%
rename from core/filter.h
rename to include/hbase/client/filter.h
index 1e870f9..6ad4443 100644
--- a/core/filter.h
+++ b/include/hbase/client/filter.h
@@ -25,9 +25,9 @@
#include <utility>
#include <vector>
-#include "if/Comparator.pb.h"
-#include "if/Filter.pb.h"
-#include "if/HBase.pb.h"
+#include "hbase/if/Comparator.pb.h"
+#include "hbase/if/Filter.pb.h"
+#include "hbase/if/HBase.pb.h"
namespace hbase {
diff --git a/core/get.h b/include/hbase/client/get.h
similarity index 96%
rename from core/get.h
rename to include/hbase/client/get.h
index c4cddfb..b6a11f9 100644
--- a/core/get.h
+++ b/include/hbase/client/get.h
@@ -24,10 +24,10 @@
#include <memory>
#include <string>
#include <vector>
-#include "core/query.h"
-#include "core/row.h"
-#include "core/time-range.h"
-#include "if/Client.pb.h"
+#include "hbase/client/query.h"
+#include "hbase/client/row.h"
+#include "hbase/client/time-range.h"
+#include "hbase/if/Client.pb.h"
namespace hbase {
diff --git a/core/hbase-configuration-loader.h b/include/hbase/client/hbase-configuration-loader.h
similarity index 98%
rename from core/hbase-configuration-loader.h
rename to include/hbase/client/hbase-configuration-loader.h
index 95b2541..8792aa2 100644
--- a/core/hbase-configuration-loader.h
+++ b/include/hbase/client/hbase-configuration-loader.h
@@ -25,8 +25,8 @@
#include <boost/optional.hpp>
-#include "core/configuration.h"
-#include "utils/optional.h"
+#include "hbase/client/configuration.h"
+#include "hbase/utils/optional.h"
namespace hbase {
diff --git a/core/hbase-rpc-controller.h b/include/hbase/client/hbase-rpc-controller.h
similarity index 100%
rename from core/hbase-rpc-controller.h
rename to include/hbase/client/hbase-rpc-controller.h
diff --git a/core/increment.h b/include/hbase/client/increment.h
similarity index 96%
rename from core/increment.h
rename to include/hbase/client/increment.h
index 330f44a..3169c19 100644
--- a/core/increment.h
+++ b/include/hbase/client/increment.h
@@ -24,8 +24,8 @@
#include <memory>
#include <string>
#include <vector>
-#include "core/cell.h"
-#include "core/mutation.h"
+#include "hbase/client/cell.h"
+#include "hbase/client/mutation.h"
namespace hbase {
diff --git a/core/keyvalue-codec.h b/include/hbase/client/keyvalue-codec.h
similarity index 98%
rename from core/keyvalue-codec.h
rename to include/hbase/client/keyvalue-codec.h
index fd58346..5a92c8c 100644
--- a/core/keyvalue-codec.h
+++ b/include/hbase/client/keyvalue-codec.h
@@ -23,8 +23,8 @@
#include <folly/io/IOBuf.h>
#include <memory>
-#include "core/cell.h"
-#include "serde/codec.h"
+#include "hbase/client/cell.h"
+#include "hbase/serde/codec.h"
namespace hbase {
diff --git a/core/location-cache.h b/include/hbase/client/location-cache.h
similarity index 96%
rename from core/location-cache.h
rename to include/hbase/client/location-cache.h
index 6eb61ef..a800642 100644
--- a/core/location-cache.h
+++ b/include/hbase/client/location-cache.h
@@ -33,13 +33,13 @@
#include <string>
#include <unordered_map>
-#include "connection/connection-pool.h"
-#include "core/async-region-locator.h"
-#include "core/configuration.h"
-#include "core/meta-utils.h"
-#include "core/region-location.h"
-#include "core/zk-util.h"
-#include "serde/table-name.h"
+#include "hbase/connection/connection-pool.h"
+#include "hbase/client/async-region-locator.h"
+#include "hbase/client/configuration.h"
+#include "hbase/client/meta-utils.h"
+#include "hbase/client/region-location.h"
+#include "hbase/client/zk-util.h"
+#include "hbase/serde/table-name.h"
namespace hbase {
// Forward
diff --git a/core/meta-utils.h b/include/hbase/client/meta-utils.h
similarity index 92%
rename from core/meta-utils.h
rename to include/hbase/client/meta-utils.h
index d178179..3fc320a 100644
--- a/core/meta-utils.h
+++ b/include/hbase/client/meta-utils.h
@@ -21,11 +21,11 @@
#include <memory>
#include <string>
-#include "connection/request.h"
-#include "connection/response.h"
-#include "core/region-location.h"
-#include "if/HBase.pb.h"
-#include "serde/table-name.h"
+#include "hbase/connection/request.h"
+#include "hbase/connection/response.h"
+#include "hbase/client/region-location.h"
+#include "hbase/if/HBase.pb.h"
+#include "hbase/serde/table-name.h"
namespace hbase {
diff --git a/core/multi-response.h b/include/hbase/client/multi-response.h
similarity index 96%
rename from core/multi-response.h
rename to include/hbase/client/multi-response.h
index d38cfd6..b9b9a38 100644
--- a/core/multi-response.h
+++ b/include/hbase/client/multi-response.h
@@ -19,15 +19,15 @@
#pragma once
-#include <core/region-result.h>
#include <folly/ExceptionWrapper.h>
#include <exception>
#include <map>
#include <memory>
#include <string>
-#include "core/result.h"
-#include "if/Client.pb.h"
+#include "hbase/client/region-result.h"
+#include "hbase/client/result.h"
+#include "hbase/if/Client.pb.h"
namespace hbase {
diff --git a/core/mutation.h b/include/hbase/client/mutation.h
similarity index 96%
rename from core/mutation.h
rename to include/hbase/client/mutation.h
index 496891e..f70e2b7 100644
--- a/core/mutation.h
+++ b/include/hbase/client/mutation.h
@@ -25,9 +25,9 @@
#include <memory>
#include <string>
#include <vector>
-#include "core/cell.h"
-#include "core/row.h"
-#include "if/Client.pb.h"
+#include "hbase/client/cell.h"
+#include "hbase/client/row.h"
+#include "hbase/if/Client.pb.h"
namespace hbase {
diff --git a/core/put.h b/include/hbase/client/put.h
similarity index 96%
rename from core/put.h
rename to include/hbase/client/put.h
index 8a43dec..cb0e17d 100644
--- a/core/put.h
+++ b/include/hbase/client/put.h
@@ -24,8 +24,8 @@
#include <memory>
#include <string>
#include <vector>
-#include "core/cell.h"
-#include "core/mutation.h"
+#include "hbase/client/cell.h"
+#include "hbase/client/mutation.h"
namespace hbase {
diff --git a/core/query.h b/include/hbase/client/query.h
similarity index 97%
rename from core/query.h
rename to include/hbase/client/query.h
index 301f448..76f613e 100644
--- a/core/query.h
+++ b/include/hbase/client/query.h
@@ -21,7 +21,7 @@
#include <memory>
-#include "core/filter.h"
+#include "hbase/client/filter.h"
namespace hbase {
diff --git a/core/raw-async-table.h b/include/hbase/client/raw-async-table.h
similarity index 90%
rename from core/raw-async-table.h
rename to include/hbase/client/raw-async-table.h
index 97eef7f..9db291e 100644
--- a/core/raw-async-table.h
+++ b/include/hbase/client/raw-async-table.h
@@ -25,18 +25,17 @@
#include <string>
#include <vector>
-#include "core/async-batch-rpc-retrying-caller.h"
-#include "core/async-client-scanner.h"
-#include "core/async-connection.h"
-#include "core/async-rpc-retrying-caller-factory.h"
-#include "core/async-rpc-retrying-caller.h"
-#include "core/connection-configuration.h"
-#include "core/delete.h"
-#include "core/get.h"
-#include "core/increment.h"
-#include "core/put.h"
-#include "core/result.h"
-#include "core/scan.h"
+#include "hbase/client/async-client-scanner.h"
+#include "hbase/client/async-connection.h"
+#include "hbase/client/async-rpc-retrying-caller-factory.h"
+#include "hbase/client/async-rpc-retrying-caller.h"
+#include "hbase/client/connection-configuration.h"
+#include "hbase/client/delete.h"
+#include "hbase/client/get.h"
+#include "hbase/client/increment.h"
+#include "hbase/client/put.h"
+#include "hbase/client/result.h"
+#include "hbase/client/scan.h"
namespace hbase {
diff --git a/core/raw-scan-result-consumer.h b/include/hbase/client/raw-scan-result-consumer.h
similarity index 97%
rename from core/raw-scan-result-consumer.h
rename to include/hbase/client/raw-scan-result-consumer.h
index b7c3c48..26ff709 100644
--- a/core/raw-scan-result-consumer.h
+++ b/include/hbase/client/raw-scan-result-consumer.h
@@ -26,9 +26,9 @@
#include <thread>
#include <vector>
-#include "core/result.h"
-#include "if/Client.pb.h"
-#include "if/HBase.pb.h"
+#include "hbase/client/result.h"
+#include "hbase/if/Client.pb.h"
+#include "hbase/if/HBase.pb.h"
namespace hbase {
diff --git a/core/region-location.h b/include/hbase/client/region-location.h
similarity index 98%
rename from core/region-location.h
rename to include/hbase/client/region-location.h
index f73999f..3eded91 100644
--- a/core/region-location.h
+++ b/include/hbase/client/region-location.h
@@ -21,7 +21,7 @@
#include <memory>
#include <string>
-#include "if/HBase.pb.h"
+#include "hbase/if/HBase.pb.h"
namespace hbase {
diff --git a/core/region-request.h b/include/hbase/client/region-request.h
similarity index 95%
rename from core/region-request.h
rename to include/hbase/client/region-request.h
index aded3a9..9e7f85e 100644
--- a/core/region-request.h
+++ b/include/hbase/client/region-request.h
@@ -22,8 +22,8 @@
#include <memory>
#include <queue>
#include <vector>
-#include "core/action.h"
-#include "core/region-location.h"
+#include "hbase/client/action.h"
+#include "hbase/client/region-location.h"
namespace hbase {
diff --git a/core/region-result.h b/include/hbase/client/region-result.h
similarity index 96%
rename from core/region-result.h
rename to include/hbase/client/region-result.h
index b961634..87398a2 100644
--- a/core/region-result.h
+++ b/include/hbase/client/region-result.h
@@ -24,8 +24,8 @@
#include <memory>
#include <string>
#include <tuple>
-#include "core/result.h"
-#include "if/Client.pb.h"
+#include "hbase/client/result.h"
+#include "hbase/if/Client.pb.h"
namespace hbase {
diff --git a/core/request-converter.h b/include/hbase/client/request-converter.h
similarity index 90%
rename from core/request-converter.h
rename to include/hbase/client/request-converter.h
index bcea278..eef6b5d 100644
--- a/core/request-converter.h
+++ b/include/hbase/client/request-converter.h
@@ -22,20 +22,20 @@
#include <memory>
#include <string>
#include <vector>
-#include "connection/request.h"
-#include "core/action.h"
-#include "core/append.h"
-#include "core/cell.h"
-#include "core/delete.h"
-#include "core/get.h"
-#include "core/increment.h"
-#include "core/mutation.h"
-#include "core/put.h"
-#include "core/region-request.h"
-#include "core/scan.h"
-#include "core/server-request.h"
-#include "if/Client.pb.h"
-#include "if/HBase.pb.h"
+#include "hbase/connection/request.h"
+#include "hbase/client/action.h"
+#include "hbase/client/append.h"
+#include "hbase/client/cell.h"
+#include "hbase/client/delete.h"
+#include "hbase/client/get.h"
+#include "hbase/client/increment.h"
+#include "hbase/client/mutation.h"
+#include "hbase/client/put.h"
+#include "hbase/client/region-request.h"
+#include "hbase/client/scan.h"
+#include "hbase/client/server-request.h"
+#include "hbase/if/Client.pb.h"
+#include "hbase/if/HBase.pb.h"
using MutationType = hbase::pb::MutationProto_MutationType;
using DeleteType = hbase::pb::MutationProto_DeleteType;
diff --git a/core/response-converter.h b/include/hbase/client/response-converter.h
similarity index 89%
rename from core/response-converter.h
rename to include/hbase/client/response-converter.h
index edd4165..14757a5 100644
--- a/core/response-converter.h
+++ b/include/hbase/client/response-converter.h
@@ -21,13 +21,13 @@
#include <memory>
#include <vector>
-#include "connection/request.h"
-#include "connection/response.h"
-#include "core/multi-response.h"
-#include "core/result.h"
-#include "core/server-request.h"
-#include "if/Client.pb.h"
-#include "serde/cell-scanner.h"
+#include "hbase/connection/request.h"
+#include "hbase/connection/response.h"
+#include "hbase/client/multi-response.h"
+#include "hbase/client/result.h"
+#include "hbase/client/server-request.h"
+#include "hbase/if/Client.pb.h"
+#include "hbase/serde/cell-scanner.h"
namespace hbase {
diff --git a/core/result-scanner.h b/include/hbase/client/result-scanner.h
similarity index 95%
rename from core/result-scanner.h
rename to include/hbase/client/result-scanner.h
index 9460521..0b22684 100644
--- a/core/result-scanner.h
+++ b/include/hbase/client/result-scanner.h
@@ -26,8 +26,8 @@
#include <string>
#include <vector>
-#include "core/cell.h"
-#include "core/result.h"
+#include "hbase/client/cell.h"
+#include "hbase/client/result.h"
namespace hbase {
diff --git a/core/result.h b/include/hbase/client/result.h
similarity index 98%
rename from core/result.h
rename to include/hbase/client/result.h
index f18071b..e5222b6 100644
--- a/core/result.h
+++ b/include/hbase/client/result.h
@@ -25,8 +25,8 @@
#include <string>
#include <vector>
-#include "core/cell.h"
-#include "utils/optional.h"
+#include "hbase/client/cell.h"
+#include "hbase/utils/optional.h"
namespace hbase {
diff --git a/core/row.h b/include/hbase/client/row.h
similarity index 100%
rename from core/row.h
rename to include/hbase/client/row.h
diff --git a/core/scan-result-cache.h b/include/hbase/client/scan-result-cache.h
similarity index 96%
rename from core/scan-result-cache.h
rename to include/hbase/client/scan-result-cache.h
index 5d3d0ab..e423108 100644
--- a/core/scan-result-cache.h
+++ b/include/hbase/client/scan-result-cache.h
@@ -26,9 +26,9 @@
#include <string>
#include <vector>
-#include "core/result.h"
-#include "if/Client.pb.h"
-#include "if/HBase.pb.h"
+#include "hbase/client/result.h"
+#include "hbase/if/Client.pb.h"
+#include "hbase/if/HBase.pb.h"
namespace hbase {
diff --git a/core/scan.h b/include/hbase/client/scan.h
similarity index 98%
rename from core/scan.h
rename to include/hbase/client/scan.h
index 1085c4b..90daa31 100644
--- a/core/scan.h
+++ b/include/hbase/client/scan.h
@@ -24,9 +24,9 @@
#include <memory>
#include <string>
#include <vector>
-#include "core/get.h"
-#include "core/time-range.h"
-#include "if/Client.pb.h"
+#include "hbase/client/get.h"
+#include "hbase/client/time-range.h"
+#include "hbase/if/Client.pb.h"
namespace hbase {
diff --git a/core/server-request.h b/include/hbase/client/server-request.h
similarity index 94%
rename from core/server-request.h
rename to include/hbase/client/server-request.h
index 85df9ed..6ad8c66 100644
--- a/core/server-request.h
+++ b/include/hbase/client/server-request.h
@@ -23,9 +23,9 @@
#include <memory>
#include <stdexcept>
#include <string>
-#include "core/action.h"
-#include "core/region-location.h"
-#include "core/region-request.h"
+#include "hbase/client/action.h"
+#include "hbase/client/region-location.h"
+#include "hbase/client/region-request.h"
namespace hbase {
diff --git a/core/table.h b/include/hbase/client/table.h
similarity index 92%
rename from core/table.h
rename to include/hbase/client/table.h
index 6340494..338d19f 100644
--- a/core/table.h
+++ b/include/hbase/client/table.h
@@ -24,17 +24,17 @@
#include <string>
#include <vector>
-#include "connection/rpc-client.h"
-#include "core/async-connection.h"
-#include "core/client.h"
-#include "core/configuration.h"
-#include "core/get.h"
-#include "core/location-cache.h"
-#include "core/put.h"
-#include "core/raw-async-table.h"
-#include "core/result-scanner.h"
-#include "core/result.h"
-#include "serde/table-name.h"
+#include "hbase/connection/rpc-client.h"
+#include "hbase/client/async-connection.h"
+#include "hbase/client/client.h"
+#include "hbase/client/configuration.h"
+#include "hbase/client/get.h"
+#include "hbase/client/location-cache.h"
+#include "hbase/client/put.h"
+#include "hbase/client/raw-async-table.h"
+#include "hbase/client/result-scanner.h"
+#include "hbase/client/result.h"
+#include "hbase/serde/table-name.h"
namespace hbase {
diff --git a/core/time-range.h b/include/hbase/client/time-range.h
similarity index 100%
rename from core/time-range.h
rename to include/hbase/client/time-range.h
diff --git a/core/zk-util.h b/include/hbase/client/zk-util.h
similarity index 97%
rename from core/zk-util.h
rename to include/hbase/client/zk-util.h
index 403fbe4..5d9a099 100644
--- a/core/zk-util.h
+++ b/include/hbase/client/zk-util.h
@@ -20,7 +20,7 @@
#include <cstdlib>
#include <string>
-#include "core/configuration.h"
+#include "hbase/client/configuration.h"
namespace hbase {
diff --git a/connection/BUCK b/include/hbase/connection/BUCK
similarity index 67%
copy from connection/BUCK
copy to include/hbase/connection/BUCK
index a87d27a..9a0b0cf 100644
--- a/connection/BUCK
+++ b/include/hbase/connection/BUCK
@@ -19,6 +19,7 @@
# to a single server.
cxx_library(
name="connection",
+ header_namespace="hbase/connection",
exported_headers=[
"client-dispatcher.h",
"client-handler.h",
@@ -38,49 +39,19 @@ cxx_library(
"rpc-fault-injector.h",
"rpc-fault-injector-inl.h",
],
- srcs=[
- "client-dispatcher.cc",
- "client-handler.cc",
- "connection-factory.cc",
- "connection-pool.cc",
- "pipeline.cc",
- "request.cc",
- "rpc-client.cc",
- "sasl-handler.cc",
- "sasl-util.cc",
- "rpc-test-server.cc",
- "rpc-test-server-handler.cc",
- "rpc-fault-injector.cc",
- ],
deps=[
- "//if:if",
- "//utils:utils",
- "//serde:serde",
- "//security:security",
+ "//src/hbase/if:if",
+ "//include/hbase/utils:utils",
+ "//include/hbase/serde:serde",
+ "//include/hbase/security:security",
"//third-party:folly",
"//third-party:wangle",
- "//exceptions:exceptions",
+ "//include/hbase/exceptions:exceptions",
],
compiler_flags=['-Weffc++'],
linker_flags=['-L/usr/local/lib', '-lsasl2', '-lkrb5'],
exported_linker_flags=['-L/usr/local/lib', '-lsasl2', '-lkrb5'],
visibility=[
- '//core/...',
- ],)
-cxx_test(
- name="connection-pool-test",
- srcs=[
- "connection-pool-test.cc",
- ],
- deps=[
- ":connection",
+ '//include/hbase/client/...',
+ '//src/hbase/connection/...',
],)
-cxx_test(
- name="rpc-test",
- srcs=[
- "rpc-test.cc",
- ],
- deps=[
- ":connection",
- ],
- run_test_separately=True,)
diff --git a/connection/client-dispatcher.h b/include/hbase/connection/client-dispatcher.h
similarity index 94%
rename from connection/client-dispatcher.h
rename to include/hbase/connection/client-dispatcher.h
index 7ef3759..33384a7 100644
--- a/connection/client-dispatcher.h
+++ b/include/hbase/connection/client-dispatcher.h
@@ -28,10 +28,10 @@
#include <mutex>
#include <string>
-#include "connection/pipeline.h"
-#include "connection/request.h"
-#include "connection/response.h"
-#include "utils/concurrent-map.h"
+#include "hbase/connection/pipeline.h"
+#include "hbase/connection/request.h"
+#include "hbase/connection/response.h"
+#include "hbase/utils/concurrent-map.h"
namespace hbase {
diff --git a/connection/client-handler.h b/include/hbase/connection/client-handler.h
similarity index 93%
rename from connection/client-handler.h
rename to include/hbase/connection/client-handler.h
index b6f19a2..b3bd2b6 100644
--- a/connection/client-handler.h
+++ b/include/hbase/connection/client-handler.h
@@ -26,11 +26,11 @@
#include <string>
#include <utility>
-#include "core/configuration.h"
-#include "exceptions/exception.h"
-#include "serde/codec.h"
-#include "serde/rpc-serde.h"
-#include "utils/concurrent-map.h"
+#include "hbase/client/configuration.h"
+#include "hbase/exceptions/exception.h"
+#include "hbase/serde/codec.h"
+#include "hbase/serde/rpc-serde.h"
+#include "hbase/utils/concurrent-map.h"
// Forward decs.
namespace hbase {
diff --git a/connection/connection-factory.h b/include/hbase/connection/connection-factory.h
similarity index 93%
rename from connection/connection-factory.h
rename to include/hbase/connection/connection-factory.h
index c4e63c2..14b7fda 100644
--- a/connection/connection-factory.h
+++ b/include/hbase/connection/connection-factory.h
@@ -26,11 +26,11 @@
#include <memory>
#include <string>
-#include "connection/pipeline.h"
-#include "connection/request.h"
-#include "connection/response.h"
-#include "connection/service.h"
-#include "security/user.h"
+#include "hbase/connection/pipeline.h"
+#include "hbase/connection/request.h"
+#include "hbase/connection/response.h"
+#include "hbase/connection/service.h"
+#include "hbase/security/user.h"
namespace hbase {
diff --git a/connection/connection-id.h b/include/hbase/connection/connection-id.h
similarity index 97%
rename from connection/connection-id.h
rename to include/hbase/connection/connection-id.h
index 065b484..98f5b47 100644
--- a/connection/connection-id.h
+++ b/include/hbase/connection/connection-id.h
@@ -24,8 +24,8 @@
#include <string>
#include <utility>
-#include "if/HBase.pb.h"
-#include "security/user.h"
+#include "hbase/if/HBase.pb.h"
+#include "hbase/security/user.h"
namespace hbase {
diff --git a/connection/connection-pool.h b/include/hbase/connection/connection-pool.h
similarity index 93%
rename from connection/connection-pool.h
rename to include/hbase/connection/connection-pool.h
index 9af1e7f..1198c33 100644
--- a/connection/connection-pool.h
+++ b/include/hbase/connection/connection-pool.h
@@ -25,11 +25,11 @@
#include <mutex>
#include <unordered_map>
-#include "connection/connection-factory.h"
-#include "connection/connection-id.h"
-#include "connection/rpc-connection.h"
-#include "connection/service.h"
-#include "if/HBase.pb.h"
+#include "hbase/connection/connection-factory.h"
+#include "hbase/connection/connection-id.h"
+#include "hbase/connection/rpc-connection.h"
+#include "hbase/connection/service.h"
+#include "hbase/if/HBase.pb.h"
namespace hbase {
diff --git a/connection/pipeline.h b/include/hbase/connection/pipeline.h
similarity index 90%
rename from connection/pipeline.h
rename to include/hbase/connection/pipeline.h
index add7fe5..63e9492 100644
--- a/connection/pipeline.h
+++ b/include/hbase/connection/pipeline.h
@@ -23,11 +23,11 @@
#include <memory>
-#include "connection/request.h"
-#include "connection/response.h"
-#include "core/configuration.h"
-#include "serde/codec.h"
-#include "utils/user-util.h"
+#include "hbase/connection/request.h"
+#include "hbase/connection/response.h"
+#include "hbase/client/configuration.h"
+#include "hbase/serde/codec.h"
+#include "hbase/utils/user-util.h"
namespace hbase {
diff --git a/connection/request.h b/include/hbase/connection/request.h
similarity index 100%
rename from connection/request.h
rename to include/hbase/connection/request.h
diff --git a/connection/response.h b/include/hbase/connection/response.h
similarity index 98%
rename from connection/response.h
rename to include/hbase/connection/response.h
index 38fdda0..5fe8b79 100644
--- a/connection/response.h
+++ b/include/hbase/connection/response.h
@@ -26,7 +26,7 @@
#include <string>
#include <utility>
-#include "serde/cell-scanner.h"
+#include "hbase/serde/cell-scanner.h"
// Forward
namespace google {
diff --git a/connection/rpc-client.h b/include/hbase/connection/rpc-client.h
similarity index 94%
rename from connection/rpc-client.h
rename to include/hbase/connection/rpc-client.h
index 93801d8..1189410 100644
--- a/connection/rpc-client.h
+++ b/include/hbase/connection/rpc-client.h
@@ -26,11 +26,11 @@
#include <string>
#include <utility>
-#include "connection/connection-id.h"
-#include "connection/connection-pool.h"
-#include "connection/request.h"
-#include "connection/response.h"
-#include "security/user.h"
+#include "hbase/connection/connection-id.h"
+#include "hbase/connection/connection-pool.h"
+#include "hbase/connection/request.h"
+#include "hbase/connection/response.h"
+#include "hbase/security/user.h"
namespace hbase {
diff --git a/connection/rpc-connection.h b/include/hbase/connection/rpc-connection.h
similarity index 92%
rename from connection/rpc-connection.h
rename to include/hbase/connection/rpc-connection.h
index 9063280..cbbac63 100644
--- a/connection/rpc-connection.h
+++ b/include/hbase/connection/rpc-connection.h
@@ -22,11 +22,11 @@
#include <mutex>
#include <utility>
-#include "connection/connection-factory.h"
-#include "connection/connection-id.h"
-#include "connection/request.h"
-#include "connection/response.h"
-#include "connection/service.h"
+#include "hbase/connection/connection-factory.h"
+#include "hbase/connection/connection-id.h"
+#include "hbase/connection/request.h"
+#include "hbase/connection/response.h"
+#include "hbase/connection/service.h"
namespace hbase {
diff --git a/connection/rpc-fault-injector-inl.h b/include/hbase/connection/rpc-fault-injector-inl.h
similarity index 100%
rename from connection/rpc-fault-injector-inl.h
rename to include/hbase/connection/rpc-fault-injector-inl.h
diff --git a/connection/rpc-fault-injector.h b/include/hbase/connection/rpc-fault-injector.h
similarity index 94%
rename from connection/rpc-fault-injector.h
rename to include/hbase/connection/rpc-fault-injector.h
index 2733b7d..1d21875 100644
--- a/connection/rpc-fault-injector.h
+++ b/include/hbase/connection/rpc-fault-injector.h
@@ -19,7 +19,7 @@
#pragma once
#include <folly/io/async/AsyncTransport.h>
-#include "connection/pipeline.h"
+#include "hbase/connection/pipeline.h"
namespace hbase {
@@ -47,4 +47,4 @@ class RpcClientFaultInjector : public RpcFaultInjector<RpcClientFaultInjector> {
};
} /* namespace hbase */
-#include "connection/rpc-fault-injector-inl.h"
+#include "hbase/connection/rpc-fault-injector-inl.h"
diff --git a/connection/rpc-test-server-handler.h b/include/hbase/connection/rpc-test-server-handler.h
similarity index 93%
rename from connection/rpc-test-server-handler.h
rename to include/hbase/connection/rpc-test-server-handler.h
index ab0264f..ef6ed4e 100644
--- a/connection/rpc-test-server-handler.h
+++ b/include/hbase/connection/rpc-test-server-handler.h
@@ -20,9 +20,9 @@
#include <wangle/channel/Handler.h>
-#include "connection/request.h"
-#include "connection/response.h"
-#include "serde/rpc-serde.h"
+#include "hbase/connection/request.h"
+#include "hbase/connection/response.h"
+#include "hbase/serde/rpc-serde.h"
using namespace hbase;
diff --git a/connection/rpc-test-server.h b/include/hbase/connection/rpc-test-server.h
similarity index 95%
rename from connection/rpc-test-server.h
rename to include/hbase/connection/rpc-test-server.h
index 955560e..76ab8ba 100644
--- a/connection/rpc-test-server.h
+++ b/include/hbase/connection/rpc-test-server.h
@@ -22,9 +22,9 @@
#include <wangle/service/ExecutorFilter.h>
#include <wangle/service/Service.h>
-#include "connection/request.h"
-#include "connection/response.h"
-#include "exceptions/exception.h"
+#include "hbase/connection/request.h"
+#include "hbase/connection/response.h"
+#include "hbase/exceptions/exception.h"
using namespace hbase;
using namespace folly;
diff --git a/connection/sasl-handler.h b/include/hbase/connection/sasl-handler.h
similarity index 95%
rename from connection/sasl-handler.h
rename to include/hbase/connection/sasl-handler.h
index 81f4e81..bb7f294 100644
--- a/connection/sasl-handler.h
+++ b/include/hbase/connection/sasl-handler.h
@@ -27,10 +27,10 @@
#include <string>
#include <vector>
-#include "connection/sasl-util.h"
-#include "connection/service.h"
-#include "security/user.h"
-#include "serde/rpc-serde.h"
+#include "hbase/connection/sasl-util.h"
+#include "hbase/connection/service.h"
+#include "hbase/security/user.h"
+#include "hbase/serde/rpc-serde.h"
namespace hbase {
diff --git a/connection/sasl-util.h b/include/hbase/connection/sasl-util.h
similarity index 97%
rename from connection/sasl-util.h
rename to include/hbase/connection/sasl-util.h
index 4d58d9ee..4d236df 100644
--- a/connection/sasl-util.h
+++ b/include/hbase/connection/sasl-util.h
@@ -22,7 +22,7 @@
#include <mutex>
#include <string>
-#include "core/configuration.h"
+#include "hbase/client/configuration.h"
class SaslUtil {
public:
diff --git a/connection/service.h b/include/hbase/connection/service.h
similarity index 92%
rename from connection/service.h
rename to include/hbase/connection/service.h
index 64d4f07..a70fd6b 100644
--- a/connection/service.h
+++ b/include/hbase/connection/service.h
@@ -22,8 +22,8 @@
#include <memory>
-#include "connection/request.h"
-#include "connection/response.h"
+#include "hbase/connection/request.h"
+#include "hbase/connection/response.h"
namespace hbase {
using HBaseService = wangle::Service<std::unique_ptr<Request>, std::unique_ptr<Response>>;
diff --git a/exceptions/BUCK b/include/hbase/exceptions/BUCK
similarity index 79%
copy from exceptions/BUCK
copy to include/hbase/exceptions/BUCK
index 3d66d72..ee296ff 100644
--- a/exceptions/BUCK
+++ b/include/hbase/exceptions/BUCK
@@ -17,23 +17,12 @@
cxx_library(
name="exceptions",
+ header_namespace="hbase/exceptions",
exported_headers=[
"exception.h",
],
- srcs=[
- "exception.cc",
- ],
deps=[
"//third-party:folly",
],
compiler_flags=['-Weffc++'],
- visibility=['//core/...', '//connection/...'],)
-cxx_test(
- name="exception-test",
- srcs=[
- "exception-test.cc",
- ],
- deps=[
- ":exceptions",
- ],
- run_test_separately=True,)
+ visibility=['//src/hbase/exceptions/...', '//include/hbase/client/...', '//include/hbase/connection/...'],)
diff --git a/exceptions/exception.h b/include/hbase/exceptions/exception.h
similarity index 100%
rename from exceptions/exception.h
rename to include/hbase/exceptions/exception.h
diff --git a/security/BUCK b/include/hbase/security/BUCK
similarity index 93%
copy from security/BUCK
copy to include/hbase/security/BUCK
index f8a5695..2e7530b 100644
--- a/security/BUCK
+++ b/include/hbase/security/BUCK
@@ -19,11 +19,11 @@
# to a single server.
cxx_library(
name="security",
+ header_namespace="hbase/security",
exported_headers=[
"user.h",
],
- srcs=[],
- deps=["//core:conf"],
+ deps=["//src/hbase/client:conf"],
compiler_flags=['-Weffc++'],
visibility=[
'PUBLIC',
diff --git a/security/user.h b/include/hbase/security/user.h
similarity index 97%
rename from security/user.h
rename to include/hbase/security/user.h
index 307fc61..c958d61 100644
--- a/security/user.h
+++ b/include/hbase/security/user.h
@@ -21,7 +21,7 @@
#include <glog/logging.h>
#include <mutex>
#include <string>
-#include "core/configuration.h"
+#include "hbase/client/configuration.h"
namespace hbase {
namespace security {
diff --git a/security/BUCK b/include/hbase/serde/BUCK
similarity index 70%
copy from security/BUCK
copy to include/hbase/serde/BUCK
index f8a5695..ee8ef82 100644
--- a/security/BUCK
+++ b/include/hbase/serde/BUCK
@@ -15,15 +15,22 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-# This is the library dealing with a single connection
-# to a single server.
cxx_library(
- name="security",
+ name="serde",
+ header_namespace="hbase/serde",
exported_headers=[
- "user.h",
+ "cell-scanner.h",
+ "cell-outputstream.h",
+ "codec.h",
+ "region-info.h",
+ "rpc-serde.h",
+ "server-name.h",
+ "table-name.h",
+ "zk.h",
+ ],
+ deps=[
+ "//src/hbase/if:if", "//third-party:folly", "//src/hbase/utils:utils", "//src/hbase/security:security"
],
- srcs=[],
- deps=["//core:conf"],
compiler_flags=['-Weffc++'],
visibility=[
'PUBLIC',
diff --git a/serde/cell-outputstream.h b/include/hbase/serde/cell-outputstream.h
similarity index 100%
rename from serde/cell-outputstream.h
rename to include/hbase/serde/cell-outputstream.h
diff --git a/serde/cell-scanner.h b/include/hbase/serde/cell-scanner.h
similarity index 100%
rename from serde/cell-scanner.h
rename to include/hbase/serde/cell-scanner.h
diff --git a/serde/codec.h b/include/hbase/serde/codec.h
similarity index 95%
rename from serde/codec.h
rename to include/hbase/serde/codec.h
index 64807dc..adc61b2 100644
--- a/serde/codec.h
+++ b/include/hbase/serde/codec.h
@@ -22,8 +22,8 @@
#include <folly/io/IOBuf.h>
#include <memory>
-#include "serde/cell-outputstream.h"
-#include "serde/cell-scanner.h"
+#include "hbase/serde/cell-outputstream.h"
+#include "hbase/serde/cell-scanner.h"
namespace hbase {
diff --git a/serde/region-info.h b/include/hbase/serde/region-info.h
similarity index 97%
rename from serde/region-info.h
rename to include/hbase/serde/region-info.h
index 8010042..075e092 100644
--- a/serde/region-info.h
+++ b/include/hbase/serde/region-info.h
@@ -24,7 +24,7 @@
#include <string>
-#include "if/HBase.pb.h"
+#include "hbase/if/HBase.pb.h"
namespace hbase {
namespace pb {
diff --git a/serde/rpc-serde.h b/include/hbase/serde/rpc-serde.h
similarity index 97%
rename from serde/rpc-serde.h
rename to include/hbase/serde/rpc-serde.h
index 6941f62..7c89347 100644
--- a/serde/rpc-serde.h
+++ b/include/hbase/serde/rpc-serde.h
@@ -22,9 +22,9 @@
#include <string>
#include <folly/ExceptionWrapper.h>
-#include "if/HBase.pb.h"
-#include "serde/cell-scanner.h"
-#include "serde/codec.h"
+#include "hbase/if/HBase.pb.h"
+#include "hbase/serde/cell-scanner.h"
+#include "hbase/serde/codec.h"
using namespace folly;
// Forward
diff --git a/serde/server-name.h b/include/hbase/serde/server-name.h
similarity index 97%
rename from serde/server-name.h
rename to include/hbase/serde/server-name.h
index 41e3c77..e650914 100644
--- a/serde/server-name.h
+++ b/include/hbase/serde/server-name.h
@@ -24,7 +24,7 @@
#include <string>
-#include "if/HBase.pb.h"
+#include "hbase/if/HBase.pb.h"
namespace hbase {
namespace pb {
diff --git a/serde/table-name.h b/include/hbase/serde/table-name.h
similarity index 98%
rename from serde/table-name.h
rename to include/hbase/serde/table-name.h
index 3594802..756f2aa 100644
--- a/serde/table-name.h
+++ b/include/hbase/serde/table-name.h
@@ -25,7 +25,7 @@
#include <string>
#include <vector>
-#include "if/HBase.pb.h"
+#include "hbase/if/HBase.pb.h"
namespace hbase {
namespace pb {
diff --git a/serde/zk.h b/include/hbase/serde/zk.h
similarity index 100%
rename from serde/zk.h
rename to include/hbase/serde/zk.h
diff --git a/utils/BUCK b/include/hbase/utils/BUCK
similarity index 74%
copy from utils/BUCK
copy to include/hbase/utils/BUCK
index 96f02b8..d44e908 100644
--- a/utils/BUCK
+++ b/include/hbase/utils/BUCK
@@ -17,6 +17,7 @@
cxx_library(
name="utils",
+ header_namespace="hbase/utils",
exported_headers=[
"bytes-util.h",
"connection-util.h",
@@ -27,7 +28,6 @@ cxx_library(
"user-util.h",
"version.h",
],
- srcs=["bytes-util.cc", "connection-util.cc", "user-util.cc"],
deps=[
'//third-party:folly',
],
@@ -38,27 +38,3 @@ cxx_library(
'PUBLIC',
],
compiler_flags=['-Weffc++'],)
-cxx_test(
- name="bytes-util-test",
- srcs=[
- "bytes-util-test.cc",
- ],
- deps=[
- ":utils",
- ],)
-cxx_test(
- name="concurrent-map-test",
- srcs=[
- "concurrent-map-test.cc",
- ],
- deps=[
- ":utils",
- ],)
-cxx_test(
- name="user-util-test",
- srcs=[
- "user-util-test.cc",
- ],
- deps=[
- ":utils",
- ],)
\ No newline at end of file
diff --git a/utils/bytes-util.h b/include/hbase/utils/bytes-util.h
similarity index 100%
rename from utils/bytes-util.h
rename to include/hbase/utils/bytes-util.h
diff --git a/utils/concurrent-map.h b/include/hbase/utils/concurrent-map.h
similarity index 100%
rename from utils/concurrent-map.h
rename to include/hbase/utils/concurrent-map.h
diff --git a/utils/connection-util.h b/include/hbase/utils/connection-util.h
similarity index 98%
rename from utils/connection-util.h
rename to include/hbase/utils/connection-util.h
index f52c2f9..b6a53e6 100644
--- a/utils/connection-util.h
+++ b/include/hbase/utils/connection-util.h
@@ -24,7 +24,7 @@
#include <cstdlib>
#include <memory>
#include <vector>
-#include "utils/time-util.h"
+#include "hbase/utils/time-util.h"
namespace hbase {
class ConnectionUtils {
diff --git a/utils/optional.h b/include/hbase/utils/optional.h
similarity index 100%
rename from utils/optional.h
rename to include/hbase/utils/optional.h
diff --git a/utils/sys-util.h b/include/hbase/utils/sys-util.h
similarity index 100%
rename from utils/sys-util.h
rename to include/hbase/utils/sys-util.h
diff --git a/utils/time-util.h b/include/hbase/utils/time-util.h
similarity index 100%
rename from utils/time-util.h
rename to include/hbase/utils/time-util.h
diff --git a/utils/user-util.h b/include/hbase/utils/user-util.h
similarity index 100%
rename from utils/user-util.h
rename to include/hbase/utils/user-util.h
diff --git a/core/BUCK b/src/hbase/client/BUCK
similarity index 66%
rename from core/BUCK
rename to src/hbase/client/BUCK
index 76c836b..1a8f434 100644
--- a/core/BUCK
+++ b/src/hbase/client/BUCK
@@ -17,53 +17,7 @@
# This is the main library.
cxx_library(
- name="core",
- exported_headers=[
- "async-client-scanner.h",
- "async-connection.h",
- "async-region-locator.h",
- "async-rpc-retrying-caller-factory.h",
- "async-rpc-retrying-caller.h",
- "async-table-result-scanner.h",
- "client.h",
- "cell.h",
- "hbase-macros.h",
- "filter.h",
- "query.h",
- "keyvalue-codec.h",
- "region-location.h",
- "location-cache.h",
- "connection-configuration.h",
- # TODO: move this out of exported
- # Once meta lookup works
- "meta-utils.h",
- "get.h",
- "increment.h",
- "mutation.h",
- "put.h",
- "delete.h",
- "scan.h",
- "append.h",
- "result.h",
- "result-scanner.h",
- "request-converter.h",
- "response-converter.h",
- "table.h",
- "async-scan-rpc-retrying-caller.h",
- "raw-async-table.h",
- "raw-scan-result-consumer.h",
- "scan-result-cache.h",
- "hbase-rpc-controller.h",
- "time-range.h",
- "zk-util.h",
- "action.h",
- "multi-response.h",
- "region-request.h",
- "region-result.h",
- "row.h",
- "server-request.h",
- "async-batch-rpc-retrying-caller.h",
- ],
+ name="client",
srcs=[
"async-client-scanner.cc",
"async-connection.cc",
@@ -97,12 +51,13 @@ cxx_library(
"async-batch-rpc-retrying-caller.cc",
],
deps=[
- "//exceptions:exceptions",
- "//utils:utils",
- "//connection:connection",
- "//core:conf",
- "//if:if",
- "//serde:serde",
+ "//include/hbase/client:client",
+ "//src/hbase/exceptions:exceptions",
+ "//src/hbase/utils:utils",
+ "//src/hbase/connection:connection",
+ "//src/hbase/client:conf",
+ "//src/hbase/if:if",
+ "//src/hbase/serde:serde",
"//third-party:folly",
"//third-party:wangle",
"//third-party:zookeeper_mt",
@@ -114,14 +69,12 @@ cxx_library(
cxx_library(
name="conf",
exported_headers=[
- "configuration.h",
- "hbase-configuration-loader.h",
],
srcs=[
"configuration.cc",
"hbase-configuration-loader.cc",
],
- deps=["//utils:utils", "//third-party:folly"],
+ deps=["//include/hbase/client:conf", "//src/hbase/utils:utils", "//third-party:folly"],
compiler_flags=['-Weffc++', '-ggdb'],
visibility=[
'PUBLIC',
@@ -132,8 +85,8 @@ cxx_test(
"location-cache-test.cc",
],
deps=[
- ":core",
- "//test-util:test-util",
+ ":client",
+ "//src/hbase/test-util:test-util",
],
run_test_separately=True,)
cxx_test(
@@ -142,10 +95,10 @@ cxx_test(
"location-cache-retry-test.cc",
],
deps=[
- ":core",
- "//if:if",
- "//serde:serde",
- "//test-util:test-util",
+ ":client",
+ "//src/hbase/if:if",
+ "//src/hbase/serde:serde",
+ "//src/hbase/test-util:test-util",
],
run_test_separately=True,)
cxx_test(
@@ -154,7 +107,7 @@ cxx_test(
"cell-test.cc",
],
deps=[
- ":core",
+ ":client",
],
run_test_separately=True,)
cxx_test(
@@ -163,10 +116,10 @@ cxx_test(
"filter-test.cc",
],
deps=[
- ":core",
- "//if:if",
- "//serde:serde",
- "//test-util:test-util",
+ ":client",
+ "//src/hbase/if:if",
+ "//src/hbase/serde:serde",
+ "//src/hbase/test-util:test-util",
],
run_test_separately=True,)
cxx_test(
@@ -175,7 +128,7 @@ cxx_test(
"get-test.cc",
],
deps=[
- ":core",
+ ":client",
],
run_test_separately=True,)
cxx_test(
@@ -184,7 +137,7 @@ cxx_test(
"delete-test.cc",
],
deps=[
- ":core",
+ ":client",
],
run_test_separately=True,)
cxx_test(
@@ -193,7 +146,7 @@ cxx_test(
"increment-test.cc",
],
deps=[
- ":core",
+ ":client",
],
run_test_separately=True,)
cxx_test(
@@ -202,7 +155,7 @@ cxx_test(
"put-test.cc",
],
deps=[
- ":core",
+ ":client",
],
run_test_separately=True,)
cxx_test(
@@ -211,7 +164,7 @@ cxx_test(
"append-test.cc",
],
deps=[
- ":core",
+ ":client",
],
run_test_separately=True,)
cxx_test(
@@ -220,9 +173,9 @@ cxx_test(
"async-rpc-retrying-test.cc",
],
deps=[
- ":core",
- "//test-util:test-util",
- "//exceptions:exceptions",
+ ":client",
+ "//src/hbase/test-util:test-util",
+ "//src/hbase/exceptions:exceptions",
],
run_test_separately=True,)
cxx_test(
@@ -231,7 +184,7 @@ cxx_test(
"time-range-test.cc",
],
deps=[
- ":core",
+ ":client",
],
run_test_separately=True,)
cxx_test(
@@ -240,7 +193,7 @@ cxx_test(
"configuration-test.cc",
],
deps=[
- ":core",
+ ":client",
],
run_test_separately=True,)
cxx_test(
@@ -249,7 +202,7 @@ cxx_test(
"hbase-configuration-test.cc",
],
deps=[
- ":core",
+ ":client",
],
run_test_separately=True,)
cxx_test(
@@ -258,7 +211,7 @@ cxx_test(
"scan-test.cc",
],
deps=[
- ":core",
+ ":client",
],
run_test_separately=True,)
cxx_test(
@@ -267,7 +220,7 @@ cxx_test(
"result-test.cc",
],
deps=[
- ":core",
+ ":client",
],
run_test_separately=True,)
cxx_test(
@@ -276,9 +229,9 @@ cxx_test(
"request-converter-test.cc",
],
deps=[
- ":core",
- "//connection:connection",
- "//if:if",
+ ":client",
+ "//src/hbase/connection:connection",
+ "//src/hbase/if:if",
],
run_test_separately=True,)
cxx_test(
@@ -287,10 +240,10 @@ cxx_test(
"client-test.cc",
],
deps=[
- ":core",
- "//if:if",
- "//serde:serde",
- "//test-util:test-util",
+ ":client",
+ "//src/hbase/if:if",
+ "//src/hbase/serde:serde",
+ "//src/hbase/test-util:test-util",
],
run_test_separately=True,)
cxx_test(
@@ -299,7 +252,7 @@ cxx_test(
"scan-result-cache-test.cc",
],
deps=[
- ":core",
+ ":client",
],
run_test_separately=True,)
cxx_test(
@@ -308,10 +261,10 @@ cxx_test(
"scanner-test.cc",
],
deps=[
- ":core",
- "//if:if",
- "//serde:serde",
- "//test-util:test-util",
+ ":client",
+ "//src/hbase/if:if",
+ "//src/hbase/serde:serde",
+ "//src/hbase/test-util:test-util",
],
run_test_separately=True,)
cxx_test(
@@ -320,7 +273,7 @@ cxx_test(
"zk-util-test.cc",
],
deps=[
- ":core",
+ ":client",
],
run_test_separately=True,)
cxx_test(
@@ -329,9 +282,9 @@ cxx_test(
"async-batch-rpc-retrying-test.cc",
],
deps=[
- ":core",
- "//test-util:test-util",
- "//exceptions:exceptions",
+ ":client",
+ "//src/hbase/test-util:test-util",
+ "//src/hbase/exceptions:exceptions",
],
run_test_separately=True,)
cxx_binary(
@@ -339,10 +292,10 @@ cxx_binary(
srcs=[
"simple-client.cc",
],
- deps=[":core", "//connection:connection"],)
+ deps=[":client", "//src/hbase/connection:connection"],)
cxx_binary(
name="load-client",
srcs=[
"load-client.cc",
],
- deps=[":core", "//connection:connection"],)
+ deps=[":client", "//src/hbase/connection:connection"],)
diff --git a/core/append-test.cc b/src/hbase/client/append-test.cc
similarity index 96%
rename from core/append-test.cc
rename to src/hbase/client/append-test.cc
index 2216034..1af138d 100644
--- a/core/append-test.cc
+++ b/src/hbase/client/append-test.cc
@@ -19,9 +19,9 @@
#include <glog/logging.h>
#include <gtest/gtest.h>
-#include "core/append.h"
-#include "core/mutation.h"
-#include "utils/time-util.h"
+#include "hbase/client/append.h"
+#include "hbase/client/mutation.h"
+#include "hbase/utils/time-util.h"
using hbase::Append;
using hbase::Cell;
diff --git a/core/append.cc b/src/hbase/client/append.cc
similarity index 98%
rename from core/append.cc
rename to src/hbase/client/append.cc
index 95349ae..431a398 100644
--- a/core/append.cc
+++ b/src/hbase/client/append.cc
@@ -19,7 +19,7 @@
*
*/
-#include "core/append.h"
+#include "hbase/client/append.h"
#include <folly/Conv.h>
#include <algorithm>
#include <limits>
diff --git a/core/async-batch-rpc-retrying-caller.cc b/src/hbase/client/async-batch-rpc-retrying-caller.cc
similarity index 99%
rename from core/async-batch-rpc-retrying-caller.cc
rename to src/hbase/client/async-batch-rpc-retrying-caller.cc
index dfbf7e7..6699b90 100644
--- a/core/async-batch-rpc-retrying-caller.cc
+++ b/src/hbase/client/async-batch-rpc-retrying-caller.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/async-batch-rpc-retrying-caller.h"
+#include "hbase/client/async-batch-rpc-retrying-caller.h"
#include <glog/logging.h>
#include <limits>
diff --git a/core/async-batch-rpc-retrying-test.cc b/src/hbase/client/async-batch-rpc-retrying-test.cc
similarity index 97%
rename from core/async-batch-rpc-retrying-test.cc
rename to src/hbase/client/async-batch-rpc-retrying-test.cc
index 00cf2b8..b0f4afb 100644
--- a/core/async-batch-rpc-retrying-test.cc
+++ b/src/hbase/client/async-batch-rpc-retrying-test.cc
@@ -29,18 +29,18 @@
#include <functional>
#include <string>
-#include "connection/rpc-client.h"
-#include "core/async-batch-rpc-retrying-caller.h"
-#include "core/async-connection.h"
-#include "core/async-rpc-retrying-caller-factory.h"
-#include "core/client.h"
-#include "core/connection-configuration.h"
-#include "core/keyvalue-codec.h"
-#include "core/region-location.h"
-#include "core/result.h"
-#include "exceptions/exception.h"
-#include "test-util/test-util.h"
-#include "utils/time-util.h"
+#include "hbase/connection/rpc-client.h"
+#include "hbase/client/async-batch-rpc-retrying-caller.h"
+#include "hbase/client/async-connection.h"
+#include "hbase/client/async-rpc-retrying-caller-factory.h"
+#include "hbase/client/client.h"
+#include "hbase/client/connection-configuration.h"
+#include "hbase/client/keyvalue-codec.h"
+#include "hbase/client/region-location.h"
+#include "hbase/client/result.h"
+#include "hbase/exceptions/exception.h"
+#include "hbase/test-util/test-util.h"
+#include "hbase/utils/time-util.h"
using hbase::AsyncRpcRetryingCallerFactory;
using hbase::AsyncConnection;
diff --git a/core/async-client-scanner.cc b/src/hbase/client/async-client-scanner.cc
similarity index 99%
rename from core/async-client-scanner.cc
rename to src/hbase/client/async-client-scanner.cc
index 720ab25..50c01ee 100644
--- a/core/async-client-scanner.cc
+++ b/src/hbase/client/async-client-scanner.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/async-client-scanner.h"
+#include "hbase/client/async-client-scanner.h"
#include <algorithm>
#include <iterator>
diff --git a/core/async-connection.cc b/src/hbase/client/async-connection.cc
similarity index 96%
rename from core/async-connection.cc
rename to src/hbase/client/async-connection.cc
index 850fb8f..f1bdebc 100644
--- a/core/async-connection.cc
+++ b/src/hbase/client/async-connection.cc
@@ -17,8 +17,8 @@
*
*/
-#include "core/async-connection.h"
-#include "core/async-rpc-retrying-caller-factory.h"
+#include "hbase/client/async-connection.h"
+#include "hbase/client/async-rpc-retrying-caller-factory.h"
namespace hbase {
diff --git a/core/async-rpc-retrying-caller-factory.cc b/src/hbase/client/async-rpc-retrying-caller-factory.cc
similarity index 93%
rename from core/async-rpc-retrying-caller-factory.cc
rename to src/hbase/client/async-rpc-retrying-caller-factory.cc
index 0ac9cac..42b4eac 100644
--- a/core/async-rpc-retrying-caller-factory.cc
+++ b/src/hbase/client/async-rpc-retrying-caller-factory.cc
@@ -17,6 +17,6 @@
*
*/
-#include "core/async-rpc-retrying-caller-factory.h"
+#include "hbase/client/async-rpc-retrying-caller-factory.h"
namespace hbase {} // namespace hbase
diff --git a/core/async-rpc-retrying-caller.cc b/src/hbase/client/async-rpc-retrying-caller.cc
similarity index 95%
rename from core/async-rpc-retrying-caller.cc
rename to src/hbase/client/async-rpc-retrying-caller.cc
index 8e60991..4c39f05 100644
--- a/core/async-rpc-retrying-caller.cc
+++ b/src/hbase/client/async-rpc-retrying-caller.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/async-rpc-retrying-caller.h"
+#include "hbase/client/async-rpc-retrying-caller.h"
#include <folly/Conv.h>
#include <folly/ExceptionWrapper.h>
@@ -25,16 +25,16 @@
#include <folly/Logging.h>
#include <folly/Unit.h>
-#include "connection/rpc-client.h"
-#include "core/async-connection.h"
-#include "core/hbase-rpc-controller.h"
-#include "core/region-location.h"
-#include "core/result.h"
-#include "exceptions/exception.h"
-#include "if/HBase.pb.h"
-#include "utils/connection-util.h"
-#include "utils/sys-util.h"
-#include "utils/time-util.h"
+#include "hbase/connection/rpc-client.h"
+#include "hbase/client/async-connection.h"
+#include "hbase/client/hbase-rpc-controller.h"
+#include "hbase/client/region-location.h"
+#include "hbase/client/result.h"
+#include "hbase/exceptions/exception.h"
+#include "hbase/if/HBase.pb.h"
+#include "hbase/utils/connection-util.h"
+#include "hbase/utils/sys-util.h"
+#include "hbase/utils/time-util.h"
using folly::exception_wrapper;
diff --git a/core/async-rpc-retrying-test.cc b/src/hbase/client/async-rpc-retrying-test.cc
similarity index 95%
rename from core/async-rpc-retrying-test.cc
rename to src/hbase/client/async-rpc-retrying-test.cc
index 2eb82a9..6782d05 100644
--- a/core/async-rpc-retrying-test.cc
+++ b/src/hbase/client/async-rpc-retrying-test.cc
@@ -30,25 +30,25 @@
#include <functional>
#include <string>
-#include "connection/request.h"
-#include "connection/response.h"
-#include "connection/rpc-client.h"
-#include "core/async-connection.h"
-#include "core/async-rpc-retrying-caller-factory.h"
-#include "core/async-rpc-retrying-caller.h"
-#include "core/client.h"
-#include "core/connection-configuration.h"
-#include "core/hbase-rpc-controller.h"
-#include "core/keyvalue-codec.h"
-#include "core/region-location.h"
-#include "core/request-converter.h"
-#include "core/response-converter.h"
-#include "core/result.h"
-#include "exceptions/exception.h"
-#include "if/Client.pb.h"
-#include "if/HBase.pb.h"
-#include "test-util/test-util.h"
-#include "utils/time-util.h"
+#include "hbase/connection/request.h"
+#include "hbase/connection/response.h"
+#include "hbase/connection/rpc-client.h"
+#include "hbase/client/async-connection.h"
+#include "hbase/client/async-rpc-retrying-caller-factory.h"
+#include "hbase/client/async-rpc-retrying-caller.h"
+#include "hbase/client/client.h"
+#include "hbase/client/connection-configuration.h"
+#include "hbase/client/hbase-rpc-controller.h"
+#include "hbase/client/keyvalue-codec.h"
+#include "hbase/client/region-location.h"
+#include "hbase/client/request-converter.h"
+#include "hbase/client/response-converter.h"
+#include "hbase/client/result.h"
+#include "hbase/exceptions/exception.h"
+#include "hbase/if/Client.pb.h"
+#include "hbase/if/HBase.pb.h"
+#include "hbase/test-util/test-util.h"
+#include "hbase/utils/time-util.h"
using hbase::AsyncRpcRetryingCallerFactory;
using hbase::AsyncConnection;
diff --git a/core/async-scan-rpc-retrying-caller.cc b/src/hbase/client/async-scan-rpc-retrying-caller.cc
similarity index 99%
rename from core/async-scan-rpc-retrying-caller.cc
rename to src/hbase/client/async-scan-rpc-retrying-caller.cc
index a1e8362..2189128 100644
--- a/core/async-scan-rpc-retrying-caller.cc
+++ b/src/hbase/client/async-scan-rpc-retrying-caller.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/async-scan-rpc-retrying-caller.h"
+#include "hbase/client/async-scan-rpc-retrying-caller.h"
namespace hbase {
diff --git a/core/async-table-result-scanner.cc b/src/hbase/client/async-table-result-scanner.cc
similarity index 98%
rename from core/async-table-result-scanner.cc
rename to src/hbase/client/async-table-result-scanner.cc
index b1935ae..3812b3a 100644
--- a/core/async-table-result-scanner.cc
+++ b/src/hbase/client/async-table-result-scanner.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/async-table-result-scanner.h"
+#include "hbase/client/async-table-result-scanner.h"
#include <vector>
diff --git a/core/cell-test.cc b/src/hbase/client/cell-test.cc
similarity index 99%
rename from core/cell-test.cc
rename to src/hbase/client/cell-test.cc
index 4611473..8a65aa5 100644
--- a/core/cell-test.cc
+++ b/src/hbase/client/cell-test.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/cell.h"
+#include "hbase/client/cell.h"
#include <glog/logging.h>
#include <gtest/gtest.h>
diff --git a/core/cell.cc b/src/hbase/client/cell.cc
similarity index 98%
rename from core/cell.cc
rename to src/hbase/client/cell.cc
index e475d49..da2c11f 100644
--- a/core/cell.cc
+++ b/src/hbase/client/cell.cc
@@ -17,13 +17,13 @@
*
*/
-#include "core/cell.h"
+#include "hbase/client/cell.h"
#include <climits>
#include <limits>
#include <stdexcept>
#include "folly/Conv.h"
-#include "utils/bytes-util.h"
+#include "hbase/utils/bytes-util.h"
namespace hbase {
diff --git a/core/client-test.cc b/src/hbase/client/client-test.cc
similarity index 97%
rename from core/client-test.cc
rename to src/hbase/client/client-test.cc
index 3f72880..7141047 100644
--- a/core/client-test.cc
+++ b/src/hbase/client/client-test.cc
@@ -19,21 +19,21 @@
#include <gtest/gtest.h>
-#include "core/append.h"
-#include "core/cell.h"
-#include "core/client.h"
-#include "core/configuration.h"
-#include "core/delete.h"
-#include "core/get.h"
-#include "core/hbase-configuration-loader.h"
-#include "core/increment.h"
-#include "core/put.h"
-#include "core/result.h"
-#include "core/table.h"
-#include "exceptions/exception.h"
-#include "serde/table-name.h"
-#include "test-util/test-util.h"
-#include "utils/bytes-util.h"
+#include "hbase/client/append.h"
+#include "hbase/client/cell.h"
+#include "hbase/client/client.h"
+#include "hbase/client/configuration.h"
+#include "hbase/client/delete.h"
+#include "hbase/client/get.h"
+#include "hbase/client/hbase-configuration-loader.h"
+#include "hbase/client/increment.h"
+#include "hbase/client/put.h"
+#include "hbase/client/result.h"
+#include "hbase/client/table.h"
+#include "hbase/exceptions/exception.h"
+#include "hbase/serde/table-name.h"
+#include "hbase/test-util/test-util.h"
+#include "hbase/utils/bytes-util.h"
using hbase::Cell;
using hbase::Configuration;
diff --git a/core/client.cc b/src/hbase/client/client.cc
similarity index 98%
rename from core/client.cc
rename to src/hbase/client/client.cc
index e23aeae..2a63c03 100644
--- a/core/client.cc
+++ b/src/hbase/client/client.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/client.h"
+#include "hbase/client/client.h"
#include <glog/logging.h>
#include <chrono>
diff --git a/core/configuration-test.cc b/src/hbase/client/configuration-test.cc
similarity index 98%
rename from core/configuration-test.cc
rename to src/hbase/client/configuration-test.cc
index abdf0c7..4201018 100644
--- a/core/configuration-test.cc
+++ b/src/hbase/client/configuration-test.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/configuration.h"
+#include "hbase/client/configuration.h"
#include <gtest/gtest.h>
using hbase::Configuration;
diff --git a/core/configuration.cc b/src/hbase/client/configuration.cc
similarity index 99%
rename from core/configuration.cc
rename to src/hbase/client/configuration.cc
index 1fd2851..d829edc 100644
--- a/core/configuration.cc
+++ b/src/hbase/client/configuration.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/configuration.h"
+#include "hbase/client/configuration.h"
#include <memory>
#include <stdexcept>
diff --git a/core/delete-test.cc b/src/hbase/client/delete-test.cc
similarity index 97%
rename from core/delete-test.cc
rename to src/hbase/client/delete-test.cc
index ec1e3a9..4af0c40 100644
--- a/core/delete-test.cc
+++ b/src/hbase/client/delete-test.cc
@@ -19,9 +19,9 @@
#include <glog/logging.h>
#include <gtest/gtest.h>
-#include "core/delete.h"
-#include "core/mutation.h"
-#include "utils/time-util.h"
+#include "hbase/client/delete.h"
+#include "hbase/client/mutation.h"
+#include "hbase/utils/time-util.h"
using hbase::Delete;
using hbase::Cell;
diff --git a/core/delete.cc b/src/hbase/client/delete.cc
similarity index 99%
rename from core/delete.cc
rename to src/hbase/client/delete.cc
index 5f48782..eed096d 100644
--- a/core/delete.cc
+++ b/src/hbase/client/delete.cc
@@ -19,7 +19,7 @@
*
*/
-#include "core/delete.h"
+#include "hbase/client/delete.h"
#include <folly/Conv.h>
#include <algorithm>
#include <limits>
diff --git a/core/filter-test.cc b/src/hbase/client/filter-test.cc
similarity index 92%
rename from core/filter-test.cc
rename to src/hbase/client/filter-test.cc
index 7276dfb..8a23913 100644
--- a/core/filter-test.cc
+++ b/src/hbase/client/filter-test.cc
@@ -18,16 +18,16 @@
*/
#include <gtest/gtest.h>
-#include "core/client.h"
-#include "core/configuration.h"
-#include "core/get.h"
-#include "core/put.h"
-#include "core/result.h"
-#include "core/table.h"
-#include "if/Comparator.pb.h"
-#include "if/HBase.pb.h"
-#include "serde/table-name.h"
-#include "test-util/test-util.h"
+#include "hbase/client/client.h"
+#include "hbase/client/configuration.h"
+#include "hbase/client/get.h"
+#include "hbase/client/put.h"
+#include "hbase/client/result.h"
+#include "hbase/client/table.h"
+#include "hbase/if/Comparator.pb.h"
+#include "hbase/if/HBase.pb.h"
+#include "hbase/serde/table-name.h"
+#include "hbase/test-util/test-util.h"
using hbase::Configuration;
using hbase::Get;
diff --git a/core/get-test.cc b/src/hbase/client/get-test.cc
similarity index 99%
rename from core/get-test.cc
rename to src/hbase/client/get-test.cc
index 4a44a26..676d798 100644
--- a/core/get-test.cc
+++ b/src/hbase/client/get-test.cc
@@ -17,8 +17,8 @@
*
*/
-#include "core/get.h"
-#include "core/cell.h"
+#include "hbase/client/get.h"
+#include "hbase/client/cell.h"
#include <glog/logging.h>
#include <gtest/gtest.h>
diff --git a/core/get.cc b/src/hbase/client/get.cc
similarity index 99%
rename from core/get.cc
rename to src/hbase/client/get.cc
index bc0d446..ba18a58 100644
--- a/core/get.cc
+++ b/src/hbase/client/get.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/get.h"
+#include "hbase/client/get.h"
#include <algorithm>
#include <limits>
#include <stdexcept>
diff --git a/core/hbase-configuration-loader.cc b/src/hbase/client/hbase-configuration-loader.cc
similarity index 99%
rename from core/hbase-configuration-loader.cc
rename to src/hbase/client/hbase-configuration-loader.cc
index 4d6a3e9..98ef20e 100644
--- a/core/hbase-configuration-loader.cc
+++ b/src/hbase/client/hbase-configuration-loader.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/hbase-configuration-loader.h"
+#include "hbase/client/hbase-configuration-loader.h"
#include <glog/logging.h>
#include <boost/foreach.hpp>
diff --git a/core/hbase-configuration-test.cc b/src/hbase/client/hbase-configuration-test.cc
similarity index 99%
rename from core/hbase-configuration-test.cc
rename to src/hbase/client/hbase-configuration-test.cc
index 56b4463..173c4df 100644
--- a/core/hbase-configuration-test.cc
+++ b/src/hbase/client/hbase-configuration-test.cc
@@ -23,8 +23,8 @@
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <boost/filesystem.hpp>
-#include "core/configuration.h"
-#include "core/hbase-configuration-loader.h"
+#include "hbase/client/configuration.h"
+#include "hbase/client/hbase-configuration-loader.h"
using namespace hbase;
using std::experimental::nullopt;
diff --git a/core/hbase-rpc-controller.cc b/src/hbase/client/hbase-rpc-controller.cc
similarity index 94%
rename from core/hbase-rpc-controller.cc
rename to src/hbase/client/hbase-rpc-controller.cc
index bc53781..c2ae6cc 100644
--- a/core/hbase-rpc-controller.cc
+++ b/src/hbase/client/hbase-rpc-controller.cc
@@ -17,6 +17,6 @@
*
*/
-#include "core/hbase-rpc-controller.h"
+#include "hbase/client/hbase-rpc-controller.h"
namespace hbase {} /* namespace hbase */
diff --git a/core/increment-test.cc b/src/hbase/client/increment-test.cc
similarity index 96%
rename from core/increment-test.cc
rename to src/hbase/client/increment-test.cc
index aa9b864..3c11fbe 100644
--- a/core/increment-test.cc
+++ b/src/hbase/client/increment-test.cc
@@ -19,10 +19,10 @@
#include <glog/logging.h>
#include <gtest/gtest.h>
-#include "core/increment.h"
-#include "core/mutation.h"
-#include "core/put.h"
-#include "utils/time-util.h"
+#include "hbase/client/increment.h"
+#include "hbase/client/mutation.h"
+#include "hbase/client/put.h"
+#include "hbase/utils/time-util.h"
using hbase::Increment;
using hbase::Increment;
diff --git a/core/increment.cc b/src/hbase/client/increment.cc
similarity index 96%
rename from core/increment.cc
rename to src/hbase/client/increment.cc
index a21a764..a8ac49a 100644
--- a/core/increment.cc
+++ b/src/hbase/client/increment.cc
@@ -19,14 +19,14 @@
*
*/
-#include "core/increment.h"
+#include "hbase/client/increment.h"
#include <folly/Conv.h>
#include <algorithm>
#include <limits>
#include <stdexcept>
#include <utility>
-#include "utils/bytes-util.h"
+#include "hbase/utils/bytes-util.h"
namespace hbase {
diff --git a/core/keyvalue-codec.cc b/src/hbase/client/keyvalue-codec.cc
similarity index 98%
rename from core/keyvalue-codec.cc
rename to src/hbase/client/keyvalue-codec.cc
index 1b526b9..9b6e689 100644
--- a/core/keyvalue-codec.cc
+++ b/src/hbase/client/keyvalue-codec.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/keyvalue-codec.h"
+#include "hbase/client/keyvalue-codec.h"
#include <string>
diff --git a/core/load-client.cc b/src/hbase/client/load-client.cc
similarity index 98%
rename from core/load-client.cc
rename to src/hbase/client/load-client.cc
index 8cceeef..a321845 100644
--- a/core/load-client.cc
+++ b/src/hbase/client/load-client.cc
@@ -26,12 +26,12 @@
#include <iostream>
#include <thread>
-#include "core/client.h"
-#include "core/get.h"
-#include "core/put.h"
-#include "core/table.h"
-#include "serde/table-name.h"
-#include "utils/time-util.h"
+#include "hbase/client/client.h"
+#include "hbase/client/get.h"
+#include "hbase/client/put.h"
+#include "hbase/client/table.h"
+#include "hbase/serde/table-name.h"
+#include "hbase/utils/time-util.h"
using hbase::Client;
using hbase::Configuration;
diff --git a/core/location-cache-retry-test.cc b/src/hbase/client/location-cache-retry-test.cc
similarity index 85%
rename from core/location-cache-retry-test.cc
rename to src/hbase/client/location-cache-retry-test.cc
index f154b69..283cf85 100644
--- a/core/location-cache-retry-test.cc
+++ b/src/hbase/client/location-cache-retry-test.cc
@@ -19,22 +19,22 @@
#include <gtest/gtest.h>
-#include "core/append.h"
-#include "core/cell.h"
-#include "core/client.h"
-#include "core/configuration.h"
-#include "core/delete.h"
-#include "core/get.h"
-#include "core/hbase-configuration-loader.h"
-#include "core/increment.h"
-#include "core/meta-utils.h"
-#include "core/put.h"
-#include "core/result.h"
-#include "core/table.h"
-#include "exceptions/exception.h"
-#include "serde/table-name.h"
-#include "test-util/test-util.h"
-#include "utils/bytes-util.h"
+#include "hbase/client/append.h"
+#include "hbase/client/cell.h"
+#include "hbase/client/client.h"
+#include "hbase/client/configuration.h"
+#include "hbase/client/delete.h"
+#include "hbase/client/get.h"
+#include "hbase/client/hbase-configuration-loader.h"
+#include "hbase/client/increment.h"
+#include "hbase/client/meta-utils.h"
+#include "hbase/client/put.h"
+#include "hbase/client/result.h"
+#include "hbase/client/table.h"
+#include "hbase/exceptions/exception.h"
+#include "hbase/serde/table-name.h"
+#include "hbase/test-util/test-util.h"
+#include "hbase/utils/bytes-util.h"
using hbase::Cell;
using hbase::Configuration;
diff --git a/core/location-cache-test.cc b/src/hbase/client/location-cache-test.cc
similarity index 96%
rename from core/location-cache-test.cc
rename to src/hbase/client/location-cache-test.cc
index fd96ff3..af25902 100644
--- a/core/location-cache-test.cc
+++ b/src/hbase/client/location-cache-test.cc
@@ -16,17 +16,17 @@
* limitations under the License.
*
*/
-#include "core/location-cache.h"
+#include "hbase/client/location-cache.h"
#include <folly/Memory.h>
#include <gtest/gtest.h>
#include <chrono>
-#include "core/keyvalue-codec.h"
-#include "if/HBase.pb.h"
-#include "serde/table-name.h"
-#include "test-util/test-util.h"
+#include "hbase/client/keyvalue-codec.h"
+#include "hbase/if/HBase.pb.h"
+#include "hbase/serde/table-name.h"
+#include "hbase/test-util/test-util.h"
using hbase::Cell;
using hbase::Configuration;
diff --git a/core/location-cache.cc b/src/hbase/client/location-cache.cc
similarity index 96%
rename from core/location-cache.cc
rename to src/hbase/client/location-cache.cc
index b728d95..2be2b94 100644
--- a/core/location-cache.cc
+++ b/src/hbase/client/location-cache.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*
*/
-#include "core/location-cache.h"
+#include "hbase/client/location-cache.h"
#include <folly/Conv.h>
#include <folly/Logging.h>
@@ -28,15 +28,15 @@
#include <shared_mutex>
#include <utility>
-#include "connection/response.h"
-#include "connection/rpc-connection.h"
-#include "core/meta-utils.h"
-#include "exceptions/exception.h"
-#include "if/Client.pb.h"
-#include "if/ZooKeeper.pb.h"
-#include "serde/region-info.h"
-#include "serde/server-name.h"
-#include "serde/zk.h"
+#include "hbase/connection/response.h"
+#include "hbase/connection/rpc-connection.h"
+#include "hbase/client/meta-utils.h"
+#include "hbase/exceptions/exception.h"
+#include "hbase/if/Client.pb.h"
+#include "hbase/if/ZooKeeper.pb.h"
+#include "hbase/serde/region-info.h"
+#include "hbase/serde/server-name.h"
+#include "hbase/serde/zk.h"
using hbase::pb::MetaRegionServer;
using hbase::pb::ServerName;
diff --git a/core/meta-utils.cc b/src/hbase/client/meta-utils.cc
similarity index 92%
rename from core/meta-utils.cc
rename to src/hbase/client/meta-utils.cc
index 31349a5..338c43e 100644
--- a/core/meta-utils.cc
+++ b/src/hbase/client/meta-utils.cc
@@ -17,21 +17,21 @@
*
*/
-#include "core/meta-utils.h"
+#include "hbase/client/meta-utils.h"
#include <folly/Conv.h>
#include <memory>
#include <utility>
#include <vector>
-#include "connection/request.h"
-#include "connection/response.h"
-#include "core/response-converter.h"
-#include "exceptions/exception.h"
-#include "if/Client.pb.h"
-#include "serde/region-info.h"
-#include "serde/server-name.h"
-#include "serde/table-name.h"
+#include "hbase/connection/request.h"
+#include "hbase/connection/response.h"
+#include "hbase/client/response-converter.h"
+#include "hbase/exceptions/exception.h"
+#include "hbase/if/Client.pb.h"
+#include "hbase/serde/region-info.h"
+#include "hbase/serde/server-name.h"
+#include "hbase/serde/table-name.h"
using hbase::pb::TableName;
using hbase::pb::RegionInfo;
diff --git a/core/multi-response.cc b/src/hbase/client/multi-response.cc
similarity index 97%
rename from core/multi-response.cc
rename to src/hbase/client/multi-response.cc
index a4c2108..564e356 100644
--- a/core/multi-response.cc
+++ b/src/hbase/client/multi-response.cc
@@ -17,9 +17,9 @@
*
*/
-#include "core/multi-response.h"
+#include "hbase/client/multi-response.h"
#include <glog/logging.h>
-#include "core/region-result.h"
+#include "hbase/client/region-result.h"
using hbase::pb::RegionLoadStats;
diff --git a/core/mutation.cc b/src/hbase/client/mutation.cc
similarity index 98%
rename from core/mutation.cc
rename to src/hbase/client/mutation.cc
index 7182202..5a43b2f 100644
--- a/core/mutation.cc
+++ b/src/hbase/client/mutation.cc
@@ -19,7 +19,7 @@
*
*/
-#include "core/mutation.h"
+#include "hbase/client/mutation.h"
#include <algorithm>
#include <limits>
#include <stdexcept>
diff --git a/core/put-test.cc b/src/hbase/client/put-test.cc
similarity index 97%
rename from core/put-test.cc
rename to src/hbase/client/put-test.cc
index d4ed00a..0657bb6 100644
--- a/core/put-test.cc
+++ b/src/hbase/client/put-test.cc
@@ -19,9 +19,9 @@
#include <glog/logging.h>
#include <gtest/gtest.h>
-#include "core/mutation.h"
-#include "core/put.h"
-#include "utils/time-util.h"
+#include "hbase/client/mutation.h"
+#include "hbase/client/put.h"
+#include "hbase/utils/time-util.h"
using hbase::Put;
using hbase::Cell;
diff --git a/core/put.cc b/src/hbase/client/put.cc
similarity index 98%
rename from core/put.cc
rename to src/hbase/client/put.cc
index bb20c5c..d942d3d 100644
--- a/core/put.cc
+++ b/src/hbase/client/put.cc
@@ -19,7 +19,7 @@
*
*/
-#include "core/put.h"
+#include "hbase/client/put.h"
#include <folly/Conv.h>
#include <algorithm>
#include <limits>
diff --git a/core/raw-async-table.cc b/src/hbase/client/raw-async-table.cc
similarity index 98%
rename from core/raw-async-table.cc
rename to src/hbase/client/raw-async-table.cc
index 409883f..96361e4 100644
--- a/core/raw-async-table.cc
+++ b/src/hbase/client/raw-async-table.cc
@@ -18,9 +18,10 @@
*/
#include <utility>
-#include "core/raw-async-table.h"
-#include "core/request-converter.h"
-#include "core/response-converter.h"
+#include "hbase/client/async-batch-rpc-retrying-caller.h"
+#include "hbase/client/raw-async-table.h"
+#include "hbase/client/request-converter.h"
+#include "hbase/client/response-converter.h"
using hbase::security::User;
diff --git a/core/region-result.cc b/src/hbase/client/region-result.cc
similarity index 98%
rename from core/region-result.cc
rename to src/hbase/client/region-result.cc
index 206c876..28c4861 100644
--- a/core/region-result.cc
+++ b/src/hbase/client/region-result.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/region-result.h"
+#include "hbase/client/region-result.h"
#include <glog/logging.h>
#include <stdexcept>
diff --git a/core/request-converter-test.cc b/src/hbase/client/request-converter-test.cc
similarity index 96%
rename from core/request-converter-test.cc
rename to src/hbase/client/request-converter-test.cc
index 6c07a19..0878519 100644
--- a/core/request-converter-test.cc
+++ b/src/hbase/client/request-converter-test.cc
@@ -17,13 +17,13 @@
*
*/
-#include "core/request-converter.h"
+#include "hbase/client/request-converter.h"
#include <gtest/gtest.h>
#include <limits>
-#include "connection/request.h"
-#include "core/get.h"
-#include "core/scan.h"
+#include "hbase/connection/request.h"
+#include "hbase/client/get.h"
+#include "hbase/client/scan.h"
using hbase::Get;
using hbase::Scan;
diff --git a/core/request-converter.cc b/src/hbase/client/request-converter.cc
similarity index 99%
rename from core/request-converter.cc
rename to src/hbase/client/request-converter.cc
index f48f228..a57ac31 100644
--- a/core/request-converter.cc
+++ b/src/hbase/client/request-converter.cc
@@ -17,12 +17,12 @@
*
*/
-#include "core/request-converter.h"
+#include "hbase/client/request-converter.h"
#include <folly/Conv.h>
#include <utility>
-#include "if/Client.pb.h"
+#include "hbase/if/Client.pb.h"
using hbase::pb::GetRequest;
using hbase::pb::MutationProto;
diff --git a/core/response-converter.cc b/src/hbase/client/response-converter.cc
similarity index 98%
rename from core/response-converter.cc
rename to src/hbase/client/response-converter.cc
index 960c487..f3b78fd 100644
--- a/core/response-converter.cc
+++ b/src/hbase/client/response-converter.cc
@@ -17,15 +17,15 @@
*
*/
-#include "core/response-converter.h"
+#include "hbase/client/response-converter.h"
#include <glog/logging.h>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
-#include "core/cell.h"
-#include "core/multi-response.h"
-#include "exceptions/exception.h"
+#include "hbase/client/cell.h"
+#include "hbase/client/multi-response.h"
+#include "hbase/exceptions/exception.h"
using hbase::pb::GetResponse;
using hbase::pb::MutateResponse;
diff --git a/core/result-test.cc b/src/hbase/client/result-test.cc
similarity index 99%
rename from core/result-test.cc
rename to src/hbase/client/result-test.cc
index dd60aeb..684c08d 100644
--- a/core/result-test.cc
+++ b/src/hbase/client/result-test.cc
@@ -24,8 +24,8 @@
#include <string>
#include <vector>
-#include "core/cell.h"
-#include "core/result.h"
+#include "hbase/client/cell.h"
+#include "hbase/client/result.h"
using hbase::Cell;
using hbase::CellType;
diff --git a/core/result.cc b/src/hbase/client/result.cc
similarity index 99%
rename from core/result.cc
rename to src/hbase/client/result.cc
index 44b4c86..a2f56aa 100644
--- a/core/result.cc
+++ b/src/hbase/client/result.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/result.h"
+#include "hbase/client/result.h"
namespace hbase {
diff --git a/core/scan-result-cache-test.cc b/src/hbase/client/scan-result-cache-test.cc
similarity index 98%
rename from core/scan-result-cache-test.cc
rename to src/hbase/client/scan-result-cache-test.cc
index 0bf83ce..4c10a05 100644
--- a/core/scan-result-cache-test.cc
+++ b/src/hbase/client/scan-result-cache-test.cc
@@ -23,9 +23,9 @@
#include <vector>
-#include "core/cell.h"
-#include "core/result.h"
-#include "core/scan-result-cache.h"
+#include "hbase/client/cell.h"
+#include "hbase/client/result.h"
+#include "hbase/client/scan-result-cache.h"
using hbase::ScanResultCache;
using hbase::Result;
diff --git a/core/scan-result-cache.cc b/src/hbase/client/scan-result-cache.cc
similarity index 99%
rename from core/scan-result-cache.cc
rename to src/hbase/client/scan-result-cache.cc
index 62a51e0..e74a7d6 100644
--- a/core/scan-result-cache.cc
+++ b/src/hbase/client/scan-result-cache.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/scan-result-cache.h"
+#include "hbase/client/scan-result-cache.h"
#include <algorithm>
#include <iterator>
#include <limits>
diff --git a/core/scan-test.cc b/src/hbase/client/scan-test.cc
similarity index 99%
rename from core/scan-test.cc
rename to src/hbase/client/scan-test.cc
index 0ee054c..ba3a029 100644
--- a/core/scan-test.cc
+++ b/src/hbase/client/scan-test.cc
@@ -20,7 +20,7 @@
#include <gtest/gtest.h>
#include <limits>
-#include "core/scan.h"
+#include "hbase/client/scan.h"
using hbase::Get;
using hbase::Scan;
diff --git a/core/scan.cc b/src/hbase/client/scan.cc
similarity index 99%
rename from core/scan.cc
rename to src/hbase/client/scan.cc
index 6dcc51b..0cadcf0 100644
--- a/core/scan.cc
+++ b/src/hbase/client/scan.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/scan.h"
+#include "hbase/client/scan.h"
#include <algorithm>
#include <iterator>
diff --git a/core/scanner-test.cc b/src/hbase/client/scanner-test.cc
similarity index 95%
rename from core/scanner-test.cc
rename to src/hbase/client/scanner-test.cc
index 1ecbd02..8a58e34 100644
--- a/core/scanner-test.cc
+++ b/src/hbase/client/scanner-test.cc
@@ -24,23 +24,23 @@
#include <thread>
#include <vector>
-#include "core/async-client-scanner.h"
-#include "core/async-table-result-scanner.h"
-#include "core/cell.h"
-#include "core/client.h"
-#include "core/configuration.h"
-#include "core/filter.h"
-#include "core/get.h"
-#include "core/hbase-configuration-loader.h"
-#include "core/put.h"
-#include "core/result.h"
-#include "core/row.h"
-#include "core/table.h"
-#include "if/Comparator.pb.h"
-#include "if/Filter.pb.h"
-#include "serde/table-name.h"
-#include "test-util/test-util.h"
-#include "utils/time-util.h"
+#include "hbase/client/async-client-scanner.h"
+#include "hbase/client/async-table-result-scanner.h"
+#include "hbase/client/cell.h"
+#include "hbase/client/client.h"
+#include "hbase/client/configuration.h"
+#include "hbase/client/filter.h"
+#include "hbase/client/get.h"
+#include "hbase/client/hbase-configuration-loader.h"
+#include "hbase/client/put.h"
+#include "hbase/client/result.h"
+#include "hbase/client/row.h"
+#include "hbase/client/table.h"
+#include "hbase/if/Comparator.pb.h"
+#include "hbase/if/Filter.pb.h"
+#include "hbase/serde/table-name.h"
+#include "hbase/test-util/test-util.h"
+#include "hbase/utils/time-util.h"
using hbase::Cell;
using hbase::ComparatorFactory;
diff --git a/core/simple-client.cc b/src/hbase/client/simple-client.cc
similarity index 94%
rename from core/simple-client.cc
rename to src/hbase/client/simple-client.cc
index 6730248..25f203a 100644
--- a/core/simple-client.cc
+++ b/src/hbase/client/simple-client.cc
@@ -26,16 +26,16 @@
#include <iostream>
#include <thread>
-#include "connection/rpc-client.h"
-#include "core/client.h"
-#include "core/get.h"
-#include "core/hbase-configuration-loader.h"
-#include "core/put.h"
-#include "core/scan.h"
-#include "core/table.h"
-#include "serde/server-name.h"
-#include "serde/table-name.h"
-#include "utils/time-util.h"
+#include "hbase/connection/rpc-client.h"
+#include "hbase/client/client.h"
+#include "hbase/client/get.h"
+#include "hbase/client/hbase-configuration-loader.h"
+#include "hbase/client/put.h"
+#include "hbase/client/scan.h"
+#include "hbase/client/table.h"
+#include "hbase/serde/server-name.h"
+#include "hbase/serde/table-name.h"
+#include "hbase/utils/time-util.h"
using hbase::Client;
using hbase::Configuration;
diff --git a/core/table.cc b/src/hbase/client/table.cc
similarity index 93%
rename from core/table.cc
rename to src/hbase/client/table.cc
index f93a029..e877ec8 100644
--- a/core/table.cc
+++ b/src/hbase/client/table.cc
@@ -17,21 +17,21 @@
*
*/
-#include "core/table.h"
+#include "hbase/client/table.h"
#include <chrono>
#include <limits>
#include <utility>
#include <vector>
-#include "core/async-connection.h"
-#include "core/async-table-result-scanner.h"
-#include "core/request-converter.h"
-#include "core/response-converter.h"
-#include "if/Client.pb.h"
-#include "security/user.h"
-#include "serde/server-name.h"
-#include "utils/time-util.h"
+#include "hbase/client/async-connection.h"
+#include "hbase/client/async-table-result-scanner.h"
+#include "hbase/client/request-converter.h"
+#include "hbase/client/response-converter.h"
+#include "hbase/if/Client.pb.h"
+#include "hbase/security/user.h"
+#include "hbase/serde/server-name.h"
+#include "hbase/utils/time-util.h"
using hbase::pb::TableName;
using hbase::security::User;
diff --git a/core/time-range-test.cc b/src/hbase/client/time-range-test.cc
similarity index 97%
rename from core/time-range-test.cc
rename to src/hbase/client/time-range-test.cc
index 136ab54..a3db877 100644
--- a/core/time-range-test.cc
+++ b/src/hbase/client/time-range-test.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/time-range.h"
+#include "hbase/client/time-range.h"
#include <glog/logging.h>
#include <gtest/gtest.h>
diff --git a/core/time-range.cc b/src/hbase/client/time-range.cc
similarity index 98%
rename from core/time-range.cc
rename to src/hbase/client/time-range.cc
index e6b789b..b53e6f9 100644
--- a/core/time-range.cc
+++ b/src/hbase/client/time-range.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/time-range.h"
+#include "hbase/client/time-range.h"
#include <limits>
#include <stdexcept>
#include <string>
diff --git a/core/zk-util-test.cc b/src/hbase/client/zk-util-test.cc
similarity index 98%
rename from core/zk-util-test.cc
rename to src/hbase/client/zk-util-test.cc
index b0cefdc..83d22fb 100644
--- a/core/zk-util-test.cc
+++ b/src/hbase/client/zk-util-test.cc
@@ -18,7 +18,7 @@
*/
#include <gtest/gtest.h>
-#include "core/zk-util.h"
+#include "hbase/client/zk-util.h"
using hbase::Configuration;
using hbase::ZKUtil;
diff --git a/core/zk-util.cc b/src/hbase/client/zk-util.cc
similarity index 98%
rename from core/zk-util.cc
rename to src/hbase/client/zk-util.cc
index d29c8c3..38a0a2e 100644
--- a/core/zk-util.cc
+++ b/src/hbase/client/zk-util.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/zk-util.h"
+#include "hbase/client/zk-util.h"
#include <folly/Conv.h>
#include <boost/algorithm/string.hpp>
diff --git a/connection/BUCK b/src/hbase/connection/BUCK
similarity index 73%
rename from connection/BUCK
rename to src/hbase/connection/BUCK
index a87d27a..1a856fb 100644
--- a/connection/BUCK
+++ b/src/hbase/connection/BUCK
@@ -19,25 +19,6 @@
# to a single server.
cxx_library(
name="connection",
- exported_headers=[
- "client-dispatcher.h",
- "client-handler.h",
- "sasl-handler.h",
- "connection-factory.h",
- "connection-pool.h",
- "connection-id.h",
- "pipeline.h",
- "request.h",
- "rpc-connection.h",
- "response.h",
- "service.h",
- "rpc-client.h",
- "sasl-util.h",
- "rpc-test-server.h",
- "rpc-test-server-handler.h",
- "rpc-fault-injector.h",
- "rpc-fault-injector-inl.h",
- ],
srcs=[
"client-dispatcher.cc",
"client-handler.cc",
@@ -53,19 +34,20 @@ cxx_library(
"rpc-fault-injector.cc",
],
deps=[
- "//if:if",
- "//utils:utils",
- "//serde:serde",
- "//security:security",
+ "//include/hbase/connection:connection",
+ "//src/hbase/if:if",
+ "//src/hbase/utils:utils",
+ "//src/hbase/serde:serde",
+ "//src/hbase/security:security",
"//third-party:folly",
"//third-party:wangle",
- "//exceptions:exceptions",
+ "//src/hbase/exceptions:exceptions",
],
compiler_flags=['-Weffc++'],
linker_flags=['-L/usr/local/lib', '-lsasl2', '-lkrb5'],
exported_linker_flags=['-L/usr/local/lib', '-lsasl2', '-lkrb5'],
visibility=[
- '//core/...',
+ '//src/hbase/client/...',
],)
cxx_test(
name="connection-pool-test",
diff --git a/connection/client-dispatcher.cc b/src/hbase/connection/client-dispatcher.cc
similarity index 96%
rename from connection/client-dispatcher.cc
rename to src/hbase/connection/client-dispatcher.cc
index fc8eb16..302ab6b 100644
--- a/connection/client-dispatcher.cc
+++ b/src/hbase/connection/client-dispatcher.cc
@@ -16,15 +16,15 @@
* limitations under the License.
*
*/
-#include "connection/client-dispatcher.h"
+#include "hbase/connection/client-dispatcher.h"
#include <folly/ExceptionWrapper.h>
#include <folly/Format.h>
#include <folly/io/async/AsyncSocketException.h>
#include <utility>
-#include "connection/rpc-connection.h"
-#include "exceptions/exception.h"
+#include "hbase/connection/rpc-connection.h"
+#include "hbase/exceptions/exception.h"
using std::unique_ptr;
diff --git a/connection/client-handler.cc b/src/hbase/connection/client-handler.cc
similarity index 96%
rename from connection/client-handler.cc
rename to src/hbase/connection/client-handler.cc
index 983a68c..c963c20 100644
--- a/connection/client-handler.cc
+++ b/src/hbase/connection/client-handler.cc
@@ -17,7 +17,7 @@
*
*/
-#include "connection/client-handler.h"
+#include "hbase/connection/client-handler.h"
#include <folly/ExceptionWrapper.h>
#include <folly/Likely.h>
@@ -25,10 +25,10 @@
#include <glog/logging.h>
#include <string>
-#include "connection/request.h"
-#include "connection/response.h"
-#include "if/Client.pb.h"
-#include "if/RPC.pb.h"
+#include "hbase/connection/request.h"
+#include "hbase/connection/response.h"
+#include "hbase/if/Client.pb.h"
+#include "hbase/if/RPC.pb.h"
using google::protobuf::Message;
diff --git a/connection/connection-factory.cc b/src/hbase/connection/connection-factory.cc
similarity index 92%
rename from connection/connection-factory.cc
rename to src/hbase/connection/connection-factory.cc
index 751073e..14ac22b 100644
--- a/connection/connection-factory.cc
+++ b/src/hbase/connection/connection-factory.cc
@@ -27,12 +27,12 @@
#include <folly/SocketAddress.h>
#include <folly/io/async/AsyncSocketException.h>
-#include "connection/client-dispatcher.h"
-#include "connection/connection-factory.h"
-#include "connection/pipeline.h"
-#include "connection/sasl-handler.h"
-#include "connection/service.h"
-#include "exceptions/exception.h"
+#include "hbase/connection/client-dispatcher.h"
+#include "hbase/connection/connection-factory.h"
+#include "hbase/connection/pipeline.h"
+#include "hbase/connection/sasl-handler.h"
+#include "hbase/connection/service.h"
+#include "hbase/exceptions/exception.h"
using std::chrono::milliseconds;
using std::chrono::nanoseconds;
diff --git a/connection/connection-pool-test.cc b/src/hbase/connection/connection-pool-test.cc
similarity index 96%
rename from connection/connection-pool-test.cc
rename to src/hbase/connection/connection-pool-test.cc
index 0dc8e14..5886a42 100644
--- a/connection/connection-pool-test.cc
+++ b/src/hbase/connection/connection-pool-test.cc
@@ -20,11 +20,11 @@
#include <folly/Logging.h>
#include <gmock/gmock.h>
-#include "connection/connection-factory.h"
-#include "connection/connection-id.h"
-#include "connection/connection-pool.h"
-#include "if/HBase.pb.h"
-#include "serde/server-name.h"
+#include "hbase/connection/connection-factory.h"
+#include "hbase/connection/connection-id.h"
+#include "hbase/connection/connection-pool.h"
+#include "hbase/if/HBase.pb.h"
+#include "hbase/serde/server-name.h"
using hbase::pb::ServerName;
using ::testing::Return;
diff --git a/connection/connection-pool.cc b/src/hbase/connection/connection-pool.cc
similarity index 98%
rename from connection/connection-pool.cc
rename to src/hbase/connection/connection-pool.cc
index e1f6358..92e87f8 100644
--- a/connection/connection-pool.cc
+++ b/src/hbase/connection/connection-pool.cc
@@ -17,7 +17,7 @@
*
*/
-#include "connection/connection-pool.h"
+#include "hbase/connection/connection-pool.h"
#include <folly/Conv.h>
#include <folly/Logging.h>
diff --git a/connection/pipeline.cc b/src/hbase/connection/pipeline.cc
similarity index 94%
rename from connection/pipeline.cc
rename to src/hbase/connection/pipeline.cc
index 9c790b6..45ac0c0 100644
--- a/connection/pipeline.cc
+++ b/src/hbase/connection/pipeline.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*
*/
-#include "connection/pipeline.h"
+#include "hbase/connection/pipeline.h"
#include <folly/Logging.h>
#include <wangle/channel/AsyncSocketHandler.h>
@@ -24,8 +24,8 @@
#include <wangle/channel/OutputBufferingHandler.h>
#include <wangle/codec/LengthFieldBasedFrameDecoder.h>
-#include "connection/client-handler.h"
-#include "connection/sasl-handler.h"
+#include "hbase/connection/client-handler.h"
+#include "hbase/connection/sasl-handler.h"
namespace hbase {
diff --git a/connection/request.cc b/src/hbase/connection/request.cc
similarity index 96%
rename from connection/request.cc
rename to src/hbase/connection/request.cc
index 8983726..91a6119 100644
--- a/connection/request.cc
+++ b/src/hbase/connection/request.cc
@@ -17,9 +17,9 @@
*
*/
-#include "connection/request.h"
+#include "hbase/connection/request.h"
-#include "if/Client.pb.h"
+#include "hbase/if/Client.pb.h"
namespace hbase {
diff --git a/connection/rpc-client.cc b/src/hbase/connection/rpc-client.cc
similarity index 98%
rename from connection/rpc-client.cc
rename to src/hbase/connection/rpc-client.cc
index 51c9c63..d73829e 100644
--- a/connection/rpc-client.cc
+++ b/src/hbase/connection/rpc-client.cc
@@ -17,14 +17,14 @@
*
*/
-#include "connection/rpc-client.h"
+#include "hbase/connection/rpc-client.h"
#include <folly/Format.h>
#include <folly/Logging.h>
#include <folly/futures/Future.h>
#include <unistd.h>
#include <wangle/concurrent/IOThreadPoolExecutor.h>
-#include "exceptions/exception.h"
+#include "hbase/exceptions/exception.h"
using hbase::security::User;
using std::chrono::nanoseconds;
diff --git a/connection/rpc-fault-injector.cc b/src/hbase/connection/rpc-fault-injector.cc
similarity index 94%
rename from connection/rpc-fault-injector.cc
rename to src/hbase/connection/rpc-fault-injector.cc
index 16e2034..202b21d 100644
--- a/connection/rpc-fault-injector.cc
+++ b/src/hbase/connection/rpc-fault-injector.cc
@@ -16,6 +16,6 @@
* limitations under the License.
*
*/
-#include "rpc-fault-injector.h"
+#include "hbase/connection/rpc-fault-injector.h"
namespace hbase {} /* namespace hbase */
diff --git a/connection/rpc-test-server-handler.cc b/src/hbase/connection/rpc-test-server-handler.cc
similarity index 96%
rename from connection/rpc-test-server-handler.cc
rename to src/hbase/connection/rpc-test-server-handler.cc
index 8e405ef..b371ba9 100644
--- a/connection/rpc-test-server-handler.cc
+++ b/src/hbase/connection/rpc-test-server-handler.cc
@@ -17,9 +17,9 @@
*
*/
-#include "connection/rpc-test-server-handler.h"
-#include "if/RPC.pb.h"
-#include "if/test.pb.h"
+#include "hbase/connection/rpc-test-server-handler.h"
+#include "hbase/if/RPC.pb.h"
+#include "hbase/if/test.pb.h"
namespace hbase {
diff --git a/connection/rpc-test-server.cc b/src/hbase/connection/rpc-test-server.cc
similarity index 96%
rename from connection/rpc-test-server.cc
rename to src/hbase/connection/rpc-test-server.cc
index 157ea71..337266e 100644
--- a/connection/rpc-test-server.cc
+++ b/src/hbase/connection/rpc-test-server.cc
@@ -22,9 +22,9 @@
#include <wangle/codec/LengthFieldPrepender.h>
#include <wangle/service/ServerDispatcher.h>
-#include "connection/rpc-test-server-handler.h"
-#include "connection/rpc-test-server.h"
-#include "if/test.pb.h"
+#include "hbase/connection/rpc-test-server-handler.h"
+#include "hbase/connection/rpc-test-server.h"
+#include "hbase/if/test.pb.h"
namespace hbase {
diff --git a/connection/rpc-test.cc b/src/hbase/connection/rpc-test.cc
similarity index 97%
rename from connection/rpc-test.cc
rename to src/hbase/connection/rpc-test.cc
index 8624e72..64f8f99 100644
--- a/connection/rpc-test.cc
+++ b/src/hbase/connection/rpc-test.cc
@@ -32,12 +32,12 @@
#include <boost/thread.hpp>
#include <chrono>
-#include "connection/rpc-client.h"
-#include "exceptions/exception.h"
-#include "if/test.pb.h"
-#include "rpc-test-server.h"
-#include "security/user.h"
-#include "serde/rpc-serde.h"
+#include "hbase/connection/rpc-client.h"
+#include "hbase/exceptions/exception.h"
+#include "hbase/if/test.pb.h"
+#include "hbase/connection/rpc-test-server.h"
+#include "hbase/security/user.h"
+#include "hbase/serde/rpc-serde.h"
using namespace wangle;
using namespace folly;
diff --git a/connection/sasl-handler.cc b/src/hbase/connection/sasl-handler.cc
similarity index 98%
rename from connection/sasl-handler.cc
rename to src/hbase/connection/sasl-handler.cc
index 9afe1e2..242665f 100644
--- a/connection/sasl-handler.cc
+++ b/src/hbase/connection/sasl-handler.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*
*/
-#include "connection/sasl-handler.h"
+#include "hbase/connection/sasl-handler.h"
#include <glog/logging.h>
#include <sasl/sasl.h>
@@ -33,8 +33,8 @@
#include <string>
#include <utility>
-#include "connection/service.h"
-#include "security/user.h"
+#include "hbase/connection/service.h"
+#include "hbase/security/user.h"
using hbase::security::User;
using std::chrono::nanoseconds;
diff --git a/connection/sasl-util.cc b/src/hbase/connection/sasl-util.cc
similarity index 98%
rename from connection/sasl-util.cc
rename to src/hbase/connection/sasl-util.cc
index ecaf015..7e7403e 100644
--- a/connection/sasl-util.cc
+++ b/src/hbase/connection/sasl-util.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*
*/
-#include "connection/sasl-util.h"
+#include "hbase/connection/sasl-util.h"
#include <glog/logging.h>
#include <sasl/sasl.h>
diff --git a/exceptions/BUCK b/src/hbase/exceptions/BUCK
similarity index 89%
rename from exceptions/BUCK
rename to src/hbase/exceptions/BUCK
index 3d66d72..00ed344 100644
--- a/exceptions/BUCK
+++ b/src/hbase/exceptions/BUCK
@@ -17,17 +17,15 @@
cxx_library(
name="exceptions",
- exported_headers=[
- "exception.h",
- ],
srcs=[
"exception.cc",
],
deps=[
+ "//include/hbase/exceptions:exceptions",
"//third-party:folly",
],
compiler_flags=['-Weffc++'],
- visibility=['//core/...', '//connection/...'],)
+ visibility=['//src/hbase/client/...', '//src/hbase/connection/...'],)
cxx_test(
name="exception-test",
srcs=[
diff --git a/exceptions/exception-test.cc b/src/hbase/exceptions/exception-test.cc
similarity index 98%
rename from exceptions/exception-test.cc
rename to src/hbase/exceptions/exception-test.cc
index 583240e..e28f084 100644
--- a/exceptions/exception-test.cc
+++ b/src/hbase/exceptions/exception-test.cc
@@ -18,7 +18,7 @@
*/
#include <gtest/gtest.h>
-#include "exceptions/exception.h"
+#include "hbase/exceptions/exception.h"
#include "folly/ExceptionWrapper.h"
diff --git a/exceptions/exception.cc b/src/hbase/exceptions/exception.cc
similarity index 99%
rename from exceptions/exception.cc
rename to src/hbase/exceptions/exception.cc
index c25acb4..91ff818 100644
--- a/exceptions/exception.cc
+++ b/src/hbase/exceptions/exception.cc
@@ -16,7 +16,7 @@
* limitations under the License.
*
*/
-#include "exceptions/exception.h"
+#include "hbase/exceptions/exception.h"
namespace hbase {
const std::vector<const char*> ExceptionUtil::kAllDoNotRetryIOExceptions = {
diff --git a/if/AccessControl.proto b/src/hbase/if/AccessControl.proto
similarity index 100%
rename from if/AccessControl.proto
rename to src/hbase/if/AccessControl.proto
diff --git a/if/Admin.proto b/src/hbase/if/Admin.proto
similarity index 100%
rename from if/Admin.proto
rename to src/hbase/if/Admin.proto
diff --git a/if/Aggregate.proto b/src/hbase/if/Aggregate.proto
similarity index 100%
rename from if/Aggregate.proto
rename to src/hbase/if/Aggregate.proto
diff --git a/if/Authentication.proto b/src/hbase/if/Authentication.proto
similarity index 100%
rename from if/Authentication.proto
rename to src/hbase/if/Authentication.proto
diff --git a/if/BUCK b/src/hbase/if/BUCK
similarity index 97%
rename from if/BUCK
rename to src/hbase/if/BUCK
index 1a9721d..c8d51f2 100644
--- a/if/BUCK
+++ b/src/hbase/if/BUCK
@@ -41,6 +41,7 @@ for cc_filename in CC_FILENAMES:
out=cc_filename, )
cxx_library(name='if',
+ header_namespace="hbase/if",
exported_headers=[':' + x for x in HEADER_FILENAMES],
srcs=[':' + x for x in CC_FILENAMES],
deps=['//third-party:protobuf'],
diff --git a/if/Cell.proto b/src/hbase/if/Cell.proto
similarity index 100%
rename from if/Cell.proto
rename to src/hbase/if/Cell.proto
diff --git a/if/Client.proto b/src/hbase/if/Client.proto
similarity index 100%
rename from if/Client.proto
rename to src/hbase/if/Client.proto
diff --git a/if/ClusterId.proto b/src/hbase/if/ClusterId.proto
similarity index 100%
rename from if/ClusterId.proto
rename to src/hbase/if/ClusterId.proto
diff --git a/if/ClusterStatus.proto b/src/hbase/if/ClusterStatus.proto
similarity index 100%
rename from if/ClusterStatus.proto
rename to src/hbase/if/ClusterStatus.proto
diff --git a/if/Comparator.proto b/src/hbase/if/Comparator.proto
similarity index 100%
rename from if/Comparator.proto
rename to src/hbase/if/Comparator.proto
diff --git a/if/Encryption.proto b/src/hbase/if/Encryption.proto
similarity index 100%
rename from if/Encryption.proto
rename to src/hbase/if/Encryption.proto
diff --git a/if/ErrorHandling.proto b/src/hbase/if/ErrorHandling.proto
similarity index 100%
rename from if/ErrorHandling.proto
rename to src/hbase/if/ErrorHandling.proto
diff --git a/if/FS.proto b/src/hbase/if/FS.proto
similarity index 100%
rename from if/FS.proto
rename to src/hbase/if/FS.proto
diff --git a/if/Filter.proto b/src/hbase/if/Filter.proto
similarity index 100%
rename from if/Filter.proto
rename to src/hbase/if/Filter.proto
diff --git a/if/HBase.proto b/src/hbase/if/HBase.proto
similarity index 100%
rename from if/HBase.proto
rename to src/hbase/if/HBase.proto
diff --git a/if/HFile.proto b/src/hbase/if/HFile.proto
similarity index 100%
rename from if/HFile.proto
rename to src/hbase/if/HFile.proto
diff --git a/if/LoadBalancer.proto b/src/hbase/if/LoadBalancer.proto
similarity index 100%
rename from if/LoadBalancer.proto
rename to src/hbase/if/LoadBalancer.proto
diff --git a/if/MapReduce.proto b/src/hbase/if/MapReduce.proto
similarity index 100%
rename from if/MapReduce.proto
rename to src/hbase/if/MapReduce.proto
diff --git a/if/Master.proto b/src/hbase/if/Master.proto
similarity index 100%
rename from if/Master.proto
rename to src/hbase/if/Master.proto
diff --git a/if/MasterProcedure.proto b/src/hbase/if/MasterProcedure.proto
similarity index 100%
rename from if/MasterProcedure.proto
rename to src/hbase/if/MasterProcedure.proto
diff --git a/if/MultiRowMutation.proto b/src/hbase/if/MultiRowMutation.proto
similarity index 100%
rename from if/MultiRowMutation.proto
rename to src/hbase/if/MultiRowMutation.proto
diff --git a/if/Procedure.proto b/src/hbase/if/Procedure.proto
similarity index 100%
rename from if/Procedure.proto
rename to src/hbase/if/Procedure.proto
diff --git a/if/Quota.proto b/src/hbase/if/Quota.proto
similarity index 100%
rename from if/Quota.proto
rename to src/hbase/if/Quota.proto
diff --git a/if/RPC.proto b/src/hbase/if/RPC.proto
similarity index 100%
rename from if/RPC.proto
rename to src/hbase/if/RPC.proto
diff --git a/if/RSGroup.proto b/src/hbase/if/RSGroup.proto
similarity index 100%
rename from if/RSGroup.proto
rename to src/hbase/if/RSGroup.proto
diff --git a/if/RSGroupAdmin.proto b/src/hbase/if/RSGroupAdmin.proto
similarity index 100%
rename from if/RSGroupAdmin.proto
rename to src/hbase/if/RSGroupAdmin.proto
diff --git a/if/RegionNormalizer.proto b/src/hbase/if/RegionNormalizer.proto
similarity index 100%
rename from if/RegionNormalizer.proto
rename to src/hbase/if/RegionNormalizer.proto
diff --git a/if/RegionServerStatus.proto b/src/hbase/if/RegionServerStatus.proto
similarity index 100%
rename from if/RegionServerStatus.proto
rename to src/hbase/if/RegionServerStatus.proto
diff --git a/if/RowProcessor.proto b/src/hbase/if/RowProcessor.proto
similarity index 100%
rename from if/RowProcessor.proto
rename to src/hbase/if/RowProcessor.proto
diff --git a/if/SecureBulkLoad.proto b/src/hbase/if/SecureBulkLoad.proto
similarity index 100%
rename from if/SecureBulkLoad.proto
rename to src/hbase/if/SecureBulkLoad.proto
diff --git a/if/Snapshot.proto b/src/hbase/if/Snapshot.proto
similarity index 100%
rename from if/Snapshot.proto
rename to src/hbase/if/Snapshot.proto
diff --git a/if/Tracing.proto b/src/hbase/if/Tracing.proto
similarity index 100%
rename from if/Tracing.proto
rename to src/hbase/if/Tracing.proto
diff --git a/if/VisibilityLabels.proto b/src/hbase/if/VisibilityLabels.proto
similarity index 100%
rename from if/VisibilityLabels.proto
rename to src/hbase/if/VisibilityLabels.proto
diff --git a/if/WAL.proto b/src/hbase/if/WAL.proto
similarity index 100%
rename from if/WAL.proto
rename to src/hbase/if/WAL.proto
diff --git a/if/ZooKeeper.proto b/src/hbase/if/ZooKeeper.proto
similarity index 100%
rename from if/ZooKeeper.proto
rename to src/hbase/if/ZooKeeper.proto
diff --git a/if/test.proto b/src/hbase/if/test.proto
similarity index 100%
rename from if/test.proto
rename to src/hbase/if/test.proto
diff --git a/if/test_rpc_service.proto b/src/hbase/if/test_rpc_service.proto
similarity index 100%
rename from if/test_rpc_service.proto
rename to src/hbase/if/test_rpc_service.proto
diff --git a/security/BUCK b/src/hbase/security/BUCK
similarity index 92%
rename from security/BUCK
rename to src/hbase/security/BUCK
index f8a5695..c329f30 100644
--- a/security/BUCK
+++ b/src/hbase/security/BUCK
@@ -19,11 +19,8 @@
# to a single server.
cxx_library(
name="security",
- exported_headers=[
- "user.h",
- ],
srcs=[],
- deps=["//core:conf"],
+ deps=["//include/hbase/security:security", "//src/hbase/client:conf"],
compiler_flags=['-Weffc++'],
visibility=[
'PUBLIC',
diff --git a/serde/BUCK b/src/hbase/serde/BUCK
similarity index 86%
rename from serde/BUCK
rename to src/hbase/serde/BUCK
index a765884..6b39e0b 100644
--- a/serde/BUCK
+++ b/src/hbase/serde/BUCK
@@ -17,22 +17,12 @@
cxx_library(
name="serde",
- exported_headers=[
- "cell-scanner.h",
- "cell-outputstream.h",
- "codec.h",
- "region-info.h",
- "rpc-serde.h",
- "server-name.h",
- "table-name.h",
- "zk.h",
- ],
srcs=[
"rpc-serde.cc",
"zk.cc",
],
deps=[
- "//if:if", "//third-party:folly", "//utils:utils", "//security:security"
+ "//include/hbase/serde:serde", "//src/hbase/if:if", "//third-party:folly", "//src/hbase/utils:utils", "//src/hbase/security:security"
],
tests=[
":client-deserializer-test",
diff --git a/serde/client-deserializer-test.cc b/src/hbase/serde/client-deserializer-test.cc
similarity index 96%
rename from serde/client-deserializer-test.cc
rename to src/hbase/serde/client-deserializer-test.cc
index 1856047..3e4c42b 100644
--- a/serde/client-deserializer-test.cc
+++ b/src/hbase/serde/client-deserializer-test.cc
@@ -19,8 +19,8 @@
#include <folly/io/IOBuf.h>
#include <gtest/gtest.h>
-#include "if/Client.pb.h"
-#include "rpc-serde.h"
+#include "hbase/if/Client.pb.h"
+#include "hbase/serde/rpc-serde.h"
using namespace hbase;
using folly::IOBuf;
diff --git a/serde/client-serializer-test.cc b/src/hbase/serde/client-serializer-test.cc
similarity index 96%
rename from serde/client-serializer-test.cc
rename to src/hbase/serde/client-serializer-test.cc
index 306f2c2..8279caa 100644
--- a/serde/client-serializer-test.cc
+++ b/src/hbase/serde/client-serializer-test.cc
@@ -22,9 +22,9 @@
#include <string>
-#include "if/HBase.pb.h"
-#include "if/RPC.pb.h"
-#include "rpc-serde.h"
+#include "hbase/if/HBase.pb.h"
+#include "hbase/if/RPC.pb.h"
+#include "hbase/serde/rpc-serde.h"
using namespace hbase;
using namespace hbase::pb;
diff --git a/serde/region-info-deserializer-test.cc b/src/hbase/serde/region-info-deserializer-test.cc
similarity index 93%
rename from serde/region-info-deserializer-test.cc
rename to src/hbase/serde/region-info-deserializer-test.cc
index 5cb8482..84219d8 100644
--- a/serde/region-info-deserializer-test.cc
+++ b/src/hbase/serde/region-info-deserializer-test.cc
@@ -17,14 +17,14 @@
*
*/
-#include "serde/region-info.h"
+#include "hbase/serde/region-info.h"
#include <gtest/gtest.h>
#include <string>
-#include "if/HBase.pb.h"
-#include "serde/table-name.h"
+#include "hbase/if/HBase.pb.h"
+#include "hbase/serde/table-name.h"
using std::string;
using hbase::pb::RegionInfo;
diff --git a/serde/rpc-serde.cc b/src/hbase/serde/rpc-serde.cc
similarity index 98%
rename from serde/rpc-serde.cc
rename to src/hbase/serde/rpc-serde.cc
index 70a57e8..5f2920d 100644
--- a/serde/rpc-serde.cc
+++ b/src/hbase/serde/rpc-serde.cc
@@ -27,9 +27,9 @@
#include <utility>
-#include "if/RPC.pb.h"
-#include "rpc-serde.h"
-#include "utils/version.h"
+#include "hbase/if/RPC.pb.h"
+#include "hbase/serde/rpc-serde.h"
+#include "hbase/utils/version.h"
using folly::IOBuf;
using folly::io::RWPrivateCursor;
diff --git a/serde/server-name-test.cc b/src/hbase/serde/server-name-test.cc
similarity index 97%
rename from serde/server-name-test.cc
rename to src/hbase/serde/server-name-test.cc
index 87c493a..12d3287 100644
--- a/serde/server-name-test.cc
+++ b/src/hbase/serde/server-name-test.cc
@@ -17,7 +17,7 @@
*
*/
-#include "serde/server-name.h"
+#include "hbase/serde/server-name.h"
#include <gtest/gtest.h>
#include <string>
diff --git a/serde/table-name-test.cc b/src/hbase/serde/table-name-test.cc
similarity index 97%
rename from serde/table-name-test.cc
rename to src/hbase/serde/table-name-test.cc
index 877d522..77bd6c2 100644
--- a/serde/table-name-test.cc
+++ b/src/hbase/serde/table-name-test.cc
@@ -22,7 +22,7 @@
#include <string>
-#include "serde/table-name.h"
+#include "hbase/serde/table-name.h"
using namespace hbase;
using hbase::pb::TableName;
diff --git a/serde/zk-deserializer-test.cc b/src/hbase/serde/zk-deserializer-test.cc
similarity index 98%
rename from serde/zk-deserializer-test.cc
rename to src/hbase/serde/zk-deserializer-test.cc
index f07eecf..141efce 100644
--- a/serde/zk-deserializer-test.cc
+++ b/src/hbase/serde/zk-deserializer-test.cc
@@ -17,14 +17,14 @@
*
*/
-#include "serde/zk.h"
+#include "hbase/serde/zk.h"
#include <folly/Logging.h>
#include <folly/io/Cursor.h>
#include <folly/io/IOBuf.h>
#include <gtest/gtest.h>
-#include "if/ZooKeeper.pb.h"
+#include "hbase/if/ZooKeeper.pb.h"
using namespace hbase;
using namespace hbase::pb;
diff --git a/serde/zk.cc b/src/hbase/serde/zk.cc
similarity index 98%
rename from serde/zk.cc
rename to src/hbase/serde/zk.cc
index a71eb87..b962cc5 100644
--- a/serde/zk.cc
+++ b/src/hbase/serde/zk.cc
@@ -17,7 +17,7 @@
*
*/
-#include "serde/zk.h"
+#include "hbase/serde/zk.h"
#include <folly/io/Cursor.h>
#include <folly/io/IOBuf.h>
diff --git a/test-util/BUCK b/src/hbase/test-util/BUCK
similarity index 96%
rename from test-util/BUCK
rename to src/hbase/test-util/BUCK
index 7c92841..f1aedab 100644
--- a/test-util/BUCK
+++ b/src/hbase/test-util/BUCK
@@ -18,11 +18,12 @@ import os
cxx_library(
name="test-util",
+ header_namespace="hbase/test-util",
exported_headers=["test-util.h", "mini-cluster.h"],
srcs=["test-util.cc", "mini-cluster.cc"],
deps=[
"//third-party:folly",
- "//core:core",
+ "//src/hbase/client:client",
],
preprocessor_flags=[
'-I' + os.environ['JAVA_HOME'] + '/include',
diff --git a/test-util/mini-cluster.cc b/src/hbase/test-util/mini-cluster.cc
similarity index 99%
rename from test-util/mini-cluster.cc
rename to src/hbase/test-util/mini-cluster.cc
index 9dd2f12..1e491a2 100644
--- a/test-util/mini-cluster.cc
+++ b/src/hbase/test-util/mini-cluster.cc
@@ -17,7 +17,7 @@
*
*/
-#include "test-util/mini-cluster.h"
+#include "hbase/test-util/mini-cluster.h"
#include <fcntl.h>
#include <glog/logging.h>
#include <boost/filesystem/fstream.hpp>
diff --git a/test-util/mini-cluster.h b/src/hbase/test-util/mini-cluster.h
similarity index 100%
rename from test-util/mini-cluster.h
rename to src/hbase/test-util/mini-cluster.h
diff --git a/test-util/test-util.cc b/src/hbase/test-util/test-util.cc
similarity index 97%
rename from test-util/test-util.cc
rename to src/hbase/test-util/test-util.cc
index ea18b84..ebaf701 100644
--- a/test-util/test-util.cc
+++ b/src/hbase/test-util/test-util.cc
@@ -17,12 +17,12 @@
*
*/
-#include "test-util/test-util.h"
+#include "hbase/test-util/test-util.h"
#include <string.h>
#include <folly/Format.h>
-#include "core/zk-util.h"
+#include "hbase/client/zk-util.h"
using hbase::TestUtil;
using folly::Random;
diff --git a/test-util/test-util.h b/src/hbase/test-util/test-util.h
similarity index 96%
rename from test-util/test-util.h
rename to src/hbase/test-util/test-util.h
index 40e99d1..7c57c28 100644
--- a/test-util/test-util.h
+++ b/src/hbase/test-util/test-util.h
@@ -25,8 +25,8 @@
#include <memory>
#include <string>
#include <vector>
-#include "core/configuration.h"
-#include "test-util/mini-cluster.h"
+#include "hbase/client/configuration.h"
+#include "hbase/test-util/mini-cluster.h"
namespace hbase {
/**
diff --git a/utils/BUCK b/src/hbase/utils/BUCK
similarity index 88%
rename from utils/BUCK
rename to src/hbase/utils/BUCK
index 96f02b8..ab55d8f 100644
--- a/utils/BUCK
+++ b/src/hbase/utils/BUCK
@@ -18,17 +18,10 @@
cxx_library(
name="utils",
exported_headers=[
- "bytes-util.h",
- "connection-util.h",
- "concurrent-map.h",
- "optional.h",
- "sys-util.h",
- "time-util.h",
- "user-util.h",
- "version.h",
],
srcs=["bytes-util.cc", "connection-util.cc", "user-util.cc"],
deps=[
+ '//include/hbase/utils:utils',
'//third-party:folly',
],
tests=[":user-util-test"],
@@ -61,4 +54,4 @@ cxx_test(
],
deps=[
":utils",
- ],)
\ No newline at end of file
+ ],)
diff --git a/utils/bytes-util-test.cc b/src/hbase/utils/bytes-util-test.cc
similarity index 98%
rename from utils/bytes-util-test.cc
rename to src/hbase/utils/bytes-util-test.cc
index 4a49593..e601d2d 100644
--- a/utils/bytes-util-test.cc
+++ b/src/hbase/utils/bytes-util-test.cc
@@ -21,7 +21,7 @@
#include <gtest/gtest.h>
#include <string>
-#include "utils/bytes-util.h"
+#include "hbase/utils/bytes-util.h"
using hbase::BytesUtil;
diff --git a/utils/bytes-util.cc b/src/hbase/utils/bytes-util.cc
similarity index 98%
rename from utils/bytes-util.cc
rename to src/hbase/utils/bytes-util.cc
index 12037c3..144b866 100644
--- a/utils/bytes-util.cc
+++ b/src/hbase/utils/bytes-util.cc
@@ -17,7 +17,7 @@
*
*/
-#include "utils/bytes-util.h"
+#include "hbase/utils/bytes-util.h"
#include <bits/stdc++.h>
#include <boost/predef.h>
diff --git a/utils/concurrent-map-test.cc b/src/hbase/utils/concurrent-map-test.cc
similarity index 96%
rename from utils/concurrent-map-test.cc
rename to src/hbase/utils/concurrent-map-test.cc
index 588bd08..b95c442 100644
--- a/utils/concurrent-map-test.cc
+++ b/src/hbase/utils/concurrent-map-test.cc
@@ -21,7 +21,7 @@
#include <gtest/gtest.h>
#include <string>
-#include "utils/concurrent-map.h"
+#include "hbase/utils/concurrent-map.h"
using hbase::concurrent_map;
diff --git a/utils/connection-util.cc b/src/hbase/utils/connection-util.cc
similarity index 96%
rename from utils/connection-util.cc
rename to src/hbase/utils/connection-util.cc
index 76689bf..d78507c 100644
--- a/utils/connection-util.cc
+++ b/src/hbase/utils/connection-util.cc
@@ -17,7 +17,7 @@
*
*/
-#include "utils/connection-util.h"
+#include "hbase/utils/connection-util.h"
namespace hbase {
diff --git a/utils/user-util-test.cc b/src/hbase/utils/user-util-test.cc
similarity index 96%
rename from utils/user-util-test.cc
rename to src/hbase/utils/user-util-test.cc
index aa3fa45..6a022ef 100644
--- a/utils/user-util-test.cc
+++ b/src/hbase/utils/user-util-test.cc
@@ -21,7 +21,7 @@
#include <gtest/gtest.h>
#include <string>
-#include "utils/user-util.h"
+#include "hbase/utils/user-util.h"
using namespace std;
using namespace hbase;
diff --git a/utils/user-util.cc b/src/hbase/utils/user-util.cc
similarity index 98%
rename from utils/user-util.cc
rename to src/hbase/utils/user-util.cc
index 092d54c..ec8bb27 100644
--- a/utils/user-util.cc
+++ b/src/hbase/utils/user-util.cc
@@ -17,7 +17,7 @@
*
*/
-#include "utils/user-util.h"
+#include "hbase/utils/user-util.h"
#include <folly/Logging.h>
#include <krb5/krb5.h>
[hbase] 125/137: HBASE-18507 [C++] Support for MultiPuts in
AsyncBatchRpcRetryingCaller class (Sudeep Sunthankar)
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 01bb387c099507029c1fc35da3a9fce0dc4f73ad
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Fri Sep 1 10:55:57 2017 -0700
HBASE-18507 [C++] Support for MultiPuts in AsyncBatchRpcRetryingCaller class (Sudeep Sunthankar)
---
core/action.h | 9 +-
core/async-batch-rpc-retrying-caller.cc | 128 ++++++++------
core/async-batch-rpc-retrying-caller.h | 12 +-
core/async-batch-rpc-retrying-test.cc | 275 ++++++++++++++++++++++---------
core/async-rpc-retrying-caller-factory.h | 20 ++-
core/client-test.cc | 148 +++++++++++++++++
core/raw-async-table.cc | 31 +++-
core/raw-async-table.h | 7 +-
core/request-converter.cc | 18 +-
core/table.cc | 13 ++
core/table.h | 6 +-
11 files changed, 506 insertions(+), 161 deletions(-)
diff --git a/core/action.h b/core/action.h
index 21a0181..a00f079 100644
--- a/core/action.h
+++ b/core/action.h
@@ -20,22 +20,21 @@
#pragma once
#include <memory>
-#include "core/get.h"
+#include "core/row.h"
namespace hbase {
-
class Action {
public:
- Action(std::shared_ptr<hbase::Get> action, int32_t original_index)
+ Action(std::shared_ptr<hbase::Row> action, int32_t original_index)
: action_(action), original_index_(original_index) {}
~Action() {}
int32_t original_index() const { return original_index_; }
- std::shared_ptr<hbase::Get> action() const { return action_; }
+ std::shared_ptr<hbase::Row> action() const { return action_; }
private:
- std::shared_ptr<hbase::Get> action_;
+ std::shared_ptr<hbase::Row> action_;
int32_t original_index_;
int64_t nonce_ = -1;
int32_t replica_id_ = -1;
diff --git a/core/async-batch-rpc-retrying-caller.cc b/core/async-batch-rpc-retrying-caller.cc
index 0d67b17..dfbf7e7 100644
--- a/core/async-batch-rpc-retrying-caller.cc
+++ b/core/async-batch-rpc-retrying-caller.cc
@@ -32,11 +32,12 @@ using std::chrono::milliseconds;
namespace hbase {
-AsyncBatchRpcRetryingCaller::AsyncBatchRpcRetryingCaller(
+template <typename REQ, typename RESP>
+AsyncBatchRpcRetryingCaller<REQ, RESP>::AsyncBatchRpcRetryingCaller(
std::shared_ptr<AsyncConnection> conn, std::shared_ptr<folly::HHWheelTimer> retry_timer,
- std::shared_ptr<TableName> table_name, const std::vector<hbase::Get> &actions,
- nanoseconds pause_ns, int32_t max_attempts, nanoseconds operation_timeout_ns,
- nanoseconds rpc_timeout_ns, int32_t start_log_errors_count)
+ std::shared_ptr<TableName> table_name, const std::vector<REQ> &actions, nanoseconds pause_ns,
+ int32_t max_attempts, nanoseconds operation_timeout_ns, nanoseconds rpc_timeout_ns,
+ int32_t start_log_errors_count)
: conn_(conn),
retry_timer_(retry_timer),
table_name_(table_name),
@@ -56,29 +57,31 @@ AsyncBatchRpcRetryingCaller::AsyncBatchRpcRetryingCaller(
max_attempts_ = ConnectionUtils::Retries2Attempts(max_attempts);
uint32_t index = 0;
for (auto row : actions) {
- actions_.push_back(std::make_shared<Action>(std::make_shared<hbase::Get>(row), index));
- Promise<std::shared_ptr<Result>> prom{};
- action2promises_.insert(
- std::pair<uint64_t, Promise<std::shared_ptr<Result>>>(index, std::move(prom)));
+ actions_.push_back(std::make_shared<Action>(row, index));
+ Promise<RESP> prom{};
+ action2promises_.insert(std::pair<uint64_t, Promise<RESP>>(index, std::move(prom)));
action2futures_.push_back(action2promises_[index++].getFuture());
}
}
-AsyncBatchRpcRetryingCaller::~AsyncBatchRpcRetryingCaller() {}
+template <typename REQ, typename RESP>
+AsyncBatchRpcRetryingCaller<REQ, RESP>::~AsyncBatchRpcRetryingCaller() {}
-Future<std::vector<Try<std::shared_ptr<Result>>>> AsyncBatchRpcRetryingCaller::Call() {
+template <typename REQ, typename RESP>
+Future<std::vector<Try<RESP>>> AsyncBatchRpcRetryingCaller<REQ, RESP>::Call() {
GroupAndSend(actions_, 1);
return collectAll(action2futures_);
}
-int64_t AsyncBatchRpcRetryingCaller::RemainingTimeNs() {
+template <typename REQ, typename RESP>
+int64_t AsyncBatchRpcRetryingCaller<REQ, RESP>::RemainingTimeNs() {
return operation_timeout_ns_.count() - (TimeUtil::GetNowNanos() - start_ns_);
}
-void AsyncBatchRpcRetryingCaller::LogException(int32_t tries,
- std::shared_ptr<RegionRequest> region_request,
- const folly::exception_wrapper &ew,
- std::shared_ptr<ServerName> server_name) {
+template <typename REQ, typename RESP>
+void AsyncBatchRpcRetryingCaller<REQ, RESP>::LogException(
+ int32_t tries, std::shared_ptr<RegionRequest> region_request,
+ const folly::exception_wrapper &ew, std::shared_ptr<ServerName> server_name) {
if (tries > start_log_errors_count_) {
std::string regions;
regions += region_request->region_location()->region_name() + ", ";
@@ -88,7 +91,8 @@ void AsyncBatchRpcRetryingCaller::LogException(int32_t tries,
}
}
-void AsyncBatchRpcRetryingCaller::LogException(
+template <typename REQ, typename RESP>
+void AsyncBatchRpcRetryingCaller<REQ, RESP>::LogException(
int32_t tries, const std::vector<std::shared_ptr<RegionRequest>> ®ion_requests,
const folly::exception_wrapper &ew, std::shared_ptr<ServerName> server_name) {
if (tries > start_log_errors_count_) {
@@ -102,29 +106,35 @@ void AsyncBatchRpcRetryingCaller::LogException(
}
}
-const std::string AsyncBatchRpcRetryingCaller::GetExtraContextForError(
+template <typename REQ, typename RESP>
+const std::string AsyncBatchRpcRetryingCaller<REQ, RESP>::GetExtraContextForError(
std::shared_ptr<ServerName> server_name) {
return server_name ? server_name->ShortDebugString() : "";
}
-void AsyncBatchRpcRetryingCaller::AddError(const std::shared_ptr<Action> &action,
- const folly::exception_wrapper &ew,
- std::shared_ptr<ServerName> server_name) {
+template <typename REQ, typename RESP>
+void AsyncBatchRpcRetryingCaller<REQ, RESP>::AddError(const std::shared_ptr<Action> &action,
+ const folly::exception_wrapper &ew,
+ std::shared_ptr<ServerName> server_name) {
ThrowableWithExtraContext twec(ew, TimeUtil::GetNowNanos(), GetExtraContextForError(server_name));
AddAction2Error(action->original_index(), twec);
}
-void AsyncBatchRpcRetryingCaller::AddError(const std::vector<std::shared_ptr<Action>> &actions,
- const folly::exception_wrapper &ew,
- std::shared_ptr<ServerName> server_name) {
+template <typename REQ, typename RESP>
+void AsyncBatchRpcRetryingCaller<REQ, RESP>::AddError(
+ const std::vector<std::shared_ptr<Action>> &actions, const folly::exception_wrapper &ew,
+ std::shared_ptr<ServerName> server_name) {
for (const auto action : actions) {
AddError(action, ew, server_name);
}
}
-void AsyncBatchRpcRetryingCaller::FailOne(const std::shared_ptr<Action> &action, int32_t tries,
- const folly::exception_wrapper &ew, int64_t current_time,
- const std::string extras) {
+template <typename REQ, typename RESP>
+void AsyncBatchRpcRetryingCaller<REQ, RESP>::FailOne(const std::shared_ptr<Action> &action,
+ int32_t tries,
+ const folly::exception_wrapper &ew,
+ int64_t current_time,
+ const std::string extras) {
auto action_index = action->original_index();
auto itr = action2promises_.find(action_index);
if (itr != action2promises_.end()) {
@@ -138,16 +148,18 @@ void AsyncBatchRpcRetryingCaller::FailOne(const std::shared_ptr<Action> &action,
RetriesExhaustedException(tries - 1, action2errors_[action_index]));
}
-void AsyncBatchRpcRetryingCaller::FailAll(const std::vector<std::shared_ptr<Action>> &actions,
- int32_t tries, const folly::exception_wrapper &ew,
- std::shared_ptr<ServerName> server_name) {
+template <typename REQ, typename RESP>
+void AsyncBatchRpcRetryingCaller<REQ, RESP>::FailAll(
+ const std::vector<std::shared_ptr<Action>> &actions, int32_t tries,
+ const folly::exception_wrapper &ew, std::shared_ptr<ServerName> server_name) {
for (const auto action : actions) {
FailOne(action, tries, ew, TimeUtil::GetNowNanos(), GetExtraContextForError(server_name));
}
}
-void AsyncBatchRpcRetryingCaller::FailAll(const std::vector<std::shared_ptr<Action>> &actions,
- int32_t tries) {
+template <typename REQ, typename RESP>
+void AsyncBatchRpcRetryingCaller<REQ, RESP>::FailAll(
+ const std::vector<std::shared_ptr<Action>> &actions, int32_t tries) {
for (const auto action : actions) {
auto action_index = action->original_index();
auto itr = action2promises_.find(action_index);
@@ -159,8 +171,9 @@ void AsyncBatchRpcRetryingCaller::FailAll(const std::vector<std::shared_ptr<Acti
}
}
-void AsyncBatchRpcRetryingCaller::AddAction2Error(uint64_t action_index,
- const ThrowableWithExtraContext &twec) {
+template <typename REQ, typename RESP>
+void AsyncBatchRpcRetryingCaller<REQ, RESP>::AddAction2Error(
+ uint64_t action_index, const ThrowableWithExtraContext &twec) {
auto erritr = action2errors_.find(action_index);
if (erritr != action2errors_.end()) {
erritr->second->push_back(twec);
@@ -171,9 +184,11 @@ void AsyncBatchRpcRetryingCaller::AddAction2Error(uint64_t action_index,
return;
}
-void AsyncBatchRpcRetryingCaller::OnError(const ActionsByRegion &actions_by_region, int32_t tries,
- const folly::exception_wrapper &ew,
- std::shared_ptr<ServerName> server_name) {
+template <typename REQ, typename RESP>
+void AsyncBatchRpcRetryingCaller<REQ, RESP>::OnError(const ActionsByRegion &actions_by_region,
+ int32_t tries,
+ const folly::exception_wrapper &ew,
+ std::shared_ptr<ServerName> server_name) {
std::vector<std::shared_ptr<Action>> copied_actions;
std::vector<std::shared_ptr<RegionRequest>> region_requests;
for (const auto &action_by_region : actions_by_region) {
@@ -192,8 +207,9 @@ void AsyncBatchRpcRetryingCaller::OnError(const ActionsByRegion &actions_by_regi
TryResubmit(copied_actions, tries);
}
-void AsyncBatchRpcRetryingCaller::TryResubmit(const std::vector<std::shared_ptr<Action>> &actions,
- int32_t tries) {
+template <typename REQ, typename RESP>
+void AsyncBatchRpcRetryingCaller<REQ, RESP>::TryResubmit(
+ const std::vector<std::shared_ptr<Action>> &actions, int32_t tries) {
int64_t delay_ns;
if (operation_timeout_ns_.count() > 0) {
int64_t max_delay_ns = RemainingTimeNs() - ConnectionUtils::kSleepDeltaNs;
@@ -213,9 +229,10 @@ void AsyncBatchRpcRetryingCaller::TryResubmit(const std::vector<std::shared_ptr<
});
}
+template <typename REQ, typename RESP>
Future<std::vector<Try<std::shared_ptr<RegionLocation>>>>
-AsyncBatchRpcRetryingCaller::GetRegionLocations(const std::vector<std::shared_ptr<Action>> &actions,
- int64_t locate_timeout_ns) {
+AsyncBatchRpcRetryingCaller<REQ, RESP>::GetRegionLocations(
+ const std::vector<std::shared_ptr<Action>> &actions, int64_t locate_timeout_ns) {
auto locs = std::vector<Future<std::shared_ptr<RegionLocation>>>{};
for (auto const &action : actions) {
locs.push_back(location_cache_->LocateRegion(*table_name_, action->action()->row(),
@@ -225,8 +242,9 @@ AsyncBatchRpcRetryingCaller::GetRegionLocations(const std::vector<std::shared_pt
return collectAll(locs);
}
-void AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr<Action>> &actions,
- int32_t tries) {
+template <typename REQ, typename RESP>
+void AsyncBatchRpcRetryingCaller<REQ, RESP>::GroupAndSend(
+ const std::vector<std::shared_ptr<Action>> &actions, int32_t tries) {
int64_t locate_timeout_ns;
if (operation_timeout_ns_.count() > 0) {
locate_timeout_ns = RemainingTimeNs();
@@ -300,8 +318,9 @@ void AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr
return;
}
-Future<std::vector<Try<std::unique_ptr<Response>>>> AsyncBatchRpcRetryingCaller::GetMultiResponse(
- const ActionsByServer &actions_by_server) {
+template <typename REQ, typename RESP>
+Future<std::vector<Try<std::unique_ptr<Response>>>>
+AsyncBatchRpcRetryingCaller<REQ, RESP>::GetMultiResponse(const ActionsByServer &actions_by_server) {
auto multi_calls = std::vector<Future<std::unique_ptr<hbase::Response>>>{};
auto user = User::defaultUser();
for (const auto &action_by_server : actions_by_server) {
@@ -315,7 +334,9 @@ Future<std::vector<Try<std::unique_ptr<Response>>>> AsyncBatchRpcRetryingCaller:
return collectAll(multi_calls);
}
-void AsyncBatchRpcRetryingCaller::Send(const ActionsByServer &actions_by_server, int32_t tries) {
+template <typename REQ, typename RESP>
+void AsyncBatchRpcRetryingCaller<REQ, RESP>::Send(const ActionsByServer &actions_by_server,
+ int32_t tries) {
int64_t remaining_ns;
if (operation_timeout_ns_.count() > 0) {
remaining_ns = RemainingTimeNs();
@@ -371,7 +392,8 @@ void AsyncBatchRpcRetryingCaller::Send(const ActionsByServer &actions_by_server,
return;
}
-void AsyncBatchRpcRetryingCaller::OnComplete(
+template <typename REQ, typename RESP>
+void AsyncBatchRpcRetryingCaller<REQ, RESP>::OnComplete(
const ActionsByRegion &actions_by_region, int32_t tries,
const std::shared_ptr<ServerName> server_name,
const std::unique_ptr<hbase::MultiResponse> multi_response) {
@@ -418,12 +440,12 @@ void AsyncBatchRpcRetryingCaller::OnComplete(
return;
}
-void AsyncBatchRpcRetryingCaller::OnComplete(const std::shared_ptr<Action> &action,
- const std::shared_ptr<RegionRequest> ®ion_request,
- int32_t tries,
- const std::shared_ptr<ServerName> &server_name,
- const std::shared_ptr<RegionResult> ®ion_result,
- std::vector<std::shared_ptr<Action>> &failed_actions) {
+template <typename REQ, typename RESP>
+void AsyncBatchRpcRetryingCaller<REQ, RESP>::OnComplete(
+ const std::shared_ptr<Action> &action, const std::shared_ptr<RegionRequest> ®ion_request,
+ int32_t tries, const std::shared_ptr<ServerName> &server_name,
+ const std::shared_ptr<RegionResult> ®ion_result,
+ std::vector<std::shared_ptr<Action>> &failed_actions) {
std::string err_msg;
try {
auto result_or_exc = region_result->ResultOrException(action->original_index());
@@ -461,4 +483,6 @@ void AsyncBatchRpcRetryingCaller::OnComplete(const std::shared_ptr<Action> &acti
return;
}
+template class AsyncBatchRpcRetryingCaller<std::shared_ptr<hbase::Row>,
+ std::shared_ptr<hbase::Result>>;
} /* namespace hbase */
diff --git a/core/async-batch-rpc-retrying-caller.h b/core/async-batch-rpc-retrying-caller.h
index 194c439..9194b04 100644
--- a/core/async-batch-rpc-retrying-caller.h
+++ b/core/async-batch-rpc-retrying-caller.h
@@ -84,6 +84,7 @@ struct ServerNameHash {
}
};
+template <typename REQ, typename RESP>
class AsyncBatchRpcRetryingCaller {
public:
using ActionsByServer =
@@ -94,15 +95,14 @@ class AsyncBatchRpcRetryingCaller {
AsyncBatchRpcRetryingCaller(std::shared_ptr<AsyncConnection> conn,
std::shared_ptr<folly::HHWheelTimer> retry_timer,
std::shared_ptr<pb::TableName> table_name,
- const std::vector<hbase::Get> &actions,
- std::chrono::nanoseconds pause_ns, int32_t max_attempts,
- std::chrono::nanoseconds operation_timeout_ns,
+ const std::vector<REQ> &actions, std::chrono::nanoseconds pause_ns,
+ int32_t max_attempts, std::chrono::nanoseconds operation_timeout_ns,
std::chrono::nanoseconds rpc_timeout_ns,
int32_t start_log_errors_count);
~AsyncBatchRpcRetryingCaller();
- folly::Future<std::vector<folly::Try<std::shared_ptr<Result>>>> Call();
+ folly::Future<std::vector<folly::Try<RESP>>> Call();
private:
int64_t RemainingTimeNs();
@@ -172,8 +172,8 @@ class AsyncBatchRpcRetryingCaller {
int64_t start_ns_ = TimeUtil::GetNowNanos();
int32_t tries_ = 1;
- std::map<uint64_t, folly::Promise<std::shared_ptr<Result>>> action2promises_;
- std::vector<folly::Future<std::shared_ptr<Result>>> action2futures_;
+ std::map<uint64_t, folly::Promise<RESP>> action2promises_;
+ std::vector<folly::Future<RESP>> action2futures_;
std::map<uint64_t, std::shared_ptr<std::vector<ThrowableWithExtraContext>>> action2errors_;
std::shared_ptr<AsyncRegionLocator> location_cache_ = nullptr;
diff --git a/core/async-batch-rpc-retrying-test.cc b/core/async-batch-rpc-retrying-test.cc
index cad03e1..b8a0b81 100644
--- a/core/async-batch-rpc-retrying-test.cc
+++ b/core/async-batch-rpc-retrying-test.cc
@@ -230,7 +230,12 @@ class MockAsyncConnection : public AsyncConnection,
return retry_executor_;
}
- void Close() override {}
+ void Close() override {
+ retry_timer_->destroy();
+ retry_executor_->stop();
+ io_executor_->stop();
+ cpu_executor_->stop();
+ }
std::shared_ptr<HBaseRpcController> CreateRpcController() override {
return std::make_shared<HBaseRpcController>();
}
@@ -254,15 +259,15 @@ class MockRawAsyncTableImpl {
virtual ~MockRawAsyncTableImpl() = default;
/* implement this in real RawAsyncTableImpl. */
- folly::Future<std::vector<folly::Try<std::shared_ptr<Result>>>> Gets(
- const std::vector<hbase::Get> &gets) {
+ template <typename REQ, typename RESP>
+ folly::Future<std::vector<folly::Try<RESP>>> Batch(const std::vector<REQ> &rows) {
/* init request caller builder */
- auto builder = conn_->caller_factory()->Batch();
+ auto builder = conn_->caller_factory()->Batch<REQ, RESP>();
/* call with retry to get result */
auto async_caller =
builder->table(tn_)
- ->actions(std::make_shared<std::vector<hbase::Get>>(gets))
+ ->actions(std::make_shared<std::vector<REQ>>(rows))
->rpc_timeout(conn_->connection_conf()->read_rpc_timeout())
->operation_timeout(conn_->connection_conf()->operation_timeout())
->pause(conn_->connection_conf()->pause())
@@ -278,9 +283,7 @@ class MockRawAsyncTableImpl {
std::shared_ptr<hbase::pb::TableName> tn_;
};
-void runMultiTest(std::shared_ptr<AsyncRegionLocatorBase> region_locator,
- const std::string &table_name, bool split_regions, uint32_t tries = 3,
- uint32_t operation_timeout_millis = 600000, uint32_t num_rows = 1000) {
+std::string createTestTable(bool split_regions, const std::string &table_name) {
std::vector<std::string> keys{"test0", "test100", "test200", "test300", "test400",
"test500", "test600", "test700", "test800", "test900"};
std::string tableName = (split_regions) ? ("split-" + table_name) : table_name;
@@ -289,30 +292,12 @@ void runMultiTest(std::shared_ptr<AsyncRegionLocatorBase> region_locator,
} else {
AsyncBatchRpcRetryTest::test_util->CreateTable(tableName, "d");
}
+ return tableName;
+}
- // Create TableName and Row to be fetched from HBase
- auto tn = folly::to<hbase::pb::TableName>(tableName);
-
- // Create a client
- Client client(*AsyncBatchRpcRetryTest::test_util->conf());
-
- // Get connection to HBase Table
- auto table = client.Table(tn);
-
- for (uint64_t i = 0; i < num_rows; i++) {
- table->Put(Put{"test" + std::to_string(i)}.AddColumn("d", std::to_string(i),
- "value" + std::to_string(i)));
- }
-
- std::map<std::string, std::shared_ptr<RegionLocation>> region_locations;
- std::vector<hbase::Get> gets;
- for (uint64_t i = 0; i < num_rows; ++i) {
- auto row = "test" + std::to_string(i);
- hbase::Get get(row);
- gets.push_back(get);
- region_locations[row] = table->GetRegionLocation(row);
- }
-
+std::shared_ptr<MockAsyncConnection> getAsyncConnection(
+ Client &client, uint32_t operation_timeout_millis, uint32_t tries,
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator) {
/* init region location and rpc channel */
auto cpu_executor_ = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
auto io_executor_ = client.async_connection()->io_executor();
@@ -332,35 +317,90 @@ void runMultiTest(std::shared_ptr<AsyncRegionLocatorBase> region_locator,
tries, // max retries
1); // start log errors count
- /* set region locator */
- region_locator->set_region_location(region_locations);
-
- /* init hbase client connection */
- auto conn = std::make_shared<MockAsyncConnection>(connection_conf, retry_timer, cpu_executor_,
- io_executor_, retry_executor_, rpc_client,
- region_locator);
- conn->Init();
-
- /* init retry caller factory */
- auto tableImpl =
- std::make_shared<MockRawAsyncTableImpl>(conn, std::make_shared<hbase::pb::TableName>(tn));
+ return std::make_shared<MockAsyncConnection>(connection_conf, retry_timer, cpu_executor_,
+ io_executor_, retry_executor_, rpc_client,
+ region_locator);
+}
- auto tresults = tableImpl->Gets(gets).get(milliseconds(operation_timeout_millis));
+template <typename ACTION>
+std::vector<std::shared_ptr<hbase::Row>> getRows(std::vector<ACTION> actions) {
+ std::vector<std::shared_ptr<hbase::Row>> rows;
+ for (auto action : actions) {
+ std::shared_ptr<hbase::Row> srow = std::make_shared<ACTION>(action);
+ rows.push_back(srow);
+ }
+ return rows;
+}
- ASSERT_TRUE(!tresults.empty()) << "tresults shouldn't be empty.";
+template <typename REQ, typename RESP>
+std::vector<std::shared_ptr<hbase::Result>> getResults(std::vector<REQ> &actions,
+ std::vector<folly::Try<RESP>> &tresults) {
std::vector<std::shared_ptr<hbase::Result>> results{};
- uint32_t num = 0;
+ uint64_t num = 0;
for (auto tresult : tresults) {
if (tresult.hasValue()) {
results.push_back(tresult.value());
} else if (tresult.hasException()) {
folly::exception_wrapper ew = tresult.exception();
- LOG(ERROR) << "Caught exception:- " << ew.what().toStdString() << " for " << gets[num].row();
+ LOG(ERROR) << "Caught exception:- " << ew.what().toStdString() << " for "
+ << actions[num].row();
throw ew;
}
++num;
}
+ return results;
+}
+
+template <typename ACTION>
+std::map<std::string, std::shared_ptr<RegionLocation>> getRegionLocationsAndActions(
+ uint64_t num_rows, std::vector<ACTION> &actions, std::shared_ptr<Table> table) {
+ std::map<std::string, std::shared_ptr<RegionLocation>> region_locations;
+ for (uint64_t i = 0; i < num_rows; ++i) {
+ auto row = "test" + std::to_string(i);
+ ACTION action(row);
+ actions.push_back(action);
+ region_locations[row] = table->GetRegionLocation(row);
+ }
+ return region_locations;
+}
+
+void runMultiGets(std::shared_ptr<AsyncRegionLocatorBase> region_locator,
+ const std::string &table_name, bool split_regions, uint32_t tries = 3,
+ uint32_t operation_timeout_millis = 600000, uint64_t num_rows = 1000) {
+ auto tableName = createTestTable(split_regions, table_name);
+ // Create TableName and Row to be fetched from HBase
+ auto tn = folly::to<hbase::pb::TableName>(tableName);
+
+ // Create a client
+ Client client(*AsyncBatchRpcRetryTest::test_util->conf());
+
+ // Get connection to HBase Table
+ std::shared_ptr<Table> table = client.Table(tn);
+
+ for (uint64_t i = 0; i < num_rows; i++) {
+ table->Put(Put{"test" + std::to_string(i)}.AddColumn("d", std::to_string(i),
+ "value" + std::to_string(i)));
+ }
+ std::vector<hbase::Get> gets;
+ auto region_locations = getRegionLocationsAndActions<hbase::Get>(num_rows, gets, table);
+
+ /* set region locator */
+ region_locator->set_region_location(region_locations);
+ /* init hbase client connection */
+ auto conn = getAsyncConnection(client, operation_timeout_millis, tries, region_locator);
+ conn->Init();
+
+ /* init retry caller factory */
+ auto tableImpl =
+ std::make_shared<MockRawAsyncTableImpl>(conn, std::make_shared<hbase::pb::TableName>(tn));
+
+ std::vector<std::shared_ptr<hbase::Row>> rows = getRows<hbase::Get>(gets);
+ auto tresults = tableImpl->Batch<std::shared_ptr<hbase::Row>, std::shared_ptr<Result>>(rows).get(
+ milliseconds(operation_timeout_millis));
+ ASSERT_TRUE(!tresults.empty()) << "tresults shouldn't be empty.";
+
+ auto results = getResults<hbase::Get, std::shared_ptr<Result>>(gets, tresults);
// Test the values, should be same as in put executed on hbase shell
ASSERT_TRUE(!results.empty()) << "Results shouldn't be empty.";
uint32_t i = 0;
@@ -371,101 +411,184 @@ void runMultiTest(std::shared_ptr<AsyncRegionLocatorBase> region_locator,
EXPECT_EQ("value" + std::to_string(i), results[i]->Value("d", std::to_string(i)).value());
}
- retry_timer->destroy();
table->Close();
client.Close();
- retry_executor_->stop();
- io_executor_->stop();
- cpu_executor_->stop();
+ conn->Close();
+}
+
+void runMultiPuts(std::shared_ptr<AsyncRegionLocatorBase> region_locator,
+ const std::string &table_name, bool split_regions, uint32_t tries = 3,
+ uint32_t operation_timeout_millis = 600000, uint32_t num_rows = 1000) {
+ auto tableName = createTestTable(split_regions, table_name);
+
+ // Create TableName and Row to be fetched from HBase
+ auto tn = folly::to<hbase::pb::TableName>(tableName);
+
+ // Create a client
+ Client client(*AsyncBatchRpcRetryTest::test_util->conf());
+
+ // Get connection to HBase Table
+ std::shared_ptr<Table> table = client.Table(tn);
+
+ std::vector<hbase::Put> puts;
+ auto region_locations = getRegionLocationsAndActions<hbase::Put>(num_rows, puts, table);
+
+ /* set region locator */
+ region_locator->set_region_location(region_locations);
+
+ /* init hbase client connection */
+ auto conn = getAsyncConnection(client, operation_timeout_millis, tries, region_locator);
+ conn->Init();
+
+ /* init retry caller factory */
+ auto tableImpl =
+ std::make_shared<MockRawAsyncTableImpl>(conn, std::make_shared<hbase::pb::TableName>(tn));
+
+ std::vector<std::shared_ptr<hbase::Row>> rows = getRows<hbase::Put>(puts);
+ auto tresults = tableImpl->Batch<std::shared_ptr<hbase::Row>, std::shared_ptr<Result>>(rows).get(
+ milliseconds(operation_timeout_millis));
+ ASSERT_TRUE(!tresults.empty()) << "tresults shouldn't be empty.";
+
+ auto results = getResults<hbase::Put, std::shared_ptr<Result>>(puts, tresults);
+ // Test the values, should be same as in put executed on hbase shell
+ ASSERT_TRUE(!results.empty()) << "Results shouldn't be empty.";
+
+ table->Close();
+ client.Close();
+ conn->Close();
}
// Test successful case
TEST_F(AsyncBatchRpcRetryTest, MultiGets) {
std::shared_ptr<AsyncRegionLocatorBase> region_locator(
std::make_shared<MockAsyncRegionLocator>());
- runMultiTest(region_locator, "table1", false);
+ runMultiGets(region_locator, "table1", false);
}
// Tests the RPC failing 3 times, then succeeding
TEST_F(AsyncBatchRpcRetryTest, HandleException) {
std::shared_ptr<AsyncRegionLocatorBase> region_locator(
std::make_shared<MockWrongRegionAsyncRegionLocator>(3));
- runMultiTest(region_locator, "table2", false, 5);
+ runMultiGets(region_locator, "table2", false, 5);
}
// Tests the RPC failing 4 times, throwing an exception
TEST_F(AsyncBatchRpcRetryTest, FailWithException) {
std::shared_ptr<AsyncRegionLocatorBase> region_locator(
std::make_shared<MockWrongRegionAsyncRegionLocator>(4));
- EXPECT_ANY_THROW(runMultiTest(region_locator, "table3", false));
+ EXPECT_ANY_THROW(runMultiGets(region_locator, "table3", false));
}
// Tests the region location lookup failing 3 times, then succeeding
TEST_F(AsyncBatchRpcRetryTest, HandleExceptionFromRegionLocationLookup) {
std::shared_ptr<AsyncRegionLocatorBase> region_locator(
std::make_shared<MockFailingAsyncRegionLocator>(3));
- runMultiTest(region_locator, "table4", false);
+ runMultiGets(region_locator, "table4", false);
}
// Tests the region location lookup failing 5 times, throwing an exception
TEST_F(AsyncBatchRpcRetryTest, FailWithExceptionFromRegionLocationLookup) {
std::shared_ptr<AsyncRegionLocatorBase> region_locator(
std::make_shared<MockFailingAsyncRegionLocator>(4));
- EXPECT_ANY_THROW(runMultiTest(region_locator, "table5", false, 3));
+ EXPECT_ANY_THROW(runMultiGets(region_locator, "table5", false, 3));
}
// Tests hitting operation timeout, thus not retrying anymore
TEST_F(AsyncBatchRpcRetryTest, FailWithOperationTimeout) {
std::shared_ptr<AsyncRegionLocatorBase> region_locator(
std::make_shared<MockFailingAsyncRegionLocator>(6));
- EXPECT_ANY_THROW(runMultiTest(region_locator, "table6", false, 5, 100, 1000));
+ EXPECT_ANY_THROW(runMultiGets(region_locator, "table6", false, 5, 100, 1000));
}
-/*
- TODO: Below tests are failing with frequently with segfaults coming from
- JNI internals indicating that we are doing something wrong in the JNI boundary.
- However, we were not able to debug furhter yet. Disable the tests for now, and
- come back later to fix the issue.
-
+//////////////////////
// Test successful case
-TEST_F(AsyncBatchRpcRetryTest, MultiGetsSplitRegions) {
+TEST_F(AsyncBatchRpcRetryTest, MultiPuts) {
std::shared_ptr<AsyncRegionLocatorBase> region_locator(
std::make_shared<MockAsyncRegionLocator>());
- runMultiTest(region_locator, "table7", true);
+ runMultiPuts(region_locator, "table1", false);
}
// Tests the RPC failing 3 times, then succeeding
-TEST_F(AsyncBatchRpcRetryTest, HandleExceptionSplitRegions) {
+TEST_F(AsyncBatchRpcRetryTest, PutsHandleException) {
std::shared_ptr<AsyncRegionLocatorBase> region_locator(
std::make_shared<MockWrongRegionAsyncRegionLocator>(3));
- runMultiTest(region_locator, "table8", true, 5);
+ runMultiPuts(region_locator, "table2", false, 5);
}
// Tests the RPC failing 4 times, throwing an exception
-TEST_F(AsyncBatchRpcRetryTest, FailWithExceptionSplitRegions) {
+TEST_F(AsyncBatchRpcRetryTest, PutsFailWithException) {
std::shared_ptr<AsyncRegionLocatorBase> region_locator(
std::make_shared<MockWrongRegionAsyncRegionLocator>(4));
- EXPECT_ANY_THROW(runMultiTest(region_locator, "table9", true));
+ EXPECT_ANY_THROW(runMultiPuts(region_locator, "table3", false));
}
// Tests the region location lookup failing 3 times, then succeeding
-TEST_F(AsyncBatchRpcRetryTest, HandleExceptionFromRegionLocationLookupSplitRegions) {
+TEST_F(AsyncBatchRpcRetryTest, PutsHandleExceptionFromRegionLocationLookup) {
std::shared_ptr<AsyncRegionLocatorBase> region_locator(
std::make_shared<MockFailingAsyncRegionLocator>(3));
- runMultiTest(region_locator, "table10", true);
+ runMultiPuts(region_locator, "table4", false);
}
// Tests the region location lookup failing 5 times, throwing an exception
-TEST_F(AsyncBatchRpcRetryTest, FailWithExceptionFromRegionLocationLookupSplitRegions) {
+TEST_F(AsyncBatchRpcRetryTest, PutsFailWithExceptionFromRegionLocationLookup) {
std::shared_ptr<AsyncRegionLocatorBase> region_locator(
std::make_shared<MockFailingAsyncRegionLocator>(4));
- EXPECT_ANY_THROW(runMultiTest(region_locator, "table11", true, 3));
+ EXPECT_ANY_THROW(runMultiPuts(region_locator, "table5", false, 3));
}
// Tests hitting operation timeout, thus not retrying anymore
-TEST_F(AsyncBatchRpcRetryTest, FailWithOperationTimeoutSplitRegions) {
+TEST_F(AsyncBatchRpcRetryTest, PutsFailWithOperationTimeout) {
std::shared_ptr<AsyncRegionLocatorBase> region_locator(
std::make_shared<MockFailingAsyncRegionLocator>(6));
- EXPECT_ANY_THROW(runMultiTest(region_locator, "table12", true, 5, 100, 1000));
+ EXPECT_ANY_THROW(runMultiPuts(region_locator, "table6", false, 5, 100, 1000));
}
-*/
+
+//////////////////////
+/*
+ TODO: Below tests are failing with frequently with segfaults coming from
+ JNI internals indicating that we are doing something wrong in the JNI boundary.
+ However, we were not able to debug furhter yet. Disable the tests for now, and
+ come back later to fix the issue.
+
+ // Test successful case
+ TEST_F(AsyncBatchRpcRetryTest, MultiGetsSplitRegions) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockAsyncRegionLocator>());
+ runMultiGets(region_locator, "table7", true);
+ }
+
+ // Tests the RPC failing 3 times, then succeeding
+ TEST_F(AsyncBatchRpcRetryTest, HandleExceptionSplitRegions) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockWrongRegionAsyncRegionLocator>(3));
+ runMultiGets(region_locator, "table8", true, 5);
+ }
+
+ // Tests the RPC failing 4 times, throwing an exception
+ TEST_F(AsyncBatchRpcRetryTest, FailWithExceptionSplitRegions) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockWrongRegionAsyncRegionLocator>(4));
+ EXPECT_ANY_THROW(runMultiGets(region_locator, "table9", true));
+ }
+
+ // Tests the region location lookup failing 3 times, then succeeding
+ TEST_F(AsyncBatchRpcRetryTest, HandleExceptionFromRegionLocationLookupSplitRegions) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockFailingAsyncRegionLocator>(3));
+ runMultiGets(region_locator, "table10", true);
+ }
+
+ // Tests the region location lookup failing 5 times, throwing an exception
+ TEST_F(AsyncBatchRpcRetryTest, FailWithExceptionFromRegionLocationLookupSplitRegions) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockFailingAsyncRegionLocator>(4));
+ EXPECT_ANY_THROW(runMultiGets(region_locator, "table11", true, 3));
+ }
+
+ // Tests hitting operation timeout, thus not retrying anymore
+ TEST_F(AsyncBatchRpcRetryTest, FailWithOperationTimeoutSplitRegions) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockFailingAsyncRegionLocator>(6));
+ EXPECT_ANY_THROW(runMultiGets(region_locator, "table12", true, 5, 100, 1000));
+ }
+ */
diff --git a/core/async-rpc-retrying-caller-factory.h b/core/async-rpc-retrying-caller-factory.h
index 1af6e72..188f469 100644
--- a/core/async-rpc-retrying-caller-factory.h
+++ b/core/async-rpc-retrying-caller-factory.h
@@ -135,7 +135,8 @@ class SingleRequestCallerBuilder
Callable<RESP> callable_;
}; // end of SingleRequestCallerBuilder
-class BatchCallerBuilder : public std::enable_shared_from_this<BatchCallerBuilder> {
+template <typename REQ, typename RESP>
+class BatchCallerBuilder : public std::enable_shared_from_this<BatchCallerBuilder<REQ, RESP>> {
public:
explicit BatchCallerBuilder(std::shared_ptr<AsyncConnection> conn,
std::shared_ptr<folly::HHWheelTimer> retry_timer)
@@ -143,14 +144,14 @@ class BatchCallerBuilder : public std::enable_shared_from_this<BatchCallerBuilde
virtual ~BatchCallerBuilder() = default;
- typedef std::shared_ptr<BatchCallerBuilder> SharedThisPtr;
+ typedef std::shared_ptr<BatchCallerBuilder<REQ, RESP>> SharedThisPtr;
SharedThisPtr table(std::shared_ptr<pb::TableName> table_name) {
table_name_ = table_name;
return shared_this();
}
- SharedThisPtr actions(std::shared_ptr<std::vector<hbase::Get>> actions) {
+ SharedThisPtr actions(std::shared_ptr<std::vector<REQ>> actions) {
actions_ = actions;
return shared_this();
}
@@ -180,10 +181,10 @@ class BatchCallerBuilder : public std::enable_shared_from_this<BatchCallerBuilde
return shared_this();
}
- folly::Future<std::vector<folly::Try<std::shared_ptr<Result>>>> Call() { return Build()->Call(); }
+ folly::Future<std::vector<folly::Try<RESP>>> Call() { return Build()->Call(); }
- std::shared_ptr<AsyncBatchRpcRetryingCaller> Build() {
- return std::make_shared<AsyncBatchRpcRetryingCaller>(
+ std::shared_ptr<AsyncBatchRpcRetryingCaller<REQ, RESP>> Build() {
+ return std::make_shared<AsyncBatchRpcRetryingCaller<REQ, RESP>>(
conn_, retry_timer_, table_name_, *actions_, pause_ns_, max_attempts_,
operation_timeout_nanos_, rpc_timeout_nanos_, start_log_errors_count_);
}
@@ -197,7 +198,7 @@ class BatchCallerBuilder : public std::enable_shared_from_this<BatchCallerBuilde
std::shared_ptr<AsyncConnection> conn_;
std::shared_ptr<folly::HHWheelTimer> retry_timer_;
std::shared_ptr<hbase::pb::TableName> table_name_ = nullptr;
- std::shared_ptr<std::vector<hbase::Get>> actions_ = nullptr;
+ std::shared_ptr<std::vector<REQ>> actions_ = nullptr;
std::chrono::nanoseconds pause_ns_;
int32_t max_attempts_ = 0;
std::chrono::nanoseconds operation_timeout_nanos_;
@@ -329,8 +330,9 @@ class AsyncRpcRetryingCallerFactory {
return std::make_shared<SingleRequestCallerBuilder<RESP>>(conn_, retry_timer_);
}
- std::shared_ptr<BatchCallerBuilder> Batch() {
- return std::make_shared<BatchCallerBuilder>(conn_, retry_timer_);
+ template <typename REQ, typename RESP>
+ std::shared_ptr<BatchCallerBuilder<REQ, RESP>> Batch() {
+ return std::make_shared<BatchCallerBuilder<REQ, RESP>>(conn_, retry_timer_);
}
std::shared_ptr<ScanCallerBuilder> Scan() {
diff --git a/core/client-test.cc b/core/client-test.cc
index 1c9b709..3f72880 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -547,3 +547,151 @@ TEST_F(ClientTest, MultiGetsWithRegionSplits) {
table->Close();
client.Close();
}
+
+void PerformMultiPuts(uint64_t num_rows, std::shared_ptr<hbase::Client> client,
+ const std::string &table_name) {
+ auto tn = folly::to<hbase::pb::TableName>(table_name);
+ auto table = client->Table(tn);
+ ASSERT_TRUE(table) << "Unable to get connection to Table.";
+ std::vector<hbase::Put> puts;
+ // Perform Puts
+ for (uint64_t i = 0; i < num_rows; i++) {
+ puts.push_back(Put{"test" + std::to_string(i)}.AddColumn("d", std::to_string(i),
+ "value" + std::to_string(i)));
+ }
+ table->Put(puts);
+}
+
+void PerformMultiPuts(std::vector<hbase::Put> &puts, std::shared_ptr<Table> table) {
+ table->Put(puts);
+}
+
+TEST_F(ClientTest, MultiGetsWithMultiPuts) {
+ std::string table_name = "t";
+ // Using TestUtil to populate test data
+ ClientTest::test_util->CreateTable(table_name, "d");
+
+ // Create TableName and Row to be fetched from HBase
+ auto tn = folly::to<hbase::pb::TableName>(table_name);
+
+ SetClientParams();
+ // Create a client
+ hbase::Client client(*ClientTest::test_util->conf());
+
+ uint64_t num_rows = 50000;
+ PerformMultiPuts(num_rows, std::make_shared<hbase::Client>(client), table_name);
+
+ // Get connection to HBase Table
+ auto table = client.Table(tn);
+ ASSERT_TRUE(table) << "Unable to get connection to Table.";
+
+ std::vector<hbase::Get> gets;
+ MakeGets(num_rows, "test", gets);
+
+ auto results = table->Get(gets);
+
+ TestMultiResults(num_rows, results, gets);
+
+ table->Close();
+ client.Close();
+}
+
+TEST_F(ClientTest, MultiGetsWithMultiPutsAndSplitRegions) {
+ // Using TestUtil to populate test data
+ std::vector<std::string> keys{"test0", "test100", "test200", "test300", "test400",
+ "test500", "test600", "test700", "test800", "test900"};
+ std::string table_name = "t";
+ ClientTest::test_util->CreateTable(table_name, "d", keys);
+
+ // Create TableName and Row to be fetched from HBase
+ auto tn = folly::to<hbase::pb::TableName>(table_name);
+
+ SetClientParams();
+
+ // Create a client
+ hbase::Client client(*ClientTest::test_util->conf());
+
+ uint64_t num_rows = 50000;
+ PerformMultiPuts(num_rows, std::make_shared<hbase::Client>(client), table_name);
+
+ // Get connection to HBase Table
+ auto table = client.Table(tn);
+ ASSERT_TRUE(table) << "Unable to get connection to Table.";
+
+ std::vector<hbase::Get> gets;
+ MakeGets(num_rows, "test", gets);
+
+ auto results = table->Get(gets);
+
+ TestMultiResults(num_rows, results, gets);
+
+ table->Close();
+ client.Close();
+}
+
+TEST_F(ClientTest, MultiPuts) {
+ std::string table_name = "t";
+ // Using TestUtil to populate test data
+ ClientTest::test_util->CreateTable(table_name, "d");
+
+ // Create TableName and Row to be fetched from HBase
+ auto tn = folly::to<hbase::pb::TableName>(table_name);
+
+ SetClientParams();
+ // Create a client
+ hbase::Client client(*ClientTest::test_util->conf());
+ std::shared_ptr<Table> table = client.Table(tn);
+ ASSERT_TRUE(table) << "Unable to get connection to Table.";
+
+ uint64_t num_rows = 80000;
+ uint64_t batch_num_rows = 10000;
+ std::vector<hbase::Put> puts;
+ for (uint64_t i = 0; i < num_rows;) {
+ puts.clear();
+ // accumulate batch_num_rows at a time
+ for (uint64_t j = 0; j < batch_num_rows && i < num_rows; ++j) {
+ hbase::Put put("test" + std::to_string(i));
+ put.AddColumn("d", std::to_string(i), "value" + std::to_string(i));
+ puts.push_back(put);
+ i++;
+ }
+ PerformMultiPuts(puts, table);
+ }
+ table->Close();
+ client.Close();
+}
+
+TEST_F(ClientTest, MultiPutsWithRegionSplits) {
+ // Using TestUtil to populate test data
+ std::vector<std::string> keys{"test0", "test100", "test200", "test300", "test400",
+ "test500", "test600", "test700", "test800", "test900"};
+ std::string table_name = "t";
+ ClientTest::test_util->CreateTable(table_name, "d", keys);
+
+ // Create TableName and Row to be fetched from HBase
+ auto tn = folly::to<hbase::pb::TableName>(table_name);
+
+ SetClientParams();
+
+ // Create a client
+ hbase::Client client(*ClientTest::test_util->conf());
+ std::shared_ptr<Table> table = client.Table(tn);
+ ASSERT_TRUE(table) << "Unable to get connection to Table.";
+
+ uint64_t num_rows = 80000;
+ uint64_t batch_num_rows = 10000;
+ std::vector<hbase::Put> puts;
+ for (uint64_t i = 0; i < num_rows;) {
+ puts.clear();
+ // accumulate batch_num_rows at a time
+ for (uint64_t j = 0; j < batch_num_rows && i < num_rows; ++j) {
+ hbase::Put put("test" + std::to_string(i));
+ put.AddColumn("d", std::to_string(i), "value" + std::to_string(i));
+ puts.push_back(put);
+ i++;
+ }
+ PerformMultiPuts(puts, table);
+ }
+ table->Close();
+ client.Close();
+}
diff --git a/core/raw-async-table.cc b/core/raw-async-table.cc
index 53ab526..409883f 100644
--- a/core/raw-async-table.cc
+++ b/core/raw-async-table.cc
@@ -197,18 +197,26 @@ folly::Future<std::shared_ptr<Result>> RawAsyncTable::Append(const hbase::Append
return caller->Call().then([caller](const auto r) { return r; });
}
+
folly::Future<std::vector<folly::Try<std::shared_ptr<Result>>>> RawAsyncTable::Get(
const std::vector<hbase::Get>& gets) {
- return this->Batch(gets);
+ std::vector<std::shared_ptr<hbase::Row>> rows;
+ for (auto get : gets) {
+ std::shared_ptr<hbase::Row> srow = std::make_shared<hbase::Get>(get);
+ rows.push_back(srow);
+ }
+ return this->Batch<std::shared_ptr<hbase::Row>, std::shared_ptr<Result>>(
+ rows, connection_conf_->read_rpc_timeout());
}
-folly::Future<std::vector<folly::Try<std::shared_ptr<Result>>>> RawAsyncTable::Batch(
- const std::vector<hbase::Get>& gets) {
+template <typename REQ, typename RESP>
+folly::Future<std::vector<folly::Try<RESP>>> RawAsyncTable::Batch(
+ const std::vector<REQ>& rows, std::chrono::nanoseconds timeout) {
auto caller = connection_->caller_factory()
- ->Batch()
+ ->Batch<REQ, RESP>()
->table(table_name_)
- ->actions(std::make_shared<std::vector<hbase::Get>>(gets))
- ->rpc_timeout(connection_conf_->read_rpc_timeout())
+ ->actions(std::make_shared<std::vector<REQ>>(rows))
+ ->rpc_timeout(timeout)
->operation_timeout(connection_conf_->operation_timeout())
->pause(connection_conf_->pause())
->max_attempts(connection_conf_->max_retries())
@@ -237,4 +245,15 @@ std::shared_ptr<hbase::Scan> RawAsyncTable::SetDefaultScanConfig(const hbase::Sc
}
return new_scan;
}
+
+folly::Future<std::vector<folly::Try<std::shared_ptr<Result>>>> RawAsyncTable::Put(
+ const std::vector<hbase::Put>& puts) {
+ std::vector<std::shared_ptr<hbase::Row>> rows;
+ for (auto put : puts) {
+ std::shared_ptr<hbase::Row> srow = std::make_shared<hbase::Put>(put);
+ rows.push_back(srow);
+ }
+ return this->Batch<std::shared_ptr<hbase::Row>, std::shared_ptr<Result>>(
+ rows, connection_conf_->write_rpc_timeout());
+}
} // namespace hbase
diff --git a/core/raw-async-table.h b/core/raw-async-table.h
index e651f8a..97eef7f 100644
--- a/core/raw-async-table.h
+++ b/core/raw-async-table.h
@@ -83,8 +83,11 @@ class RawAsyncTable {
folly::Future<std::vector<folly::Try<std::shared_ptr<Result>>>> Get(
const std::vector<hbase::Get>& gets);
- folly::Future<std::vector<folly::Try<std::shared_ptr<Result>>>> Batch(
- const std::vector<hbase::Get>& gets);
+ template <typename REQ, typename RESP>
+ folly::Future<std::vector<folly::Try<RESP>>> Batch(const std::vector<REQ>& rows,
+ std::chrono::nanoseconds timeout);
+ folly::Future<std::vector<folly::Try<std::shared_ptr<Result>>>> Put(
+ const std::vector<hbase::Put>& puts);
private:
/* Data */
diff --git a/core/request-converter.cc b/core/request-converter.cc
index 47c09d1..f48f228 100644
--- a/core/request-converter.cc
+++ b/core/request-converter.cc
@@ -173,14 +173,23 @@ std::unique_ptr<Request> RequestConverter::ToMultiRequest(
auto pb_action = pb_region_action->add_action();
auto pget = region_action->action();
// We store only hbase::Get in hbase::Action as of now. It will be changed later on.
- CHECK(pget) << "Unexpected. action can't be null";
- auto pb_get = RequestConverter::ToGet(*pget);
- pb_action->set_allocated_get(pb_get.release());
+ CHECK(pget) << "Unexpected. action can't be null.";
+ std::string error_msg("");
+ if (typeid(*pget) == typeid(hbase::Get)) {
+ auto getp = dynamic_cast<hbase::Get *>(pget.get());
+ pb_action->set_allocated_get(RequestConverter::ToGet(*getp).release());
+ } else if (typeid(*pget) == typeid(hbase::Put)) {
+ auto putp = dynamic_cast<hbase::Put *>(pget.get());
+ pb_action->set_allocated_mutation(
+ RequestConverter::ToMutation(MutationType::MutationProto_MutationType_PUT, *putp, -1)
+ .release());
+ } else {
+ throw std::runtime_error("Unexpected action type encountered.");
+ }
pb_action->set_index(action_num);
action_num++;
}
}
-
return pb_req;
}
@@ -355,4 +364,5 @@ std::unique_ptr<Request> RequestConverter::AppendToMutateRequest(const Append &a
VLOG(3) << "Req is " << pb_req->req_msg()->ShortDebugString();
return pb_req;
}
+
} /* namespace hbase */
diff --git a/core/table.cc b/core/table.cc
index 3b7a87b..f93a029 100644
--- a/core/table.cc
+++ b/core/table.cc
@@ -128,4 +128,17 @@ std::vector<std::shared_ptr<hbase::Result>> Table::Get(const std::vector<hbase::
return results;
}
+void Table::Put(const std::vector<hbase::Put> &puts) {
+ auto tresults = async_table_->Put(puts).get(operation_timeout());
+ uint32_t num = 0;
+ for (auto tresult : tresults) {
+ if (tresult.hasException()) {
+ LOG(ERROR) << "Caught exception:- " << tresult.exception().what() << " for "
+ << puts[num++].row();
+ throw tresult.exception();
+ }
+ }
+ return;
+}
+
} /* namespace hbase */
diff --git a/core/table.h b/core/table.h
index cc37182..6340494 100644
--- a/core/table.h
+++ b/core/table.h
@@ -119,11 +119,15 @@ class Table {
* @param - append Append object to perform HBase Append operation.
*/
std::shared_ptr<hbase::Result> Append(const hbase::Append &append);
- // TODO: Batch Puts
std::shared_ptr<ResultScanner> Scan(const hbase::Scan &scan);
/**
+ * @brief - Multi Puts.
+ * @param - puts vector of hbase::Put.
+ */
+ void Put(const std::vector<hbase::Put> &puts);
+ /**
* @brief - Close the client connection.
*/
void Close();
[hbase] 89/137: HBASE-15903 Delete Object - addnedum moves
misplaced sleep
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 0cb84520b9f9c93465af47bb5e86134bb5c99092
Author: tedyu <yu...@gmail.com>
AuthorDate: Thu Jun 1 17:13:03 2017 -0700
HBASE-15903 Delete Object - addnedum moves misplaced sleep
---
core/client-test.cc | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/core/client-test.cc b/core/client-test.cc
index d0791b8..c378f81 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -138,8 +138,8 @@ TEST_F(ClientTest, PutGetDelete) {
table->Put(Put{row}.AddColumn("d", "1", "value1"));
// Put two values for column "extra"
table->Put(Put{row}.AddColumn("d", "extra", "1st val extra"));
- table->Put(Put{row}.AddColumn("d", "extra", valExtra));
usleep(1000);
+ table->Put(Put{row}.AddColumn("d", "extra", valExtra));
table->Put(Put{row}.AddColumn("d", "ext", valExt));
// Perform the Get
@@ -153,6 +153,7 @@ TEST_F(ClientTest, PutGetDelete) {
EXPECT_EQ(valExtra, *(result->Value("d", "extra")));
auto cell = *(result->ColumnCells("d", "extra"))[0];
auto tsExtra = cell.Timestamp();
+ auto tsExt = (*(result->ColumnCells("d", "ext"))[0]).Timestamp();
// delete column "1"
table->Delete(hbase::Delete{row}.AddColumn("d", "1"));
[hbase] 39/137: HBASE-15945 Patch for Cell
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 26dd78c824bea8e4ddf7e3d060ced43d73d7a3d7
Author: Sudeep Sunthankar <su...@gmail.com>
AuthorDate: Mon Jul 4 21:02:25 2016 +1000
HBASE-15945 Patch for Cell
This patch consists of Cell implementation without additional interfaces.
Signed-off-by: Elliott Clark <ec...@apache.org>
---
core/cell-test.cc | 175 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
core/cell.cc | 77 ++++++++++++++++++++++++
core/cell.h | 60 +++++++++++++++++++
3 files changed, 312 insertions(+)
diff --git a/core/cell-test.cc b/core/cell-test.cc
new file mode 100644
index 0000000..cbe50eb
--- /dev/null
+++ b/core/cell-test.cc
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/cell.h"
+
+#include <memory>
+#include <gtest/gtest.h>
+#include <glog/logging.h>
+
+using namespace hbase;
+TEST(CellTest, CellFailureTest) {
+ CellType cell_type = CellType::PUT;
+ std::string row = "row";
+ std::string family = "family";
+ std::string column = "column";
+ std::string value = "value";
+ long timestamp = std::numeric_limits<long>::max();
+ std::string tags = "";
+ std::unique_ptr<Cell> cell(
+ new Cell(row, family, column, timestamp, value, cell_type));
+ if (cell.get()) {
+ EXPECT_NE("row-value", cell.get()->Row());
+ EXPECT_NE("family-value", cell.get()->Family());
+ EXPECT_NE("column-value", cell.get()->Qualifier());
+ EXPECT_NE("value-value", cell.get()->Value());
+ EXPECT_NE(8975431260, cell.get()->Timestamp());
+ EXPECT_NE(CellType::MAXIMUM, cell.get()->Type());
+ }
+}
+
+TEST(CellTest, CellSuceessTest) {
+ std::string row = "row-value";
+ std::string family = "family-value";
+ std::string column = "column-value";
+ std::string value = "value-value";
+ long timestamp = std::numeric_limits<long>::max();
+ CellType cell_type = CellType::PUT;
+ const std::unique_ptr<Cell> cell(
+ new Cell(row, family, column, timestamp, value, cell_type));
+ if (cell.get()) {
+ EXPECT_EQ(row, cell.get()->Row());
+ EXPECT_EQ(family, cell.get()->Family());
+ EXPECT_EQ(column, cell.get()->Qualifier());
+ EXPECT_EQ(value, cell.get()->Value());
+ EXPECT_EQ(timestamp, cell.get()->Timestamp());
+ EXPECT_EQ(cell_type, cell.get()->Type());
+ }
+}
+
+TEST(CellTest, MultipleCellsTest) {
+ std::vector<const Cell *> cells;
+ for (int i = 0; i < 5; i++) {
+ std::string row = "row-value";
+ std::string family = "family-value";
+ std::string column = "column-value";
+ std::string value = "value-value";
+ long timestamp = std::numeric_limits<long>::max();
+ row += std::to_string(i);
+ value += std::to_string(i);
+ CellType cell_type = CellType::PUT;
+ const Cell *cell = new Cell(row, family, column, timestamp, value,
+ cell_type);
+ cells.push_back(cell);
+ }
+ int i = 0;
+ for (const auto cell : cells) {
+ std::string row = "row-value";
+ std::string value = "value-value";
+ row += std::to_string(i);
+ value += std::to_string(i);
+ EXPECT_EQ(row, cell->Row());
+ EXPECT_EQ("family-value", cell->Family());
+ EXPECT_EQ("column-value", cell->Qualifier());
+ EXPECT_EQ(value, cell->Value());
+ EXPECT_EQ(std::numeric_limits<long>::max(), cell->Timestamp());
+ EXPECT_EQ(CellType::PUT, cell->Type());
+ i += 1;
+ }
+ for (const auto cell : cells) {
+ delete cell;
+ }
+ cells.clear();
+}
+
+TEST(CellTest, CellRowTest) {
+ std::string row = "only-row";
+ std::string family = "";
+ std::string column = "";
+ std::string value = "";
+ long timestamp = std::numeric_limits<long>::max();
+ CellType cell_type = CellType::PUT;
+ std::unique_ptr<Cell> cell(
+ new Cell(row, family, column, timestamp, value, cell_type));
+ if (cell.get()) {
+ EXPECT_EQ(row, cell.get()->Row());
+ EXPECT_EQ(family, cell.get()->Family());
+ EXPECT_EQ(column, cell.get()->Qualifier());
+ EXPECT_EQ(value, cell.get()->Value());
+ EXPECT_EQ(timestamp, cell.get()->Timestamp());
+ EXPECT_EQ(cell_type, cell.get()->Type());
+ }
+}
+
+TEST(CellTest, CellRowFamilyTest) {
+ std::string row = "only-row";
+ std::string family = "only-family";
+ std::string column = "";
+ std::string value = "";
+ long timestamp = std::numeric_limits<long>::max();
+ CellType cell_type = CellType::PUT;
+ const std::unique_ptr<Cell> cell(
+ new Cell(row, family, column, timestamp, value, cell_type));
+ if (cell.get()) {
+ EXPECT_EQ(row, cell.get()->Row());
+ EXPECT_EQ(family, cell.get()->Family());
+ EXPECT_EQ(column, cell.get()->Qualifier());
+ EXPECT_EQ(value, cell.get()->Value());
+ EXPECT_EQ(timestamp, cell.get()->Timestamp());
+ EXPECT_EQ(cell_type, cell.get()->Type());
+ }
+}
+
+TEST(CellTest, CellRowFamilyValueTest) {
+ std::string row = "only-row";
+ std::string family = "only-family";
+ std::string column = "";
+ std::string value = "only-value";
+ long timestamp = std::numeric_limits<long>::max();
+ CellType cell_type = CellType::PUT;
+ const std::unique_ptr<Cell> cell(
+ new Cell(row, family, column, timestamp, value, cell_type));
+ if (cell.get()) {
+ EXPECT_EQ(row, cell.get()->Row());
+ EXPECT_EQ(family, cell.get()->Family());
+ EXPECT_EQ(column, cell.get()->Qualifier());
+ EXPECT_EQ(value, cell.get()->Value());
+ EXPECT_EQ(timestamp, cell.get()->Timestamp());
+ EXPECT_EQ(cell_type, cell.get()->Type());
+ }
+}
+
+TEST(CellTest, CellRowFamilyColumnValueTest) {
+ std::string row = "only-row";
+ std::string family = "only-family";
+ std::string column = "only-column";
+ std::string value = "only-value";
+ long timestamp = std::numeric_limits<long>::max();
+ CellType cell_type = CellType::PUT;
+ std::unique_ptr<Cell> cell(
+ new Cell(row, family, column, timestamp, value, cell_type));
+ if (cell.get()) {
+ EXPECT_EQ(row, cell.get()->Row());
+ EXPECT_EQ(family, cell.get()->Family());
+ EXPECT_EQ(column, cell.get()->Qualifier());
+ EXPECT_EQ(value, cell.get()->Value());
+ EXPECT_EQ(timestamp, cell.get()->Timestamp());
+ EXPECT_EQ(cell_type, cell.get()->Type());
+ }
+}
diff --git a/core/cell.cc b/core/cell.cc
new file mode 100644
index 0000000..f214479
--- /dev/null
+++ b/core/cell.cc
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/cell.h"
+#include <stdexcept>
+
+namespace hbase {
+
+Cell::Cell(const std::string &row, const std::string &family,
+ const std::string &qualifier, const long ×tamp,
+ const std::string &value, const hbase::CellType &cell_type)
+ : row_(row),
+ family_(family),
+ qualifier_(qualifier),
+ timestamp_(timestamp),
+ cell_type_(cell_type),
+ value_(value),
+ sequence_id_(0) {
+
+ if (0 == row.size())
+ throw std::runtime_error("Row size should be greater than 0");
+
+ if (0 == family.size())
+ throw std::runtime_error("Column family size should be greater than 0");
+
+ if (0 >= timestamp)
+ throw std::runtime_error("Timestamp should be greater than 0");
+}
+
+Cell::~Cell() {
+}
+
+const std::string &Cell::Row() const {
+ return row_;
+}
+
+const std::string &Cell::Family() const {
+ return family_;
+}
+
+const std::string &Cell::Qualifier() const {
+ return qualifier_;
+}
+
+unsigned long Cell::Timestamp() const {
+ return timestamp_;
+}
+
+const std::string &Cell::Value() const {
+ return value_;
+}
+
+hbase::CellType Cell::Type() const {
+ return cell_type_;
+}
+
+long Cell::SequenceId() const {
+ return sequence_id_;
+}
+
+} /* namespace hbase */
diff --git a/core/cell.h b/core/cell.h
new file mode 100644
index 0000000..16ed280
--- /dev/null
+++ b/core/cell.h
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <string>
+
+namespace hbase {
+
+enum CellType {
+ MINIMUM = 0,
+ PUT = 4,
+ DELETE = 8,
+ DELETEFAMILYVERSION = 10,
+ DELETE_COLUMN = 12,
+ DELETE_FAMILY = 14,
+ MAXIMUM = 255
+};
+
+class Cell {
+ public:
+ Cell(const std::string &row, const std::string &family,
+ const std::string &qualifier, const long ×tamp,
+ const std::string &value, const hbase::CellType &cell_type);
+ virtual ~Cell();
+ const std::string &Row() const;
+ const std::string &Family() const;
+ const std::string &Qualifier() const;
+ unsigned long Timestamp() const;
+ const std::string &Value() const;
+ CellType Type() const;
+ long SequenceId() const;
+
+ private:
+ std::string row_;
+ std::string family_;
+ std::string qualifier_;
+ unsigned long timestamp_;
+ hbase::CellType cell_type_;
+ std::string value_;
+ long sequence_id_;
+};
+
+} /* namespace hbase */
[hbase] 12/137: HBASE-15045 Keep hbase-native-client/if and
hbase-protocol in sync.
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 36d79d9808637322b3e8a956aa11503eabd2685e
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Wed Apr 13 15:59:13 2016 -0700
HBASE-15045 Keep hbase-native-client/if and hbase-protocol in sync.
Summary: On every start of docker make sure that protos are up to date.
Test Plan: Added the script and it updated the protos, but didn't delete BUCK.
Differential Revision: https://reviews.facebook.net/D56763
---
bin/copy-protobuf.sh | 8 +++
bin/start-docker.sh | 22 ++++++--
if/Admin.proto | 1 +
if/Client.proto | 8 ++-
if/ClusterStatus.proto | 3 ++
if/Filter.proto | 1 +
if/Master.proto | 47 ++++++++++++++--
if/MasterProcedure.proto | 40 ++++++++++++++
if/RSGroup.proto | 34 ++++++++++++
if/RSGroupAdmin.proto | 136 +++++++++++++++++++++++++++++++++++++++++++++++
if/ZooKeeper.proto | 13 +++++
11 files changed, 304 insertions(+), 9 deletions(-)
diff --git a/bin/copy-protobuf.sh b/bin/copy-protobuf.sh
new file mode 100755
index 0000000..c9e70f2
--- /dev/null
+++ b/bin/copy-protobuf.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+
+BIN_DIR=$(dirname "$0")
+PB_SOURCE_DIR="${BIN_DIR}/../../hbase-protocol/src/main/protobuf/"
+PB_DEST_DIR="${BIN_DIR}/../if/"
+rsync -r --delete --exclude BUCK ${PB_SOURCE_DIR} ${PB_DEST_DIR}
diff --git a/bin/start-docker.sh b/bin/start-docker.sh
index 725ed6a..9cbd8b7 100755
--- a/bin/start-docker.sh
+++ b/bin/start-docker.sh
@@ -23,22 +23,38 @@ set -x
eval "$(docker-machine env docker-vm)"
eval "$(docker-machine env dinghy)"
-# Build the image
-docker build -t hbase_native .
+BIN_DIR=$(pushd `dirname "$0"` 2>&1 > /dev/null && pwd && popd 2>&1 > /dev/null)
+BASE_DIR=$(pushd "${BIN_DIR}/../" 2>&1 > /dev/null && pwd && popd 2>&1 > /dev/null)
+${BIN_DIR}/copy-protobuf.sh
+# Go into the base dir. This just makes things cleaner.
+pushd ${BASE_DIR}
+
+# Make sure that there is a thrid-party dir.
mkdir third-party || true
+
+# Get gtest
+# TODO(eclark): Remove this ( see HBASE-15427 )
if [[ ! -d third-party/googletest ]]; then
git clone https://github.com/google/googletest.git third-party/googletest
fi
+# We don't want to have to re-download all the jars in docker if we can help it
if [[ ! -d ~/.m2 ]]; then
echo "~/.m2 directory doesn't exist. Check Apache Maven is installed."
exit 1
fi;
+# Build the image
+#
+# This shouldn't be needed after the development environment is a little more stable.
+docker build -t hbase_native .
+
+# After the image is built run the thing
docker run -p 16010:16010/tcp \
-e "JAVA_HOME=/usr/lib/jvm/java-8-oracle" \
- -v ${PWD}/..:/usr/src/hbase \
+ -v ${BASE_DIR}/..:/usr/src/hbase \
-v ~/.m2:/root/.m2 \
-it hbase_native /bin/bash
+popd
diff --git a/if/Admin.proto b/if/Admin.proto
index e905340..a1905a4 100644
--- a/if/Admin.proto
+++ b/if/Admin.proto
@@ -25,6 +25,7 @@ option java_generic_services = true;
option java_generate_equals_and_hash = true;
option optimize_for = SPEED;
+import "Client.proto";
import "HBase.proto";
import "WAL.proto";
diff --git a/if/Client.proto b/if/Client.proto
index ca9abdc..8a4d459 100644
--- a/if/Client.proto
+++ b/if/Client.proto
@@ -402,6 +402,11 @@ message RegionLoadStats {
optional int32 compactionPressure = 3 [default = 0];
}
+message MultiRegionLoadStats{
+ repeated RegionSpecifier region = 1;
+ repeated RegionLoadStats stat = 2;
+}
+
/**
* Either a Result or an Exception NameBytesPair (keyed by
* exception name whose value is the exception stringified)
@@ -416,7 +421,7 @@ message ResultOrException {
// result if this was a coprocessor service call
optional CoprocessorServiceResult service_result = 4;
// current load on the region
- optional RegionLoadStats loadStats = 5;
+ optional RegionLoadStats loadStats = 5 [deprecated=true];
}
/**
@@ -445,6 +450,7 @@ message MultiResponse {
repeated RegionActionResult regionActionResult = 1;
// used for mutate to indicate processed only
optional bool processed = 2;
+ optional MultiRegionLoadStats regionStatistics = 3;
}
diff --git a/if/ClusterStatus.proto b/if/ClusterStatus.proto
index 228be7e..54bc0c3 100644
--- a/if/ClusterStatus.proto
+++ b/if/ClusterStatus.proto
@@ -136,6 +136,9 @@ message RegionLoad {
/** the most recent sequence Id of store from cache flush */
repeated StoreSequenceId store_complete_sequence_id = 18;
+
+ /** the current total filtered read requests made to region */
+ optional uint64 filtered_read_requests_count = 19;
}
/* Server-level protobufs */
diff --git a/if/Filter.proto b/if/Filter.proto
index 67d5717..1fa6697 100644
--- a/if/Filter.proto
+++ b/if/Filter.proto
@@ -146,6 +146,7 @@ message SkipFilter {
message TimestampsFilter {
repeated int64 timestamps = 1 [packed=true];
+ optional bool can_hint = 2;
}
message ValueFilter {
diff --git a/if/Master.proto b/if/Master.proto
index 4d3a2e1..1f7a3b7 100644
--- a/if/Master.proto
+++ b/if/Master.proto
@@ -186,6 +186,7 @@ message CreateNamespaceRequest {
}
message CreateNamespaceResponse {
+ optional uint64 proc_id = 1;
}
message DeleteNamespaceRequest {
@@ -195,6 +196,7 @@ message DeleteNamespaceRequest {
}
message DeleteNamespaceResponse {
+ optional uint64 proc_id = 1;
}
message ModifyNamespaceRequest {
@@ -204,6 +206,7 @@ message ModifyNamespaceRequest {
}
message ModifyNamespaceResponse {
+ optional uint64 proc_id = 1;
}
message GetNamespaceDescriptorRequest {
@@ -276,6 +279,29 @@ message IsBalancerEnabledResponse {
required bool enabled = 1;
}
+enum MasterSwitchType {
+ SPLIT = 0;
+ MERGE = 1;
+}
+
+message SetSplitOrMergeEnabledRequest {
+ required bool enabled = 1;
+ optional bool synchronous = 2;
+ repeated MasterSwitchType switch_types = 3;
+}
+
+message SetSplitOrMergeEnabledResponse {
+ repeated bool prev_value = 1;
+}
+
+message IsSplitOrMergeEnabledRequest {
+ required MasterSwitchType switch_type = 1;
+}
+
+message IsSplitOrMergeEnabledResponse {
+ required bool enabled = 1;
+}
+
message NormalizeRequest {
}
@@ -344,9 +370,12 @@ message DeleteSnapshotResponse {
message RestoreSnapshotRequest {
required SnapshotDescription snapshot = 1;
+ optional uint64 nonce_group = 2 [default = 0];
+ optional uint64 nonce = 3 [default = 0];
}
message RestoreSnapshotResponse {
+ required uint64 proc_id = 1;
}
/* if you don't send the snapshot, then you will get it back
@@ -630,6 +659,19 @@ service MasterService {
returns(IsBalancerEnabledResponse);
/**
+ * Turn the split or merge switch on or off.
+ * If synchronous is true, it waits until current operation call, if outstanding, to return.
+ */
+ rpc SetSplitOrMergeEnabled(SetSplitOrMergeEnabledRequest)
+ returns(SetSplitOrMergeEnabledResponse);
+
+ /**
+ * Query whether the split or merge switch is on/off.
+ */
+ rpc IsSplitOrMergeEnabled(IsSplitOrMergeEnabledRequest)
+ returns(IsSplitOrMergeEnabledResponse);
+
+ /**
* Run region normalizer. Can NOT run for various reasons. Check logs.
*/
rpc Normalize(NormalizeRequest)
@@ -696,11 +738,6 @@ service MasterService {
rpc RestoreSnapshot(RestoreSnapshotRequest) returns(RestoreSnapshotResponse);
/**
- * Determine if the snapshot restore is done yet.
- */
- rpc IsRestoreSnapshotDone(IsRestoreSnapshotDoneRequest) returns(IsRestoreSnapshotDoneResponse);
-
- /**
* Execute a distributed procedure.
*/
rpc ExecProcedure(ExecProcedureRequest) returns(ExecProcedureResponse);
diff --git a/if/MasterProcedure.proto b/if/MasterProcedure.proto
index 2d2aff4..87aae6a 100644
--- a/if/MasterProcedure.proto
+++ b/if/MasterProcedure.proto
@@ -222,6 +222,46 @@ message DisableTableStateData {
required bool skip_table_state_check = 3;
}
+message RestoreParentToChildRegionsPair {
+ required string parent_region_name = 1;
+ required string child1_region_name = 2;
+ required string child2_region_name = 3;
+}
+
+enum CloneSnapshotState {
+ CLONE_SNAPSHOT_PRE_OPERATION = 1;
+ CLONE_SNAPSHOT_WRITE_FS_LAYOUT = 2;
+ CLONE_SNAPSHOT_ADD_TO_META = 3;
+ CLONE_SNAPSHOT_ASSIGN_REGIONS = 4;
+ CLONE_SNAPSHOT_UPDATE_DESC_CACHE = 5;
+ CLONE_SNAPSHOT_POST_OPERATION = 6;
+}
+
+message CloneSnapshotStateData {
+ required UserInformation user_info = 1;
+ required SnapshotDescription snapshot = 2;
+ required TableSchema table_schema = 3;
+ repeated RegionInfo region_info = 4;
+ repeated RestoreParentToChildRegionsPair parent_to_child_regions_pair_list = 5;
+}
+
+enum RestoreSnapshotState {
+ RESTORE_SNAPSHOT_PRE_OPERATION = 1;
+ RESTORE_SNAPSHOT_UPDATE_TABLE_DESCRIPTOR = 2;
+ RESTORE_SNAPSHOT_WRITE_FS_LAYOUT = 3;
+ RESTORE_SNAPSHOT_UPDATE_META = 4;
+}
+
+message RestoreSnapshotStateData {
+ required UserInformation user_info = 1;
+ required SnapshotDescription snapshot = 2;
+ required TableSchema modified_table_schema = 3;
+ repeated RegionInfo region_info_for_restore = 4;
+ repeated RegionInfo region_info_for_remove = 5;
+ repeated RegionInfo region_info_for_add = 6;
+ repeated RestoreParentToChildRegionsPair parent_to_child_regions_pair_list = 7;
+}
+
message ServerCrashStateData {
required ServerName server_name = 1;
optional bool distributed_log_replay = 2;
diff --git a/if/RSGroup.proto b/if/RSGroup.proto
new file mode 100644
index 0000000..7358941
--- /dev/null
+++ b/if/RSGroup.proto
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "RSGroupProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+
+message RSGroupInfo {
+ required string name = 1;
+ repeated ServerName servers = 4;
+ repeated TableName tables = 3;
+}
+
diff --git a/if/RSGroupAdmin.proto b/if/RSGroupAdmin.proto
new file mode 100644
index 0000000..fda9b09
--- /dev/null
+++ b/if/RSGroupAdmin.proto
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "RSGroupAdminProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "RSGroup.proto";
+
+/** Group level protobufs */
+
+message ListTablesOfRSGroupRequest {
+ required string r_s_group_name = 1;
+}
+
+message ListTablesOfRSGroupResponse {
+ repeated TableName table_name = 1;
+}
+
+message GetRSGroupInfoRequest {
+ required string r_s_group_name = 1;
+}
+
+message GetRSGroupInfoResponse {
+ optional RSGroupInfo r_s_group_info = 1;
+}
+
+message GetRSGroupInfoOfTableRequest {
+ required TableName table_name = 1;
+}
+
+message GetRSGroupInfoOfTableResponse {
+ optional RSGroupInfo r_s_group_info = 1;
+}
+
+message MoveServersRequest {
+ required string target_group = 1;
+ repeated ServerName servers = 3;
+}
+
+message MoveServersResponse {
+}
+
+message MoveTablesRequest {
+ required string target_group = 1;
+ repeated TableName table_name = 2;
+}
+
+message MoveTablesResponse {
+}
+
+message AddRSGroupRequest {
+ required string r_s_group_name = 1;
+}
+
+message AddRSGroupResponse {
+}
+
+message RemoveRSGroupRequest {
+ required string r_s_group_name = 1;
+}
+
+message RemoveRSGroupResponse {
+}
+
+message BalanceRSGroupRequest {
+ required string r_s_group_name = 1;
+}
+
+message BalanceRSGroupResponse {
+ required bool balanceRan = 1;
+}
+
+message ListRSGroupInfosRequest {
+}
+
+message ListRSGroupInfosResponse {
+ repeated RSGroupInfo r_s_group_info = 1;
+}
+
+message GetRSGroupInfoOfServerRequest {
+ required ServerName server = 2;
+}
+
+message GetRSGroupInfoOfServerResponse {
+ optional RSGroupInfo r_s_group_info = 1;
+}
+
+service RSGroupAdminService {
+ rpc GetRSGroupInfo(GetRSGroupInfoRequest)
+ returns (GetRSGroupInfoResponse);
+
+ rpc GetRSGroupInfoOfTable(GetRSGroupInfoOfTableRequest)
+ returns (GetRSGroupInfoOfTableResponse);
+
+ rpc GetRSGroupInfoOfServer(GetRSGroupInfoOfServerRequest)
+ returns (GetRSGroupInfoOfServerResponse);
+
+ rpc MoveServers(MoveServersRequest)
+ returns (MoveServersResponse);
+
+ rpc MoveTables(MoveTablesRequest)
+ returns (MoveTablesResponse);
+
+ rpc AddRSGroup(AddRSGroupRequest)
+ returns (AddRSGroupResponse);
+
+ rpc RemoveRSGroup(RemoveRSGroupRequest)
+ returns (RemoveRSGroupResponse);
+
+ rpc BalanceRSGroup(BalanceRSGroupRequest)
+ returns (BalanceRSGroupResponse);
+
+ rpc ListRSGroupInfos(ListRSGroupInfosRequest)
+ returns (ListRSGroupInfosResponse);
+}
diff --git a/if/ZooKeeper.proto b/if/ZooKeeper.proto
index 54652af..186d183 100644
--- a/if/ZooKeeper.proto
+++ b/if/ZooKeeper.proto
@@ -105,6 +105,11 @@ message DeprecatedTableState {
required State state = 1 [default = ENABLED];
}
+message TableCF {
+ optional TableName table_name = 1;
+ repeated bytes families = 2;
+}
+
/**
* Used by replication. Holds a replication peer key.
*/
@@ -115,6 +120,7 @@ message ReplicationPeer {
optional string replicationEndpointImpl = 2;
repeated BytesBytesPair data = 3;
repeated NameStringPair configuration = 4;
+ repeated TableCF table_cfs = 5;
}
/**
@@ -153,3 +159,10 @@ message TableLock {
optional string purpose = 5;
optional int64 create_time = 6;
}
+
+/**
+ * State of the switch.
+ */
+message SwitchState {
+ optional bool enabled = 1;
+}
\ No newline at end of file
[hbase] 64/137: HBASE-17278 [C++] Cell Scanner and KeyValueCodec
for encoding cells in RPC (Sudeep Sunthankar and enis)
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 719615ae398028b8d01707309e9f1087c3d98188
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Thu Feb 9 11:26:47 2017 -0800
HBASE-17278 [C++] Cell Scanner and KeyValueCodec for encoding cells in RPC (Sudeep Sunthankar and enis)
---
.gitignore | 3 +-
core/BUCK | 2 +
core/keyvalue-codec.cc | 69 ++++++++++++++++++++++
core/keyvalue-codec.h | 147 ++++++++++++++++++++++++++++++++++++++++++++++
serde/BUCK | 3 +
serde/cell-outputstream.h | 53 +++++++++++++++++
serde/cell-scanner.h | 50 ++++++++++++++++
serde/codec.h | 50 ++++++++++++++++
8 files changed, 376 insertions(+), 1 deletion(-)
diff --git a/.gitignore b/.gitignore
index b172fe9..c0032e4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -22,4 +22,5 @@ buck-out
*.swp
# Thirdparty dirs
-third-party/googletest*
+third-party/*
+/gcc-debug/
diff --git a/core/BUCK b/core/BUCK
index f1880a4..d8d15a9 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -22,6 +22,7 @@ cxx_library(
"client.h",
"cell.h",
"hbase_macros.h",
+ "keyvalue-codec.h",
"region-location.h",
"location-cache.h",
# TODO: move this out of exported
@@ -40,6 +41,7 @@ cxx_library(
srcs=[
"cell.cc",
"client.cc",
+ "keyvalue-codec.cc",
"location-cache.cc",
"meta-utils.cc",
"get.cc",
diff --git a/core/keyvalue-codec.cc b/core/keyvalue-codec.cc
new file mode 100644
index 0000000..1b526b9
--- /dev/null
+++ b/core/keyvalue-codec.cc
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/keyvalue-codec.h"
+
+#include <string>
+
+namespace hbase {
+
+KeyValueCodec::KVDecoder::KVDecoder(std::unique_ptr<folly::IOBuf> cell_block, uint32_t offset,
+ uint32_t length)
+ : cell_block_(std::move(cell_block)), offset_(offset), length_(length) {}
+
+KeyValueCodec::KVDecoder::~KVDecoder() {}
+
+std::shared_ptr<Cell> KeyValueCodec::KVDecoder::Decode(folly::io::Cursor &cursor) {
+ uint32_t key_length = cursor.readBE<uint32_t>();
+ uint32_t value_length = cursor.readBE<uint32_t>();
+ uint16_t row_length = cursor.readBE<uint16_t>();
+ std::string row = cursor.readFixedString(row_length);
+ uint8_t column_family_length = cursor.readBE<uint8_t>();
+ std::string column_family = cursor.readFixedString(column_family_length);
+ int qualifier_length =
+ key_length - (row_length + column_family_length + kHBaseSizeOfKeyInfrastructure_);
+ std::string column_qualifier = cursor.readFixedString(qualifier_length);
+ uint64_t timestamp = cursor.readBE<uint64_t>();
+ uint8_t key_type = cursor.readBE<uint8_t>();
+ std::string value = cursor.readFixedString(value_length);
+
+ return std::make_shared<Cell>(row, column_family, column_qualifier, timestamp, value,
+ static_cast<hbase::CellType>(key_type));
+}
+
+bool KeyValueCodec::KVDecoder::Advance() {
+ if (end_of_cell_block_) {
+ return false;
+ }
+
+ if (cur_pos_ == length_) {
+ end_of_cell_block_ = true;
+ return false;
+ }
+
+ folly::io::Cursor cursor(cell_block_.get());
+ cursor.skip(offset_ + cur_pos_);
+ uint32_t current_cell_size = cursor.readBE<uint32_t>();
+ current_cell_ = Decode(cursor);
+ cur_pos_ += kHBaseSizeOfInt_ + current_cell_size;
+ return true;
+}
+
+uint32_t KeyValueCodec::KVDecoder::CellBlockLength() const { return length_; }
+} /* namespace hbase */
diff --git a/core/keyvalue-codec.h b/core/keyvalue-codec.h
new file mode 100644
index 0000000..fd58346
--- /dev/null
+++ b/core/keyvalue-codec.h
@@ -0,0 +1,147 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <folly/io/Cursor.h>
+#include <folly/io/IOBuf.h>
+#include <memory>
+
+#include "core/cell.h"
+#include "serde/codec.h"
+
+namespace hbase {
+
+/**
+ * @brief Class for parsing sequence of Cells based on org.apache.hadoop.hbase.KeyValueCodec.java
+ *
+ * KeyValueCodec implements CellScanner interface. Sequence of cells are obtained from cell_block.
+ * We have CreateEncoder and CreateDecoder public methods which will return Encoder/Decoder
+ * instances which will be used to obtain individual cells in cell_block.
+ * Usage:-
+ * 1) Cell Decoding:-
+ * unique_ptr<CellScanner> cell_scanner = KeyValueCodec::CreateDecoder(cell_block, cb_start_offset,
+ * cb_length);
+ * while (cell_scanner->Advance()) {
+ * auto current_cell = cell_scanner->Current
+ * }
+ */
+class KeyValueCodec : public Codec {
+ public:
+ /**
+ * Constructor
+ */
+ KeyValueCodec() {}
+
+ std::unique_ptr<Codec::Encoder> CreateEncoder() override { return std::make_unique<KVEncoder>(); }
+ std::unique_ptr<Codec::Decoder> CreateDecoder(std::unique_ptr<folly::IOBuf> cell_block,
+ uint32_t offset, uint32_t length) override {
+ return std::make_unique<KVDecoder>(std::move(cell_block), offset, length);
+ }
+
+ /** @brief returns the java class name corresponding to this Codec implementation */
+ virtual const char* java_class_name() const override { return kJavaClassName; }
+
+ static constexpr const char* kJavaClassName = "org.apache.hadoop.hbase.codec.KeyValueCodec";
+
+ private:
+ class KVEncoder : public Codec::Encoder {
+ public:
+ KVEncoder() {}
+
+ void Write(const Cell& cell) {
+ // TODO: Encode Cells using KeyValueCodec wire format
+ }
+
+ void Flush() {}
+ };
+
+ class KVDecoder : public Codec::Decoder {
+ public:
+ KVDecoder(std::unique_ptr<folly::IOBuf> cell_block, uint32_t cell_block_start_offset,
+ uint32_t cell_block_length);
+ ~KVDecoder();
+
+ /**
+ * @brief Overridden from CellScanner. This method parses cell_block and stores the current in
+ * current_cell_. Current cell can be obtained using cell_scanner.Current();
+ */
+ bool Advance();
+
+ /**
+ * @brief returns the current cell
+ */
+ const std::shared_ptr<Cell> Current() const { return current_cell_; }
+
+ /**
+ * @brief returns the total length of cell_meta_block
+ */
+ uint32_t CellBlockLength() const;
+
+ private:
+ std::shared_ptr<Cell> Decode(folly::io::Cursor& cursor);
+
+ /**
+ * Size of boolean in bytes
+ */
+ const int kHBaseSizeOfBoolean_ = sizeof(uint8_t) / sizeof(uint8_t);
+
+ /**
+ * Size of byte in bytes
+ */
+ const uint8_t kHBaseSizeOfByte_ = kHBaseSizeOfBoolean_;
+
+ /**
+ * Size of int in bytes
+ */
+ const uint32_t kHBaseSizeOfInt_ = sizeof(uint32_t) / kHBaseSizeOfByte_;
+
+ /**
+ * Size of long in bytes
+ */
+ const uint64_t kHBaseSizeOfLong_ = sizeof(uint64_t) / kHBaseSizeOfByte_;
+
+ /**
+ * Size of Short in bytes
+ */
+ const uint16_t kHBaseSizeOfShort_ = sizeof(uint16_t) / kHBaseSizeOfByte_;
+
+ const uint32_t kHBaseSizeOfKeyLength_ = kHBaseSizeOfInt_;
+ const uint32_t kHBaseSizeOfValueLength_ = kHBaseSizeOfInt_;
+ const uint16_t kHBaseSizeOfRowLength_ = kHBaseSizeOfShort_;
+ const uint8_t kHBaseSizeOfFamilyLength_ = kHBaseSizeOfByte_;
+ const uint64_t kHBaseSizeOfTimestamp_ = kHBaseSizeOfLong_;
+ const uint8_t kHBaseSizeOfKeyType_ = kHBaseSizeOfByte_;
+ const uint32_t kHBaseSizeOfTimestampAndKey_ = kHBaseSizeOfTimestamp_ + kHBaseSizeOfKeyType_;
+ const uint32_t kHBaseSizeOfKeyInfrastructure_ =
+ kHBaseSizeOfRowLength_ + kHBaseSizeOfFamilyLength_ + kHBaseSizeOfTimestampAndKey_;
+ const uint32_t kHBaseSizeOfKeyValueInfrastructure_ =
+ kHBaseSizeOfKeyLength_ + kHBaseSizeOfValueLength_;
+
+ std::unique_ptr<folly::IOBuf> cell_block_ = nullptr;
+ uint32_t offset_ = 0;
+ uint32_t length_ = 0;
+ uint32_t cur_pos_ = 0;
+ bool end_of_cell_block_ = false;
+
+ std::shared_ptr<Cell> current_cell_ = nullptr;
+ };
+};
+
+} /* namespace hbase */
diff --git a/serde/BUCK b/serde/BUCK
index b5aad3d..c6809a8 100644
--- a/serde/BUCK
+++ b/serde/BUCK
@@ -18,6 +18,9 @@
cxx_library(
name="serde",
exported_headers=[
+ "cell-scanner.h",
+ "cell-outputstream.h",
+ "codec.h",
"region-info.h",
"rpc.h",
"server-name.h",
diff --git a/serde/cell-outputstream.h b/serde/cell-outputstream.h
new file mode 100644
index 0000000..963dd31
--- /dev/null
+++ b/serde/cell-outputstream.h
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <memory>
+
+namespace hbase {
+
+class Cell;
+
+/**
+ * @brief Encoder / Decoder for Cells.
+ */
+class CellOutputStream {
+ public:
+ virtual ~CellOutputStream() {}
+
+ /**
+ * Implementation must copy the entire state of the Cell. If the written Cell is modified
+ * immediately after the write method returns, the modifications must have absolutely no effect
+ * on the copy of the Cell that was added in the write.
+ * @param cell Cell to write out
+ * @throws IOException
+ */
+ virtual void Write(const Cell& cell) = 0;
+
+ /**
+ * Let the implementation decide what to do. Usually means writing accumulated data into a
+ * byte[] that can then be read from the implementation to be sent to disk, put in the block
+ * cache, or sent over the network.
+ * @throws IOException
+ */
+ virtual void Flush() = 0;
+};
+
+} /* namespace hbase */
diff --git a/serde/cell-scanner.h b/serde/cell-scanner.h
new file mode 100644
index 0000000..fe4a249
--- /dev/null
+++ b/serde/cell-scanner.h
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+#include <folly/io/IOBuf.h>
+#include <memory>
+
+namespace hbase {
+
+class Cell;
+
+/**
+ * @brief Interface for iterating over a sequence of Cells
+ */
+class CellScanner {
+ public:
+ virtual ~CellScanner() {}
+
+ /**
+ * @brief This method will be used to iterate the cells.
+ * Typical usage will be :-
+ * while(cell_scanner.Advance()){
+ * auto current_cell = cell_scanner.Current();
+ * }
+ */
+ virtual bool Advance() = 0;
+
+ /**
+ * @brief returns the current cell
+ */
+ virtual const std::shared_ptr<Cell> Current() const = 0;
+};
+
+} /* namespace hbase */
diff --git a/serde/codec.h b/serde/codec.h
new file mode 100644
index 0000000..64807dc
--- /dev/null
+++ b/serde/codec.h
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <folly/io/IOBuf.h>
+#include <memory>
+
+#include "serde/cell-outputstream.h"
+#include "serde/cell-scanner.h"
+
+namespace hbase {
+
+/**
+ * @brief Encoder / Decoder for Cells.
+ */
+class Codec {
+ public:
+ virtual ~Codec() {}
+
+ class Encoder : public CellOutputStream {};
+
+ class Decoder : public CellScanner {};
+
+ virtual std::unique_ptr<Encoder> CreateEncoder() = 0;
+ virtual std::unique_ptr<Decoder> CreateDecoder(std::unique_ptr<folly::IOBuf> cell_block,
+ uint32_t cell_block_start_offset,
+ uint32_t cell_block_length) = 0;
+
+ /** @brief returns the java class name corresponding to this Codec implementation */
+ virtual const char* java_class_name() const = 0;
+};
+
+} /* namespace hbase */
[hbase] 94/137: HBASE-18126 Increment class - addendum changes
table name for test
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 14c2d8730888bc4b7a371501a0a93f993fa7fdae
Author: tedyu <yu...@gmail.com>
AuthorDate: Tue Jun 6 16:28:32 2017 -0700
HBASE-18126 Increment class - addendum changes table name for test
---
core/client-test.cc | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/core/client-test.cc b/core/client-test.cc
index 743e928..d166f1c 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -191,10 +191,10 @@ TEST_F(ClientTest, PutGetDelete) {
TEST_F(ClientTest, Increment) {
// Using TestUtil to populate test data
- ClientTest::test_util->CreateTable("t", "d");
+ ClientTest::test_util->CreateTable("t1", "d");
// Create TableName and Row to be fetched from HBase
- auto tn = folly::to<hbase::pb::TableName>("t");
+ auto tn = folly::to<hbase::pb::TableName>("t1");
auto row = "test1";
// Create a client
[hbase] 34/137: HBASE-15823 Use call once for user util
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 32a5e54544cfa78b5f4dd97fa997d4da3c35dd2e
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Fri May 13 13:07:03 2016 -0700
HBASE-15823 Use call once for user util
---
utils/user-util.cc | 13 ++-----------
utils/user-util.h | 6 ++----
2 files changed, 4 insertions(+), 15 deletions(-)
diff --git a/utils/user-util.cc b/utils/user-util.cc
index 3d963b3..9e170e0 100644
--- a/utils/user-util.cc
+++ b/utils/user-util.cc
@@ -27,22 +27,14 @@
using namespace hbase;
using namespace std;
-UserUtil::UserUtil() : init_{false}, user_name_{"drwho"}, m_() {}
+UserUtil::UserUtil() : once_flag_{}, user_name_{"drwho"} {}
string UserUtil::user_name() {
- if (!init_) {
- compute_user_name();
- }
+ std::call_once(once_flag_, [this]() { compute_user_name(); });
return user_name_;
}
void UserUtil::compute_user_name() {
- lock_guard<mutex> lock(m_);
-
- if (init_) {
- return;
- }
-
// According to the man page of getpwuid
// this should never be free'd
//
@@ -52,6 +44,5 @@ void UserUtil::compute_user_name() {
// make sure that we got something.
if (passwd && passwd->pw_name) {
user_name_ = string{passwd->pw_name};
- init_ = true;
}
}
diff --git a/utils/user-util.h b/utils/user-util.h
index 0b4cc73..fdfc0c8 100644
--- a/utils/user-util.h
+++ b/utils/user-util.h
@@ -19,8 +19,7 @@
#pragma once
-#include <atomic>
-#include <memory>
+#include <string>
#include <mutex>
namespace hbase {
@@ -49,8 +48,7 @@ private:
* Compute the username. This will block.
*/
void compute_user_name();
- std::atomic<bool> init_;
+ std::once_flag once_flag_;
std::string user_name_;
- std::mutex m_;
};
} // namespace hbase
[hbase] 13/137: HBASE-14855 Connect to regionserver
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 5c167a04929697da49ed1658192a1f61215e9c51
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Mon Apr 4 13:52:05 2016 -0700
HBASE-14855 Connect to regionserver
Summary:
* Client dispatcher to put request and response together with call id. Though right now this is a pretty empty version of that.
* pipeline. The Serialization pipeline.
* client-serilization-handler.h This does the work of sending the preamable, the connection header, and length prepending data. This will obviously need to be split up into h and cc files
Test Plan: Use simple-client to connect to a standalone server
Differential Revision: https://reviews.facebook.net/D56385
---
core/BUCK | 30 ++++++--
core/client-dispatcher.cc | 54 +++++++++++++
core/{client.h => client-dispatcher.h} | 22 +++++-
core/client-serialize-handler.cc | 104 +++++++++++++++++++++++++
core/{client.h => client-serialize-handler.h} | 26 ++++++-
core/client.cc | 16 ++--
core/client.h | 18 ++++-
core/connection-factory.cc | 57 ++++++++++++++
core/{client.h => connection-factory.h} | 23 +++++-
core/{client.h => get-request.cc} | 9 +--
core/{client.h => get-request.h} | 17 ++++-
core/{client.h => get-result.cc} | 9 +--
core/{client.h => get-result.h} | 14 +++-
core/location-cache-test.cc | 25 +++++-
core/location-cache.cc | 62 ++++++++++++---
core/location-cache.h | 29 +++++--
core/native-client-test-env.cc | 6 +-
core/{client.cc => pipeline.cc} | 34 ++++-----
core/{client.h => pipeline.h} | 16 +++-
core/{client.h => request.h} | 15 +++-
core/{client.h => response.h} | 16 +++-
core/{client.h => service.h} | 10 +--
core/{client.cc => simple-client.cc} | 31 ++++++--
core/simple-native-client-test.cc | 5 +-
core/{client.h => table-name.cc} | 9 +--
core/{client.h => table-name.h} | 14 +++-
if/BUCK | 24 +++++-
third-party/BUCK | 105 +++++++++++++-------------
28 files changed, 615 insertions(+), 185 deletions(-)
diff --git a/core/BUCK b/core/BUCK
index d1e89d1..2b00d66 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -16,35 +16,52 @@
# limitations under the License.
cxx_library(name="core",
- headers=[
+ exported_headers=[
"admin.h",
+ "client-dispatcher.h",
+ "client-serialize-handler.h",
"client.h",
+ "connection-factory.h",
"connection.h",
"connection_attr.h",
"delete.h",
+ "get-request.h",
+ "get-result.h",
"get.h",
"hbase_macros.h",
+ "location-cache.h",
"mutation.h",
+ "pipeline.h",
"put.h",
+ "request.h",
+ "response.h",
"scanner.h",
- "location-cache.h",
+ "service.h",
+ "table-name.h",
],
srcs=[
"admin.cc",
+ "client-dispatcher.cc",
+ "client-serialize-handler.cc",
"client.cc",
+ "connection-factory.cc",
"connection.cc",
+ "delete.cc",
+ "get-request.cc",
+ "get-result.cc",
"get.cc",
+ "location-cache.cc",
"mutation.cc",
+ "pipeline.cc",
"put.cc",
- "delete.cc",
"scanner.cc",
- "location-cache.cc",
+ "table-name.cc",
],
deps=[
"//if:if",
- "//third-party:zookeeper_mt",
"//third-party:folly",
"//third-party:wangle",
+ "//third-party:zookeeper_mt",
],
visibility=[
'PUBLIC',
@@ -68,3 +85,6 @@ cxx_test(name="location-cache-test",
":core",
],
run_test_separately=True, )
+cxx_binary(name="simple-client",
+ srcs=["simple-client.cc", ],
+ deps=[":core", ], )
diff --git a/core/client-dispatcher.cc b/core/client-dispatcher.cc
new file mode 100644
index 0000000..d356759
--- /dev/null
+++ b/core/client-dispatcher.cc
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include "core/client-dispatcher.h"
+
+using namespace folly;
+using namespace hbase;
+using namespace wangle;
+
+void ClientDispatcher::read(Context *ctx, Response in) {
+ auto call_id = in.call_id();
+ auto search = requests_.find(call_id);
+ CHECK(search != requests_.end());
+ auto p = std::move(search->second);
+ requests_.erase(call_id);
+
+ // TODO(eclark): check if the response
+ // is an exception. If it is then set that.
+ p.setValue(in);
+}
+
+Future<Response> ClientDispatcher::operator()(Request arg) {
+ auto call_id = ++current_call_id_;
+ arg.set_call_id(call_id);
+ auto &p = requests_[call_id];
+ auto f = p.getFuture();
+ p.setInterruptHandler([call_id, this](const folly::exception_wrapper &e) {
+ this->requests_.erase(call_id);
+ });
+ this->pipeline_->write(arg);
+
+ return f;
+}
+
+Future<Unit> ClientDispatcher::close() { return ClientDispatcherBase::close(); }
+
+Future<Unit> ClientDispatcher::close(Context *ctx) {
+ return ClientDispatcherBase::close(ctx);
+}
diff --git a/core/client.h b/core/client-dispatcher.h
similarity index 56%
copy from core/client.h
copy to core/client-dispatcher.h
index 35a3bd8..4b9d35a 100644
--- a/core/client.h
+++ b/core/client-dispatcher.h
@@ -19,8 +19,24 @@
#pragma once
-#include <folly/io/IOBuf.h>
+#include <wangle/service/ClientDispatcher.h>
-#include "if/Cell.pb.h"
+#include "core/pipeline.h"
+#include "core/request.h"
+#include "core/response.h"
-class Client {};
+namespace hbase {
+class ClientDispatcher
+ : public wangle::ClientDispatcherBase<SerializePipeline, Request,
+ Response> {
+public:
+ void read(Context *ctx, Response in) override;
+ folly::Future<Response> operator()(Request arg) override;
+ folly::Future<folly::Unit> close(Context *ctx) override;
+ folly::Future<folly::Unit> close() override;
+
+private:
+ std::unordered_map<int32_t, folly::Promise<Response>> requests_;
+ uint32_t current_call_id_ = 1;
+};
+} // namespace hbase
diff --git a/core/client-serialize-handler.cc b/core/client-serialize-handler.cc
new file mode 100644
index 0000000..cad1308
--- /dev/null
+++ b/core/client-serialize-handler.cc
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/client-serialize-handler.h"
+
+#include <string>
+
+using namespace hbase;
+using namespace folly;
+using namespace wangle;
+
+static const std::string PREAMBLE = "HBas";
+static const std::string INTERFACE = "ClientService";
+static const uint8_t RPC_VERSION = 0;
+static const uint8_t AUTH_TYPE = 80;
+
+// TODO(eclark): Make this actually do ANYTHING.
+void ClientSerializeHandler::read(Context *ctx, std::unique_ptr<IOBuf> msg) {
+ Response received;
+ ctx->fireRead(received);
+}
+
+Future<Unit> ClientSerializeHandler::write(Context *ctx, Request r) {
+ // Keep track of if we have sent the header.
+ if (need_send_header_) {
+ need_send_header_ = false;
+
+ // Should this be replacing the IOBuf rather than
+ // sending several different calls?
+ write_preamble(ctx);
+ write_header(ctx);
+ }
+
+ // Send out the actual request and not just a test string.
+ std::string out{"test"};
+ return ctx->fireWrite(prepend_length(IOBuf::copyBuffer(out)));
+}
+
+Future<Unit> ClientSerializeHandler::write_preamble(Context *ctx) {
+ auto magic = IOBuf::copyBuffer(PREAMBLE);
+ auto buf = IOBuf::create(2);
+ buf->append(2);
+ folly::io::RWPrivateCursor c(buf.get());
+
+ // Version
+ c.write(RPC_VERSION);
+ // Standard security aka Please don't lie to me.
+ c.write(AUTH_TYPE);
+ magic->appendChain(std::move(buf));
+ return ctx->fireWrite(std::move(magic));
+}
+
+Future<Unit> ClientSerializeHandler::write_header(Context *ctx) {
+ pb::ConnectionHeader h;
+
+ // TODO(eclark): Make this not a total lie.
+ h.mutable_user_info()->set_effective_user("elliott");
+ // The service name that we want to talk to.
+ //
+ // Right now we're completely ignoring the service interface.
+ // That may or may not be the correct thing to do.
+ // It worked for a while with the java client; until it
+ // didn't.
+ h.set_service_name(INTERFACE);
+ // TODO(eclark): Make this 1 copy.
+ auto msg = IOBuf::copyBuffer(h.SerializeAsString());
+ return ctx->fireWrite(prepend_length(std::move(msg)));
+}
+
+// Our own simple version of LengthFieldPrepender
+std::unique_ptr<IOBuf>
+ClientSerializeHandler::prepend_length(std::unique_ptr<IOBuf> msg) {
+ // Java ints are 4 long. So create a buffer that large
+ auto len_buf = IOBuf::create(4);
+ // Then make those bytes visible.
+ len_buf->append(4);
+
+ io::RWPrivateCursor c(len_buf.get());
+ // Get the size of the data to be pushed out the network.
+ auto size = msg->computeChainDataLength();
+
+ // Write the length to this IOBuf.
+ c.writeBE(static_cast<uint32_t>(size));
+
+ // Then attach the origional to the back of len_buf
+ len_buf->appendChain(std::move(msg));
+ return len_buf;
+}
diff --git a/core/client.h b/core/client-serialize-handler.h
similarity index 50%
copy from core/client.h
copy to core/client-serialize-handler.h
index 35a3bd8..961a03b 100644
--- a/core/client.h
+++ b/core/client-serialize-handler.h
@@ -16,11 +16,29 @@
* limitations under the License.
*
*/
-
#pragma once
-#include <folly/io/IOBuf.h>
+#include <wangle/channel/Handler.h>
+
+#include "if/HBase.pb.h"
+#include "if/RPC.pb.h"
+#include "core/request.h"
+#include "core/response.h"
-#include "if/Cell.pb.h"
+namespace hbase {
+class ClientSerializeHandler
+ : public wangle::Handler<std::unique_ptr<folly::IOBuf>, Response, Request,
+ std::unique_ptr<folly::IOBuf>> {
+public:
+ void read(Context *ctx, std::unique_ptr<folly::IOBuf> msg) override;
+ folly::Future<folly::Unit> write(Context *ctx, Request r) override;
-class Client {};
+private:
+ folly::Future<folly::Unit> write_preamble(Context *ctx);
+ folly::Future<folly::Unit> write_header(Context *ctx);
+ // Our own simple version of LengthFieldPrepender
+ std::unique_ptr<folly::IOBuf>
+ prepend_length(std::unique_ptr<folly::IOBuf> msg);
+ bool need_send_header_ = true;
+};
+} // namespace hbase
diff --git a/core/client.cc b/core/client.cc
index a04daee..893894f 100644
--- a/core/client.cc
+++ b/core/client.cc
@@ -23,20 +23,18 @@
#include <folly/Random.h>
#include <glog/logging.h>
#include <gflags/gflags.h>
+#include <wangle/concurrent/GlobalExecutor.h>
+
+#include <string>
#include "if/ZooKeeper.pb.h"
using namespace folly;
+using namespace std;
using namespace hbase::pb;
-int main(int argc, char *argv[]) {
- MetaRegionServer mrs;
- google::ParseCommandLineFlags(&argc, &argv, true);
- google::InitGoogleLogging(argv[0]);
+namespace hbase {
- FB_LOG_EVERY_MS(INFO, 10000) << "Hello";
- for (long i = 0; i < 10000000; i++) {
- FB_LOG_EVERY_MS(INFO, 1) << Random::rand32();
- }
- return 0;
+Client::Client(string quorum_spec)
+ : location_cache(quorum_spec, wangle::getCPUExecutor()) {}
}
diff --git a/core/client.h b/core/client.h
index 35a3bd8..818bc6b 100644
--- a/core/client.h
+++ b/core/client.h
@@ -20,7 +20,23 @@
#pragma once
#include <folly/io/IOBuf.h>
+#include <folly/futures/Future.h>
+#include <string>
+
+#include "core/get-request.h"
+#include "core/get-result.h"
+#include "core/location-cache.h"
#include "if/Cell.pb.h"
-class Client {};
+namespace hbase {
+class Client {
+public:
+ explicit Client(std::string quorum_spec);
+ folly::Future<GetResult> get(const GetRequest &get_request);
+
+private:
+ LocationCache location_cache;
+};
+
+} // namespace hbase
diff --git a/core/connection-factory.cc b/core/connection-factory.cc
new file mode 100644
index 0000000..785b239
--- /dev/null
+++ b/core/connection-factory.cc
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/connection-factory.h"
+
+#include <wangle/channel/AsyncSocketHandler.h>
+#include <wangle/channel/EventBaseHandler.h>
+#include <wangle/channel/OutputBufferingHandler.h>
+#include <wangle/service/ClientDispatcher.h>
+#include <wangle/service/ExpiringFilter.h>
+#include <folly/futures/Future.h>
+
+#include <string>
+
+#include "core/client-dispatcher.h"
+#include "core/pipeline.h"
+#include "core/request.h"
+#include "core/response.h"
+#include "core/service.h"
+
+using namespace folly;
+using namespace hbase;
+using namespace wangle;
+
+ConnectionFactory::ConnectionFactory() {
+ bootstrap_.group(std::make_shared<wangle::IOThreadPoolExecutor>(2));
+ bootstrap_.pipelineFactory(std::make_shared<RpcPipelineFactory>());
+}
+
+Future<ClientDispatcher> ConnectionFactory::make_connection(std::string host,
+ int port) {
+ // Connect to a given server
+ // Then when connected create a ClientDispactcher.
+ auto srv = bootstrap_.connect(SocketAddress(host, port, true))
+ .then([](SerializePipeline *pipeline) {
+ ClientDispatcher dispatcher;
+ dispatcher.setPipeline(pipeline);
+ return dispatcher;
+ });
+ return srv;
+}
diff --git a/core/client.h b/core/connection-factory.h
similarity index 62%
copy from core/client.h
copy to core/connection-factory.h
index 35a3bd8..6f450c2 100644
--- a/core/client.h
+++ b/core/connection-factory.h
@@ -16,11 +16,26 @@
* limitations under the License.
*
*/
-
#pragma once
-#include <folly/io/IOBuf.h>
+#include <wangle/bootstrap/ClientBootstrap.h>
+#include <wangle/service/Service.h>
+
+#include <string>
+
+#include "core/service.h"
+#include "core/pipeline.h"
+#include "core/client-dispatcher.h"
+#include "core/request.h"
+#include "core/response.h"
-#include "if/Cell.pb.h"
+namespace hbase {
+class ConnectionFactory {
+public:
+ ConnectionFactory();
+ folly::Future<ClientDispatcher> make_connection(std::string host, int port);
-class Client {};
+private:
+ wangle::ClientBootstrap<SerializePipeline> bootstrap_;
+};
+} // namespace hbase
diff --git a/core/client.h b/core/get-request.cc
similarity index 90%
copy from core/client.h
copy to core/get-request.cc
index 35a3bd8..e927ccc 100644
--- a/core/client.h
+++ b/core/get-request.cc
@@ -16,11 +16,4 @@
* limitations under the License.
*
*/
-
-#pragma once
-
-#include <folly/io/IOBuf.h>
-
-#include "if/Cell.pb.h"
-
-class Client {};
+#include "core/get-request.h"
diff --git a/core/client.h b/core/get-request.h
similarity index 78%
copy from core/client.h
copy to core/get-request.h
index 35a3bd8..c9113ad 100644
--- a/core/client.h
+++ b/core/get-request.h
@@ -16,11 +16,20 @@
* limitations under the License.
*
*/
-
#pragma once
-#include <folly/io/IOBuf.h>
+#include <string>
+
+#include "core/table-name.h"
+
+namespace hbase {
-#include "if/Cell.pb.h"
+class GetRequest {
+public:
+ GetRequest(TableName table_name, std::string key);
-class Client {};
+private:
+ TableName table_name_;
+ std::string key_;
+};
+} // namespace hbase
diff --git a/core/client.h b/core/get-result.cc
similarity index 90%
copy from core/client.h
copy to core/get-result.cc
index 35a3bd8..7eea483 100644
--- a/core/client.h
+++ b/core/get-result.cc
@@ -16,11 +16,4 @@
* limitations under the License.
*
*/
-
-#pragma once
-
-#include <folly/io/IOBuf.h>
-
-#include "if/Cell.pb.h"
-
-class Client {};
+#include "core/get-result.h"
diff --git a/core/client.h b/core/get-result.h
similarity index 84%
copy from core/client.h
copy to core/get-result.h
index 35a3bd8..e021316 100644
--- a/core/client.h
+++ b/core/get-result.h
@@ -16,11 +16,17 @@
* limitations under the License.
*
*/
-
#pragma once
-#include <folly/io/IOBuf.h>
+#include <string>
+
+namespace hbase {
-#include "if/Cell.pb.h"
+class GetResult {
+public:
+ explicit GetResult(std::string key);
-class Client {};
+private:
+ std::string key_;
+};
+} // namespace hbase
diff --git a/core/location-cache-test.cc b/core/location-cache-test.cc
index 3106e36..70ca6f1 100644
--- a/core/location-cache-test.cc
+++ b/core/location-cache-test.cc
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
#include <gtest/gtest.h>
#include <folly/Memory.h>
#include <wangle/concurrent/GlobalExecutor.h>
@@ -8,7 +26,8 @@ using namespace hbase;
TEST(LocationCacheTest, TestGetMetaNodeContents) {
// TODO(elliott): need to make a test utility for this.
LocationCache cache{"localhost:2181", wangle::getCPUExecutor()};
- auto result = cache.LocateMeta();
- result.wait();
- ASSERT_FALSE(result.hasException());
+ auto f = cache.LocateMeta();
+ auto result = f.get();
+ ASSERT_FALSE(f.hasException());
+ ASSERT_TRUE(result.has_port());
}
diff --git a/core/location-cache.cc b/core/location-cache.cc
index cf61e24..34e3236 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
#include "location-cache.h"
#include <folly/Logging.h>
@@ -7,11 +25,10 @@
using namespace std;
using namespace folly;
using namespace hbase::pb;
+using namespace hbase;
-namespace hbase {
-
-// TODO(elliott): make this configurable on client creation
-const static string META_LOCATION = "/hbase/meta-region-server";
+// TODO(eclark): make this configurable on client creation
+static const char META_LOCATION[] = "/hbase/meta-region-server";
LocationCache::LocationCache(string quorum_spec,
shared_ptr<folly::Executor> executor)
@@ -50,18 +67,45 @@ void LocationCache::RefreshMetaLocation() {
ServerName LocationCache::ReadMetaLocation() {
char contents[4096];
+ // This needs to be int rather than size_t as that's what ZK expects.
int len = sizeof(contents);
// TODO(elliott): handle disconnects/reconntion as needed.
int zk_result =
- zoo_get(this->zk_, META_LOCATION.c_str(), 0, contents, &len, nullptr);
-
- if (zk_result != ZOK) {
+ zoo_get(this->zk_, META_LOCATION, 0, contents, &len, nullptr);
+ if (zk_result != ZOK || len < 9) {
LOG(ERROR) << "Error getting meta location.";
throw runtime_error("Error getting meta location");
}
+ // There should be a magic number for recoverable zk
+ if (static_cast<uint8_t>(contents[0]) != 255) {
+ LOG(ERROR) << "Magic number not in ZK znode data expected 255 got ="
+ << unsigned(static_cast<uint8_t>(contents[0]));
+ throw runtime_error("Magic number not in znode data");
+ }
+ // pos will keep track of skipped bytes.
+ int pos = 1;
+ // How long is the id?
+ int id_len = 0;
+ for (int i = 0; i < 4; ++i) {
+ id_len = id_len << 8;
+ id_len = id_len | static_cast<uint8_t>(contents[pos]);
+ ++pos;
+ }
+ // Skip the id
+ pos += id_len;
+ // Then all protobuf's for HBase are prefixed with a magic string.
+ // PBUF, so we skip that.
+ // TODO(eclark): check to make sure that the magic string is correct
+ // though I am not sure that will get us much.
+ pos += 4;
MetaRegionServer mrs;
- mrs.ParseFromArray(contents, len);
+ // Try to decode the protobuf.
+ // If there's an error bail out.
+ if (mrs.ParseFromArray(contents + pos, len - pos) == false) {
+ LOG(ERROR) << "Error parsing Protobuf Message";
+ throw runtime_error("Error parsing protobuf");
+ }
+
return mrs.server();
}
-}
diff --git a/core/location-cache.h b/core/location-cache.h
index 8dc2760..efcfde5 100644
--- a/core/location-cache.h
+++ b/core/location-cache.h
@@ -1,13 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
#pragma once
-#include <memory>
-#include <mutex>
-
#include <zookeeper/zookeeper.h>
#include <folly/futures/Future.h>
#include <folly/futures/SharedPromise.h>
-
#include <folly/Executor.h>
+
+#include <memory>
+#include <mutex>
+#include <string>
+
#include "if/HBase.pb.h"
namespace hbase {
@@ -32,4 +51,4 @@ private:
zhandle_t *zk_;
};
-} // hbase
+} // namespace hbase
diff --git a/core/native-client-test-env.cc b/core/native-client-test-env.cc
index 07f30a6..0269a43 100644
--- a/core/native-client-test-env.cc
+++ b/core/native-client-test-env.cc
@@ -22,7 +22,7 @@
namespace {
class NativeClientTestEnv : public ::testing::Environment {
- public:
+public:
void SetUp() override {
// start local HBase cluster to be reused by all tests
auto result = system("bin/start_local_hbase_and_wait.sh");
@@ -36,9 +36,9 @@ class NativeClientTestEnv : public ::testing::Environment {
}
};
-} // anonymous
+} // anonymous
-int main(int argc, char** argv) {
+int main(int argc, char **argv) {
testing::InitGoogleTest(&argc, argv);
::testing::AddGlobalTestEnvironment(new NativeClientTestEnv());
return RUN_ALL_TESTS();
diff --git a/core/client.cc b/core/pipeline.cc
similarity index 55%
copy from core/client.cc
copy to core/pipeline.cc
index a04daee..30d14ff 100644
--- a/core/client.cc
+++ b/core/pipeline.cc
@@ -16,27 +16,27 @@
* limitations under the License.
*
*/
-
-#include "core/client.h"
+#include "core/pipeline.h"
#include <folly/Logging.h>
-#include <folly/Random.h>
-#include <glog/logging.h>
-#include <gflags/gflags.h>
+#include <wangle/channel/AsyncSocketHandler.h>
+#include <wangle/channel/EventBaseHandler.h>
+#include <wangle/channel/OutputBufferingHandler.h>
+#include <wangle/codec/LengthFieldBasedFrameDecoder.h>
-#include "if/ZooKeeper.pb.h"
+#include "core/client-serialize-handler.h"
using namespace folly;
-using namespace hbase::pb;
-
-int main(int argc, char *argv[]) {
- MetaRegionServer mrs;
- google::ParseCommandLineFlags(&argc, &argv, true);
- google::InitGoogleLogging(argv[0]);
+using namespace hbase;
+using namespace wangle;
- FB_LOG_EVERY_MS(INFO, 10000) << "Hello";
- for (long i = 0; i < 10000000; i++) {
- FB_LOG_EVERY_MS(INFO, 1) << Random::rand32();
- }
- return 0;
+SerializePipeline::Ptr
+RpcPipelineFactory::newPipeline(std::shared_ptr<AsyncTransportWrapper> sock) {
+ auto pipeline = SerializePipeline::create();
+ pipeline->addBack(AsyncSocketHandler(sock));
+ pipeline->addBack(EventBaseHandler());
+ pipeline->addBack(LengthFieldBasedFrameDecoder());
+ pipeline->addBack(ClientSerializeHandler());
+ pipeline->finalize();
+ return pipeline;
}
diff --git a/core/client.h b/core/pipeline.h
similarity index 65%
copy from core/client.h
copy to core/pipeline.h
index 35a3bd8..d199d08 100644
--- a/core/client.h
+++ b/core/pipeline.h
@@ -16,11 +16,19 @@
* limitations under the License.
*
*/
-
#pragma once
-#include <folly/io/IOBuf.h>
+#include <wangle/service/Service.h>
+#include <folly/io/IOBufQueue.h>
+#include "core/request.h"
+#include "core/response.h"
-#include "if/Cell.pb.h"
+namespace hbase {
+using SerializePipeline = wangle::Pipeline<folly::IOBufQueue &, Request>;
-class Client {};
+class RpcPipelineFactory : public wangle::PipelineFactory<SerializePipeline> {
+public:
+ SerializePipeline::Ptr
+ newPipeline(std::shared_ptr<folly::AsyncTransportWrapper> sock) override;
+};
+} // namespace hbase
diff --git a/core/client.h b/core/request.h
similarity index 76%
copy from core/client.h
copy to core/request.h
index 35a3bd8..39083ed 100644
--- a/core/client.h
+++ b/core/request.h
@@ -16,11 +16,18 @@
* limitations under the License.
*
*/
-
#pragma once
-#include <folly/io/IOBuf.h>
+#include <cstdint>
-#include "if/Cell.pb.h"
+namespace hbase {
+class Request {
+public:
+ Request() : call_id_(0) {}
+ uint32_t call_id() { return call_id_; }
+ void set_call_id(uint32_t call_id) { call_id_ = call_id; }
-class Client {};
+private:
+ uint32_t call_id_;
+};
+} // namespace hbase
diff --git a/core/client.h b/core/response.h
similarity index 76%
copy from core/client.h
copy to core/response.h
index 35a3bd8..34a284d 100644
--- a/core/client.h
+++ b/core/response.h
@@ -16,11 +16,19 @@
* limitations under the License.
*
*/
-
#pragma once
-#include <folly/io/IOBuf.h>
+#include <cstdint>
+
+namespace hbase {
-#include "if/Cell.pb.h"
+class Response {
+public:
+ Response() : call_id_(0) {}
+ uint32_t call_id() { return call_id_; }
+ void set_call_id(uint32_t call_id) { call_id_ = call_id; }
-class Client {};
+private:
+ uint32_t call_id_;
+};
+} // namespace hbase
diff --git a/core/client.h b/core/service.h
similarity index 84%
copy from core/client.h
copy to core/service.h
index 35a3bd8..880e65f 100644
--- a/core/client.h
+++ b/core/service.h
@@ -16,11 +16,11 @@
* limitations under the License.
*
*/
-
#pragma once
-#include <folly/io/IOBuf.h>
-
-#include "if/Cell.pb.h"
+#include "core/request.h"
+#include "core/response.h"
-class Client {};
+namespace hbase {
+using HBaseService = wangle::Service<Request, Response>;
+} // namespace hbase
diff --git a/core/client.cc b/core/simple-client.cc
similarity index 67%
copy from core/client.cc
copy to core/simple-client.cc
index a04daee..08e886a 100644
--- a/core/client.cc
+++ b/core/simple-client.cc
@@ -17,26 +17,41 @@
*
*/
-#include "core/client.h"
-
#include <folly/Logging.h>
#include <folly/Random.h>
-#include <glog/logging.h>
#include <gflags/gflags.h>
+#include <glog/logging.h>
+#include <wangle/concurrent/GlobalExecutor.h>
+
+#include <iostream>
+#include "core/client.h"
+#include "core/connection-factory.h"
#include "if/ZooKeeper.pb.h"
using namespace folly;
+using namespace std;
+using namespace hbase;
using namespace hbase::pb;
int main(int argc, char *argv[]) {
- MetaRegionServer mrs;
google::ParseCommandLineFlags(&argc, &argv, true);
google::InitGoogleLogging(argv[0]);
- FB_LOG_EVERY_MS(INFO, 10000) << "Hello";
- for (long i = 0; i < 10000000; i++) {
- FB_LOG_EVERY_MS(INFO, 1) << Random::rand32();
- }
+ // Create a connection factory
+ ConnectionFactory cf;
+
+ LocationCache cache{"localhost:2181", wangle::getCPUExecutor()};
+
+ auto result = cache.LocateMeta().get();
+ cout << "ServerName = " << result.host_name() << ":" << result.port() << endl;
+
+ // Create a connection to the local host
+ auto conn = cf.make_connection(result.host_name(), result.port()).get();
+
+ // Send the request
+ Request r;
+ conn(r).get();
+
return 0;
}
diff --git a/core/simple-native-client-test.cc b/core/simple-native-client-test.cc
index ef564f7..ee39986 100644
--- a/core/simple-native-client-test.cc
+++ b/core/simple-native-client-test.cc
@@ -22,7 +22,4 @@
/**
* Sample test.
*/
-TEST(SampleTest, sample) {
- EXPECT_TRUE(true);
-}
-
+TEST(SampleTest, sample) { EXPECT_TRUE(true); }
diff --git a/core/client.h b/core/table-name.cc
similarity index 90%
copy from core/client.h
copy to core/table-name.cc
index 35a3bd8..ffaaed0 100644
--- a/core/client.h
+++ b/core/table-name.cc
@@ -16,11 +16,4 @@
* limitations under the License.
*
*/
-
-#pragma once
-
-#include <folly/io/IOBuf.h>
-
-#include "if/Cell.pb.h"
-
-class Client {};
+#include "core/table-name.h"
diff --git a/core/client.h b/core/table-name.h
similarity index 75%
copy from core/client.h
copy to core/table-name.h
index 35a3bd8..796115b 100644
--- a/core/client.h
+++ b/core/table-name.h
@@ -16,11 +16,17 @@
* limitations under the License.
*
*/
-
#pragma once
-#include <folly/io/IOBuf.h>
+#include <memory>
+#include <string>
-#include "if/Cell.pb.h"
+namespace hbase {
-class Client {};
+// This is the core class of a HBase client.
+class TableName {
+public:
+ explicit TableName(std::string tableName);
+ explicit TableName(std::string namespaceName, std::string tableName);
+};
+} // namespace hbase
diff --git a/if/BUCK b/if/BUCK
index 9b989b5..5ff617d 100644
--- a/if/BUCK
+++ b/if/BUCK
@@ -1,3 +1,21 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
PROTO_SRCS = glob(['*.proto'])
HEADER_FILENAMES = [ x.replace('.proto','.pb.h') for x in PROTO_SRCS]
CC_FILENAMES = [ x.replace('.proto', '.pb.cc') for x in PROTO_SRCS]
@@ -20,7 +38,7 @@ for cc_filename in CC_FILENAMES:
genrule(
name = cc_filename,
cmd = 'mkdir -p `dirname $OUT` '
- ' && cp $(location :generate-proto-sources)/{} $OUT '
+ ' && cp $(location :generate-proto-sources)/*.cc `dirname $OUT` '
' && cp $(location :generate-proto-sources)/*.h `dirname $OUT`'.format(cc_filename),
out = cc_filename,
)
@@ -29,9 +47,7 @@ cxx_library(
name = 'if',
exported_headers = [':' + x for x in HEADER_FILENAMES],
srcs = [':' + x for x in CC_FILENAMES],
- deps = [ '//third-party:protobuf']
- + [':' + x for x in CC_FILENAMES]
- + [ ':' + x for x in HEADER_FILENAMES ],
+ deps = [ '//third-party:protobuf'],
visibility = [ 'PUBLIC', ],
exported_deps = ['//third-party:protobuf']
)
diff --git a/third-party/BUCK b/third-party/BUCK
index 6548695..4327530 100644
--- a/third-party/BUCK
+++ b/third-party/BUCK
@@ -23,19 +23,30 @@ def add_system_libs(names=[],
exported_linker_flags=[]):
rules = []
for name in names:
+ rule_visibility = ['PUBLIC']
gen_rule_name = "gen_lib{}".format(name)
- genrule(name=gen_rule_name,
- out=gen_rule_name,
- bash="mkdir -p $OUT && cp {}/lib{}.a $OUT".format(lib_dir,
- name), )
+ genrule(
+ name=gen_rule_name,
+ out=gen_rule_name,
+ bash="mkdir -p $OUT && cp {}/lib{}.a $OUT".format(lib_dir, name), )
prebuilt_cxx_library(name=name,
lib_name=name,
lib_dir='$(location :{})'.format(gen_rule_name),
deps=deps,
- force_static = True,
+ force_static=True,
exported_deps=exported_deps,
- visibility=['PUBLIC'],
- exported_linker_flags=exported_linker_flags, )
+ visibility=rule_visibility, )
+ rules.append(":" + name)
+ return rules
+
+
+def add_dynamic_libs(names=[]):
+ rules = []
+ for name in names:
+ prebuilt_cxx_library(name=name,
+ header_only=True,
+ exported_linker_flags=["-l" + name],
+ visibility=["PUBLIC"], )
rules.append(":" + name)
return rules
@@ -54,58 +65,46 @@ local_libs = [
"glog",
"protobuf",
]
+dynamic_libs = ["stdc++", "pthread", "ssl", "crypto", "dl", "atomic", ]
+dynamic_rules = add_dynamic_libs(dynamic_libs)
+tp_dep_rules = add_system_libs(system_libs,) \
+ + add_system_libs(local_libs, lib_dir = "/usr/local/lib") \
+ + dynamic_rules
-
-
-tp_dep_rules = add_system_libs(system_libs) \
- + add_system_libs(local_libs, lib_dir = "/usr/local/lib")
-
-zookeeper = add_system_libs(["zookeeper_mt"], lib_dir = "/usr/local/lib")
+zookeeper = add_system_libs(["zookeeper_mt"], lib_dir="/usr/local/lib")
folly = add_system_libs(['folly'],
lib_dir='/usr/local/lib',
- exported_deps=tp_dep_rules,
- exported_linker_flags=["-pthread",
- "-lstdc++", ])
+ exported_deps=tp_dep_rules, )
folly_bench = add_system_libs(['follybenchmark'],
lib_dir='/usr/local/lib',
- exported_deps=tp_dep_rules + folly,
- exported_linker_flags=["-pthread",
- "-lstdc++", ])
+ exported_deps=folly + tp_dep_rules, )
wangle = add_system_libs(['wangle'],
lib_dir='/usr/local/lib',
- exported_deps=tp_dep_rules + folly,
- exported_linker_flags=["-pthread",
- "-lstdc++", ])
-
+ exported_deps=folly + tp_dep_rules)
genrule(
-name = "gen_zk",
-out = "gen_zk",
-bash = "mkdir -p $OUT && wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && tar zxf zookeeper-3.4.8.tar.gz && rm -rf zookeeper-3.4.8.tar.gz && cd zookeeper-3.4.8 && cd src/c && ./configure --prefix=$OUT && make && make install && cd $OUT && rm -rf zookeeper-3.4.8*"
-)
-cxx_library(
- name = 'google-test',
- srcs = [
- 'googletest/googletest/src/gtest-all.cc',
- 'googletest/googlemock/src/gmock-all.cc',
- 'googletest/googlemock/src/gmock_main.cc',
- ],
- header_namespace = '',
- exported_headers = subdir_glob([
- ('googletest/googletest/include', '**/*.h'),
- ('googletest/googlemock/include', '**/*.h'),
- ]),
- headers = subdir_glob([
- ('googletest/googletest', 'src/*.h'),
- ('googletest/googletest', 'src/*.cc'),
- ('googletest/googlemock', 'src/*.h'),
- ('googletest/googlemock', 'src/*.cc'),
- ]),
- exported_linker_flags = [
- "-pthread",
- "-lstdc++",
- ],
- visibility = [
- 'PUBLIC',
- ],
-)
+ name="gen_zk",
+ out="gen_zk",
+ bash=
+ "mkdir -p $OUT && wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && tar zxf zookeeper-3.4.8.tar.gz && rm -rf zookeeper-3.4.8.tar.gz && cd zookeeper-3.4.8 && cd src/c && ./configure --prefix=$OUT && make && make install && cd $OUT && rm -rf zookeeper-3.4.8*")
+cxx_library(name='google-test',
+ srcs=[
+ 'googletest/googletest/src/gtest-all.cc',
+ 'googletest/googlemock/src/gmock-all.cc',
+ 'googletest/googlemock/src/gmock_main.cc',
+ ],
+ header_namespace='',
+ exported_headers=subdir_glob([
+ ('googletest/googletest/include', '**/*.h'),
+ ('googletest/googlemock/include', '**/*.h'),
+ ]),
+ headers=subdir_glob([
+ ('googletest/googletest', 'src/*.h'),
+ ('googletest/googletest', 'src/*.cc'),
+ ('googletest/googlemock', 'src/*.h'),
+ ('googletest/googlemock', 'src/*.cc'),
+ ]),
+ exported_deps=dynamic_rules,
+ visibility=[
+ 'PUBLIC',
+ ], )
[hbase] 126/137: HBASE-18720 [C++] Update wangle dependency to pick
up the new release with Apache License v2
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 0084543450fbfa4b44b8707404884d0a5c50abf4
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Wed Sep 6 16:29:18 2017 -0700
HBASE-18720 [C++] Update wangle dependency to pick up the new release with Apache License v2
---
connection/rpc-test-server.cc | 2 +-
docker-files/Dockerfile | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/connection/rpc-test-server.cc b/connection/rpc-test-server.cc
index 707bca7..157ea71 100644
--- a/connection/rpc-test-server.cc
+++ b/connection/rpc-test-server.cc
@@ -85,7 +85,7 @@ Future<std::unique_ptr<Response>> RpcTestService::operator()(std::unique_ptr<Req
response->set_resp_msg(pb_resp_msg);
VLOG(1) << "RPC server:"
<< " error called.";
- response->set_exception(folly::make_exception_wrapper<RpcTestException>("server error!"));
+ response->set_exception(RpcTestException("server error!"));
} else if (method_name == "pause") {
auto pb_resp_msg = std::make_shared<EmptyResponseProto>();
diff --git a/docker-files/Dockerfile b/docker-files/Dockerfile
index efd9a9d..1265b83 100644
--- a/docker-files/Dockerfile
+++ b/docker-files/Dockerfile
@@ -86,7 +86,7 @@ RUN git clone https://github.com/google/protobuf.git /usr/src/protobuf && \
# Update folly
RUN cd /usr/src/ && \
- ver=2017.06.19.00 && \
+ ver=2017.09.04.00 && \
wget https://github.com/facebook/folly/archive/v$ver.tar.gz && \
tar zxf v$ver.tar.gz && \
rm -rf v$ver.tar.gz && \
@@ -106,7 +106,7 @@ RUN cd /usr/src/ && \
# Update wangle
RUN cd /usr/src/ && \
- ver=2017.06.26.00 && \
+ ver=2017.09.04.00 && \
wget https://github.com/facebook/wangle/archive/v$ver.tar.gz && \
tar zxf v$ver.tar.gz && \
rm -rf v$ver.tar.gz && \
[hbase] 35/137: HBASE-15821 Document TestUtil
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 4f651b7c514b01d17b24a7193f25488a05c742bf
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Fri May 13 09:17:49 2016 -0700
HBASE-15821 Document TestUtil
---
bin/format-code.sh | 2 +-
connection/connection-pool.h | 4 ++--
core/location-cache.cc | 2 +-
core/location-cache.h | 2 +-
test-util/test-util.cc | 20 +++++++++++++-------
test-util/test-util.h | 20 +++++++++++++++-----
6 files changed, 33 insertions(+), 17 deletions(-)
diff --git a/bin/format-code.sh b/bin/format-code.sh
index 36a114d..cc8b368 100755
--- a/bin/format-code.sh
+++ b/bin/format-code.sh
@@ -19,5 +19,5 @@ set -euo pipefail
IFS=$'\n\t'
-find core connection serde utils -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i
+find core connection serde utils test-util -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i
find core connection serde utils third-party -name "BUCK" | xargs -P8 yapf -i
diff --git a/connection/connection-pool.h b/connection/connection-pool.h
index 605a81b..b8c950b 100644
--- a/connection/connection-pool.h
+++ b/connection/connection-pool.h
@@ -88,8 +88,8 @@ public:
void Close(const hbase::pb::ServerName &sn);
private:
- std::shared_ptr<HBaseService> GetCached(const hbase::pb::ServerName& sn);
- std::shared_ptr<HBaseService> GetNew(const hbase::pb::ServerName& sn);
+ std::shared_ptr<HBaseService> GetCached(const hbase::pb::ServerName &sn);
+ std::shared_ptr<HBaseService> GetNew(const hbase::pb::ServerName &sn);
std::unordered_map<hbase::pb::ServerName, std::shared_ptr<HBaseService>,
ServerNameHash, ServerNameEquals>
connections_;
diff --git a/core/location-cache.cc b/core/location-cache.cc
index 6ba8add..efd2210 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -116,7 +116,7 @@ LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
return this->LocateMeta()
.via(cpu_executor_.get())
.then([this](ServerName sn) { return this->cp_.Get(sn); })
- .then([tn, row, this](std::shared_ptr<HBaseService> service) {
+ .then([tn, row, this](std::shared_ptr<HBaseService> service) {
return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
})
.then([this](Response resp) {
diff --git a/core/location-cache.h b/core/location-cache.h
index d435530..830cd96 100644
--- a/core/location-cache.h
+++ b/core/location-cache.h
@@ -55,7 +55,7 @@ public:
* @param io_executor executor used to talk to the network
*/
LocationCache(std::string quorum_spec,
- std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_exector,
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor);
/**
* Destructor.
diff --git a/test-util/test-util.cc b/test-util/test-util.cc
index e5fba48..88ce7c8 100644
--- a/test-util/test-util.cc
+++ b/test-util/test-util.cc
@@ -24,13 +24,18 @@
using hbase::TestUtil;
using folly::Random;
-const static int STR_LEN = 32;
-
-std::string TestUtil::RandString() {
- auto s = std::string(STR_LEN, 'z');
-
- for (int i = 0; i < STR_LEN; i++) {
+std::string TestUtil::RandString(int len) {
+ // Create the whole string.
+ // Filling everything with z's
+ auto s = std::string(len, 'z');
+
+ // Now pick a bunch of random numbers
+ for (int i = 0; i < len; i++) {
+ // use Folly's random to get the numbers
+ // as I don't want to have to learn
+ // all the cpp rand invocation magic.
auto r = Random::rand32('a', 'z');
+ // Cast that to ascii.
s[i] = static_cast<char>(r);
}
return s;
@@ -42,12 +47,13 @@ TestUtil::TestUtil() : temp_dir_(TestUtil::RandString()) {
auto res_code = std::system(cmd.c_str());
CHECK(res_code == 0);
}
+
TestUtil::~TestUtil() {
auto res_code = std::system("bin/stop-local-hbase.sh");
CHECK(res_code == 0);
}
-void TestUtil::RunShellCmd(const std::string& command) {
+void TestUtil::RunShellCmd(const std::string &command) {
auto cmd_string = folly::sformat("echo \"{}\" | ../bin/hbase shell", command);
auto res_code = std::system(cmd_string.c_str());
CHECK(res_code == 0);
diff --git a/test-util/test-util.h b/test-util/test-util.h
index 395b157..20e4981 100644
--- a/test-util/test-util.h
+++ b/test-util/test-util.h
@@ -30,20 +30,30 @@ namespace hbase {
*/
class TestUtil {
public:
-
/**
* Creating a TestUtil will spin up a cluster.
*/
TestUtil();
+
/**
- * Destroying a TestUtil will spin down a cluster.
+ * Destroying a TestUtil will spin down a cluster and remove the test dir.
*/
~TestUtil();
+
+ /**
+ * Run a command in the hbase shell. Command should not include any double
+ * quotes.
+ *
+ * This should only be used until there is a good Admin support from the
+ * native client
+ */
+ void RunShellCmd(const std::string &command);
+
/**
- * Run a command in the hbase shell.
+ * Create a random string. This random string is all letters, as such it is
+ * very good for use as a directory name.
*/
- void RunShellCmd(const std::string& command);
- static std::string RandString();
+ static std::string RandString(int len = 32);
private:
folly::test::TemporaryDirectory temp_dir_;
[hbase] 47/137: HBASE-15902 Scan Object
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 21e728be9c47232c19067d7091d745cbb21fd51c
Author: Sudeep Sunthankar <su...@hashmapinc.com>
AuthorDate: Sat Dec 3 11:35:11 2016 +1100
HBASE-15902 Scan Object
Signed-off-by: Enis Soztutar <en...@apache.org>
---
core/BUCK | 8 ++
core/get-test.cc | 253 ++++++++++++++++++++++++++++++++++--------------------
core/get.cc | 109 ++++++++++-------------
core/get.h | 40 +++------
core/scan-test.cc | 233 +++++++++++++++++++++++++++++++++++++++++++++++++
core/scan.cc | 239 +++++++++++++++++++++++++++++++++++++++++++++++++++
core/scan.h | 253 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
7 files changed, 953 insertions(+), 182 deletions(-)
diff --git a/core/BUCK b/core/BUCK
index 9dbc82e..fa01eb3 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -31,6 +31,7 @@ cxx_library(
"time_range.h",
"configuration.h",
"hbase_configuration_loader.h",
+ "scan.h",
],
srcs=[
"cell.cc",
@@ -41,6 +42,7 @@ cxx_library(
"time_range.cc",
"configuration.cc",
"hbase_configuration_loader.cc",
+ "scan.cc",
],
deps=[
"//connection:connection",
@@ -85,6 +87,12 @@ cxx_test(name="hbase_configuration-test",
],
deps=[":core", ],
run_test_separately=True, )
+cxx_test(name="scan-test",
+ srcs=[
+ "scan-test.cc",
+ ],
+ deps=[":core", ],
+ run_test_separately=True, )
cxx_binary(name="simple-client",
srcs=["simple-client.cc", ],
deps=[":core", "//connection:connection"], )
diff --git a/core/get-test.cc b/core/get-test.cc
index 8ed5f2f..9a09a23 100644
--- a/core/get-test.cc
+++ b/core/get-test.cc
@@ -21,129 +21,200 @@
#include <gtest/gtest.h>
#include <glog/logging.h>
-
using namespace hbase;
const int NUMBER_OF_GETS = 5;
-TEST (Get, SingleGet) {
-
- std::string row_str = "row-test";
- ASSERT_NO_THROW(Get tmp = Get(row_str));
- Get get = Get(row_str);
-
- get.SetCacheBlocks(true);
- get.SetConsistency(hbase::pb::Consistency::STRONG);
- get.SetMaxResultsPerColumnFamily(1);
-
- ASSERT_THROW(get.SetMaxVersions(0), std::runtime_error);
- ASSERT_NO_THROW(get.SetMaxVersions(-10));
- ASSERT_THROW(get.SetMaxVersions(std::numeric_limits<unsigned int>::max() + 1),
- std::runtime_error);
-
- ASSERT_THROW(get.SetTimeRange(-100, 2000), std::runtime_error);
- ASSERT_THROW(get.SetTimeRange(100, -2000), std::runtime_error);
- ASSERT_THROW(get.SetTimeRange(1000, 200), std::runtime_error);
-
- ASSERT_NO_THROW(get.SetMaxVersions());
- ASSERT_NO_THROW(get.SetMaxVersions(2));
- ASSERT_NO_THROW(get.SetTimeRange(0, std::numeric_limits<long>::max()));
-
- EXPECT_EQ(true, get.CacheBlocks());
- EXPECT_EQ(hbase::pb::Consistency::STRONG, get.Consistency());
- EXPECT_EQ(1, get.MaxResultsPerColumnFamily());
- EXPECT_EQ(2, get.MaxVersions());
-
- TimeRange tr = get.Timerange();
- EXPECT_EQ(0, tr.MinTimeStamp());
- EXPECT_EQ(std::numeric_limits<long>::max(), tr.MaxTimeStamp());
-
- EXPECT_EQ("row-test", get.Row());
-
+void CheckFamilies(Get &get) {
EXPECT_EQ(false, get.HasFamilies());
-
get.AddFamily("family-1");
EXPECT_EQ(true, get.HasFamilies());
+ EXPECT_EQ(1, get.Family().size());
+ for (const auto &family : get.Family()) {
+ EXPECT_STREQ("family-1", family.first.c_str());
+ EXPECT_EQ(0, family.second.size());
+ }
+ // Not allowed to add the same CF.
+ get.AddFamily("family-1");
+ EXPECT_EQ(1, get.Family().size());
+ get.AddFamily("family-2");
+ EXPECT_EQ(2, get.Family().size());
+ get.AddFamily("family-3");
+ EXPECT_EQ(3, get.Family().size());
+ int i = 1;
+ for (const auto &family : get.Family()) {
+ std::string family_name = "family-" + std::to_string(i);
+ EXPECT_STREQ(family_name.c_str(), family.first.c_str());
+ EXPECT_EQ(0, family.second.size());
+ i += 1;
+ }
get.AddColumn("family-1", "column-1");
get.AddColumn("family-1", "column-2");
get.AddColumn("family-1", "");
get.AddColumn("family-1", "column-3");
- get.AddFamily("family-1");
- get.AddFamily("family-2");
- get.AddFamily("family-3");
-
+ get.AddColumn("family-2", "column-X");
+
+ EXPECT_EQ(3, get.Family().size());
+ auto it = get.Family().begin();
+ EXPECT_STREQ("family-1", it->first.c_str());
+ EXPECT_EQ(4, it->second.size());
+ EXPECT_STREQ("column-1", it->second[0].c_str());
+ EXPECT_STREQ("column-2", it->second[1].c_str());
+ EXPECT_STREQ("", it->second[2].c_str());
+ EXPECT_STREQ("column-3", it->second[3].c_str());
+ ++it;
+ EXPECT_STREQ("family-2", it->first.c_str());
+ EXPECT_EQ(1, it->second.size());
+ EXPECT_STREQ("column-X", it->second[0].c_str());
+ ++it;
+ EXPECT_STREQ("family-3", it->first.c_str());
+ EXPECT_EQ(0, it->second.size());
+ ++it;
+ EXPECT_EQ(it, get.Family().end());
}
-TEST (Get, MultiGet) {
-
- std::vector<Get *> gets;
- for (int i = 0; i < NUMBER_OF_GETS; i++) {
- std::string row_str = "row-test";
- row_str += std::to_string(i);
- ASSERT_NO_THROW(Get tmp = Get(row_str));
- Get *get = new Get(row_str);
-
- get->SetCacheBlocks(true);
- get->SetConsistency(hbase::pb::Consistency::STRONG);
- get->SetMaxResultsPerColumnFamily(1);
-
- ASSERT_THROW(get->SetMaxVersions(0), std::runtime_error);
- ASSERT_NO_THROW(get->SetMaxVersions(-10));
- ASSERT_THROW(
- get->SetMaxVersions(std::numeric_limits<unsigned int>::max() + 1),
- std::runtime_error);
+void CheckFamiliesAfterCopy(Get &get) {
+ EXPECT_EQ(true, get.HasFamilies());
+ EXPECT_EQ(3, get.Family().size());
+ int i = 1;
+ for (const auto &family : get.Family()) {
+ std::string family_name = "family-" + std::to_string(i);
+ EXPECT_STREQ(family_name.c_str(), family.first.c_str());
+ i += 1;
+ }
+ // Check if the alreaday added CF's and CQ's are as expected
+ auto it = get.Family().begin();
+ EXPECT_STREQ("family-1", it->first.c_str());
+ EXPECT_EQ(4, it->second.size());
+ EXPECT_STREQ("column-1", it->second[0].c_str());
+ EXPECT_STREQ("column-2", it->second[1].c_str());
+ EXPECT_STREQ("", it->second[2].c_str());
+ EXPECT_STREQ("column-3", it->second[3].c_str());
+ ++it;
+ EXPECT_STREQ("family-2", it->first.c_str());
+ EXPECT_EQ(1, it->second.size());
+ EXPECT_STREQ("column-X", it->second[0].c_str());
+ ++it;
+ EXPECT_STREQ("family-3", it->first.c_str());
+ EXPECT_EQ(0, it->second.size());
+ ++it;
+ EXPECT_EQ(it, get.Family().end());
+}
- ASSERT_THROW(get->SetTimeRange(-100, 2000), std::runtime_error);
- ASSERT_THROW(get->SetTimeRange(100, -2000), std::runtime_error);
- ASSERT_THROW(get->SetTimeRange(1000, 200), std::runtime_error);
+void GetMethods(Get &get, const std::string &row) {
+ EXPECT_EQ(row, get.Row());
- get->SetMaxVersions();
- get->SetMaxVersions(2);
- get->SetTimeRange(0, std::numeric_limits<long>::max());
+ CheckFamilies(get);
+ EXPECT_EQ(true, get.CacheBlocks());
+ get.SetCacheBlocks(false);
+ EXPECT_EQ(false, get.CacheBlocks());
- EXPECT_EQ(true, get->CacheBlocks());
- EXPECT_EQ(hbase::pb::Consistency::STRONG, get->Consistency());
- EXPECT_EQ(1, get->MaxResultsPerColumnFamily());
- EXPECT_EQ(2, get->MaxVersions());
+ EXPECT_EQ(hbase::pb::Consistency::STRONG, get.Consistency());
+ get.SetConsistency(hbase::pb::Consistency::TIMELINE);
+ EXPECT_EQ(hbase::pb::Consistency::TIMELINE, get.Consistency());
- TimeRange tr = get->Timerange();
- EXPECT_EQ(0, tr.MinTimeStamp());
- EXPECT_EQ(std::numeric_limits<long>::max(), tr.MaxTimeStamp());
+ EXPECT_EQ(1, get.MaxVersions());
+ get.SetMaxVersions(2);
+ EXPECT_EQ(2, get.MaxVersions());
+ get.SetMaxVersions();
+ EXPECT_EQ(1, get.MaxVersions());
+
+ // Test initial values
+ EXPECT_EQ(0, get.Timerange().MinTimeStamp());
+ EXPECT_EQ(std::numeric_limits<long>::max(), get.Timerange().MaxTimeStamp());
+
+ // Set & Test new values using TimeRange and TimeStamp
+ get.SetTimeRange(1000, 2000);
+ EXPECT_EQ(1000, get.Timerange().MinTimeStamp());
+ EXPECT_EQ(2000, get.Timerange().MaxTimeStamp());
+ get.SetTimeStamp(0);
+ EXPECT_EQ(0, get.Timerange().MinTimeStamp());
+ EXPECT_EQ(1, get.Timerange().MaxTimeStamp());
+
+ // Test some exceptions
+ ASSERT_THROW(get.SetTimeRange(-1000, 2000), std::runtime_error);
+ ASSERT_THROW(get.SetTimeRange(1000, -2000), std::runtime_error);
+ ASSERT_THROW(get.SetTimeRange(1000, 200), std::runtime_error);
+ ASSERT_THROW(get.SetTimeStamp(std::numeric_limits<long>::max()), std::runtime_error);
- EXPECT_EQ(false, get->HasFamilies());
+ //Test some exceptions
+ ASSERT_THROW(get.SetMaxVersions(0), std::runtime_error);
+ ASSERT_THROW(get.SetMaxVersions(std::numeric_limits<unsigned int>::max() + 1),
+ std::runtime_error);
+}
- get->AddFamily("family-1");
- EXPECT_EQ(true, get->HasFamilies());
+TEST (Get, SingleGet) {
+ std::string row_str = "row-test";
+ Get get(row_str);
+ GetMethods(get, row_str);
+
+ Get get_tmp(row_str);
+ Get getcp(get_tmp);
+ GetMethods(getcp, row_str);
+
+ Get geteq("test");
+ geteq = get_tmp;
+ GetMethods(geteq, row_str);
+
+ // Adding the below tests as there were some concerns raised that the same vector of qualifiers in FamilyMap is being shared between copied objects
+ // Verify the source object's family map size before using it to copy.
+ EXPECT_EQ(3, get.Family().size());
+
+ Get getcp_fam(get);
+ // address of family maps should be different.
+ EXPECT_NE(&(get.Family()), &(getcp_fam.Family()));
+
+ // Add family to the source object
+ get.AddColumn("family-4", "column-A");
+ get.AddColumn("family-4", "column-B");
+ // Verify the source object's family map size
+ EXPECT_EQ(4, get.Family().size());
+ // Verify the source object's family elements
+ auto it = get.Family().begin();
+ EXPECT_STREQ("family-1", it->first.c_str());
+ EXPECT_EQ(4, it->second.size());
+ EXPECT_STREQ("column-1", it->second[0].c_str());
+ EXPECT_STREQ("column-2", it->second[1].c_str());
+ EXPECT_STREQ("", it->second[2].c_str());
+ EXPECT_STREQ("column-3", it->second[3].c_str());
+ ++it;
+ EXPECT_STREQ("family-2", it->first.c_str());
+ EXPECT_EQ(1, it->second.size());
+ EXPECT_STREQ("column-X", it->second[0].c_str());
+ ++it;
+ EXPECT_STREQ("family-3", it->first.c_str());
+ EXPECT_EQ(0, it->second.size());
+ ++it;
+ EXPECT_STREQ("family-4", it->first.c_str());
+ EXPECT_EQ(2, it->second.size());
+ EXPECT_STREQ("column-A", it->second[0].c_str());
+ EXPECT_STREQ("column-B", it->second[1].c_str());
+ ++it;
+ EXPECT_EQ(it, get.Family().end());
+
+ //Verifying the copied object's families. It will remain unchanged and below tests should pass
+ CheckFamiliesAfterCopy(getcp_fam);
+}
- get->AddColumn("family-1", "column-1");
- get->AddColumn("family-1", "column-2");
- get->AddColumn("family-1", "");
- get->AddColumn("family-1", "column-3");
- get->AddFamily("family-1");
- get->AddFamily("family-2");
- get->AddFamily("family-3");
+TEST (Get, MultiGet) {
+ std::vector<Get *> gets;
+ for (int i = 0; i < NUMBER_OF_GETS; i++) {
+ std::string row_str = "row-test";
+ row_str += std::to_string(i);
+ Get *get = new Get(row_str);
+ GetMethods(*get, row_str);
gets.push_back(get);
}
EXPECT_EQ(NUMBER_OF_GETS, gets.size());
- int i = 0;
- for (const auto &get : gets) {
- std::string row_str = "row-test";
- row_str += std::to_string(i);
- EXPECT_EQ(row_str, get->Row());
- i++;
- }
-
for (const auto &get : gets) {
delete get;
}
gets.clear();
-
}
TEST (Get, Exception) {
-
std::string row(std::numeric_limits<short>::max() + 1, 'X');
ASSERT_THROW(Get tmp = Get(row), std::runtime_error);
ASSERT_THROW(Get tmp = Get(""), std::runtime_error);
diff --git a/core/get.cc b/core/get.cc
index b0d91ec..b412a4b 100644
--- a/core/get.cc
+++ b/core/get.cc
@@ -25,42 +25,31 @@
namespace hbase {
Get::~Get() {
-
}
Get::Get(const std::string &row)
- : row_(row),
- max_versions_(1),
- cache_blocks_(true),
- store_limit_(-1),
- store_offset_(0),
- check_existence_only_(false),
- consistency_(hbase::pb::Consistency::STRONG),
- tr_(TimeRange()) {
- Get::CheckRow(&row_);
- family_map_.clear();
-}
-
-Get::Get(const Get &cget) {
- this->row_ = cget.row_;
- this->max_versions_ = cget.max_versions_;
- this->cache_blocks_ = cget.cache_blocks_;
- this->store_limit_ = cget.store_limit_;
- this->store_offset_ = cget.store_offset_;
- this->check_existence_only_ = cget.check_existence_only_;
- this->consistency_ = cget.consistency_;
- this->tr_ = cget.tr_;
-}
-
-Get& Get::operator=(const Get &cget) {
- this->row_ = cget.row_;
- this->max_versions_ = cget.max_versions_;
- this->cache_blocks_ = cget.cache_blocks_;
- this->store_limit_ = cget.store_limit_;
- this->store_offset_ = cget.store_offset_;
- this->check_existence_only_ = cget.check_existence_only_;
- this->consistency_ = cget.consistency_;
- this->tr_ = cget.tr_;
+ : row_(row) {
+ CheckRow(row_);
+}
+
+Get::Get(const Get &get) {
+ row_ = get.row_;
+ max_versions_ = get.max_versions_;
+ cache_blocks_ = get.cache_blocks_;
+ check_existence_only_ = get.check_existence_only_;
+ consistency_ = get.consistency_;
+ tr_.reset(new TimeRange(get.Timerange().MinTimeStamp(), get.Timerange().MaxTimeStamp()));
+ family_map_.insert(get.family_map_.begin(), get.family_map_.end());
+}
+
+Get& Get::operator=(const Get &get) {
+ row_ = get.row_;
+ max_versions_ = get.max_versions_;
+ cache_blocks_ = get.cache_blocks_;
+ check_existence_only_ = get.check_existence_only_;
+ consistency_ = get.consistency_;
+ tr_.reset(new TimeRange(get.Timerange().MinTimeStamp(), get.Timerange().MaxTimeStamp()));
+ family_map_.insert(get.family_map_.begin(), get.family_map_.end());
return *this;
}
@@ -80,15 +69,14 @@ Get& Get::AddFamily(const std::string &family) {
}
Get& Get::AddColumn(const std::string &family, const std::string &qualifier) {
- const auto &it = std::find(this->family_map_[family].begin(),
- this->family_map_[family].end(), qualifier);
+ const auto &it = std::find(family_map_[family].begin(), family_map_[family].end(), qualifier);
/**
* Check if any qualifiers are already present or not.
* Add only if qualifiers for a given family are not present
*/
- if (it == this->family_map_[family].end()) {
- this->family_map_[family].push_back(qualifier);
+ if (it == family_map_[family].end()) {
+ family_map_[family].push_back(qualifier);
}
return *this;
}
@@ -98,76 +86,67 @@ const std::string& Get::Row() const {
}
hbase::pb::Consistency Get::Consistency() const {
- return this->consistency_;
+ return consistency_;
}
Get &Get::SetConsistency(hbase::pb::Consistency consistency) {
- this->consistency_ = consistency;
+ consistency_ = consistency;
return *this;
}
-bool Get::HasFamilies() {
- return !this->family_map_.empty();
+bool Get::HasFamilies() const {
+ return !family_map_.empty();
}
-const FAMILY_MAP &Get::FamilyMap() const {
- return this->family_map_;
+const FamilyMap &Get::Family() const {
+ return family_map_;
}
int Get::MaxVersions() const {
- return this->max_versions_;
+ return max_versions_;
}
Get& Get::SetMaxVersions(uint32_t max_versions) {
if (0 == max_versions)
throw std::runtime_error("max_versions must be positive");
- this->max_versions_ = max_versions;
+ max_versions_ = max_versions;
return *this;
}
bool Get::CacheBlocks() const {
- return this->cache_blocks_;
+ return cache_blocks_;
}
Get & Get::SetCacheBlocks(bool cache_blocks) {
- this->cache_blocks_ = cache_blocks;
- return *this;
-}
-
-int Get::MaxResultsPerColumnFamily() const {
- return this->store_limit_;
-}
-
-Get& Get::SetMaxResultsPerColumnFamily(int store_limit) {
- this->store_limit_ = store_limit;
+ cache_blocks_ = cache_blocks;
return *this;
}
Get& Get::SetTimeRange(long min_timestamp, long max_timestamp) {
- this->tr_ = TimeRange(min_timestamp, max_timestamp);
+ tr_.reset(new TimeRange(min_timestamp, max_timestamp));
return *this;
}
Get& Get::SetTimeStamp(long timestamp) {
- this->tr_ = TimeRange(timestamp, timestamp + 1);
+ tr_.reset(new TimeRange(timestamp, timestamp + 1));
return *this;
}
const TimeRange& Get::Timerange() const {
- return this->tr_;
+ return *tr_;
}
-void Get::CheckRow(const std::string *row) {
- int MAX_ROW_LENGTH = std::numeric_limits<short>::max();
- int row_length = row->size();
+void Get::CheckRow(const std::string &row) {
+ const int kMaxRowLength = std::numeric_limits<short>::max();
+ int row_length = row.size();
if (0 == row_length) {
throw std::runtime_error("Row length can't be 0");
}
- if (row_length > MAX_ROW_LENGTH) {
+ if (row_length > kMaxRowLength) {
throw std::runtime_error(
- "Length of " + *row + " is greater than max row size: "
- + std::to_string(MAX_ROW_LENGTH));
+ "Length of " + row + " is greater than max row size: " + std::to_string(kMaxRowLength));
}
}
+
}
diff --git a/core/get.h b/core/get.h
index 28141f2..9704aa1 100644
--- a/core/get.h
+++ b/core/get.h
@@ -20,6 +20,7 @@
#pragma once
#include <map>
+#include <memory>
#include <string>
#include <vector>
#include <cstdint>
@@ -31,7 +32,7 @@ namespace hbase {
/**
* @brief Map consisting of column families and qualifiers to be used for Get operation
*/
-using FAMILY_MAP = std::map<std::string, std::vector<std::string>>;
+using FamilyMap = std::map<std::string, std::vector<std::string>>;
class Get {
@@ -69,20 +70,9 @@ class Get {
Get& SetCacheBlocks(bool cache_blocks);
/**
- * @brief Method for retrieving the get's maximum number of values to return per Column Family
+ * @brief Returns the Get family map (FamilyMap) for this Get operation. Used for constructing Scan object with an already constructed Get
*/
- int MaxResultsPerColumnFamily() const;
-
- /**
- * @brief Set the maximum number of values to return per row per Column Family
- * @param the store_limit to be set
- */
- Get& SetMaxResultsPerColumnFamily(int store_limit);
-
- /**
- * @brief Returns the Get family map (FAMILY_MAP) for this Get operation.
- */
- const FAMILY_MAP &FamilyMap() const;
+ const FamilyMap &Family() const;
/**
* @brief Returns the timerange for this Get
@@ -121,9 +111,9 @@ class Get {
const std::string& Row() const;
/**
- * @brief Returns true if family map (FAMILY_MAP) is non empty false otherwise
+ * @brief Returns true if family map (FamilyMap) is non empty false otherwise
*/
- bool HasFamilies();
+ bool HasFamilies() const;
/**
* @brief Returns the consistency level for this Get operation
@@ -137,21 +127,19 @@ class Get {
Get& SetConsistency(hbase::pb::Consistency consistency);
private:
- std::string row_;
- uint32_t max_versions_;
- bool cache_blocks_;
- int store_limit_;
- int store_offset_;
- bool check_existence_only_;
- FAMILY_MAP family_map_;
- hbase::pb::Consistency consistency_;
- TimeRange tr_;
+ std::string row_ = "";
+ uint32_t max_versions_ = 1;
+ bool cache_blocks_ = true;
+ bool check_existence_only_ = false;
+ FamilyMap family_map_;
+ hbase::pb::Consistency consistency_ = hbase::pb::Consistency::STRONG;
+ std::unique_ptr<TimeRange> tr_ = std::make_unique<TimeRange>();
/**
* @brief Checks if the row for this Get operation is proper or not
* @param row Row to check
* @throws std::runtime_error if row is empty or greater than MAX_ROW_LENGTH(i.e. std::numeric_limits<short>::max())
*/
- void CheckRow(const std::string *row);
+ void CheckRow(const std::string &row);
};
}
diff --git a/core/scan-test.cc b/core/scan-test.cc
new file mode 100644
index 0000000..a114502
--- /dev/null
+++ b/core/scan-test.cc
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/scan.h"
+
+#include <limits>
+
+#include <gtest/gtest.h>
+
+using namespace hbase;
+
+void CheckFamilies(Scan &scan) {
+ EXPECT_EQ(false, scan.HasFamilies());
+ scan.AddFamily("family-1");
+ EXPECT_EQ(true, scan.HasFamilies());
+ EXPECT_EQ(1, scan.Family().size());
+ for (const auto &family : scan.Family()) {
+ EXPECT_STREQ("family-1", family.first.c_str());
+ EXPECT_EQ(0, family.second.size());
+ }
+ // Not allowed to add the same CF.
+ scan.AddFamily("family-1");
+ EXPECT_EQ(1, scan.Family().size());
+ scan.AddFamily("family-2");
+ EXPECT_EQ(2, scan.Family().size());
+ scan.AddFamily("family-3");
+ EXPECT_EQ(3, scan.Family().size());
+ int i = 1;
+ for (const auto &family : scan.Family()) {
+ std::string family_name = "family-" + std::to_string(i);
+ EXPECT_STREQ(family_name.c_str(), family.first.c_str());
+ EXPECT_EQ(0, family.second.size());
+ i += 1;
+ }
+
+ scan.AddColumn("family-1", "column-1");
+ scan.AddColumn("family-1", "column-2");
+ scan.AddColumn("family-1", "");
+ scan.AddColumn("family-1", "column-3");
+ scan.AddColumn("family-2", "column-X");
+
+ EXPECT_EQ(3, scan.Family().size());
+ auto it = scan.Family().begin();
+ EXPECT_STREQ("family-1", it->first.c_str());
+ EXPECT_EQ(4, it->second.size());
+ EXPECT_STREQ("column-1", it->second[0].c_str());
+ EXPECT_STREQ("column-2", it->second[1].c_str());
+ EXPECT_STREQ("", it->second[2].c_str());
+ EXPECT_STREQ("column-3", it->second[3].c_str());
+ ++it;
+ EXPECT_STREQ("family-2", it->first.c_str());
+ EXPECT_EQ(1, it->second.size());
+ EXPECT_STREQ("column-X", it->second[0].c_str());
+ ++it;
+ EXPECT_STREQ("family-3", it->first.c_str());
+ EXPECT_EQ(0, it->second.size());
+ ++it;
+ EXPECT_EQ(it, scan.Family().end());
+}
+
+void CheckFamiliesAfterCopy(Scan &scan) {
+ EXPECT_EQ(true, scan.HasFamilies());
+ EXPECT_EQ(3, scan.Family().size());
+ int i = 1;
+ for (const auto &family : scan.Family()) {
+ std::string family_name = "family-" + std::to_string(i);
+ EXPECT_STREQ(family_name.c_str(), family.first.c_str());
+ i += 1;
+ }
+ // Check if the alreaday added CF's and CQ's are as expected
+ auto it = scan.Family().begin();
+ EXPECT_STREQ("family-1", it->first.c_str());
+ EXPECT_EQ(4, it->second.size());
+ EXPECT_STREQ("column-1", it->second[0].c_str());
+ EXPECT_STREQ("column-2", it->second[1].c_str());
+ EXPECT_STREQ("", it->second[2].c_str());
+ EXPECT_STREQ("column-3", it->second[3].c_str());
+ ++it;
+ EXPECT_STREQ("family-2", it->first.c_str());
+ EXPECT_EQ(1, it->second.size());
+ EXPECT_STREQ("column-X", it->second[0].c_str());
+ ++it;
+ EXPECT_STREQ("family-3", it->first.c_str());
+ EXPECT_EQ(0, it->second.size());
+ ++it;
+ EXPECT_EQ(it, scan.Family().end());
+}
+
+void ScanMethods(Scan &scan) {
+ scan.SetReversed(true);
+ EXPECT_EQ(true, scan.IsReversed());
+ scan.SetReversed(false);
+ EXPECT_EQ(false, scan.IsReversed());
+
+ std::string start_row("start-row");
+ std::string stop_row("stop-row");
+ scan.SetStartRow(start_row);
+ EXPECT_EQ(start_row, scan.StartRow());
+
+ scan.SetStopRow(stop_row);
+ EXPECT_EQ(stop_row, scan.StopRow());
+
+ scan.SetSmall(true);
+ EXPECT_EQ(true, scan.IsSmall());
+ scan.SetSmall(false);
+ EXPECT_EQ(false, scan.IsSmall());
+
+ scan.SetCaching(3);
+ EXPECT_EQ(3, scan.Caching());
+
+ scan.SetConsistency(hbase::pb::Consistency::STRONG);
+ EXPECT_EQ(hbase::pb::Consistency::STRONG, scan.Consistency());
+ scan.SetConsistency(hbase::pb::Consistency::TIMELINE);
+ EXPECT_EQ(hbase::pb::Consistency::TIMELINE, scan.Consistency());
+
+ scan.SetCacheBlocks(true);
+ EXPECT_EQ(true, scan.CacheBlocks());
+ scan.SetCacheBlocks(false);
+ EXPECT_EQ(false, scan.CacheBlocks());
+
+ scan.SetAllowPartialResults(true);
+ EXPECT_EQ(true, scan.AllowPartialResults());
+ scan.SetAllowPartialResults(false);
+ EXPECT_EQ(false, scan.AllowPartialResults());
+
+ scan.SetLoadColumnFamiliesOnDemand(true);
+ EXPECT_EQ(true, scan.LoadColumnFamiliesOnDemand());
+ scan.SetLoadColumnFamiliesOnDemand(false);
+ EXPECT_EQ(false, scan.LoadColumnFamiliesOnDemand());
+
+ scan.SetMaxVersions();
+ EXPECT_EQ(1, scan.MaxVersions());
+ scan.SetMaxVersions(20);
+ EXPECT_EQ(20, scan.MaxVersions());
+
+ scan.SetMaxResultSize(1024);
+ EXPECT_EQ(1024, scan.MaxResultSize());
+
+ // Test initial values
+ EXPECT_EQ(0, scan.Timerange().MinTimeStamp());
+ EXPECT_EQ(std::numeric_limits<long>::max(), scan.Timerange().MaxTimeStamp());
+
+ // Set & Test new values using TimeRange and TimeStamp
+ scan.SetTimeRange(1000, 2000);
+ EXPECT_EQ(1000, scan.Timerange().MinTimeStamp());
+ EXPECT_EQ(2000, scan.Timerange().MaxTimeStamp());
+ scan.SetTimeStamp(0);
+ EXPECT_EQ(0, scan.Timerange().MinTimeStamp());
+ EXPECT_EQ(1, scan.Timerange().MaxTimeStamp());
+
+ // Test some exceptions
+ ASSERT_THROW(scan.SetTimeRange(-1000, 2000), std::runtime_error);
+ ASSERT_THROW(scan.SetTimeRange(1000, -2000), std::runtime_error);
+ ASSERT_THROW(scan.SetTimeRange(1000, 200), std::runtime_error);
+ ASSERT_THROW(scan.SetTimeStamp(std::numeric_limits<long>::max()), std::runtime_error);
+}
+
+TEST (Scan, Object) {
+ Scan scan;
+ ScanMethods(scan);
+ CheckFamilies(scan);
+
+ // Resetting TimeRange values so that the copy construction and assignment operator tests pass.
+ scan.SetTimeRange(0, std::numeric_limits<long>::max());
+ Scan scancp(scan);
+ ScanMethods(scancp);
+ CheckFamiliesAfterCopy(scancp);
+
+ Scan scaneq;
+ scaneq = scan;
+ ScanMethods(scaneq);
+ CheckFamiliesAfterCopy(scaneq);
+}
+
+TEST (Scan, WithStartRow) {
+ Scan("row-test");
+ Scan scan("row-test");
+ ScanMethods(scan);
+ CheckFamilies(scan);
+}
+
+TEST (Scan, WithStartAndStopRow) {
+ Scan("start-row", "stop-row");
+ Scan scan("start-row", "stop-row");
+ ScanMethods(scan);
+ CheckFamilies(scan);
+}
+
+TEST (Scan,FromGet) {
+
+ std::string row_str = "row-test";
+ Get get = Get(row_str);
+
+ get.SetCacheBlocks(true);
+ get.SetMaxVersions(5);
+ get.AddFamily("family-1");
+ get.AddFamily("family-1");
+ get.AddFamily("family-2");
+ get.AddFamily("family-3");
+ get.AddColumn("family-1", "column-1");
+ get.AddColumn("family-1", "column-2");
+ get.AddColumn("family-1", "");
+ get.AddColumn("family-1", "column-3");
+ get.AddColumn("family-2", "column-X");
+
+ EXPECT_EQ(3, get.Family().size());
+
+ Scan scan(get);
+ ScanMethods(scan);
+ CheckFamiliesAfterCopy(scan);
+}
+
+TEST (Scan, Exception) {
+ std::string row(std::numeric_limits<short>::max() + 1, 'X');
+ ASSERT_THROW(Scan tmp(row), std::runtime_error);
+ ASSERT_THROW(Scan tmp(""), std::runtime_error);
+}
diff --git a/core/scan.cc b/core/scan.cc
new file mode 100644
index 0000000..318cdb7
--- /dev/null
+++ b/core/scan.cc
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/scan.h"
+
+#include <algorithm>
+#include <iterator>
+#include <limits>
+#include <stdexcept>
+
+namespace hbase {
+
+Scan::Scan() {
+}
+
+Scan::~Scan() {
+}
+
+Scan::Scan(const std::string &start_row)
+ : start_row_(start_row) {
+ CheckRow(start_row_);
+}
+
+Scan::Scan(const std::string &start_row, const std::string &stop_row)
+ : start_row_(start_row),
+ stop_row_(stop_row) {
+ CheckRow(start_row_);
+ CheckRow(stop_row_);
+}
+
+Scan::Scan(const Scan &scan) {
+ start_row_ = scan.start_row_;
+ stop_row_ = scan.stop_row_;
+ max_versions_ = scan.max_versions_;
+ caching_ = scan.caching_;
+ max_result_size_ = scan.max_result_size_;
+ cache_blocks_ = scan.cache_blocks_;
+ load_column_families_on_demand_ = scan.load_column_families_on_demand_;
+ reversed_ = scan.reversed_;
+ small_ = scan.small_;
+ allow_partial_results_ = scan.allow_partial_results_;
+ consistency_ = scan.consistency_;
+ tr_.reset(new TimeRange(scan.tr_->MinTimeStamp(), scan.tr_->MaxTimeStamp()));
+ family_map_.insert(scan.family_map_.begin(), scan.family_map_.end());
+}
+
+Scan& Scan::operator=(const Scan &scan) {
+ start_row_ = scan.start_row_;
+ stop_row_ = scan.stop_row_;
+ max_versions_ = scan.max_versions_;
+ caching_ = scan.caching_;
+ max_result_size_ = scan.max_result_size_;
+ cache_blocks_ = scan.cache_blocks_;
+ load_column_families_on_demand_ = scan.load_column_families_on_demand_;
+ reversed_ = scan.reversed_;
+ small_ = scan.small_;
+ allow_partial_results_ = scan.allow_partial_results_;
+ consistency_ = scan.consistency_;
+ tr_.reset(new TimeRange(scan.tr_->MinTimeStamp(), scan.tr_->MaxTimeStamp()));
+ family_map_.insert(scan.family_map_.begin(), scan.family_map_.end());
+ return *this;
+}
+
+Scan::Scan(const Get &get) {
+ cache_blocks_ = get.CacheBlocks();
+ max_versions_ = get.MaxVersions();
+ tr_.reset(new TimeRange(get.Timerange().MinTimeStamp(), get.Timerange().MaxTimeStamp()));
+ family_map_.insert(get.Family().begin(), get.Family().end());
+}
+
+Scan& Scan::AddFamily(const std::string &family) {
+ const auto &it = family_map_.find(family);
+ /**
+ * Check if any qualifiers are already present or not.
+ * Remove all existing qualifiers if the given family is already present in the map
+ */
+ if (family_map_.end() != it) {
+ it->second.clear();
+ } else {
+ family_map_[family];
+ }
+ return *this;
+}
+
+Scan& Scan::AddColumn(const std::string &family, const std::string &qualifier) {
+ const auto &it = std::find(family_map_[family].begin(), family_map_[family].end(), qualifier);
+ /**
+ * Check if any qualifiers are already present or not.
+ * Add only if qualifiers for a given family are not present
+ */
+ if (it == family_map_[family].end()) {
+ family_map_[family].push_back(qualifier);
+ }
+ return *this;
+}
+
+void Scan::SetReversed(bool reversed) {
+ reversed_ = reversed;
+}
+
+bool Scan::IsReversed() const {
+ return reversed_;
+}
+
+void Scan::SetStartRow(std::string &start_row) {
+ CheckRow(start_row);
+ start_row_ = start_row;
+}
+
+const std::string& Scan::StartRow() const {
+ return start_row_;
+}
+
+void Scan::SetStopRow(std::string &stop_row) {
+ CheckRow(stop_row);
+ stop_row_ = stop_row;
+}
+
+const std::string& Scan::StopRow() const {
+ return stop_row_;
+}
+
+void Scan::SetSmall(bool small) {
+ small_ = small;
+}
+
+bool Scan::IsSmall() const {
+ return small_;
+}
+
+void Scan::SetCaching(int caching) {
+ caching_ = caching;
+}
+
+int Scan::Caching() const {
+ return caching_;
+}
+
+Scan& Scan::SetConsistency(const hbase::pb::Consistency consistency) {
+ consistency_ = consistency;
+ return *this;
+}
+
+hbase::pb::Consistency Scan::Consistency() const {
+ return consistency_;
+}
+
+void Scan::SetCacheBlocks(bool cache_blocks) {
+ cache_blocks_ = cache_blocks;
+}
+
+bool Scan::CacheBlocks() const {
+ return cache_blocks_;
+}
+
+void Scan::SetAllowPartialResults(bool allow_partial_results) {
+ allow_partial_results_ = allow_partial_results;
+}
+
+bool Scan::AllowPartialResults() const {
+ return allow_partial_results_;
+}
+
+void Scan::SetLoadColumnFamiliesOnDemand(bool load_column_families_on_demand) {
+ load_column_families_on_demand_ = load_column_families_on_demand;
+}
+
+bool Scan::LoadColumnFamiliesOnDemand() const {
+ return load_column_families_on_demand_;
+}
+
+Scan& Scan::SetMaxVersions(uint32_t max_versions) {
+ max_versions_ = max_versions;
+ return *this;
+}
+
+int Scan::MaxVersions() const {
+ return max_versions_;
+}
+
+void Scan::SetMaxResultSize(long max_result_size) {
+ max_result_size_ = max_result_size;
+}
+
+long Scan::MaxResultSize() const {
+ return max_result_size_;
+}
+
+Scan &Scan::SetTimeRange(long min_stamp, long max_stamp) {
+ tr_.reset(new TimeRange(min_stamp, max_stamp));
+ return *this;
+}
+
+Scan& Scan::SetTimeStamp(long timestamp) {
+ tr_.reset(new TimeRange(timestamp, timestamp + 1));
+ return *this;
+}
+
+const TimeRange& Scan::Timerange() const {
+ return *tr_;
+}
+
+void Scan::CheckRow(const std::string &row) {
+ const int kMaxRowLength = std::numeric_limits<short>::max();
+ int row_length = row.size();
+ if (0 == row_length) {
+ throw std::runtime_error("Row length can't be 0");
+ }
+ if (row_length > kMaxRowLength) {
+ throw std::runtime_error(
+ "Length of " + row + " is greater than max row size: " + std::to_string(kMaxRowLength));
+ }
+}
+
+bool Scan::HasFamilies() const {
+ return !family_map_.empty();
+}
+
+const FamilyMap &Scan::Family() const {
+ return family_map_;
+}
+}
+/* namespace hbase */
diff --git a/core/scan.h b/core/scan.h
new file mode 100644
index 0000000..bd655ce
--- /dev/null
+++ b/core/scan.h
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+#include "core/get.h"
+#include "core/time_range.h"
+#include "if/Client.pb.h"
+
+namespace hbase {
+
+/**
+ * @brief Map consisting of column families and qualifiers to be used for Get operation
+ */
+using FamilyMap = std::map<std::string, std::vector<std::string>>;
+
+class Scan {
+ public:
+ /**
+ * @brief Constructors. Create a Scan operation across all rows.
+ */
+ Scan();
+ Scan(const Scan &scan);
+ Scan& operator=(const Scan &scan);
+
+ ~Scan();
+
+ /**
+ * @brief Create a Scan operation starting at the specified row. If the specified row does not exist,
+ * the Scanner will start from the next closest row after the specified row.
+ * @param start_row - row to start scanner at or after
+ */
+ Scan(const std::string &start_row);
+
+ /**
+ * @brief Create a Scan operation for the range of rows specified.
+ * @param start_row - row to start scanner at or after (inclusive).
+ * @param stop_row - row to stop scanner before (exclusive).
+ */
+ Scan(const std::string &start_row, const std::string &stop_row);
+
+ /**
+ * @brief Builds a scan object with the same specs as get.
+ * @param get - get to model scan after
+ */
+ Scan(const Get &get);
+
+ /**
+ * @brief Get all columns from the specified family.Overrides previous calls to AddColumn for this family.
+ * @param family - family name
+ */
+ Scan& AddFamily(const std::string &family);
+
+ /**
+ * @brief Get the column from the specified family with the specified qualifier.Overrides previous calls to AddFamily for this family.
+ * @param family - family name.
+ * @param qualifier - column qualifier.
+ */
+ Scan& AddColumn(const std::string &family, const std::string &qualifier);
+
+ /**
+ * @brief Set whether this scan is a reversed one. This is false by default which means forward(normal) scan.
+ * @param reversed - if true, scan will be backward order
+ */
+ void SetReversed(bool reversed);
+
+ /**
+ * @brief Get whether this scan is a reversed one. Returns true if backward scan, false if forward(default) scan
+ */
+ bool IsReversed() const;
+
+ /**
+ * @brief Set the start row of the scan.If the specified row does not exist, the Scanner will start from the next closest row after the specified row.
+ * @param start_row - row to start scanner at or after
+ * @throws std::runtime_error if start_row length is 0 or greater than MAX_ROW_LENGTH
+ */
+ void SetStartRow(std::string &start_row);
+
+ /**
+ * @brief returns start_row of the Scan.
+ */
+ const std::string& StartRow() const;
+
+ /**
+ * @brief Set the stop row of the scan. The scan will include rows that are lexicographically less than the provided stop_row.
+ * @param stop_row - row to end at (exclusive)
+ * @throws std::runtime_error if stop_row length is 0 or greater than MAX_ROW_LENGTH
+ */
+ void SetStopRow(std::string &stop_row);
+
+ /**
+ * @brief returns stop_row of the Scan.
+ */
+ const std::string& StopRow() const;
+
+ /**
+ * @brief Set whether this scan is a small scan.
+ */
+ void SetSmall(bool small);
+
+ /**
+ * @brief Returns if the scan is a small scan.
+ */
+ bool IsSmall() const;
+
+ /**
+ * @brief Set the number of rows for caching that will be passed to scanners. Higher caching values will enable faster scanners but will use more memory.
+ * @param caching - the number of rows for caching.
+ */
+ void SetCaching(int caching);
+
+ /**
+ * @brief caching the number of rows fetched when calling next on a scanner.
+ */
+ int Caching() const;
+
+ /**
+ * @brief Sets the consistency level for this operation.
+ * @param consistency - the consistency level
+ */
+ Scan& SetConsistency(const hbase::pb::Consistency consistency);
+
+ /**
+ * @brief Returns the consistency level for this operation.
+ */
+ hbase::pb::Consistency Consistency() const;
+
+ /**
+ * @brief Set whether blocks should be cached for this Scan.This is true by default. When true, default settings of the table and family are used (this will never override caching blocks if the block cache is disabled for that family or entirely).
+ * @param cache_blocks - if false, default settings are overridden and blocks will not be cached
+ */
+ void SetCacheBlocks(bool cache_blocks);
+
+ /**
+ * @brief Get whether blocks should be cached for this Scan.
+ */
+ bool CacheBlocks() const;
+
+ /**
+ * @brief Setting whether the caller wants to see the partial results that may be returned from the server. By default this value is false and the complete results will be assembled client side before being delivered to the caller.
+ * @param allow_partial_results - if true partial results will be returned.
+ */
+ void SetAllowPartialResults(bool allow_partial_results);
+
+ /**
+ * @brief true when the constructor of this scan understands that the results they will see may only represent a partial portion of a row. The entire row would be retrieved by subsequent calls to ResultScanner.next()
+ */
+ bool AllowPartialResults() const;
+
+ /**
+ * @brief Set the value indicating whether loading CFs on demand should be allowed (cluster default is false). On-demand CF loading doesn't load column families until necessary.
+ * @param load_column_families_on_demand
+ */
+ void SetLoadColumnFamiliesOnDemand(bool load_column_families_on_demand);
+
+ /**
+ * @brief Get the raw loadColumnFamiliesOnDemand setting.
+ */
+ bool LoadColumnFamiliesOnDemand() const;
+
+ /**
+ * @brief Get up to the specified number of versions of each column if specified else get default i.e. one.
+ * @param max_versions - maximum versions for each column.
+ */
+ Scan &SetMaxVersions(uint32_t max_versions = 1);
+
+ /**
+ * @brief the max number of versions to fetch
+ */
+ int MaxVersions() const;
+
+ /**
+ * @brief Set the maximum result size. The default is -1; this means that no specific maximum result size will be set for this scan, and the global configured value will be used instead. (Defaults to unlimited).
+ * @param The maximum result size in bytes.
+ */
+ void SetMaxResultSize(long max_result_size);
+
+ /**
+ * @brief the maximum result size in bytes.
+ */
+ long MaxResultSize() const;
+
+ /**
+ * @brief Get versions of columns only within the specified timestamp range, [min_stamp, max_stamp). Note, default maximum versions to return is 1. If your time range spans more than one version and you want all versions returned, up the number of versions beyond the default.
+ * @param min_stamp - minimum timestamp value, inclusive.
+ * @param max_stamp - maximum timestamp value, exclusive.
+ */
+ Scan &SetTimeRange(long min_stamp, long max_stamp);
+
+ /**
+ * @brief Get versions of columns with the specified timestamp. Note, default maximum versions to return is 1. If your time range spans more than one version and you want all versions returned, up the number of versions beyond the defaut.
+ * @param timestamp - version timestamp
+ */
+ Scan &SetTimeStamp(long timestamp);
+
+ /**
+ * @brief Return Timerange
+ */
+ const TimeRange& Timerange() const;
+
+ /**
+ * @brief Returns true if family map (FamilyMap) is non empty false otherwise
+ */
+ bool HasFamilies() const;
+
+ /**
+ * @brief Returns the Scan family map (FamilyMap) for this Scan operation.
+ */
+ const FamilyMap &Family() const;
+
+ private:
+ std::string start_row_ = "";
+ std::string stop_row_ = "";
+ uint32_t max_versions_ = 1;
+ int caching_ = -1;
+ long max_result_size_ = -1;
+ bool cache_blocks_ = true;
+ bool load_column_families_on_demand_ = false;
+ bool reversed_ = false;
+ bool small_ = false;
+ bool allow_partial_results_ = false;
+ hbase::pb::Consistency consistency_ = hbase::pb::Consistency::STRONG;
+ std::unique_ptr<TimeRange> tr_ = std::make_unique<TimeRange>();
+ FamilyMap family_map_;
+
+ /**
+ * @brief Checks for row length validity, throws if length check fails, returns null otherwise.
+ * @param row - row whose validity needs to be checked
+ * @throws std::runtime_error if row length equals 0 or greater than std::numeric_limits<short>::max();
+ */
+ void CheckRow(const std::string &row);
+};
+} /* namespace hbase */
[hbase] 02/137: HBASE-10302 Fix rat check issues in
hbase-native-client.
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 2e8bfb875e9fc35be2109e77d047b0c96512d5b4
Author: eclark <ec...@unknown>
AuthorDate: Thu Jan 9 20:16:55 2014 +0000
HBASE-10302 Fix rat check issues in hbase-native-client.
git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1556936 13f79535-47bb-0310-9956-ffa450edef68
---
cmake_modules/FindGTest.cmake | 13 +++++++++++++
cmake_modules/FindLibEv.cmake | 13 +++++++++++++
src/rpc/CMakeLists.txt | 13 +++++++++++++
3 files changed, 39 insertions(+)
diff --git a/cmake_modules/FindGTest.cmake b/cmake_modules/FindGTest.cmake
index 2d7e3e9..1da6892 100644
--- a/cmake_modules/FindGTest.cmake
+++ b/cmake_modules/FindGTest.cmake
@@ -1,3 +1,16 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
set( GTEST_INCLUDE_SEARCH
${CMAKE_SOURCE_DIR}/thirdparty/gtest/include
)
diff --git a/cmake_modules/FindLibEv.cmake b/cmake_modules/FindLibEv.cmake
index f61a2a5..9a40c17 100644
--- a/cmake_modules/FindLibEv.cmake
+++ b/cmake_modules/FindLibEv.cmake
@@ -1,3 +1,16 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
set( LIBEV_INCLUDE_SEARCH
${CMAKE_SOURCE_DIR}/thirdparty/installed/include
)
diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt
index e69de29..d9c0695 100644
--- a/src/rpc/CMakeLists.txt
+++ b/src/rpc/CMakeLists.txt
@@ -0,0 +1,13 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
[hbase] 106/137: HBASE-18338 [C++] Implement RpcTestServer
(Xiaobing Zhou)
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 9918d09dab8753c1b8c6900d16f0120626d6f77f
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Fri Jul 21 16:29:44 2017 -0700
HBASE-18338 [C++] Implement RpcTestServer (Xiaobing Zhou)
---
connection/BUCK | 13 ++++++
connection/client-handler.cc | 21 +++++----
connection/client-handler.h | 7 ++-
connection/pipeline.cc | 12 +++--
connection/rpc-test-server-handler.cc | 77 +++++++++++++++++++++++++++++++
connection/rpc-test-server-handler.h | 47 +++++++++++++++++++
connection/rpc-test-server.cc | 70 ++++++++++++++++++++++++++++
connection/rpc-test-server.h | 50 ++++++++++++++++++++
connection/rpc-test.cc | 86 +++++++++++++++++++++++++++++++++++
connection/sasl-handler.h | 2 +-
if/test.proto | 43 ++++++++++++++++++
if/test_rpc_service.proto | 35 ++++++++++++++
serde/BUCK | 4 +-
serde/client-deserializer-test.cc | 3 +-
serde/client-serializer-test.cc | 2 +-
serde/{rpc.cc => rpc-serde.cc} | 16 ++++++-
serde/{rpc.h => rpc-serde.h} | 16 +++++++
17 files changed, 482 insertions(+), 22 deletions(-)
diff --git a/connection/BUCK b/connection/BUCK
index c3119eb..aaf8fdb 100644
--- a/connection/BUCK
+++ b/connection/BUCK
@@ -33,6 +33,8 @@ cxx_library(
"service.h",
"rpc-client.h",
"sasl-util.h",
+ "rpc-test-server.h",
+ "rpc-test-server-handler.h",
],
srcs=[
"client-dispatcher.cc",
@@ -44,6 +46,8 @@ cxx_library(
"rpc-client.cc",
"sasl-handler.cc",
"sasl-util.cc",
+ "rpc-test-server.cc",
+ "rpc-test-server-handler.cc",
],
deps=[
"//if:if",
@@ -68,3 +72,12 @@ cxx_test(
deps=[
":connection",
],)
+cxx_test(
+ name="rpc-test",
+ srcs=[
+ "rpc-test.cc",
+ ],
+ deps=[
+ ":connection",
+ ],
+ run_test_separately=True,)
diff --git a/connection/client-handler.cc b/connection/client-handler.cc
index 052c171..39227d3 100644
--- a/connection/client-handler.cc
+++ b/connection/client-handler.cc
@@ -35,9 +35,10 @@ using google::protobuf::Message;
namespace hbase {
ClientHandler::ClientHandler(std::string user_name, std::shared_ptr<Codec> codec,
- const std::string &server)
+ std::shared_ptr<Configuration> conf, const std::string &server)
: user_name_(user_name),
serde_(codec),
+ conf_(conf),
server_(server),
once_flag_(std::make_unique<std::once_flag>()),
resp_msgs_(
@@ -115,13 +116,17 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<folly::IOBuf> buf) {
}
folly::Future<folly::Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
- // We need to send the header once.
- // So use call_once to make sure that only one thread wins this.
- std::call_once((*once_flag_), [ctx, this]() {
- VLOG(3) << "Writing RPC Header to server: " << server_;
- auto header = serde_.Header(user_name_);
- ctx->fireWrite(std::move(header));
- });
+ /* for RPC test, there's no need to send connection header */
+ if (!conf_->GetBool(RpcSerde::HBASE_CLIENT_RPC_TEST_MODE,
+ RpcSerde::DEFAULT_HBASE_CLIENT_RPC_TEST_MODE)) {
+ // We need to send the header once.
+ // So use call_once to make sure that only one thread wins this.
+ std::call_once((*once_flag_), [ctx, this]() {
+ VLOG(3) << "Writing RPC Header to server: " << server_;
+ auto header = serde_.Header(user_name_);
+ ctx->fireWrite(std::move(header));
+ });
+ }
VLOG(3) << "Writing RPC Request:" << r->DebugString() << ", server: " << server_;
diff --git a/connection/client-handler.h b/connection/client-handler.h
index 8de3a8b..b6f19a2 100644
--- a/connection/client-handler.h
+++ b/connection/client-handler.h
@@ -26,9 +26,10 @@
#include <string>
#include <utility>
+#include "core/configuration.h"
#include "exceptions/exception.h"
#include "serde/codec.h"
-#include "serde/rpc.h"
+#include "serde/rpc-serde.h"
#include "utils/concurrent-map.h"
// Forward decs.
@@ -60,7 +61,8 @@ class ClientHandler
* Create the handler
* @param user_name the user name of the user running this process.
*/
- ClientHandler(std::string user_name, std::shared_ptr<Codec> codec, const std::string &server);
+ ClientHandler(std::string user_name, std::shared_ptr<Codec> codec,
+ std::shared_ptr<Configuration> conf, const std::string &server);
/**
* Get bytes from the wire.
@@ -79,6 +81,7 @@ class ClientHandler
std::string user_name_;
RpcSerde serde_;
std::string server_; // for logging
+ std::shared_ptr<Configuration> conf_;
// in flight requests
std::unique_ptr<concurrent_map<uint32_t, std::shared_ptr<google::protobuf::Message>>> resp_msgs_;
diff --git a/connection/pipeline.cc b/connection/pipeline.cc
index 2844752..9c790b6 100644
--- a/connection/pipeline.cc
+++ b/connection/pipeline.cc
@@ -32,7 +32,6 @@ namespace hbase {
RpcPipelineFactory::RpcPipelineFactory(std::shared_ptr<Codec> codec,
std::shared_ptr<Configuration> conf)
: user_util_(), codec_(codec), conf_(conf) {}
-
SerializePipeline::Ptr RpcPipelineFactory::newPipeline(
std::shared_ptr<folly::AsyncTransportWrapper> sock) {
folly::SocketAddress addr; // for logging
@@ -41,10 +40,15 @@ SerializePipeline::Ptr RpcPipelineFactory::newPipeline(
auto pipeline = SerializePipeline::create();
pipeline->addBack(wangle::AsyncSocketHandler{sock});
pipeline->addBack(wangle::EventBaseHandler{});
- auto secure = security::User::IsSecurityEnabled(*conf_);
- pipeline->addBack(SaslHandler{user_util_.user_name(secure), conf_});
+ bool secure = false;
+ /* for RPC test, there's no need to setup Sasl */
+ if (!conf_->GetBool(RpcSerde::HBASE_CLIENT_RPC_TEST_MODE,
+ RpcSerde::DEFAULT_HBASE_CLIENT_RPC_TEST_MODE)) {
+ secure = security::User::IsSecurityEnabled(*conf_);
+ pipeline->addBack(SaslHandler{user_util_.user_name(secure), conf_});
+ }
pipeline->addBack(wangle::LengthFieldBasedFrameDecoder{});
- pipeline->addBack(ClientHandler{user_util_.user_name(secure), codec_, addr.describe()});
+ pipeline->addBack(ClientHandler{user_util_.user_name(secure), codec_, conf_, addr.describe()});
pipeline->finalize();
return pipeline;
}
diff --git a/connection/rpc-test-server-handler.cc b/connection/rpc-test-server-handler.cc
new file mode 100644
index 0000000..7d2f407
--- /dev/null
+++ b/connection/rpc-test-server-handler.cc
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "connection/rpc-test-server-handler.h"
+#include "if/RPC.pb.h"
+#include "if/test.pb.h"
+
+namespace hbase {
+
+void RpcTestServerSerializeHandler::read(Context* ctx, std::unique_ptr<folly::IOBuf> buf) {
+ buf->coalesce();
+ pb::RequestHeader header;
+
+ int used_bytes = serde_.ParseDelimited(buf.get(), &header);
+ VLOG(3) << "Read RPC RequestHeader size=" << used_bytes << " call_id=" << header.call_id();
+
+ auto received = CreateReceivedRequest(header.method_name());
+
+ buf->trimStart(used_bytes);
+ if (header.has_request_param() && received != nullptr) {
+ used_bytes = serde_.ParseDelimited(buf.get(), received->req_msg().get());
+ VLOG(3) << "Read RPCRequest, buf length:" << buf->length()
+ << ", header PB length:" << used_bytes;
+ received->set_call_id(header.call_id());
+ }
+
+ if (received != nullptr) {
+ ctx->fireRead(std::move(received));
+ }
+}
+
+folly::Future<folly::Unit> RpcTestServerSerializeHandler::write(Context* ctx,
+ std::unique_ptr<Response> r) {
+ VLOG(3) << "Writing RPC Request";
+ // Send the data down the pipeline.
+ return ctx->fireWrite(serde_.Response(r->call_id(), r->resp_msg().get()));
+}
+
+std::unique_ptr<Request> RpcTestServerSerializeHandler::CreateReceivedRequest(
+ const std::string& method_name) {
+ std::unique_ptr<Request> result = nullptr;
+ ;
+ if (method_name == "ping") {
+ result = std::make_unique<Request>(std::make_shared<EmptyRequestProto>(),
+ std::make_shared<EmptyResponseProto>(), method_name);
+ } else if (method_name == "echo") {
+ result = std::make_unique<Request>(std::make_shared<EchoRequestProto>(),
+ std::make_shared<EchoResponseProto>(), method_name);
+ } else if (method_name == "error") {
+ result = std::make_unique<Request>(std::make_shared<EmptyRequestProto>(),
+ std::make_shared<EmptyResponseProto>(), method_name);
+ } else if (method_name == "pause") {
+ result = std::make_unique<Request>(std::make_shared<PauseRequestProto>(),
+ std::make_shared<EmptyResponseProto>(), method_name);
+ } else if (method_name == "addr") {
+ result = std::make_unique<Request>(std::make_shared<EmptyRequestProto>(),
+ std::make_shared<AddrResponseProto>(), method_name);
+ }
+ return result;
+}
+} // end of namespace hbase
diff --git a/connection/rpc-test-server-handler.h b/connection/rpc-test-server-handler.h
new file mode 100644
index 0000000..4c84615
--- /dev/null
+++ b/connection/rpc-test-server-handler.h
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <wangle/channel/Handler.h>
+
+#include "connection/request.h"
+#include "connection/response.h"
+#include "serde/rpc-serde.h"
+
+using namespace hbase;
+
+namespace hbase {
+// A real rpc server would probably use generated client/server stubs
+class RpcTestServerSerializeHandler
+ : public wangle::Handler<std::unique_ptr<folly::IOBuf>, std::unique_ptr<Request>,
+ std::unique_ptr<Response>, std::unique_ptr<folly::IOBuf>> {
+ public:
+ RpcTestServerSerializeHandler() : serde_() {}
+
+ void read(Context* ctx, std::unique_ptr<folly::IOBuf> buf) override;
+
+ folly::Future<folly::Unit> write(Context* ctx, std::unique_ptr<Response> r) override;
+
+ private:
+ std::unique_ptr<Request> CreateReceivedRequest(const std::string& method_name);
+
+ private:
+ hbase::RpcSerde serde_;
+};
+} // end of namespace hbase
diff --git a/connection/rpc-test-server.cc b/connection/rpc-test-server.cc
new file mode 100644
index 0000000..d3a30b1
--- /dev/null
+++ b/connection/rpc-test-server.cc
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include <wangle/channel/AsyncSocketHandler.h>
+#include <wangle/channel/EventBaseHandler.h>
+#include <wangle/codec/LengthFieldBasedFrameDecoder.h>
+#include <wangle/codec/LengthFieldPrepender.h>
+#include <wangle/service/ServerDispatcher.h>
+
+#include "connection/rpc-test-server-handler.h"
+#include "connection/rpc-test-server.h"
+#include "if/test.pb.h"
+
+namespace hbase {
+
+RpcTestServerSerializePipeline::Ptr RpcTestServerPipelineFactory::newPipeline(
+ std::shared_ptr<AsyncTransportWrapper> sock) {
+ auto pipeline = RpcTestServerSerializePipeline::create();
+ pipeline->addBack(AsyncSocketHandler(sock));
+ // ensure we can write from any thread
+ pipeline->addBack(EventBaseHandler());
+ pipeline->addBack(LengthFieldBasedFrameDecoder());
+ pipeline->addBack(RpcTestServerSerializeHandler());
+ pipeline->addBack(
+ MultiplexServerDispatcher<std::unique_ptr<Request>, std::unique_ptr<Response>>(&service_));
+ pipeline->finalize();
+
+ return pipeline;
+}
+
+Future<std::unique_ptr<Response>> RpcTestService::operator()(std::unique_ptr<Request> request) {
+ /* build Response */
+ auto response = std::make_unique<Response>();
+ response->set_call_id(request->call_id());
+ std::string method_name = request->method();
+
+ if (method_name == "ping") {
+ auto pb_resp_msg = std::make_shared<EmptyResponseProto>();
+ response->set_resp_msg(pb_resp_msg);
+ } else if (method_name == "echo") {
+ auto pb_resp_msg = std::make_shared<EchoResponseProto>();
+ auto pb_req_msg = std::static_pointer_cast<EchoRequestProto>(request->req_msg());
+ pb_resp_msg->set_message(pb_req_msg->message());
+ response->set_resp_msg(pb_resp_msg);
+ } else if (method_name == "error") {
+ // TODO:
+ } else if (method_name == "pause") {
+ // TODO:
+ } else if (method_name == "addr") {
+ // TODO:
+ }
+
+ return folly::makeFuture<std::unique_ptr<Response>>(std::move(response));
+}
+} // namespace hbase
diff --git a/connection/rpc-test-server.h b/connection/rpc-test-server.h
new file mode 100644
index 0000000..c3225ff
--- /dev/null
+++ b/connection/rpc-test-server.h
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+#include <wangle/concurrent/CPUThreadPoolExecutor.h>
+#include <wangle/service/ExecutorFilter.h>
+#include <wangle/service/Service.h>
+
+#include "connection/request.h"
+#include "connection/response.h"
+
+using namespace hbase;
+using namespace folly;
+using namespace wangle;
+
+namespace hbase {
+using RpcTestServerSerializePipeline = wangle::Pipeline<IOBufQueue&, std::unique_ptr<Response>>;
+
+class RpcTestService : public Service<std::unique_ptr<Request>, std::unique_ptr<Response>> {
+ public:
+ RpcTestService() {}
+ virtual ~RpcTestService() = default;
+ Future<std::unique_ptr<Response>> operator()(std::unique_ptr<Request> request) override;
+};
+
+class RpcTestServerPipelineFactory : public PipelineFactory<RpcTestServerSerializePipeline> {
+ public:
+ RpcTestServerSerializePipeline::Ptr newPipeline(
+ std::shared_ptr<AsyncTransportWrapper> sock) override;
+
+ private:
+ ExecutorFilter<std::unique_ptr<Request>, std::unique_ptr<Response>> service_{
+ std::make_shared<CPUThreadPoolExecutor>(1), std::make_shared<RpcTestService>()};
+};
+} // end of namespace hbase
diff --git a/connection/rpc-test.cc b/connection/rpc-test.cc
new file mode 100644
index 0000000..d4cd89f
--- /dev/null
+++ b/connection/rpc-test.cc
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <wangle/bootstrap/ClientBootstrap.h>
+#include <wangle/channel/Handler.h>
+
+#include <folly/Logging.h>
+#include <folly/SocketAddress.h>
+#include <folly/String.h>
+#include <folly/experimental/TestUtil.h>
+#include <gflags/gflags.h>
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+#include <boost/thread.hpp>
+
+#include "connection/rpc-client.h"
+#include "if/test.pb.h"
+#include "rpc-test-server.h"
+#include "security/user.h"
+#include "serde/rpc-serde.h"
+
+using namespace wangle;
+using namespace folly;
+using namespace hbase;
+
+DEFINE_int32(port, 0, "test server port");
+
+TEST(RpcTestServer, echo) {
+ /* create conf */
+ auto conf = std::make_shared<Configuration>();
+ conf->Set(RpcSerde::HBASE_CLIENT_RPC_TEST_MODE, "true");
+
+ /* create rpc test server */
+ auto server = std::make_shared<ServerBootstrap<RpcTestServerSerializePipeline>>();
+ server->childPipeline(std::make_shared<RpcTestServerPipelineFactory>());
+ server->bind(FLAGS_port);
+ folly::SocketAddress server_addr;
+ server->getSockets()[0]->getAddress(&server_addr);
+
+ /* create RpcClient */
+ auto io_executor = std::make_shared<wangle::IOThreadPoolExecutor>(1);
+
+ auto rpc_client = std::make_shared<RpcClient>(io_executor, nullptr, conf);
+
+ /**
+ * test echo
+ */
+ try {
+ std::string greetings = "hello, hbase server!";
+ auto request = std::make_unique<Request>(std::make_shared<EchoRequestProto>(),
+ std::make_shared<EchoResponseProto>(), "echo");
+ auto pb_msg = std::static_pointer_cast<EchoRequestProto>(request->req_msg());
+ pb_msg->set_message(greetings);
+
+ /* sending out request */
+ rpc_client
+ ->AsyncCall(server_addr.getAddressStr(), server_addr.getPort(), std::move(request),
+ hbase::security::User::defaultUser())
+ .then([=](std::unique_ptr<Response> response) {
+ auto pb_resp = std::static_pointer_cast<EchoResponseProto>(response->resp_msg());
+ VLOG(1) << "message returned: " + pb_resp->message();
+ EXPECT_EQ(greetings, pb_resp->message());
+ });
+ } catch (const std::exception& e) {
+ throw e;
+ }
+
+ server->stop();
+ server->join();
+}
diff --git a/connection/sasl-handler.h b/connection/sasl-handler.h
index f606a23..81f4e81 100644
--- a/connection/sasl-handler.h
+++ b/connection/sasl-handler.h
@@ -30,7 +30,7 @@
#include "connection/sasl-util.h"
#include "connection/service.h"
#include "security/user.h"
-#include "serde/rpc.h"
+#include "serde/rpc-serde.h"
namespace hbase {
diff --git a/if/test.proto b/if/test.proto
new file mode 100644
index 0000000..72b68e9
--- /dev/null
+++ b/if/test.proto
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.hadoop.hbase.ipc.protobuf.generated";
+option java_outer_classname = "TestProtos";
+option java_generate_equals_and_hash = true;
+
+message EmptyRequestProto {
+}
+
+message EmptyResponseProto {
+}
+
+message EchoRequestProto {
+ required string message = 1;
+}
+
+message EchoResponseProto {
+ required string message = 1;
+}
+
+message PauseRequestProto {
+ required uint32 ms = 1;
+}
+
+message AddrResponseProto {
+ required string addr = 1;
+}
diff --git a/if/test_rpc_service.proto b/if/test_rpc_service.proto
new file mode 100644
index 0000000..5f91dc4
--- /dev/null
+++ b/if/test_rpc_service.proto
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+option java_package = "org.apache.hadoop.hbase.ipc.protobuf.generated";
+option java_outer_classname = "TestRpcServiceProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+
+import "test.proto";
+
+
+/**
+ * A protobuf service for use in tests
+ */
+service TestProtobufRpcProto {
+ rpc ping(EmptyRequestProto) returns (EmptyResponseProto);
+ rpc echo(EchoRequestProto) returns (EchoResponseProto);
+ rpc error(EmptyRequestProto) returns (EmptyResponseProto);
+ rpc pause(PauseRequestProto) returns (EmptyResponseProto);
+ rpc addr(EmptyRequestProto) returns (AddrResponseProto);
+}
diff --git a/serde/BUCK b/serde/BUCK
index 18e949c..a765884 100644
--- a/serde/BUCK
+++ b/serde/BUCK
@@ -22,13 +22,13 @@ cxx_library(
"cell-outputstream.h",
"codec.h",
"region-info.h",
- "rpc.h",
+ "rpc-serde.h",
"server-name.h",
"table-name.h",
"zk.h",
],
srcs=[
- "rpc.cc",
+ "rpc-serde.cc",
"zk.cc",
],
deps=[
diff --git a/serde/client-deserializer-test.cc b/serde/client-deserializer-test.cc
index 054684d..1856047 100644
--- a/serde/client-deserializer-test.cc
+++ b/serde/client-deserializer-test.cc
@@ -16,12 +16,11 @@
* limitations under the License.
*
*/
-#include "serde/rpc.h"
-
#include <folly/io/IOBuf.h>
#include <gtest/gtest.h>
#include "if/Client.pb.h"
+#include "rpc-serde.h"
using namespace hbase;
using folly::IOBuf;
diff --git a/serde/client-serializer-test.cc b/serde/client-serializer-test.cc
index 7d8b29c..306f2c2 100644
--- a/serde/client-serializer-test.cc
+++ b/serde/client-serializer-test.cc
@@ -24,7 +24,7 @@
#include "if/HBase.pb.h"
#include "if/RPC.pb.h"
-#include "serde/rpc.h"
+#include "rpc-serde.h"
using namespace hbase;
using namespace hbase::pb;
diff --git a/serde/rpc.cc b/serde/rpc-serde.cc
similarity index 94%
rename from serde/rpc.cc
rename to serde/rpc-serde.cc
index 957a317..9e1f79a 100644
--- a/serde/rpc.cc
+++ b/serde/rpc-serde.cc
@@ -17,8 +17,6 @@
*
*/
-#include "serde/rpc.h"
-
#include <folly/Conv.h>
#include <folly/Logging.h>
#include <folly/io/Cursor.h>
@@ -30,6 +28,7 @@
#include <utility>
#include "if/RPC.pb.h"
+#include "rpc-serde.h"
#include "utils/version.h"
using folly::IOBuf;
@@ -83,6 +82,8 @@ int RpcSerde::ParseDelimited(const IOBuf *buf, Message *msg) {
return coded_stream.CurrentPosition();
}
+RpcSerde::RpcSerde() {}
+
RpcSerde::RpcSerde(std::shared_ptr<Codec> codec) : codec_(codec) {}
std::unique_ptr<IOBuf> RpcSerde::Preamble(bool secure) {
@@ -162,6 +163,17 @@ std::unique_ptr<IOBuf> RpcSerde::Request(const uint32_t call_id, const std::stri
return PrependLength(std::move(ser_header));
}
+std::unique_ptr<folly::IOBuf> RpcSerde::Response(const uint32_t call_id,
+ const google::protobuf::Message *msg) {
+ pb::ResponseHeader rh;
+ rh.set_call_id(call_id);
+ auto ser_header = SerializeDelimited(rh);
+ auto ser_resp = SerializeDelimited(*msg);
+ ser_header->appendChain(std::move(ser_resp));
+
+ return PrependLength(std::move(ser_header));
+}
+
std::unique_ptr<CellScanner> RpcSerde::CreateCellScanner(std::unique_ptr<folly::IOBuf> buf,
uint32_t offset, uint32_t length) {
if (codec_ == nullptr) {
diff --git a/serde/rpc.h b/serde/rpc-serde.h
similarity index 87%
rename from serde/rpc.h
rename to serde/rpc-serde.h
index 15aa1ee..0e1d44e 100644
--- a/serde/rpc.h
+++ b/serde/rpc-serde.h
@@ -45,6 +45,7 @@ namespace hbase {
*/
class RpcSerde {
public:
+ RpcSerde();
/**
* Constructor assumes the default auth type.
*/
@@ -98,6 +99,17 @@ class RpcSerde {
const google::protobuf::Message *msg);
/**
+ * Serialize a response message into a protobuf.
+ * Request consists of:
+ *
+ * - Big endian length
+ * - ResponseHeader object
+ * - The passed in Message object
+ */
+ std::unique_ptr<folly::IOBuf> Response(const uint32_t call_id,
+ const google::protobuf::Message *msg);
+
+ /**
* Serialize a message in the delimited format.
* Delimited format consists of the following:
*
@@ -117,6 +129,10 @@ class RpcSerde {
*/
std::unique_ptr<folly::IOBuf> PrependLength(std::unique_ptr<folly::IOBuf> msg);
+ public:
+ static constexpr const char *HBASE_CLIENT_RPC_TEST_MODE = "hbase.client.rpc.test.mode";
+ static constexpr const bool DEFAULT_HBASE_CLIENT_RPC_TEST_MODE = false;
+
private:
/* data */
std::shared_ptr<Codec> codec_;
[hbase] 48/137: HBASE-17219 [C++] Reformat the code according to
the style guidelines
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit e0a022536582667300d312ede378722da3132f0d
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Thu Dec 15 15:54:58 2016 -0800
HBASE-17219 [C++] Reformat the code according to the style guidelines
---
Dockerfile | 2 +-
bin/format-code.sh | 2 +-
connection/BUCK | 64 +++++++++++-----------
connection/client-dispatcher.h | 6 +--
connection/client-handler.cc | 3 +-
connection/client-handler.h | 6 +--
connection/connection-factory.h | 12 ++---
connection/connection-pool-test.cc | 6 +--
connection/connection-pool.cc | 6 ++-
connection/connection-pool.h | 9 ++--
connection/pipeline.cc | 4 +-
connection/pipeline.h | 10 ++--
connection/request.h | 6 +--
connection/response.h | 6 +--
connection/service.h | 2 +-
core/BUCK | 81 +++++++++++++---------------
core/cell.cc | 10 ++--
core/cell.h | 4 +-
core/client.cc | 4 +-
core/client.h | 6 +--
core/configuration.cc | 12 ++---
core/configuration.h | 2 +-
core/get-test.cc | 19 ++++---
core/get.cc | 68 ++++++++++--------------
core/get.h | 36 +++++++------
core/hbase_configuration-test.cc | 15 +++---
core/hbase_configuration_loader.cc | 7 ++-
core/hbase_configuration_loader.h | 1 -
core/hbase_macros.h | 4 +-
core/location-cache.cc | 16 +++---
core/location-cache.h | 10 ++--
core/meta-utils.h | 4 +-
core/region-location.h | 6 +--
core/scan-test.cc | 17 +++---
core/scan.cc | 106 +++++++++++++------------------------
core/scan.h | 91 +++++++++++++++++++++----------
core/time_range-test.cc | 8 ++-
core/time_range.cc | 38 +++++--------
core/time_range.h | 8 +--
serde/BUCK | 97 +++++++++++++++++----------------
serde/region-info.h | 9 ++--
serde/rpc.h | 16 +++---
serde/server-name.h | 5 +-
serde/table-name.h | 12 +++--
serde/zk.cc | 1 -
serde/zk.h | 4 +-
test-util/test-util.h | 6 +--
third-party/BUCK | 95 +++++++++++++++++----------------
utils/BUCK | 36 +++++--------
utils/user-util.h | 6 +--
50 files changed, 493 insertions(+), 511 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 4504a8e..8b56590 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -24,7 +24,7 @@ ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -p
ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
-RUN apt-get install -y vim maven inetutils-ping python-pip doxygen graphviz && \
+RUN apt-get install -y vim maven inetutils-ping python-pip doxygen graphviz clang-format && \
pip install yapf && \
apt-get -qq clean && \
apt-get -y -qq autoremove && \
diff --git a/bin/format-code.sh b/bin/format-code.sh
index 331717a..55fe69e 100755
--- a/bin/format-code.sh
+++ b/bin/format-code.sh
@@ -20,4 +20,4 @@ IFS=$'\n\t'
find core connection serde utils test-util -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i --style=Google
-find core connection serde utils third-party -name "BUCK" | xargs -P8 yapf -i
+find core connection serde utils third-party -name "BUCK" | xargs -P8 yapf -i --style=google
diff --git a/connection/BUCK b/connection/BUCK
index 39b8be5..f093d5a 100644
--- a/connection/BUCK
+++ b/connection/BUCK
@@ -17,34 +17,36 @@
# This is the library dealing with a single connection
# to a single server.
-cxx_library(name="connection",
- exported_headers=[
- "client-dispatcher.h",
- "client-handler.h",
- "connection-factory.h",
- "connection-pool.h",
- "pipeline.h",
- "request.h",
- "response.h",
- "service.h",
- ],
- srcs=[
- "client-dispatcher.cc",
- "client-handler.cc",
- "connection-factory.cc",
- "connection-pool.cc",
- "pipeline.cc",
- "request.cc",
- ],
- deps=[
- "//if:if",
- "//utils:utils",
- "//serde:serde",
- "//third-party:folly",
- "//third-party:wangle",
- ],
- compiler_flags=['-Weffc++'],
- visibility=['//core/...', ], )
-cxx_test(name="connection-pool-test",
- srcs=["connection-pool-test.cc", ],
- deps=[":connection", ], )
+cxx_library(
+ name="connection",
+ exported_headers=[
+ "client-dispatcher.h",
+ "client-handler.h",
+ "connection-factory.h",
+ "connection-pool.h",
+ "pipeline.h",
+ "request.h",
+ "response.h",
+ "service.h",
+ ],
+ srcs=[
+ "client-dispatcher.cc",
+ "client-handler.cc",
+ "connection-factory.cc",
+ "connection-pool.cc",
+ "pipeline.cc",
+ "request.cc",
+ ],
+ deps=[
+ "//if:if",
+ "//utils:utils",
+ "//serde:serde",
+ "//third-party:folly",
+ "//third-party:wangle",
+ ],
+ compiler_flags=['-Weffc++'],
+ visibility=['//core/...',],)
+cxx_test(
+ name="connection-pool-test",
+ srcs=["connection-pool-test.cc",],
+ deps=[":connection",],)
diff --git a/connection/client-dispatcher.h b/connection/client-dispatcher.h
index 2497cc7..5528a44 100644
--- a/connection/client-dispatcher.h
+++ b/connection/client-dispatcher.h
@@ -37,7 +37,7 @@ namespace hbase {
class ClientDispatcher
: public wangle::ClientDispatcherBase<SerializePipeline,
std::unique_ptr<Request>, Response> {
-public:
+ public:
/** Create a new ClientDispatcher */
ClientDispatcher();
/** Read a response off the pipeline. */
@@ -49,7 +49,7 @@ public:
/** Close the dispatcher and the associated pipeline. */
folly::Future<folly::Unit> close() override;
-private:
+ private:
folly::AtomicHashMap<uint32_t, folly::Promise<Response>> requests_;
// Start at some number way above what could
// be there for un-initialized call id counters.
@@ -61,4 +61,4 @@ private:
// not a big deal.
std::atomic<uint32_t> current_call_id_;
};
-} // namespace hbase
+} // namespace hbase
diff --git a/connection/client-handler.cc b/connection/client-handler.cc
index 3409b80..e445444 100644
--- a/connection/client-handler.cc
+++ b/connection/client-handler.cc
@@ -37,7 +37,8 @@ using hbase::pb::GetResponse;
using google::protobuf::Message;
ClientHandler::ClientHandler(std::string user_name)
- : user_name_(user_name), serde_(),
+ : user_name_(user_name),
+ serde_(),
once_flag_(std::make_unique<std::once_flag>()),
resp_msgs_(
make_unique<folly::AtomicHashMap<
diff --git a/connection/client-handler.h b/connection/client-handler.h
index 984a947..269b737 100644
--- a/connection/client-handler.h
+++ b/connection/client-handler.h
@@ -51,7 +51,7 @@ namespace hbase {
class ClientHandler : public wangle::Handler<std::unique_ptr<folly::IOBuf>,
Response, std::unique_ptr<Request>,
std::unique_ptr<folly::IOBuf>> {
-public:
+ public:
/**
* Create the handler
* @param user_name the user name of the user running this process.
@@ -71,7 +71,7 @@ public:
folly::Future<folly::Unit> write(Context *ctx,
std::unique_ptr<Request> r) override;
-private:
+ private:
std::unique_ptr<std::once_flag> once_flag_;
std::string user_name_;
RpcSerde serde_;
@@ -81,4 +81,4 @@ private:
uint32_t, std::shared_ptr<google::protobuf::Message>>>
resp_msgs_;
};
-} // namespace hbase
+} // namespace hbase
diff --git a/connection/connection-factory.h b/connection/connection-factory.h
index fb5d9fe..4627952 100644
--- a/connection/connection-factory.h
+++ b/connection/connection-factory.h
@@ -34,7 +34,7 @@ namespace hbase {
* pipeline.
*/
class ConnectionFactory {
-public:
+ public:
/**
* Constructor.
* There should only be one ConnectionFactory per client.
@@ -54,12 +54,12 @@ public:
*
* This is mostly visible so that mocks can override socket connections.
*/
- virtual std::shared_ptr<HBaseService>
- Connect(std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> client,
- const std::string &hostname, int port);
+ virtual std::shared_ptr<HBaseService> Connect(
+ std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> client,
+ const std::string &hostname, int port);
-private:
+ private:
std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool_;
std::shared_ptr<RpcPipelineFactory> pipeline_factory_;
};
-} // namespace hbase
+} // namespace hbase
diff --git a/connection/connection-pool-test.cc b/connection/connection-pool-test.cc
index bd2d585..e17c16c 100644
--- a/connection/connection-pool-test.cc
+++ b/connection/connection-pool-test.cc
@@ -33,7 +33,7 @@ using ::testing::Return;
using ::testing::_;
class MockConnectionFactory : public ConnectionFactory {
-public:
+ public:
MockConnectionFactory() : ConnectionFactory(nullptr) {}
MOCK_METHOD0(MakeBootstrap,
std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>());
@@ -46,7 +46,7 @@ public:
class MockBootstrap : public wangle::ClientBootstrap<SerializePipeline> {};
class MockServiceBase : public HBaseService {
-public:
+ public:
folly::Future<Response> operator()(std::unique_ptr<Request> req) override {
return do_operation(req.get());
}
@@ -56,7 +56,7 @@ public:
};
class MockService : public MockServiceBase {
-public:
+ public:
MOCK_METHOD1(do_operation, folly::Future<Response>(Request *));
};
diff --git a/connection/connection-pool.cc b/connection/connection-pool.cc
index aa3d094..2cb8e81 100644
--- a/connection/connection-pool.cc
+++ b/connection/connection-pool.cc
@@ -33,8 +33,10 @@ using folly::SocketAddress;
ConnectionPool::ConnectionPool(
std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor)
- : cf_(std::make_shared<ConnectionFactory>(io_executor)), clients_(),
- connections_(), map_mutex_() {}
+ : cf_(std::make_shared<ConnectionFactory>(io_executor)),
+ clients_(),
+ connections_(),
+ map_mutex_() {}
ConnectionPool::ConnectionPool(std::shared_ptr<ConnectionFactory> cf)
: cf_(cf), clients_(), connections_(), map_mutex_() {}
diff --git a/connection/connection-pool.h b/connection/connection-pool.h
index b8c950b..c89744e 100644
--- a/connection/connection-pool.h
+++ b/connection/connection-pool.h
@@ -18,8 +18,8 @@
*/
#pragma once
-#include <boost/functional/hash.hpp>
#include <folly/SharedMutex.h>
+#include <boost/functional/hash.hpp>
#include <mutex>
#include <unordered_map>
@@ -31,7 +31,6 @@ namespace hbase {
/** Equals function for server name that ignores start time */
struct ServerNameEquals {
-
/** equals */
bool operator()(const hbase::pb::ServerName &lhs,
const hbase::pb::ServerName &rhs) const {
@@ -59,7 +58,7 @@ struct ServerNameHash {
* regionserver has on it.
*/
class ConnectionPool {
-public:
+ public:
/** Create connection pool wit default connection factory */
ConnectionPool(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor);
@@ -87,7 +86,7 @@ public:
*/
void Close(const hbase::pb::ServerName &sn);
-private:
+ private:
std::shared_ptr<HBaseService> GetCached(const hbase::pb::ServerName &sn);
std::shared_ptr<HBaseService> GetNew(const hbase::pb::ServerName &sn);
std::unordered_map<hbase::pb::ServerName, std::shared_ptr<HBaseService>,
@@ -102,4 +101,4 @@ private:
std::shared_ptr<ConnectionFactory> cf_;
};
-} // namespace hbase
+} // namespace hbase
diff --git a/connection/pipeline.cc b/connection/pipeline.cc
index 39cce8b..14ad73c 100644
--- a/connection/pipeline.cc
+++ b/connection/pipeline.cc
@@ -32,8 +32,8 @@ using namespace wangle;
RpcPipelineFactory::RpcPipelineFactory() : user_util_() {}
-SerializePipeline::Ptr
-RpcPipelineFactory::newPipeline(std::shared_ptr<AsyncTransportWrapper> sock) {
+SerializePipeline::Ptr RpcPipelineFactory::newPipeline(
+ std::shared_ptr<AsyncTransportWrapper> sock) {
auto pipeline = SerializePipeline::create();
pipeline->addBack(AsyncSocketHandler{sock});
pipeline->addBack(EventBaseHandler{});
diff --git a/connection/pipeline.h b/connection/pipeline.h
index 88d7845..b969e2b 100644
--- a/connection/pipeline.h
+++ b/connection/pipeline.h
@@ -35,7 +35,7 @@ using SerializePipeline =
* Factory to create new pipelines for HBase RPC's.
*/
class RpcPipelineFactory : public wangle::PipelineFactory<SerializePipeline> {
-public:
+ public:
/**
* Constructor. This will create user util.
*/
@@ -50,10 +50,10 @@ public:
* - Length Field Based Frame Decoder
* - Client Handler
*/
- SerializePipeline::Ptr
- newPipeline(std::shared_ptr<folly::AsyncTransportWrapper> sock) override;
+ SerializePipeline::Ptr newPipeline(
+ std::shared_ptr<folly::AsyncTransportWrapper> sock) override;
-private:
+ private:
UserUtil user_util_;
};
-} // namespace hbase
+} // namespace hbase
diff --git a/connection/request.h b/connection/request.h
index 634d1ba..91c684d 100644
--- a/connection/request.h
+++ b/connection/request.h
@@ -32,7 +32,7 @@ namespace hbase {
* response object.
*/
class Request {
-public:
+ public:
/** Create a request object for a get */
static std::unique_ptr<Request> get();
/** Create a request object for a mutate */
@@ -62,10 +62,10 @@ public:
* method type to decode. */
std::string method() { return method_; }
-private:
+ private:
uint32_t call_id_;
std::shared_ptr<google::protobuf::Message> req_msg_ = nullptr;
std::shared_ptr<google::protobuf::Message> resp_msg_ = nullptr;
std::string method_ = "Get";
};
-} // namespace hbase
+} // namespace hbase
diff --git a/connection/response.h b/connection/response.h
index aac57c2..6458c6b 100644
--- a/connection/response.h
+++ b/connection/response.h
@@ -35,7 +35,7 @@ namespace hbase {
* This is the class sent to a service.
*/
class Response {
-public:
+ public:
/**
* Constructor.
* Initinalizes the call id to 0. 0 should never be a valid call id.
@@ -62,8 +62,8 @@ public:
resp_msg_ = std::move(response);
}
-private:
+ private:
uint32_t call_id_;
std::shared_ptr<google::protobuf::Message> resp_msg_;
};
-} // namespace hbase
+} // namespace hbase
diff --git a/connection/service.h b/connection/service.h
index 79f087d..226f936 100644
--- a/connection/service.h
+++ b/connection/service.h
@@ -25,4 +25,4 @@
namespace hbase {
using HBaseService = wangle::Service<std::unique_ptr<Request>, Response>;
-} // namespace hbase
+} // namespace hbase
diff --git a/core/BUCK b/core/BUCK
index fa01eb3..20e4736 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -53,46 +53,41 @@ cxx_library(
"//third-party:zookeeper_mt",
],
compiler_flags=['-Weffc++'],
- visibility=[
- 'PUBLIC',
- ], )
-cxx_test(name="location-cache-test",
- srcs=[
- "location-cache-test.cc",
- ],
- deps=[":core",
- "//test-util:test-util", ],
- run_test_separately=True, )
-cxx_test(name="cell-test",
- srcs=[
- "cell-test.cc",
- ],
- deps=[":core", ],
- run_test_separately=True, )
-cxx_test(name="get-test",
- srcs=[
- "get-test.cc",
- ],
- deps=[":core", ],
- run_test_separately=True, )
-cxx_test(name="time_range-test",
- srcs=[
- "time_range-test.cc",
- ],
- deps=[":core", ],
- run_test_separately=True, )
-cxx_test(name="hbase_configuration-test",
- srcs=[
- "hbase_configuration-test.cc",
- ],
- deps=[":core", ],
- run_test_separately=True, )
-cxx_test(name="scan-test",
- srcs=[
- "scan-test.cc",
- ],
- deps=[":core", ],
- run_test_separately=True, )
-cxx_binary(name="simple-client",
- srcs=["simple-client.cc", ],
- deps=[":core", "//connection:connection"], )
+ visibility=['PUBLIC',],)
+cxx_test(
+ name="location-cache-test",
+ srcs=["location-cache-test.cc",],
+ deps=[
+ ":core",
+ "//test-util:test-util",
+ ],
+ run_test_separately=True,)
+cxx_test(
+ name="cell-test",
+ srcs=["cell-test.cc",],
+ deps=[":core",],
+ run_test_separately=True,)
+cxx_test(
+ name="get-test",
+ srcs=["get-test.cc",],
+ deps=[":core",],
+ run_test_separately=True,)
+cxx_test(
+ name="time_range-test",
+ srcs=["time_range-test.cc",],
+ deps=[":core",],
+ run_test_separately=True,)
+cxx_test(
+ name="hbase_configuration-test",
+ srcs=["hbase_configuration-test.cc",],
+ deps=[":core",],
+ run_test_separately=True,)
+cxx_test(
+ name="scan-test",
+ srcs=["scan-test.cc",],
+ deps=[":core",],
+ run_test_separately=True,)
+cxx_binary(
+ name="simple-client",
+ srcs=["simple-client.cc",],
+ deps=[":core", "//connection:connection"],)
diff --git a/core/cell.cc b/core/cell.cc
index 5129bc9..37ebd13 100644
--- a/core/cell.cc
+++ b/core/cell.cc
@@ -25,9 +25,13 @@ namespace hbase {
Cell::Cell(const std::string &row, const std::string &family,
const std::string &qualifier, const long ×tamp,
const std::string &value, const hbase::CellType &cell_type)
- : row_(row), family_(family), qualifier_(qualifier), timestamp_(timestamp),
- cell_type_(cell_type), value_(value), sequence_id_(0) {
-
+ : row_(row),
+ family_(family),
+ qualifier_(qualifier),
+ timestamp_(timestamp),
+ cell_type_(cell_type),
+ value_(value),
+ sequence_id_(0) {
if (0 == row.size())
throw std::runtime_error("Row size should be greater than 0");
diff --git a/core/cell.h b/core/cell.h
index 2b15ad6..16ed280 100644
--- a/core/cell.h
+++ b/core/cell.h
@@ -34,7 +34,7 @@ enum CellType {
};
class Cell {
-public:
+ public:
Cell(const std::string &row, const std::string &family,
const std::string &qualifier, const long ×tamp,
const std::string &value, const hbase::CellType &cell_type);
@@ -47,7 +47,7 @@ public:
CellType Type() const;
long SequenceId() const;
-private:
+ private:
std::string row_;
std::string family_;
std::string qualifier_;
diff --git a/core/client.cc b/core/client.cc
index 82ecd22..0389b24 100644
--- a/core/client.cc
+++ b/core/client.cc
@@ -21,8 +21,8 @@
#include <glog/logging.h>
-#include <string>
#include <unistd.h>
+#include <string>
using namespace folly;
using namespace std;
@@ -43,4 +43,4 @@ Client::~Client() {
io_executor_->stop();
}
-} // namespace hbase
+} // namespace hbase
diff --git a/core/client.h b/core/client.h
index ba24bb9..296a8a0 100644
--- a/core/client.h
+++ b/core/client.h
@@ -39,7 +39,7 @@ namespace hbase {
* have only one Client per cluster around.
*/
class Client {
-public:
+ public:
/**
* Create a new client.
* @param quorum_spec Where to connect to get Zookeeper bootstrap information.
@@ -47,10 +47,10 @@ public:
explicit Client(std::string quorum_spec);
~Client();
-private:
+ private:
std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_;
LocationCache location_cache_;
};
-} // namespace hbase
+} // namespace hbase
diff --git a/core/configuration.cc b/core/configuration.cc
index 9816cff..c074cf6 100644
--- a/core/configuration.cc
+++ b/core/configuration.cc
@@ -21,8 +21,8 @@
#include <stdexcept>
-#include <boost/lexical_cast.hpp>
#include <glog/logging.h>
+#include <boost/lexical_cast.hpp>
namespace hbase {
@@ -129,7 +129,6 @@ optional<std::string> Configuration::GetEnv(const std::string &key) const {
}
optional<std::string> Configuration::GetProperty(const std::string &key) const {
-
auto found = hb_property_.find(key);
if (found != hb_property_.end()) {
return std::experimental::make_optional(found->second.value);
@@ -158,8 +157,7 @@ optional<int32_t> Configuration::GetInt(const std::string &key) const {
try {
return std::experimental::make_optional(
boost::lexical_cast<int32_t>(*raw));
- }
- catch (const boost::bad_lexical_cast &blex) {
+ } catch (const boost::bad_lexical_cast &blex) {
throw std::runtime_error(blex.what());
}
}
@@ -177,8 +175,7 @@ optional<int64_t> Configuration::GetLong(const std::string &key) const {
try {
return std::experimental::make_optional(
boost::lexical_cast<int64_t>(*raw));
- }
- catch (const boost::bad_lexical_cast &blex) {
+ } catch (const boost::bad_lexical_cast &blex) {
throw std::runtime_error(blex.what());
}
}
@@ -196,8 +193,7 @@ optional<double> Configuration::GetDouble(const std::string &key) const {
try {
return std::experimental::make_optional(
boost::lexical_cast<double>(*raw));
- }
- catch (const boost::bad_lexical_cast &blex) {
+ } catch (const boost::bad_lexical_cast &blex) {
throw std::runtime_error(blex.what());
}
}
diff --git a/core/configuration.h b/core/configuration.h
index 44d6cd6..d313cf6 100644
--- a/core/configuration.h
+++ b/core/configuration.h
@@ -88,7 +88,7 @@ class Configuration {
struct ConfigData {
std::string value;
bool final;
- ConfigData() : final(false) {};
+ ConfigData() : final(false){};
ConfigData(const std::string &value) : value(value), final(false) {}
void operator=(const std::string &new_value) {
value = new_value;
diff --git a/core/get-test.cc b/core/get-test.cc
index 9a09a23..53c14e3 100644
--- a/core/get-test.cc
+++ b/core/get-test.cc
@@ -19,8 +19,8 @@
#include "core/get.h"
-#include <gtest/gtest.h>
#include <glog/logging.h>
+#include <gtest/gtest.h>
using namespace hbase;
const int NUMBER_OF_GETS = 5;
@@ -135,15 +135,16 @@ void GetMethods(Get &get, const std::string &row) {
ASSERT_THROW(get.SetTimeRange(-1000, 2000), std::runtime_error);
ASSERT_THROW(get.SetTimeRange(1000, -2000), std::runtime_error);
ASSERT_THROW(get.SetTimeRange(1000, 200), std::runtime_error);
- ASSERT_THROW(get.SetTimeStamp(std::numeric_limits<long>::max()), std::runtime_error);
+ ASSERT_THROW(get.SetTimeStamp(std::numeric_limits<long>::max()),
+ std::runtime_error);
- //Test some exceptions
+ // Test some exceptions
ASSERT_THROW(get.SetMaxVersions(0), std::runtime_error);
ASSERT_THROW(get.SetMaxVersions(std::numeric_limits<unsigned int>::max() + 1),
std::runtime_error);
}
-TEST (Get, SingleGet) {
+TEST(Get, SingleGet) {
std::string row_str = "row-test";
Get get(row_str);
GetMethods(get, row_str);
@@ -156,7 +157,8 @@ TEST (Get, SingleGet) {
geteq = get_tmp;
GetMethods(geteq, row_str);
- // Adding the below tests as there were some concerns raised that the same vector of qualifiers in FamilyMap is being shared between copied objects
+ // Adding the below tests as there were some concerns raised that the same
+ // vector of qualifiers in FamilyMap is being shared between copied objects
// Verify the source object's family map size before using it to copy.
EXPECT_EQ(3, get.Family().size());
@@ -192,11 +194,12 @@ TEST (Get, SingleGet) {
++it;
EXPECT_EQ(it, get.Family().end());
- //Verifying the copied object's families. It will remain unchanged and below tests should pass
+ // Verifying the copied object's families. It will remain unchanged and below
+ // tests should pass
CheckFamiliesAfterCopy(getcp_fam);
}
-TEST (Get, MultiGet) {
+TEST(Get, MultiGet) {
std::vector<Get *> gets;
for (int i = 0; i < NUMBER_OF_GETS; i++) {
std::string row_str = "row-test";
@@ -214,7 +217,7 @@ TEST (Get, MultiGet) {
gets.clear();
}
-TEST (Get, Exception) {
+TEST(Get, Exception) {
std::string row(std::numeric_limits<short>::max() + 1, 'X');
ASSERT_THROW(Get tmp = Get(row), std::runtime_error);
ASSERT_THROW(Get tmp = Get(""), std::runtime_error);
diff --git a/core/get.cc b/core/get.cc
index b412a4b..a70da37 100644
--- a/core/get.cc
+++ b/core/get.cc
@@ -24,13 +24,9 @@
namespace hbase {
-Get::~Get() {
-}
+Get::~Get() {}
-Get::Get(const std::string &row)
- : row_(row) {
- CheckRow(row_);
-}
+Get::Get(const std::string &row) : row_(row) { CheckRow(row_); }
Get::Get(const Get &get) {
row_ = get.row_;
@@ -38,27 +34,30 @@ Get::Get(const Get &get) {
cache_blocks_ = get.cache_blocks_;
check_existence_only_ = get.check_existence_only_;
consistency_ = get.consistency_;
- tr_.reset(new TimeRange(get.Timerange().MinTimeStamp(), get.Timerange().MaxTimeStamp()));
+ tr_.reset(new TimeRange(get.Timerange().MinTimeStamp(),
+ get.Timerange().MaxTimeStamp()));
family_map_.insert(get.family_map_.begin(), get.family_map_.end());
}
-Get& Get::operator=(const Get &get) {
+Get &Get::operator=(const Get &get) {
row_ = get.row_;
max_versions_ = get.max_versions_;
cache_blocks_ = get.cache_blocks_;
check_existence_only_ = get.check_existence_only_;
consistency_ = get.consistency_;
- tr_.reset(new TimeRange(get.Timerange().MinTimeStamp(), get.Timerange().MaxTimeStamp()));
+ tr_.reset(new TimeRange(get.Timerange().MinTimeStamp(),
+ get.Timerange().MaxTimeStamp()));
family_map_.insert(get.family_map_.begin(), get.family_map_.end());
return *this;
}
-Get& Get::AddFamily(const std::string &family) {
+Get &Get::AddFamily(const std::string &family) {
const auto &it = family_map_.find(family);
/**
* Check if any qualifiers are already present or not.
- * Remove all existing qualifiers if the given family is already present in the map
+ * Remove all existing qualifiers if the given family is already present in
+ * the map
*/
if (family_map_.end() != it) {
it->second.clear();
@@ -68,8 +67,9 @@ Get& Get::AddFamily(const std::string &family) {
return *this;
}
-Get& Get::AddColumn(const std::string &family, const std::string &qualifier) {
- const auto &it = std::find(family_map_[family].begin(), family_map_[family].end(), qualifier);
+Get &Get::AddColumn(const std::string &family, const std::string &qualifier) {
+ const auto &it = std::find(family_map_[family].begin(),
+ family_map_[family].end(), qualifier);
/**
* Check if any qualifiers are already present or not.
@@ -81,32 +81,22 @@ Get& Get::AddColumn(const std::string &family, const std::string &qualifier) {
return *this;
}
-const std::string& Get::Row() const {
- return row_;
-}
+const std::string &Get::Row() const { return row_; }
-hbase::pb::Consistency Get::Consistency() const {
- return consistency_;
-}
+hbase::pb::Consistency Get::Consistency() const { return consistency_; }
Get &Get::SetConsistency(hbase::pb::Consistency consistency) {
consistency_ = consistency;
return *this;
}
-bool Get::HasFamilies() const {
- return !family_map_.empty();
-}
+bool Get::HasFamilies() const { return !family_map_.empty(); }
-const FamilyMap &Get::Family() const {
- return family_map_;
-}
+const FamilyMap &Get::Family() const { return family_map_; }
-int Get::MaxVersions() const {
- return max_versions_;
-}
+int Get::MaxVersions() const { return max_versions_; }
-Get& Get::SetMaxVersions(uint32_t max_versions) {
+Get &Get::SetMaxVersions(uint32_t max_versions) {
if (0 == max_versions)
throw std::runtime_error("max_versions must be positive");
@@ -114,28 +104,24 @@ Get& Get::SetMaxVersions(uint32_t max_versions) {
return *this;
}
-bool Get::CacheBlocks() const {
- return cache_blocks_;
-}
+bool Get::CacheBlocks() const { return cache_blocks_; }
-Get & Get::SetCacheBlocks(bool cache_blocks) {
+Get &Get::SetCacheBlocks(bool cache_blocks) {
cache_blocks_ = cache_blocks;
return *this;
}
-Get& Get::SetTimeRange(long min_timestamp, long max_timestamp) {
+Get &Get::SetTimeRange(long min_timestamp, long max_timestamp) {
tr_.reset(new TimeRange(min_timestamp, max_timestamp));
return *this;
}
-Get& Get::SetTimeStamp(long timestamp) {
+Get &Get::SetTimeStamp(long timestamp) {
tr_.reset(new TimeRange(timestamp, timestamp + 1));
return *this;
}
-const TimeRange& Get::Timerange() const {
- return *tr_;
-}
+const TimeRange &Get::Timerange() const { return *tr_; }
void Get::CheckRow(const std::string &row) {
const int kMaxRowLength = std::numeric_limits<short>::max();
@@ -144,9 +130,9 @@ void Get::CheckRow(const std::string &row) {
throw std::runtime_error("Row length can't be 0");
}
if (row_length > kMaxRowLength) {
- throw std::runtime_error(
- "Length of " + row + " is greater than max row size: " + std::to_string(kMaxRowLength));
+ throw std::runtime_error("Length of " + row +
+ " is greater than max row size: " +
+ std::to_string(kMaxRowLength));
}
}
-
}
diff --git a/core/get.h b/core/get.h
index 9704aa1..90c42ee 100644
--- a/core/get.h
+++ b/core/get.h
@@ -19,31 +19,30 @@
#pragma once
+#include <cstdint>
#include <map>
#include <memory>
#include <string>
#include <vector>
-#include <cstdint>
#include "core/time_range.h"
#include "if/Client.pb.h"
namespace hbase {
/**
- * @brief Map consisting of column families and qualifiers to be used for Get operation
+ * @brief Map consisting of column families and qualifiers to be used for Get
+ * operation
*/
using FamilyMap = std::map<std::string, std::vector<std::string>>;
class Get {
-
public:
-
/**
* Constructors
*/
- Get(const std::string &row);
- Get(const Get &cget);
- Get& operator=(const Get &cget);
+ Get(const std::string& row);
+ Get(const Get& cget);
+ Get& operator=(const Get& cget);
~Get();
@@ -53,7 +52,8 @@ class Get {
int MaxVersions() const;
/**
- * @brief Get up to the specified number of versions of each column. default is 1.
+ * @brief Get up to the specified number of versions of each column. default
+ * is 1.
* @param max_versions max_versons to set
*/
Get& SetMaxVersions(uint32_t max_versions = 1);
@@ -70,9 +70,10 @@ class Get {
Get& SetCacheBlocks(bool cache_blocks);
/**
- * @brief Returns the Get family map (FamilyMap) for this Get operation. Used for constructing Scan object with an already constructed Get
+ * @brief Returns the Get family map (FamilyMap) for this Get operation. Used
+ * for constructing Scan object with an already constructed Get
*/
- const FamilyMap &Family() const;
+ const FamilyMap& Family() const;
/**
* @brief Returns the timerange for this Get
@@ -80,7 +81,8 @@ class Get {
const TimeRange& Timerange() const;
/**
- * @brief Get versions of columns only within the specified timestamp range, [minStamp, maxStamp).
+ * @brief Get versions of columns only within the specified timestamp range,
+ * [minStamp, maxStamp).
* @param minStamp the minimum timestamp, inclusive
* @param maxStamp the maximum timestamp, exclusive
*/
@@ -96,14 +98,15 @@ class Get {
* @brief Get all columns from the specified family.
* @param family to be retrieved
*/
- Get& AddFamily(const std::string &family);
+ Get& AddFamily(const std::string& family);
/**
- * @brief Get the column from the specific family with the specified qualifier.
+ * @brief Get the column from the specific family with the specified
+ * qualifier.
* @param family to be retrieved
* @param qualifier to be retrieved
*/
- Get& AddColumn(const std::string &family, const std::string &qualifier);
+ Get& AddColumn(const std::string& family, const std::string& qualifier);
/**
* @brief Returns the row for this Get operation
@@ -138,8 +141,9 @@ class Get {
/**
* @brief Checks if the row for this Get operation is proper or not
* @param row Row to check
- * @throws std::runtime_error if row is empty or greater than MAX_ROW_LENGTH(i.e. std::numeric_limits<short>::max())
+ * @throws std::runtime_error if row is empty or greater than
+ * MAX_ROW_LENGTH(i.e. std::numeric_limits<short>::max())
*/
- void CheckRow(const std::string &row);
+ void CheckRow(const std::string& row);
};
}
diff --git a/core/hbase_configuration-test.cc b/core/hbase_configuration-test.cc
index 167f506..eec500e 100644
--- a/core/hbase_configuration-test.cc
+++ b/core/hbase_configuration-test.cc
@@ -20,16 +20,18 @@
#include <fstream>
#include <iostream>
-#include <boost/filesystem.hpp>
-#include <gtest/gtest.h>
#include <glog/logging.h>
-#include "core/hbase_configuration_loader.h"
+#include <gtest/gtest.h>
+#include <boost/filesystem.hpp>
#include "core/configuration.h"
+#include "core/hbase_configuration_loader.h"
using namespace hbase;
-const std::string kDefHBaseConfPath("./build/test-data/hbase-configuration-test/conf/");
-const std::string kHBaseConfPath("./build/test-data/hbase-configuration-test/custom-conf/");
+const std::string kDefHBaseConfPath(
+ "./build/test-data/hbase-configuration-test/conf/");
+const std::string kHBaseConfPath(
+ "./build/test-data/hbase-configuration-test/custom-conf/");
const std::string kHBaseDefaultXml("hbase-default.xml");
const std::string kHBaseSiteXml("hbase-site.xml");
@@ -131,7 +133,6 @@ void CreateHBaseConf(const std::string &dir, const std::string &file,
}
void CreateHBaseConfWithEnv() {
-
CreateHBaseConf(kDefHBaseConfPath, kHBaseDefaultXml, kHBaseDefaultXmlData);
CreateHBaseConf(kDefHBaseConfPath, kHBaseSiteXml, kHBaseSiteXmlData);
setenv("HBASE_CONF", kDefHBaseConfPath.c_str(), 1);
@@ -178,7 +179,7 @@ TEST(Configuration, LoadConfFromCustomLocation) {
/*
* Config will be loaded from hbase-defualt.xml and hbase-site.xml @
- * kDefHBaseConfPath and kHBaseConfPath respectively.
+ * kDefHBaseConfPath and kHBaseConfPath respectively.
*/
TEST(Configuration, LoadConfFromMultipleLocatons) {
// Remove already configured env if present.
diff --git a/core/hbase_configuration_loader.cc b/core/hbase_configuration_loader.cc
index 43f733a..d3aff8c 100644
--- a/core/hbase_configuration_loader.cc
+++ b/core/hbase_configuration_loader.cc
@@ -19,10 +19,10 @@
#include "core/hbase_configuration_loader.h"
+#include <glog/logging.h>
#include <boost/foreach.hpp>
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/xml_parser.hpp>
-#include <glog/logging.h>
namespace hbase {
@@ -172,7 +172,7 @@ bool HBaseConfigurationLoader::LoadProperties(const std::string &file,
// Iterate over configuration section.
// Store all found properties in ConfigMap
- BOOST_FOREACH(ptree::value_type & v, pt.get_child("configuration")) {
+ BOOST_FOREACH (ptree::value_type &v, pt.get_child("configuration")) {
if ("property" == v.first) {
std::string name_node = v.second.get<std::string>("name");
std::string value_node = v.second.get<std::string>("value");
@@ -183,8 +183,7 @@ bool HBaseConfigurationLoader::LoadProperties(const std::string &file,
}
}
}
- }
- catch (std::exception &ex) {
+ } catch (std::exception &ex) {
DLOG(WARNING) << "Exception in parsing file [" << file << "]:[" << ex.what()
<< "]";
return false;
diff --git a/core/hbase_configuration_loader.h b/core/hbase_configuration_loader.h
index 54d90a5..49dc2b9 100644
--- a/core/hbase_configuration_loader.h
+++ b/core/hbase_configuration_loader.h
@@ -34,7 +34,6 @@ template <class T>
using optional = std::experimental::optional<T>;
class HBaseConfigurationLoader {
-
public:
HBaseConfigurationLoader();
~HBaseConfigurationLoader();
diff --git a/core/hbase_macros.h b/core/hbase_macros.h
index 48304f0..d70c247 100644
--- a/core/hbase_macros.h
+++ b/core/hbase_macros.h
@@ -52,5 +52,5 @@ extern "C" {
#endif
#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
+} // extern "C"
+#endif // __cplusplus
diff --git a/core/location-cache.cc b/core/location-cache.cc
index efd2210..4359795 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -53,8 +53,12 @@ LocationCache::LocationCache(
std::string quorum_spec,
std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor)
- : quorum_spec_(quorum_spec), cpu_executor_(cpu_executor),
- meta_promise_(nullptr), meta_lock_(), cp_(io_executor), meta_util_(),
+ : quorum_spec_(quorum_spec),
+ cpu_executor_(cpu_executor),
+ meta_promise_(nullptr),
+ meta_lock_(),
+ cp_(io_executor),
+ meta_util_(),
zk_(nullptr) {
zk_ = zookeeper_init(quorum_spec.c_str(), nullptr, 1000, 0, 0, 0);
}
@@ -111,8 +115,8 @@ ServerName LocationCache::ReadMetaLocation() {
return mrs.server();
}
-Future<std::shared_ptr<RegionLocation>>
-LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
+Future<std::shared_ptr<RegionLocation>> LocationCache::LocateFromMeta(
+ const TableName &tn, const string &row) {
return this->LocateMeta()
.via(cpu_executor_.get())
.then([this](ServerName sn) { return this->cp_.Get(sn); })
@@ -139,8 +143,8 @@ LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
});
}
-std::shared_ptr<RegionLocation>
-LocationCache::CreateLocation(const Response &resp) {
+std::shared_ptr<RegionLocation> LocationCache::CreateLocation(
+ const Response &resp) {
auto resp_msg = static_pointer_cast<ScanResponse>(resp.resp_msg());
auto &results = resp_msg->results().Get(0);
auto &cells = results.cell();
diff --git a/core/location-cache.h b/core/location-cache.h
index 830cd96..53e41ab 100644
--- a/core/location-cache.h
+++ b/core/location-cache.h
@@ -46,7 +46,7 @@ class ServerName;
* Class that can look up and cache locations.
*/
class LocationCache {
-public:
+ public:
/**
* Constructor.
* @param quorum_spec Where to connect for Zookeeper.
@@ -79,15 +79,15 @@ public:
* @param row of the table to look up. This object must live until after the
* future is returned
*/
- folly::Future<std::shared_ptr<RegionLocation>>
- LocateFromMeta(const hbase::pb::TableName &tn, const std::string &row);
+ folly::Future<std::shared_ptr<RegionLocation>> LocateFromMeta(
+ const hbase::pb::TableName &tn, const std::string &row);
/**
* Remove the cached location of meta.
*/
void InvalidateMeta();
-private:
+ private:
void RefreshMetaLocation();
hbase::pb::ServerName ReadMetaLocation();
std::shared_ptr<RegionLocation> CreateLocation(const Response &resp);
@@ -103,4 +103,4 @@ private:
// TODO: migrate this to a smart pointer with a deleter.
zhandle_t *zk_;
};
-} // namespace hbase
+} // namespace hbase
diff --git a/core/meta-utils.h b/core/meta-utils.h
index 294e17d..303a273 100644
--- a/core/meta-utils.h
+++ b/core/meta-utils.h
@@ -30,7 +30,7 @@ namespace hbase {
* @brief Utility for meta operations.
*/
class MetaUtil {
-public:
+ public:
/**
* Given a table and a row give the row key from which to start a scan to find
* region locations.
@@ -45,4 +45,4 @@ public:
std::unique_ptr<Request> MetaRequest(const hbase::pb::TableName tn,
const std::string &row) const;
};
-} // namespace hbase
+} // namespace hbase
diff --git a/core/region-location.h b/core/region-location.h
index a3d832e..b3c124d 100644
--- a/core/region-location.h
+++ b/core/region-location.h
@@ -32,7 +32,7 @@ namespace hbase {
* region name, and a connection to the service used for connecting to it.
*/
class RegionLocation {
-public:
+ public:
/**
* Constructor.
* @param region_name The region name of this region.
@@ -79,11 +79,11 @@ public:
*/
void set_server_name(hbase::pb::ServerName sn) { sn_ = sn; }
-private:
+ private:
std::string region_name_;
hbase::pb::RegionInfo ri_;
hbase::pb::ServerName sn_;
std::shared_ptr<HBaseService> service_;
};
-} // namespace hbase
+} // namespace hbase
diff --git a/core/scan-test.cc b/core/scan-test.cc
index a114502..b6a6f3e 100644
--- a/core/scan-test.cc
+++ b/core/scan-test.cc
@@ -168,15 +168,17 @@ void ScanMethods(Scan &scan) {
ASSERT_THROW(scan.SetTimeRange(-1000, 2000), std::runtime_error);
ASSERT_THROW(scan.SetTimeRange(1000, -2000), std::runtime_error);
ASSERT_THROW(scan.SetTimeRange(1000, 200), std::runtime_error);
- ASSERT_THROW(scan.SetTimeStamp(std::numeric_limits<long>::max()), std::runtime_error);
+ ASSERT_THROW(scan.SetTimeStamp(std::numeric_limits<long>::max()),
+ std::runtime_error);
}
-TEST (Scan, Object) {
+TEST(Scan, Object) {
Scan scan;
ScanMethods(scan);
CheckFamilies(scan);
- // Resetting TimeRange values so that the copy construction and assignment operator tests pass.
+ // Resetting TimeRange values so that the copy construction and assignment
+ // operator tests pass.
scan.SetTimeRange(0, std::numeric_limits<long>::max());
Scan scancp(scan);
ScanMethods(scancp);
@@ -188,22 +190,21 @@ TEST (Scan, Object) {
CheckFamiliesAfterCopy(scaneq);
}
-TEST (Scan, WithStartRow) {
+TEST(Scan, WithStartRow) {
Scan("row-test");
Scan scan("row-test");
ScanMethods(scan);
CheckFamilies(scan);
}
-TEST (Scan, WithStartAndStopRow) {
+TEST(Scan, WithStartAndStopRow) {
Scan("start-row", "stop-row");
Scan scan("start-row", "stop-row");
ScanMethods(scan);
CheckFamilies(scan);
}
-TEST (Scan,FromGet) {
-
+TEST(Scan, FromGet) {
std::string row_str = "row-test";
Get get = Get(row_str);
@@ -226,7 +227,7 @@ TEST (Scan,FromGet) {
CheckFamiliesAfterCopy(scan);
}
-TEST (Scan, Exception) {
+TEST(Scan, Exception) {
std::string row(std::numeric_limits<short>::max() + 1, 'X');
ASSERT_THROW(Scan tmp(row), std::runtime_error);
ASSERT_THROW(Scan tmp(""), std::runtime_error);
diff --git a/core/scan.cc b/core/scan.cc
index 318cdb7..2dd8ef2 100644
--- a/core/scan.cc
+++ b/core/scan.cc
@@ -26,20 +26,16 @@
namespace hbase {
-Scan::Scan() {
-}
+Scan::Scan() {}
-Scan::~Scan() {
-}
+Scan::~Scan() {}
-Scan::Scan(const std::string &start_row)
- : start_row_(start_row) {
+Scan::Scan(const std::string &start_row) : start_row_(start_row) {
CheckRow(start_row_);
}
Scan::Scan(const std::string &start_row, const std::string &stop_row)
- : start_row_(start_row),
- stop_row_(stop_row) {
+ : start_row_(start_row), stop_row_(stop_row) {
CheckRow(start_row_);
CheckRow(stop_row_);
}
@@ -60,7 +56,7 @@ Scan::Scan(const Scan &scan) {
family_map_.insert(scan.family_map_.begin(), scan.family_map_.end());
}
-Scan& Scan::operator=(const Scan &scan) {
+Scan &Scan::operator=(const Scan &scan) {
start_row_ = scan.start_row_;
stop_row_ = scan.stop_row_;
max_versions_ = scan.max_versions_;
@@ -80,15 +76,17 @@ Scan& Scan::operator=(const Scan &scan) {
Scan::Scan(const Get &get) {
cache_blocks_ = get.CacheBlocks();
max_versions_ = get.MaxVersions();
- tr_.reset(new TimeRange(get.Timerange().MinTimeStamp(), get.Timerange().MaxTimeStamp()));
+ tr_.reset(new TimeRange(get.Timerange().MinTimeStamp(),
+ get.Timerange().MaxTimeStamp()));
family_map_.insert(get.Family().begin(), get.Family().end());
}
-Scan& Scan::AddFamily(const std::string &family) {
+Scan &Scan::AddFamily(const std::string &family) {
const auto &it = family_map_.find(family);
/**
* Check if any qualifiers are already present or not.
- * Remove all existing qualifiers if the given family is already present in the map
+ * Remove all existing qualifiers if the given family is already present in
+ * the map
*/
if (family_map_.end() != it) {
it->second.clear();
@@ -98,8 +96,9 @@ Scan& Scan::AddFamily(const std::string &family) {
return *this;
}
-Scan& Scan::AddColumn(const std::string &family, const std::string &qualifier) {
- const auto &it = std::find(family_map_[family].begin(), family_map_[family].end(), qualifier);
+Scan &Scan::AddColumn(const std::string &family, const std::string &qualifier) {
+ const auto &it = std::find(family_map_[family].begin(),
+ family_map_[family].end(), qualifier);
/**
* Check if any qualifiers are already present or not.
* Add only if qualifiers for a given family are not present
@@ -110,72 +109,48 @@ Scan& Scan::AddColumn(const std::string &family, const std::string &qualifier) {
return *this;
}
-void Scan::SetReversed(bool reversed) {
- reversed_ = reversed;
-}
+void Scan::SetReversed(bool reversed) { reversed_ = reversed; }
-bool Scan::IsReversed() const {
- return reversed_;
-}
+bool Scan::IsReversed() const { return reversed_; }
void Scan::SetStartRow(std::string &start_row) {
CheckRow(start_row);
start_row_ = start_row;
}
-const std::string& Scan::StartRow() const {
- return start_row_;
-}
+const std::string &Scan::StartRow() const { return start_row_; }
void Scan::SetStopRow(std::string &stop_row) {
CheckRow(stop_row);
stop_row_ = stop_row;
}
-const std::string& Scan::StopRow() const {
- return stop_row_;
-}
+const std::string &Scan::StopRow() const { return stop_row_; }
-void Scan::SetSmall(bool small) {
- small_ = small;
-}
+void Scan::SetSmall(bool small) { small_ = small; }
-bool Scan::IsSmall() const {
- return small_;
-}
+bool Scan::IsSmall() const { return small_; }
-void Scan::SetCaching(int caching) {
- caching_ = caching;
-}
+void Scan::SetCaching(int caching) { caching_ = caching; }
-int Scan::Caching() const {
- return caching_;
-}
+int Scan::Caching() const { return caching_; }
-Scan& Scan::SetConsistency(const hbase::pb::Consistency consistency) {
+Scan &Scan::SetConsistency(const hbase::pb::Consistency consistency) {
consistency_ = consistency;
return *this;
}
-hbase::pb::Consistency Scan::Consistency() const {
- return consistency_;
-}
+hbase::pb::Consistency Scan::Consistency() const { return consistency_; }
-void Scan::SetCacheBlocks(bool cache_blocks) {
- cache_blocks_ = cache_blocks;
-}
+void Scan::SetCacheBlocks(bool cache_blocks) { cache_blocks_ = cache_blocks; }
-bool Scan::CacheBlocks() const {
- return cache_blocks_;
-}
+bool Scan::CacheBlocks() const { return cache_blocks_; }
void Scan::SetAllowPartialResults(bool allow_partial_results) {
allow_partial_results_ = allow_partial_results;
}
-bool Scan::AllowPartialResults() const {
- return allow_partial_results_;
-}
+bool Scan::AllowPartialResults() const { return allow_partial_results_; }
void Scan::SetLoadColumnFamiliesOnDemand(bool load_column_families_on_demand) {
load_column_families_on_demand_ = load_column_families_on_demand;
@@ -185,36 +160,30 @@ bool Scan::LoadColumnFamiliesOnDemand() const {
return load_column_families_on_demand_;
}
-Scan& Scan::SetMaxVersions(uint32_t max_versions) {
+Scan &Scan::SetMaxVersions(uint32_t max_versions) {
max_versions_ = max_versions;
return *this;
}
-int Scan::MaxVersions() const {
- return max_versions_;
-}
+int Scan::MaxVersions() const { return max_versions_; }
void Scan::SetMaxResultSize(long max_result_size) {
max_result_size_ = max_result_size;
}
-long Scan::MaxResultSize() const {
- return max_result_size_;
-}
+long Scan::MaxResultSize() const { return max_result_size_; }
Scan &Scan::SetTimeRange(long min_stamp, long max_stamp) {
tr_.reset(new TimeRange(min_stamp, max_stamp));
return *this;
}
-Scan& Scan::SetTimeStamp(long timestamp) {
+Scan &Scan::SetTimeStamp(long timestamp) {
tr_.reset(new TimeRange(timestamp, timestamp + 1));
return *this;
}
-const TimeRange& Scan::Timerange() const {
- return *tr_;
-}
+const TimeRange &Scan::Timerange() const { return *tr_; }
void Scan::CheckRow(const std::string &row) {
const int kMaxRowLength = std::numeric_limits<short>::max();
@@ -223,17 +192,14 @@ void Scan::CheckRow(const std::string &row) {
throw std::runtime_error("Row length can't be 0");
}
if (row_length > kMaxRowLength) {
- throw std::runtime_error(
- "Length of " + row + " is greater than max row size: " + std::to_string(kMaxRowLength));
+ throw std::runtime_error("Length of " + row +
+ " is greater than max row size: " +
+ std::to_string(kMaxRowLength));
}
}
-bool Scan::HasFamilies() const {
- return !family_map_.empty();
-}
+bool Scan::HasFamilies() const { return !family_map_.empty(); }
-const FamilyMap &Scan::Family() const {
- return family_map_;
-}
+const FamilyMap &Scan::Family() const { return family_map_; }
}
/* namespace hbase */
diff --git a/core/scan.h b/core/scan.h
index bd655ce..99ed26a 100644
--- a/core/scan.h
+++ b/core/scan.h
@@ -30,7 +30,8 @@
namespace hbase {
/**
- * @brief Map consisting of column families and qualifiers to be used for Get operation
+ * @brief Map consisting of column families and qualifiers to be used for Get
+ * operation
*/
using FamilyMap = std::map<std::string, std::vector<std::string>>;
@@ -41,12 +42,13 @@ class Scan {
*/
Scan();
Scan(const Scan &scan);
- Scan& operator=(const Scan &scan);
+ Scan &operator=(const Scan &scan);
~Scan();
/**
- * @brief Create a Scan operation starting at the specified row. If the specified row does not exist,
+ * @brief Create a Scan operation starting at the specified row. If the
+ * specified row does not exist,
* the Scanner will start from the next closest row after the specified row.
* @param start_row - row to start scanner at or after
*/
@@ -66,52 +68,60 @@ class Scan {
Scan(const Get &get);
/**
- * @brief Get all columns from the specified family.Overrides previous calls to AddColumn for this family.
+ * @brief Get all columns from the specified family.Overrides previous calls
+ * to AddColumn for this family.
* @param family - family name
*/
- Scan& AddFamily(const std::string &family);
+ Scan &AddFamily(const std::string &family);
/**
- * @brief Get the column from the specified family with the specified qualifier.Overrides previous calls to AddFamily for this family.
+ * @brief Get the column from the specified family with the specified
+ * qualifier.Overrides previous calls to AddFamily for this family.
* @param family - family name.
* @param qualifier - column qualifier.
*/
- Scan& AddColumn(const std::string &family, const std::string &qualifier);
+ Scan &AddColumn(const std::string &family, const std::string &qualifier);
/**
- * @brief Set whether this scan is a reversed one. This is false by default which means forward(normal) scan.
+ * @brief Set whether this scan is a reversed one. This is false by default
+ * which means forward(normal) scan.
* @param reversed - if true, scan will be backward order
*/
void SetReversed(bool reversed);
/**
- * @brief Get whether this scan is a reversed one. Returns true if backward scan, false if forward(default) scan
+ * @brief Get whether this scan is a reversed one. Returns true if backward
+ * scan, false if forward(default) scan
*/
bool IsReversed() const;
/**
- * @brief Set the start row of the scan.If the specified row does not exist, the Scanner will start from the next closest row after the specified row.
+ * @brief Set the start row of the scan.If the specified row does not exist,
+ * the Scanner will start from the next closest row after the specified row.
* @param start_row - row to start scanner at or after
- * @throws std::runtime_error if start_row length is 0 or greater than MAX_ROW_LENGTH
+ * @throws std::runtime_error if start_row length is 0 or greater than
+ * MAX_ROW_LENGTH
*/
void SetStartRow(std::string &start_row);
/**
* @brief returns start_row of the Scan.
*/
- const std::string& StartRow() const;
+ const std::string &StartRow() const;
/**
- * @brief Set the stop row of the scan. The scan will include rows that are lexicographically less than the provided stop_row.
+ * @brief Set the stop row of the scan. The scan will include rows that are
+ * lexicographically less than the provided stop_row.
* @param stop_row - row to end at (exclusive)
- * @throws std::runtime_error if stop_row length is 0 or greater than MAX_ROW_LENGTH
+ * @throws std::runtime_error if stop_row length is 0 or greater than
+ * MAX_ROW_LENGTH
*/
void SetStopRow(std::string &stop_row);
/**
* @brief returns stop_row of the Scan.
*/
- const std::string& StopRow() const;
+ const std::string &StopRow() const;
/**
* @brief Set whether this scan is a small scan.
@@ -124,7 +134,8 @@ class Scan {
bool IsSmall() const;
/**
- * @brief Set the number of rows for caching that will be passed to scanners. Higher caching values will enable faster scanners but will use more memory.
+ * @brief Set the number of rows for caching that will be passed to scanners.
+ * Higher caching values will enable faster scanners but will use more memory.
* @param caching - the number of rows for caching.
*/
void SetCaching(int caching);
@@ -138,7 +149,7 @@ class Scan {
* @brief Sets the consistency level for this operation.
* @param consistency - the consistency level
*/
- Scan& SetConsistency(const hbase::pb::Consistency consistency);
+ Scan &SetConsistency(const hbase::pb::Consistency consistency);
/**
* @brief Returns the consistency level for this operation.
@@ -146,8 +157,12 @@ class Scan {
hbase::pb::Consistency Consistency() const;
/**
- * @brief Set whether blocks should be cached for this Scan.This is true by default. When true, default settings of the table and family are used (this will never override caching blocks if the block cache is disabled for that family or entirely).
- * @param cache_blocks - if false, default settings are overridden and blocks will not be cached
+ * @brief Set whether blocks should be cached for this Scan.This is true by
+ * default. When true, default settings of the table and family are used (this
+ * will never override caching blocks if the block cache is disabled for that
+ * family or entirely).
+ * @param cache_blocks - if false, default settings are overridden and blocks
+ * will not be cached
*/
void SetCacheBlocks(bool cache_blocks);
@@ -157,18 +172,25 @@ class Scan {
bool CacheBlocks() const;
/**
- * @brief Setting whether the caller wants to see the partial results that may be returned from the server. By default this value is false and the complete results will be assembled client side before being delivered to the caller.
+ * @brief Setting whether the caller wants to see the partial results that may
+ * be returned from the server. By default this value is false and the
+ * complete results will be assembled client side before being delivered to
+ * the caller.
* @param allow_partial_results - if true partial results will be returned.
*/
void SetAllowPartialResults(bool allow_partial_results);
/**
- * @brief true when the constructor of this scan understands that the results they will see may only represent a partial portion of a row. The entire row would be retrieved by subsequent calls to ResultScanner.next()
+ * @brief true when the constructor of this scan understands that the results
+ * they will see may only represent a partial portion of a row. The entire row
+ * would be retrieved by subsequent calls to ResultScanner.next()
*/
bool AllowPartialResults() const;
/**
- * @brief Set the value indicating whether loading CFs on demand should be allowed (cluster default is false). On-demand CF loading doesn't load column families until necessary.
+ * @brief Set the value indicating whether loading CFs on demand should be
+ * allowed (cluster default is false). On-demand CF loading doesn't load
+ * column families until necessary.
* @param load_column_families_on_demand
*/
void SetLoadColumnFamiliesOnDemand(bool load_column_families_on_demand);
@@ -179,7 +201,8 @@ class Scan {
bool LoadColumnFamiliesOnDemand() const;
/**
- * @brief Get up to the specified number of versions of each column if specified else get default i.e. one.
+ * @brief Get up to the specified number of versions of each column if
+ * specified else get default i.e. one.
* @param max_versions - maximum versions for each column.
*/
Scan &SetMaxVersions(uint32_t max_versions = 1);
@@ -190,7 +213,9 @@ class Scan {
int MaxVersions() const;
/**
- * @brief Set the maximum result size. The default is -1; this means that no specific maximum result size will be set for this scan, and the global configured value will be used instead. (Defaults to unlimited).
+ * @brief Set the maximum result size. The default is -1; this means that no
+ * specific maximum result size will be set for this scan, and the global
+ * configured value will be used instead. (Defaults to unlimited).
* @param The maximum result size in bytes.
*/
void SetMaxResultSize(long max_result_size);
@@ -201,14 +226,20 @@ class Scan {
long MaxResultSize() const;
/**
- * @brief Get versions of columns only within the specified timestamp range, [min_stamp, max_stamp). Note, default maximum versions to return is 1. If your time range spans more than one version and you want all versions returned, up the number of versions beyond the default.
+ * @brief Get versions of columns only within the specified timestamp range,
+ * [min_stamp, max_stamp). Note, default maximum versions to return is 1. If
+ * your time range spans more than one version and you want all versions
+ * returned, up the number of versions beyond the default.
* @param min_stamp - minimum timestamp value, inclusive.
* @param max_stamp - maximum timestamp value, exclusive.
*/
Scan &SetTimeRange(long min_stamp, long max_stamp);
/**
- * @brief Get versions of columns with the specified timestamp. Note, default maximum versions to return is 1. If your time range spans more than one version and you want all versions returned, up the number of versions beyond the defaut.
+ * @brief Get versions of columns with the specified timestamp. Note, default
+ * maximum versions to return is 1. If your time range spans more than one
+ * version and you want all versions returned, up the number of versions
+ * beyond the defaut.
* @param timestamp - version timestamp
*/
Scan &SetTimeStamp(long timestamp);
@@ -216,7 +247,7 @@ class Scan {
/**
* @brief Return Timerange
*/
- const TimeRange& Timerange() const;
+ const TimeRange &Timerange() const;
/**
* @brief Returns true if family map (FamilyMap) is non empty false otherwise
@@ -244,9 +275,11 @@ class Scan {
FamilyMap family_map_;
/**
- * @brief Checks for row length validity, throws if length check fails, returns null otherwise.
+ * @brief Checks for row length validity, throws if length check fails,
+ * returns null otherwise.
* @param row - row whose validity needs to be checked
- * @throws std::runtime_error if row length equals 0 or greater than std::numeric_limits<short>::max();
+ * @throws std::runtime_error if row length equals 0 or greater than
+ * std::numeric_limits<short>::max();
*/
void CheckRow(const std::string &row);
};
diff --git a/core/time_range-test.cc b/core/time_range-test.cc
index c08d52a..e066c57 100644
--- a/core/time_range-test.cc
+++ b/core/time_range-test.cc
@@ -19,13 +19,12 @@
#include "core/time_range.h"
-#include <gtest/gtest.h>
#include <glog/logging.h>
+#include <gtest/gtest.h>
using namespace hbase;
-TEST (TimeRange, DefaultObject) {
-
+TEST(TimeRange, DefaultObject) {
TimeRange *timerange_def = nullptr;
ASSERT_NO_THROW(timerange_def = new TimeRange());
@@ -37,8 +36,7 @@ TEST (TimeRange, DefaultObject) {
timerange_def = nullptr;
}
-TEST (TimeRange, Exception) {
-
+TEST(TimeRange, Exception) {
// Negative Min TS
ASSERT_THROW(TimeRange(-1000, 2000), std::runtime_error);
diff --git a/core/time_range.cc b/core/time_range.cc
index 9e6ed99..2accc9a 100644
--- a/core/time_range.cc
+++ b/core/time_range.cc
@@ -27,9 +27,7 @@ namespace hbase {
TimeRange::TimeRange()
: min_timestamp_(0L),
max_timestamp_(std::numeric_limits<long>::max()),
- all_time_(true) {
-
-}
+ all_time_(true) {}
TimeRange::TimeRange(const TimeRange &tr) {
this->all_time_ = tr.all_time_;
@@ -37,16 +35,14 @@ TimeRange::TimeRange(const TimeRange &tr) {
this->min_timestamp_ = tr.min_timestamp_;
}
-TimeRange &TimeRange::operator =(const TimeRange &tr) {
+TimeRange &TimeRange::operator=(const TimeRange &tr) {
this->all_time_ = tr.all_time_;
this->max_timestamp_ = tr.max_timestamp_;
this->min_timestamp_ = tr.min_timestamp_;
return *this;
}
-TimeRange::~TimeRange() {
-
-}
+TimeRange::~TimeRange() {}
TimeRange::TimeRange(long min_timestamp) {
this->min_timestamp_ = min_timestamp;
@@ -56,16 +52,15 @@ TimeRange::TimeRange(long min_timestamp) {
TimeRange::TimeRange(long min_timestamp, long max_timestamp) {
if (min_timestamp < 0 || max_timestamp < 0) {
- throw std::runtime_error(
- "Timestamp cannot be negative. min_timestamp: "
- + std::to_string(min_timestamp) + ", max_timestamp:"
- + std::to_string(max_timestamp));
+ throw std::runtime_error("Timestamp cannot be negative. min_timestamp: " +
+ std::to_string(min_timestamp) +
+ ", max_timestamp:" +
+ std::to_string(max_timestamp));
}
if (max_timestamp < min_timestamp) {
- throw std::runtime_error(
- "max_timestamp [" + std::to_string(max_timestamp)
- + "] should be greater than min_timestamp ["
- + std::to_string(min_timestamp) + "]");
+ throw std::runtime_error("max_timestamp [" + std::to_string(max_timestamp) +
+ "] should be greater than min_timestamp [" +
+ std::to_string(min_timestamp) + "]");
}
this->min_timestamp_ = min_timestamp;
@@ -73,16 +68,9 @@ TimeRange::TimeRange(long min_timestamp, long max_timestamp) {
this->all_time_ = false;
}
-long TimeRange::MinTimeStamp() const {
- return this->min_timestamp_;
+long TimeRange::MinTimeStamp() const { return this->min_timestamp_; }
-}
-
-long TimeRange::MaxTimeStamp() const {
- return this->max_timestamp_;
-}
+long TimeRange::MaxTimeStamp() const { return this->max_timestamp_; }
-bool TimeRange::IsAllTime() const {
- return this->all_time_;
-}
+bool TimeRange::IsAllTime() const { return this->all_time_; }
}
diff --git a/core/time_range.h b/core/time_range.h
index 5c07b4b..171f3ed 100644
--- a/core/time_range.h
+++ b/core/time_range.h
@@ -21,11 +21,10 @@
namespace hbase {
class TimeRange {
-
public:
-
/**
- * @brief Default constructor. Represents interval [0, Long.MAX_VALUE) (allTime)
+ * @brief Default constructor. Represents interval [0, Long.MAX_VALUE)
+ * (allTime)
*/
TimeRange();
TimeRange(const TimeRange &tr);
@@ -39,7 +38,8 @@ class TimeRange {
* @brief Represents interval [minStamp, maxStamp)
* @param minStamp the minimum timestamp, inclusive
* @param maxStamp the maximum timestamp, exclusive
- * @throws std::runtime_error if min_timestamp < 0 or max_timestamp < 0 or max_timestamp < min_timestamp
+ * @throws std::runtime_error if min_timestamp < 0 or max_timestamp < 0 or
+ * max_timestamp < min_timestamp
*/
TimeRange(long min_timestamp, long max_timestamp);
long MinTimeStamp() const;
diff --git a/serde/BUCK b/serde/BUCK
index db15026..b5aad3d 100644
--- a/serde/BUCK
+++ b/serde/BUCK
@@ -15,49 +15,54 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-cxx_library(name="serde",
- exported_headers=[
- "region-info.h",
- "rpc.h",
- "server-name.h",
- "table-name.h",
- "zk.h",
- ],
- srcs=[
- "rpc.cc",
- "zk.cc",
- ],
- deps=[
- "//if:if",
- "//third-party:folly",
- ],
- tests=[
- ":client-deserializer-test",
- ":client-serializer-test",
- ":server-name-test",
- ":table-name-test",
- ":zk-deserializer-test",
- ":region-info-deserializer-test",
- ],
- compiler_flags=['-Weffc++'],
- visibility=[
- 'PUBLIC',
- ], )
-cxx_test(name="table-name-test",
- srcs=["table-name-test.cc", ],
- deps=[":serde", ], )
-cxx_test(name="server-name-test",
- srcs=["server-name-test.cc", ],
- deps=[":serde", ], )
-cxx_test(name="client-serializer-test",
- srcs=["client-serializer-test.cc", ],
- deps=[":serde", ], )
-cxx_test(name="client-deserializer-test",
- srcs=["client-deserializer-test.cc", ],
- deps=[":serde", ], )
-cxx_test(name="zk-deserializer-test",
- srcs=["zk-deserializer-test.cc", ],
- deps=[":serde", ], )
-cxx_test(name="region-info-deserializer-test",
- srcs=["region-info-deserializer-test.cc", ],
- deps=[":serde", ], )
+cxx_library(
+ name="serde",
+ exported_headers=[
+ "region-info.h",
+ "rpc.h",
+ "server-name.h",
+ "table-name.h",
+ "zk.h",
+ ],
+ srcs=[
+ "rpc.cc",
+ "zk.cc",
+ ],
+ deps=[
+ "//if:if",
+ "//third-party:folly",
+ ],
+ tests=[
+ ":client-deserializer-test",
+ ":client-serializer-test",
+ ":server-name-test",
+ ":table-name-test",
+ ":zk-deserializer-test",
+ ":region-info-deserializer-test",
+ ],
+ compiler_flags=['-Weffc++'],
+ visibility=['PUBLIC',],)
+cxx_test(
+ name="table-name-test",
+ srcs=["table-name-test.cc",],
+ deps=[":serde",],)
+cxx_test(
+ name="server-name-test",
+ srcs=["server-name-test.cc",],
+ deps=[":serde",],)
+cxx_test(
+ name="client-serializer-test",
+ srcs=["client-serializer-test.cc",],
+ deps=[":serde",],)
+cxx_test(
+ name="client-deserializer-test",
+ srcs=["client-deserializer-test.cc",],
+ deps=[":serde",],)
+cxx_test(
+ name="zk-deserializer-test",
+ srcs=["zk-deserializer-test.cc",],
+ deps=[":serde",],)
+cxx_test(
+ name="region-info-deserializer-test",
+ srcs=["region-info-deserializer-test.cc",],
+ deps=[":serde",],)
diff --git a/serde/region-info.h b/serde/region-info.h
index e2ecfc9..1b0a3fb 100644
--- a/serde/region-info.h
+++ b/serde/region-info.h
@@ -21,12 +21,13 @@
#include "if/HBase.pb.h"
-#include <boost/algorithm/string/predicate.hpp>
#include <folly/Conv.h>
+#include <boost/algorithm/string/predicate.hpp>
namespace hbase {
namespace pb {
-template <class String> void parseTo(String in, RegionInfo &out) {
+template <class String>
+void parseTo(String in, RegionInfo &out) {
// TODO(eclark): there has to be something better.
std::string s = folly::to<std::string>(in);
@@ -37,5 +38,5 @@ template <class String> void parseTo(String in, RegionInfo &out) {
throw std::runtime_error("Bad protobuf for RegionInfo");
}
}
-} // namespace pb
-} // namespace hbase
+} // namespace pb
+} // namespace hbase
diff --git a/serde/rpc.h b/serde/rpc.h
index b28b4a3..873d01c 100644
--- a/serde/rpc.h
+++ b/serde/rpc.h
@@ -40,7 +40,7 @@ namespace hbase {
* It should throw exceptions if anything goes wrong.
*/
class RpcSerde {
-public:
+ public:
/**
* Constructor assumes the default auth type.
*/
@@ -94,23 +94,23 @@ public:
* - A protobuf var int32
* - The message object seriailized after that.
*/
- std::unique_ptr<folly::IOBuf>
- SerializeDelimited(const google::protobuf::Message &msg);
+ std::unique_ptr<folly::IOBuf> SerializeDelimited(
+ const google::protobuf::Message &msg);
/**
* Serilalize a message. This does not add any length prepend.
*/
- std::unique_ptr<folly::IOBuf>
- SerializeMessage(const google::protobuf::Message &msg);
+ std::unique_ptr<folly::IOBuf> SerializeMessage(
+ const google::protobuf::Message &msg);
/**
* Prepend a length IOBuf to the given IOBuf chain.
* This involves no copies or moves of the passed in data.
*/
- std::unique_ptr<folly::IOBuf>
- PrependLength(std::unique_ptr<folly::IOBuf> msg);
+ std::unique_ptr<folly::IOBuf> PrependLength(
+ std::unique_ptr<folly::IOBuf> msg);
-private:
+ private:
/* data */
uint8_t auth_type_;
};
diff --git a/serde/server-name.h b/serde/server-name.h
index 6551fdc..dda9bf2 100644
--- a/serde/server-name.h
+++ b/serde/server-name.h
@@ -19,14 +19,15 @@
#pragma once
-#include "if/HBase.pb.h"
#include <folly/Conv.h>
#include <folly/String.h>
+#include "if/HBase.pb.h"
namespace hbase {
namespace pb {
-template <class String> void parseTo(String in, ServerName &out) {
+template <class String>
+void parseTo(String in, ServerName &out) {
// TODO see about getting rsplit into folly.
std::string s = folly::to<std::string>(in);
diff --git a/serde/table-name.h b/serde/table-name.h
index c81e166..707ba44 100644
--- a/serde/table-name.h
+++ b/serde/table-name.h
@@ -21,15 +21,16 @@
#include <memory>
#include <string>
-#include "if/HBase.pb.h"
#include <folly/Conv.h>
#include <folly/String.h>
+#include "if/HBase.pb.h"
namespace hbase {
namespace pb {
// Provide folly::to<std::string>(TableName);
-template <class String> void toAppend(const TableName &in, String *result) {
+template <class String>
+void toAppend(const TableName &in, String *result) {
if (!in.has_namespace_() || in.namespace_() == "default") {
folly::toAppend(in.qualifier(), result);
} else {
@@ -37,7 +38,8 @@ template <class String> void toAppend(const TableName &in, String *result) {
}
}
-template <class String> void parseTo(String in, TableName &out) {
+template <class String>
+void parseTo(String in, TableName &out) {
std::vector<std::string> v;
folly::split(":", in, v);
@@ -50,5 +52,5 @@ template <class String> void parseTo(String in, TableName &out) {
}
}
-} // namespace pb
-} // namespace hbase
+} // namespace pb
+} // namespace hbase
diff --git a/serde/zk.cc b/serde/zk.cc
index 59871a5..63c96ed 100644
--- a/serde/zk.cc
+++ b/serde/zk.cc
@@ -32,7 +32,6 @@ using google::protobuf::Message;
static const std::string MAGIC_STRING = "PBUF";
bool ZkDeserializer::Parse(IOBuf *buf, Message *out) {
-
// The format is like this
// 1 byte of magic number. 255
// 4 bytes of id length.
diff --git a/serde/zk.h b/serde/zk.h
index fa3de5b..5cadec2 100644
--- a/serde/zk.h
+++ b/serde/zk.h
@@ -35,7 +35,7 @@ namespace hbase {
*
*/
class ZkDeserializer {
-public:
+ public:
/**
* Merge the data from a buffer into a given message.
*
@@ -46,4 +46,4 @@ public:
*/
bool Parse(folly::IOBuf *buf, google::protobuf::Message *out);
};
-} // namespace hbase
+} // namespace hbase
diff --git a/test-util/test-util.h b/test-util/test-util.h
index 9050961..2fc9d69 100644
--- a/test-util/test-util.h
+++ b/test-util/test-util.h
@@ -29,7 +29,7 @@ namespace hbase {
* @brief Class to deal with a local instance cluster for testing.
*/
class TestUtil {
-public:
+ public:
/**
* Creating a TestUtil will spin up a cluster.
*/
@@ -55,7 +55,7 @@ public:
*/
static std::string RandString(int len = 32);
-private:
+ private:
folly::test::TemporaryDirectory temp_dir_;
};
-} // namespace hbase
+} // namespace hbase
diff --git a/third-party/BUCK b/third-party/BUCK
index 8c0b5f9..f37eb4e 100644
--- a/third-party/BUCK
+++ b/third-party/BUCK
@@ -28,14 +28,15 @@ def add_system_libs(names=[],
genrule(
name=gen_rule_name,
out=gen_rule_name,
- bash="mkdir -p $OUT && cp {}/lib{}.a $OUT".format(lib_dir, name), )
- prebuilt_cxx_library(name=name,
- lib_name=name,
- lib_dir='$(location :{})'.format(gen_rule_name),
- deps=deps,
- force_static=True,
- exported_deps=exported_deps,
- visibility=rule_visibility, )
+ bash="mkdir -p $OUT && cp {}/lib{}.a $OUT".format(lib_dir, name),)
+ prebuilt_cxx_library(
+ name=name,
+ lib_name=name,
+ lib_dir='$(location :{})'.format(gen_rule_name),
+ deps=deps,
+ force_static=True,
+ exported_deps=exported_deps,
+ visibility=rule_visibility,)
rules.append(":" + name)
return rules
@@ -43,16 +44,19 @@ def add_system_libs(names=[],
def add_dynamic_libs(names=[]):
rules = []
for name in names:
- prebuilt_cxx_library(name=name,
- header_only=True,
- exported_linker_flags=["-l" + name],
- visibility=["PUBLIC"], )
+ prebuilt_cxx_library(
+ name=name,
+ header_only=True,
+ exported_linker_flags=["-l" + name],
+ visibility=["PUBLIC"],)
rules.append(":" + name)
return rules
-system_libs = ["lzma",
- "event", ]
+system_libs = [
+ "lzma",
+ "event",
+]
local_libs = [
"double-conversion",
"boost_regex",
@@ -73,38 +77,39 @@ tp_dep_rules = add_system_libs(system_libs,) \
+ dynamic_rules
zookeeper = add_system_libs(["zookeeper_mt"], lib_dir="/usr/local/lib")
-folly = add_system_libs(['folly'],
- lib_dir='/usr/local/lib',
- exported_deps=tp_dep_rules, )
-folly_bench = add_system_libs(['follybenchmark'],
- lib_dir='/usr/local/lib',
- exported_deps=folly + tp_dep_rules, )
-wangle = add_system_libs(['wangle'],
- lib_dir='/usr/local/lib',
- exported_deps=folly + tp_dep_rules)
+folly = add_system_libs(
+ ['folly'],
+ lib_dir='/usr/local/lib',
+ exported_deps=tp_dep_rules,)
+folly_bench = add_system_libs(
+ ['follybenchmark'],
+ lib_dir='/usr/local/lib',
+ exported_deps=folly + tp_dep_rules,)
+wangle = add_system_libs(
+ ['wangle'], lib_dir='/usr/local/lib', exported_deps=folly + tp_dep_rules)
genrule(
name="gen_zk",
out="gen_zk",
- bash="mkdir -p $OUT && wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && tar zxf zookeeper-3.4.8.tar.gz && rm -rf zookeeper-3.4.8.tar.gz && cd zookeeper-3.4.8 && cd src/c && ./configure --prefix=$OUT && make && make install && cd $OUT && rm -rf zookeeper-3.4.8*")
-cxx_library(name='google-test',
- srcs=[
- 'googletest/googletest/src/gtest-all.cc',
- 'googletest/googlemock/src/gmock-all.cc',
- 'googletest/googlemock/src/gmock_main.cc',
- ],
- header_namespace='',
- exported_headers=subdir_glob([
- ('googletest/googletest/include', '**/*.h'),
- ('googletest/googlemock/include', '**/*.h'),
- ]),
- headers=subdir_glob([
- ('googletest/googletest', 'src/*.h'),
- ('googletest/googletest', 'src/*.cc'),
- ('googletest/googlemock', 'src/*.h'),
- ('googletest/googlemock', 'src/*.cc'),
- ]),
- exported_deps=dynamic_rules,
- visibility=[
- 'PUBLIC',
- ], )
+ bash="mkdir -p $OUT && wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && tar zxf zookeeper-3.4.8.tar.gz && rm -rf zookeeper-3.4.8.tar.gz && cd zookeeper-3.4.8 && cd src/c && ./configure --prefix=$OUT && make && make install && cd $OUT && rm -rf zookeeper-3.4.8*"
+)
+cxx_library(
+ name='google-test',
+ srcs=[
+ 'googletest/googletest/src/gtest-all.cc',
+ 'googletest/googlemock/src/gmock-all.cc',
+ 'googletest/googlemock/src/gmock_main.cc',
+ ],
+ header_namespace='',
+ exported_headers=subdir_glob([
+ ('googletest/googletest/include', '**/*.h'),
+ ('googletest/googlemock/include', '**/*.h'),
+ ]),
+ headers=subdir_glob([
+ ('googletest/googletest', 'src/*.h'),
+ ('googletest/googletest', 'src/*.cc'),
+ ('googletest/googlemock', 'src/*.h'),
+ ('googletest/googlemock', 'src/*.cc'),
+ ]),
+ exported_deps=dynamic_rules,
+ visibility=['PUBLIC',],)
diff --git a/utils/BUCK b/utils/BUCK
index 2ab6a89..120331a 100644
--- a/utils/BUCK
+++ b/utils/BUCK
@@ -15,27 +15,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-cxx_library(name="utils",
- exported_headers=[
- "user-util.h",
- ],
- srcs=[
- "user-util.cc",
- ],
- deps=[
- '//third-party:folly',
- ],
- tests=[
- ":user-util-test"
- ],
- visibility=[
- 'PUBLIC',
- ],
- compiler_flags=['-Weffc++'], )
-cxx_test(name="user-util-test",
- srcs=[
- "user-util-test.cc",
- ],
- deps=[
- ":utils",
- ], )
+cxx_library(
+ name="utils",
+ exported_headers=["user-util.h",],
+ srcs=["user-util.cc",],
+ deps=['//third-party:folly',],
+ tests=[":user-util-test"],
+ visibility=['PUBLIC',],
+ compiler_flags=['-Weffc++'],)
+cxx_test(
+ name="user-util-test",
+ srcs=["user-util-test.cc",],
+ deps=[":utils",],)
diff --git a/utils/user-util.h b/utils/user-util.h
index 0704bc6..6f8fce1 100644
--- a/utils/user-util.h
+++ b/utils/user-util.h
@@ -31,7 +31,7 @@ namespace hbase {
* the user information after the first invocation.
*/
class UserUtil {
-public:
+ public:
/**
* Constructor.
*/
@@ -43,7 +43,7 @@ public:
*/
std::string user_name();
-private:
+ private:
/**
* Compute the username. This will block.
*/
@@ -51,4 +51,4 @@ private:
std::once_flag once_flag_;
std::string user_name_;
};
-} // namespace hbase
+} // namespace hbase
[hbase] 107/137: HBASE-18407 [C++] make Configuration::Set/GetBool
work for both true/false and 1/0 (Xiaobing Zhou)
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 96d8879ebfbd852162aca4e8afbdee9cecc0d3ca
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Fri Jul 21 16:46:54 2017 -0700
HBASE-18407 [C++] make Configuration::Set/GetBool work for both true/false and 1/0 (Xiaobing Zhou)
---
core/configuration-test.cc | 58 +++++++++++++++++++++++++++++++++++++++++++++-
core/configuration.cc | 10 +++++---
2 files changed, 64 insertions(+), 4 deletions(-)
diff --git a/core/configuration-test.cc b/core/configuration-test.cc
index 192ed46..abdf0c7 100644
--- a/core/configuration-test.cc
+++ b/core/configuration-test.cc
@@ -22,6 +22,62 @@
using hbase::Configuration;
+TEST(Configuration, SetGetBool) {
+ Configuration conf;
+
+ /* test true/false */
+ conf.SetBool("bool_key1", true);
+ EXPECT_EQ(true, conf.GetBool("bool_key1", false));
+ conf.SetBool("bool_key2", false);
+ EXPECT_EQ(false, conf.GetBool("bool_key2", true));
+
+ /* test 1/0 */
+ conf.SetBool("bool_key3", 1);
+ EXPECT_EQ(true, conf.GetBool("bool_key3", false));
+ conf.SetBool("bool_key4", 0);
+ EXPECT_EQ(false, conf.GetBool("bool_key4", true));
+
+ /* test non zero integer */
+ conf.SetBool("bool_key5", 5);
+ EXPECT_EQ(true, conf.GetBool("bool_key5", false));
+ conf.SetBool("bool_key6", -1);
+ EXPECT_EQ(true, conf.GetBool("bool_key5", false));
+
+ /* test non zero float */
+ conf.SetBool("bool_key7", 5.1);
+ EXPECT_EQ(true, conf.GetBool("bool_key7", false));
+ conf.SetBool("bool_key8", -1.2);
+ EXPECT_EQ(true, conf.GetBool("bool_key8", false));
+}
+
+TEST(Configuration, SetGetForBool) {
+ Configuration conf;
+
+ /* test true/false */
+ conf.Set("bool_key1", "true");
+ EXPECT_EQ(true, conf.GetBool("bool_key1", false));
+ conf.Set("bool_key2", "false");
+ EXPECT_EQ(false, conf.GetBool("bool_key2", true));
+
+ /* test 1/0 */
+ conf.Set("bool_key3", "1");
+ EXPECT_EQ(true, conf.GetBool("bool_key3", false));
+ conf.Set("bool_key4", "0");
+ EXPECT_EQ(false, conf.GetBool("bool_key4", true));
+
+ /* test non zero integer */
+ conf.Set("bool_key5", "5");
+ EXPECT_THROW(conf.GetBool("bool_key5", false), std::runtime_error);
+ conf.Set("bool_key6", "-1");
+ EXPECT_THROW(conf.GetBool("bool_key6", false), std::runtime_error);
+
+ /* test non zero float */
+ conf.Set("bool_key7", "5.1");
+ EXPECT_THROW(conf.GetBool("bool_key7", false), std::runtime_error);
+ conf.Set("bool_key8", "-1.2");
+ EXPECT_THROW(conf.GetBool("bool_key8", false), std::runtime_error);
+}
+
TEST(Configuration, SetGet) {
Configuration conf;
@@ -54,7 +110,7 @@ TEST(Configuration, SetGetDouble) {
EXPECT_EQ(conf.GetDouble("foo", 0), 42.0);
}
-TEST(Configuration, SetGetBool) {
+TEST(Configuration, SetGetBoolBasic) {
Configuration conf;
EXPECT_EQ(conf.GetBool("foo", false), false);
diff --git a/core/configuration.cc b/core/configuration.cc
index afca122..f4fc46d 100644
--- a/core/configuration.cc
+++ b/core/configuration.cc
@@ -25,6 +25,7 @@
#include <glog/logging.h>
#include <boost/lexical_cast.hpp>
+#include <boost/format.hpp>
namespace hbase {
@@ -202,12 +203,15 @@ double Configuration::GetDouble(const std::string &key, double default_value) co
optional<bool> Configuration::GetBool(const std::string &key) const {
optional<std::string> raw = Get(key);
if (raw) {
- if (!strcasecmp((*raw).c_str(), "true")) {
+ if (!strcasecmp((*raw).c_str(), "true") || !strcasecmp((*raw).c_str(), "1")) {
return std::experimental::make_optional(true);
- } else if (!strcasecmp((*raw).c_str(), "false")) {
+ } else if (!strcasecmp((*raw).c_str(), "false") || !strcasecmp((*raw).c_str(), "0")) {
return std::experimental::make_optional(false);
} else {
- throw std::runtime_error("Unexpected value found while conversion to bool.");
+ boost::format what("Unexpected value \"%s\" found being converted to bool for key \"%s\"");
+ what % (*raw);
+ what % key;
+ throw std::runtime_error(what.str());
}
}
return optional<bool>();
[hbase] 93/137: HBASE-17907 [C++] End to end Scans from Client/Table
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit f1614d86ca9b6c20a4b30568fd88464792a6663b
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Tue Jun 6 15:16:09 2017 -0700
HBASE-17907 [C++] End to end Scans from Client/Table
---
bin/format-code.sh | 4 +-
connection/client-handler.cc | 5 +-
connection/request.h | 6 +
connection/response.h | 22 +-
connection/rpc-client.cc | 2 -
connection/rpc-client.h | 12 +-
core/BUCK | 41 ++-
core/async-client-scanner.cc | 142 ++++++++++
core/async-client-scanner.h | 119 ++++++++
core/async-rpc-retrying-caller-factory.h | 124 ++++++++-
core/async-rpc-retrying-caller.cc | 12 +-
core/async-rpc-retrying-caller.h | 2 -
core/async-scan-rpc-retrying-caller.cc | 447 +++++++++++++++++++++++++++++++
core/async-scan-rpc-retrying-caller.h | 233 ++++++++++++++++
core/async-table-result-scanner.cc | 161 +++++++++++
core/async-table-result-scanner.h | 98 +++++++
core/cell-test.cc | 24 ++
core/cell.cc | 21 +-
core/cell.h | 5 +-
core/client-test.cc | 13 +-
core/delete-test.cc | 2 +-
core/delete.cc | 46 ++--
core/get.cc | 3 +-
core/hbase-configuration-loader.h | 5 +-
core/hbase-rpc-controller.h | 8 +
core/meta-utils.cc | 15 +-
core/query.h | 19 +-
core/raw-async-table.cc | 27 +-
core/raw-async-table.h | 21 +-
core/raw-scan-result-consumer.h | 131 +++++++++
core/region-location.h | 2 +-
core/request-converter-test.cc | 3 +-
core/request-converter.cc | 96 +++++--
core/request-converter.h | 15 +-
core/response-converter.cc | 38 +--
core/response-converter.h | 5 +-
core/{query.h => result-scanner.h} | 28 +-
core/result-test.cc | 26 +-
core/result.cc | 60 +++--
core/result.h | 18 +-
core/scan-result-cache-test.cc | 177 ++++++++++++
core/scan-result-cache.cc | 160 +++++++++++
core/scan-result-cache.h | 80 ++++++
core/scan-test.cc | 15 +-
core/scan.cc | 19 +-
core/scan.h | 13 +-
core/scanner-test.cc | 368 +++++++++++++++++++++++++
core/simple-client.cc | 85 ++++--
core/table.cc | 16 ++
core/table.h | 5 +
exceptions/BUCK | 10 +-
exceptions/exception.h | 192 ++++++++-----
test-util/BUCK | 62 +++--
test-util/mini-cluster.cc | 38 ++-
test-util/mini-cluster.h | 6 +-
test-util/test-util.cc | 9 +
test-util/test-util.h | 3 +
utils/BUCK | 1 +
utils/bytes-util-test.cc | 11 +
utils/bytes-util.h | 24 +-
core/query.h => utils/optional.h | 19 +-
61 files changed, 3008 insertions(+), 366 deletions(-)
diff --git a/bin/format-code.sh b/bin/format-code.sh
index 8a19930..fe236d8 100755
--- a/bin/format-code.sh
+++ b/bin/format-code.sh
@@ -19,5 +19,5 @@ set -euo pipefail
IFS=$'\n\t'
-find core connection serde utils test-util security -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i --style='{BasedOnStyle: Google, ColumnLimit: 100}'
-find core connection serde utils third-party security -name "BUCK" | xargs -P8 yapf -i --style=google
+find core connection exceptions serde utils test-util security -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i --style='{BasedOnStyle: Google, ColumnLimit: 100}'
+find core connection exceptions serde utils test-util third-party security -name "BUCK" | xargs -P8 yapf -i --style=google
diff --git a/connection/client-handler.cc b/connection/client-handler.cc
index ac16197..894ecb3 100644
--- a/connection/client-handler.cc
+++ b/connection/client-handler.cc
@@ -87,7 +87,7 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<folly::IOBuf> buf) {
if (cell_block_length > 0) {
auto cell_scanner = serde_.CreateCellScanner(std::move(buf), used_bytes, cell_block_length);
- received->set_cell_scanner(std::move(cell_scanner));
+ received->set_cell_scanner(std::shared_ptr<CellScanner>{cell_scanner.release()});
}
received->set_resp_msg(resp_msg);
@@ -129,8 +129,7 @@ folly::Future<folly::Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Re
ctx->fireWrite(std::move(header));
});
- VLOG(3) << "Writing RPC Request with call_id:"
- << r->call_id(); // TODO: more logging for RPC Header
+ VLOG(3) << "Writing RPC Request:" << r->DebugString();
// Now store the call id to response.
resp_msgs_->insert(r->call_id(), r->resp_msg());
diff --git a/connection/request.h b/connection/request.h
index 520b380..4b652c0 100644
--- a/connection/request.h
+++ b/connection/request.h
@@ -18,6 +18,7 @@
*/
#pragma once
+#include <folly/Conv.h>
#include <google/protobuf/message.h>
#include <cstdint>
@@ -64,6 +65,11 @@ class Request {
* method type to decode. */
std::string method() { return method_; }
+ std::string DebugString() {
+ return "call_id:" + folly::to<std::string>(call_id_) + ", req_msg:" +
+ req_msg_->ShortDebugString() + ", method:" + method_;
+ }
+
private:
uint32_t call_id_;
std::shared_ptr<google::protobuf::Message> req_msg_ = nullptr;
diff --git a/connection/response.h b/connection/response.h
index c5472b0..38fdda0 100644
--- a/connection/response.h
+++ b/connection/response.h
@@ -18,12 +18,14 @@
*/
#pragma once
+#include <folly/Conv.h>
+#include <folly/ExceptionWrapper.h>
+
#include <cstdint>
#include <memory>
+#include <string>
#include <utility>
-#include <folly/ExceptionWrapper.h>
-
#include "serde/cell-scanner.h"
// Forward
@@ -66,20 +68,26 @@ class Response {
resp_msg_ = std::move(response);
}
- void set_cell_scanner(std::unique_ptr<CellScanner> cell_scanner) {
- cell_scanner_ = std::move(cell_scanner);
- }
+ void set_cell_scanner(std::shared_ptr<CellScanner> cell_scanner) { cell_scanner_ = cell_scanner; }
- const std::unique_ptr<CellScanner>& cell_scanner() const { return cell_scanner_; }
+ const std::shared_ptr<CellScanner> cell_scanner() const { return cell_scanner_; }
folly::exception_wrapper exception() { return exception_; }
void set_exception(folly::exception_wrapper value) { exception_ = value; }
+ std::string DebugString() const {
+ std::string s{"call_id:"};
+ s += folly::to<std::string>(call_id_);
+ s += ", resp_msg:";
+ s += resp_msg_->ShortDebugString();
+ return s;
+ }
+
private:
uint32_t call_id_;
std::shared_ptr<google::protobuf::Message> resp_msg_;
- std::unique_ptr<CellScanner> cell_scanner_;
+ std::shared_ptr<CellScanner> cell_scanner_;
folly::exception_wrapper exception_;
};
} // namespace hbase
diff --git a/connection/rpc-client.cc b/connection/rpc-client.cc
index fbbe3e7..10faa7a 100644
--- a/connection/rpc-client.cc
+++ b/connection/rpc-client.cc
@@ -22,8 +22,6 @@
#include <folly/Logging.h>
#include <unistd.h>
#include <wangle/concurrent/IOThreadPoolExecutor.h>
-#include <memory>
-#include <string>
using hbase::security::User;
using std::chrono::nanoseconds;
diff --git a/connection/rpc-client.h b/connection/rpc-client.h
index 8b0abe7..0ecde5b 100644
--- a/connection/rpc-client.h
+++ b/connection/rpc-client.h
@@ -18,17 +18,19 @@
*/
#pragma once
+#include <google/protobuf/service.h>
+
+#include <chrono>
+#include <memory>
+#include <string>
+#include <utility>
+
#include "connection/connection-id.h"
#include "connection/connection-pool.h"
#include "connection/request.h"
#include "connection/response.h"
#include "security/user.h"
-#include <google/protobuf/service.h>
-
-#include <chrono>
-#include <utility>
-
namespace hbase {
class RpcClient {
diff --git a/core/BUCK b/core/BUCK
index 203e00d..81fd4a7 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -19,10 +19,12 @@
cxx_library(
name="core",
exported_headers=[
+ "async-client-scanner.h",
"async-connection.h",
"async-region-locator.h",
"async-rpc-retrying-caller-factory.h",
"async-rpc-retrying-caller.h",
+ "async-table-result-scanner.h",
"client.h",
"cell.h",
"hbase-macros.h",
@@ -42,10 +44,14 @@ cxx_library(
"delete.h",
"scan.h",
"result.h",
+ "result-scanner.h",
"request-converter.h",
"response-converter.h",
"table.h",
+ "async-scan-rpc-retrying-caller.h",
"raw-async-table.h",
+ "raw-scan-result-consumer.h",
+ "scan-result-cache.h",
"hbase-rpc-controller.h",
"time-range.h",
"zk-util.h",
@@ -58,9 +64,12 @@ cxx_library(
"async-batch-rpc-retrying-caller.h",
],
srcs=[
+ "async-client-scanner.cc",
"async-connection.cc",
"async-rpc-retrying-caller-factory.cc",
"async-rpc-retrying-caller.cc",
+ "async-scan-rpc-retrying-caller.cc",
+ "async-table-result-scanner.cc",
"cell.cc",
"client.cc",
"hbase-rpc-controller.cc",
@@ -73,6 +82,7 @@ cxx_library(
"put.cc",
"delete.cc",
"scan.cc",
+ "scan-result-cache.cc",
"raw-async-table.cc",
"result.cc",
"request-converter.cc",
@@ -109,7 +119,7 @@ cxx_library(
"configuration.cc",
"hbase-configuration-loader.cc",
],
- deps=["//third-party:folly"],
+ deps=["//utils:utils", "//third-party:folly"],
compiler_flags=['-Weffc++', '-ggdb'],
visibility=[
'PUBLIC',
@@ -156,8 +166,12 @@ cxx_test(
run_test_separately=True,)
cxx_test(
name="delete-test",
- srcs=["delete-test.cc",],
- deps=[":core",],
+ srcs=[
+ "delete-test.cc",
+ ],
+ deps=[
+ ":core",
+ ],
run_test_separately=True,)
cxx_test(
name="increment-test",
@@ -257,6 +271,27 @@ cxx_test(
],
run_test_separately=True,)
cxx_test(
+ name="scan-result-cache-test",
+ srcs=[
+ "scan-result-cache-test.cc",
+ ],
+ deps=[
+ ":core",
+ ],
+ run_test_separately=True,)
+cxx_test(
+ name="scanner-test",
+ srcs=[
+ "scanner-test.cc",
+ ],
+ deps=[
+ ":core",
+ "//if:if",
+ "//serde:serde",
+ "//test-util:test-util",
+ ],
+ run_test_separately=True,)
+cxx_test(
name="zk-util-test",
srcs=[
"zk-util-test.cc",
diff --git a/core/async-client-scanner.cc b/core/async-client-scanner.cc
new file mode 100644
index 0000000..720ab25
--- /dev/null
+++ b/core/async-client-scanner.cc
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/async-client-scanner.h"
+
+#include <algorithm>
+#include <iterator>
+#include <limits>
+#include <stdexcept>
+
+namespace hbase {
+
+AsyncClientScanner::AsyncClientScanner(
+ std::shared_ptr<AsyncConnection> conn, std::shared_ptr<Scan> scan,
+ std::shared_ptr<pb::TableName> table_name, std::shared_ptr<RawScanResultConsumer> consumer,
+ nanoseconds pause, uint32_t max_retries, nanoseconds scan_timeout_nanos,
+ nanoseconds rpc_timeout_nanos, uint32_t start_log_errors_count)
+ : conn_(conn),
+ scan_(scan),
+ table_name_(table_name),
+ consumer_(consumer),
+ pause_(pause),
+ max_retries_(max_retries),
+ scan_timeout_nanos_(scan_timeout_nanos),
+ rpc_timeout_nanos_(rpc_timeout_nanos),
+ start_log_errors_count_(start_log_errors_count) {
+ results_cache_ = std::make_shared<ScanResultCache>();
+ max_attempts_ = ConnectionUtils::Retries2Attempts(max_retries);
+}
+
+void AsyncClientScanner::Start() { OpenScanner(); }
+
+folly::Future<std::shared_ptr<OpenScannerResponse>> AsyncClientScanner::CallOpenScanner(
+ std::shared_ptr<hbase::RpcClient> rpc_client,
+ std::shared_ptr<hbase::HBaseRpcController> controller,
+ std::shared_ptr<hbase::RegionLocation> loc) {
+ open_scanner_tries_++;
+
+ auto preq = RequestConverter::ToScanRequest(*scan_, loc->region_name(), scan_->Caching(), false);
+
+ auto self(shared_from_this());
+ VLOG(5) << "Calling RPC Client to open the scanner for region:" << loc->DebugString();
+ return rpc_client
+ ->AsyncCall(loc->server_name().host_name(), loc->server_name().port(), std::move(preq),
+ security::User::defaultUser(), "ClientService")
+ .then([self, loc, controller, rpc_client](const std::unique_ptr<Response>& presp) {
+ VLOG(5) << "Scan Response:" << presp->DebugString();
+ return std::make_shared<OpenScannerResponse>(rpc_client, presp, loc, controller);
+ });
+}
+
+void AsyncClientScanner::OpenScanner() {
+ auto self(shared_from_this());
+ open_scanner_tries_ = 1;
+
+ auto caller = conn_->caller_factory()
+ ->Single<std::shared_ptr<OpenScannerResponse>>()
+ ->table(table_name_)
+ ->row(scan_->StartRow())
+ ->locate_type(GetLocateType(*scan_))
+ ->rpc_timeout(rpc_timeout_nanos_)
+ ->operation_timeout(scan_timeout_nanos_)
+ ->pause(pause_)
+ ->max_retries(max_retries_)
+ ->start_log_errors_count(start_log_errors_count_)
+ ->action([&](std::shared_ptr<hbase::HBaseRpcController> controller,
+ std::shared_ptr<hbase::RegionLocation> loc,
+ std::shared_ptr<hbase::RpcClient> rpc_client)
+ -> folly::Future<std::shared_ptr<OpenScannerResponse>> {
+ return CallOpenScanner(rpc_client, controller, loc);
+ })
+ ->Build();
+
+ caller->Call()
+ .then([this, self](std::shared_ptr<OpenScannerResponse> resp) {
+ VLOG(3) << "Opened scanner with id:" << resp->scan_resp_->scanner_id()
+ << ", region:" << resp->region_location_->DebugString() << ", starting scan";
+ StartScan(resp);
+ })
+ .onError([this, self](const folly::exception_wrapper& e) {
+ VLOG(3) << "Open scan request received error:" << e.what();
+ consumer_->OnError(e);
+ })
+ .then([caller, self](const auto r) { return r; });
+}
+
+void AsyncClientScanner::StartScan(std::shared_ptr<OpenScannerResponse> resp) {
+ auto self(shared_from_this());
+ auto caller = conn_->caller_factory()
+ ->Scan()
+ ->scanner_id(resp->scan_resp_->scanner_id())
+ ->region_location(resp->region_location_)
+ ->scanner_lease_timeout(TimeUtil::MillisToNanos(resp->scan_resp_->ttl()))
+ ->scan(scan_)
+ ->rpc_client(resp->rpc_client_)
+ ->consumer(consumer_)
+ ->results_cache(results_cache_)
+ ->rpc_timeout(rpc_timeout_nanos_)
+ ->scan_timeout(scan_timeout_nanos_)
+ ->pause(pause_)
+ ->max_retries(max_retries_)
+ ->start_log_errors_count(start_log_errors_count_)
+ ->Build();
+
+ caller->Start(resp->controller_, resp->scan_resp_, resp->cell_scanner_)
+ .then([caller, self](const bool has_more) {
+ if (has_more) {
+ // open the next scanner on the next region.
+ self->OpenScanner();
+ } else {
+ self->consumer_->OnComplete();
+ }
+ })
+ .onError([caller, self](const folly::exception_wrapper& e) { self->consumer_->OnError(e); })
+ .then([caller, self](const auto r) { return r; });
+}
+
+RegionLocateType AsyncClientScanner::GetLocateType(const Scan& scan) {
+ // TODO: In C++, there is no Scan::IncludeStartRow() and Scan::IncludeStopRow().
+ // When added, this method should be modified to return other RegionLocateTypes
+ // (see ConnectionUtils.java #getLocateType())
+ // TODO: When reversed scans are implemented, return other RegionLocateTypes
+ return RegionLocateType::kCurrent;
+}
+
+} // namespace hbase
diff --git a/core/async-client-scanner.h b/core/async-client-scanner.h
new file mode 100644
index 0000000..8663468
--- /dev/null
+++ b/core/async-client-scanner.h
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <folly/Format.h>
+#include <folly/Logging.h>
+#include <folly/futures/Future.h>
+#include <folly/io/async/EventBase.h>
+#include <folly/io/async/HHWheelTimer.h>
+
+#include <algorithm>
+#include <chrono>
+#include <functional>
+#include <memory>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "connection/rpc-client.h"
+#include "core/async-connection.h"
+#include "core/async-rpc-retrying-caller-factory.h"
+#include "core/async-rpc-retrying-caller.h"
+#include "core/hbase-rpc-controller.h"
+#include "core/raw-scan-result-consumer.h"
+#include "core/region-location.h"
+#include "core/request-converter.h"
+#include "core/response-converter.h"
+#include "core/result.h"
+#include "core/scan-result-cache.h"
+#include "core/scan.h"
+#include "exceptions/exception.h"
+#include "if/Client.pb.h"
+#include "if/HBase.pb.h"
+#include "utils/connection-util.h"
+#include "utils/sys-util.h"
+#include "utils/time-util.h"
+
+using std::chrono::nanoseconds;
+using std::chrono::milliseconds;
+
+namespace hbase {
+class OpenScannerResponse {
+ public:
+ OpenScannerResponse(std::shared_ptr<hbase::RpcClient> rpc_client,
+ const std::unique_ptr<Response>& resp,
+ std::shared_ptr<RegionLocation> region_location,
+ std::shared_ptr<hbase::HBaseRpcController> controller)
+ : rpc_client_(rpc_client), region_location_(region_location), controller_(controller) {
+ scan_resp_ = std::static_pointer_cast<pb::ScanResponse>(resp->resp_msg());
+ cell_scanner_ = resp->cell_scanner();
+ }
+ std::shared_ptr<hbase::RpcClient> rpc_client_;
+ std::shared_ptr<pb::ScanResponse> scan_resp_;
+ std::shared_ptr<RegionLocation> region_location_;
+ std::shared_ptr<hbase::HBaseRpcController> controller_;
+ std::shared_ptr<CellScanner> cell_scanner_;
+};
+
+class AsyncClientScanner : public std::enable_shared_from_this<AsyncClientScanner> {
+ public:
+ template <typename... T>
+ static std::shared_ptr<AsyncClientScanner> Create(T&&... all) {
+ return std::shared_ptr<AsyncClientScanner>(new AsyncClientScanner(std::forward<T>(all)...));
+ }
+
+ void Start();
+
+ private:
+ // methods
+ AsyncClientScanner(std::shared_ptr<AsyncConnection> conn, std::shared_ptr<Scan> scan,
+ std::shared_ptr<pb::TableName> table_name,
+ std::shared_ptr<RawScanResultConsumer> consumer, nanoseconds pause,
+ uint32_t max_retries, nanoseconds scan_timeout_nanos,
+ nanoseconds rpc_timeout_nanos, uint32_t start_log_errors_count);
+
+ folly::Future<std::shared_ptr<OpenScannerResponse>> CallOpenScanner(
+ std::shared_ptr<hbase::RpcClient> rpc_client,
+ std::shared_ptr<hbase::HBaseRpcController> controller,
+ std::shared_ptr<hbase::RegionLocation> loc);
+
+ void OpenScanner();
+
+ void StartScan(std::shared_ptr<OpenScannerResponse> resp);
+
+ RegionLocateType GetLocateType(const Scan& scan);
+
+ private:
+ // data
+ std::shared_ptr<AsyncConnection> conn_;
+ std::shared_ptr<Scan> scan_;
+ std::shared_ptr<pb::TableName> table_name_;
+ std::shared_ptr<ScanResultCache> results_cache_;
+ std::shared_ptr<RawScanResultConsumer> consumer_;
+ nanoseconds pause_;
+ uint32_t max_retries_;
+ nanoseconds scan_timeout_nanos_;
+ nanoseconds rpc_timeout_nanos_;
+ uint32_t start_log_errors_count_;
+ uint32_t max_attempts_;
+ uint32_t open_scanner_tries_ = 0;
+};
+} // namespace hbase
diff --git a/core/async-rpc-retrying-caller-factory.h b/core/async-rpc-retrying-caller-factory.h
index 3c11f52..1af6e72 100644
--- a/core/async-rpc-retrying-caller-factory.h
+++ b/core/async-rpc-retrying-caller-factory.h
@@ -28,7 +28,13 @@
#include "connection/rpc-client.h"
#include "core/async-batch-rpc-retrying-caller.h"
#include "core/async-rpc-retrying-caller.h"
+#include "core/async-scan-rpc-retrying-caller.h"
+#include "core/raw-scan-result-consumer.h"
+#include "core/region-location.h"
#include "core/row.h"
+#include "core/scan-result-cache.h"
+#include "core/scan.h"
+
#include "if/Client.pb.h"
#include "if/HBase.pb.h"
@@ -54,8 +60,8 @@ class SingleRequestCallerBuilder
virtual ~SingleRequestCallerBuilder() = default;
- typedef SingleRequestCallerBuilder<RESP> GenenericThisType;
- typedef std::shared_ptr<GenenericThisType> SharedThisPtr;
+ typedef SingleRequestCallerBuilder<RESP> GenericThisType;
+ typedef std::shared_ptr<GenericThisType> SharedThisPtr;
SharedThisPtr table(std::shared_ptr<pb::TableName> table_name) {
table_name_ = table_name;
@@ -112,7 +118,7 @@ class SingleRequestCallerBuilder
private:
SharedThisPtr shared_this() {
- return std::enable_shared_from_this<GenenericThisType>::shared_from_this();
+ return std::enable_shared_from_this<GenericThisType>::shared_from_this();
}
private:
@@ -198,6 +204,114 @@ class BatchCallerBuilder : public std::enable_shared_from_this<BatchCallerBuilde
std::chrono::nanoseconds rpc_timeout_nanos_;
int32_t start_log_errors_count_ = 0;
};
+
+class ScanCallerBuilder : public std::enable_shared_from_this<ScanCallerBuilder> {
+ public:
+ explicit ScanCallerBuilder(std::shared_ptr<AsyncConnection> conn,
+ std::shared_ptr<folly::HHWheelTimer> retry_timer)
+ : conn_(conn),
+ retry_timer_(retry_timer),
+ rpc_timeout_nanos_(conn->connection_conf()->rpc_timeout()),
+ pause_(conn->connection_conf()->pause()),
+ scan_timeout_nanos_(conn->connection_conf()->scan_timeout()),
+ max_retries_(conn->connection_conf()->max_retries()),
+ start_log_errors_count_(conn->connection_conf()->start_log_errors_count()),
+ scanner_id_(-1) {}
+
+ virtual ~ScanCallerBuilder() = default;
+
+ typedef ScanCallerBuilder GenericThisType;
+ typedef std::shared_ptr<ScanCallerBuilder> SharedThisPtr;
+
+ SharedThisPtr rpc_client(std::shared_ptr<hbase::RpcClient> rpc_client) {
+ rpc_client_ = rpc_client;
+ return shared_this();
+ }
+
+ SharedThisPtr rpc_timeout(nanoseconds rpc_timeout_nanos) {
+ rpc_timeout_nanos_ = rpc_timeout_nanos;
+ return shared_this();
+ }
+
+ SharedThisPtr scan_timeout(nanoseconds scan_timeout_nanos) {
+ scan_timeout_nanos_ = scan_timeout_nanos;
+ return shared_this();
+ }
+
+ SharedThisPtr scanner_lease_timeout(nanoseconds scanner_lease_timeout_nanos) {
+ scanner_lease_timeout_nanos_ = scanner_lease_timeout_nanos;
+ return shared_this();
+ }
+
+ SharedThisPtr pause(nanoseconds pause) {
+ pause_ = pause;
+ return shared_this();
+ }
+
+ SharedThisPtr max_retries(uint32_t max_retries) {
+ max_retries_ = max_retries;
+ return shared_this();
+ }
+
+ SharedThisPtr start_log_errors_count(uint32_t start_log_errors_count) {
+ start_log_errors_count_ = start_log_errors_count;
+ return shared_this();
+ }
+
+ SharedThisPtr region_location(std::shared_ptr<RegionLocation> region_location) {
+ region_location_ = region_location;
+ return shared_this();
+ }
+
+ SharedThisPtr scanner_id(int64_t scanner_id) {
+ scanner_id_ = scanner_id;
+ return shared_this();
+ }
+
+ SharedThisPtr scan(std::shared_ptr<Scan> scan) {
+ scan_ = scan;
+ return shared_this();
+ }
+
+ SharedThisPtr results_cache(std::shared_ptr<ScanResultCache> results_cache) {
+ results_cache_ = results_cache;
+ return shared_this();
+ }
+
+ SharedThisPtr consumer(std::shared_ptr<RawScanResultConsumer> consumer) {
+ consumer_ = consumer;
+ return shared_this();
+ }
+
+ std::shared_ptr<AsyncScanRpcRetryingCaller> Build() {
+ return std::make_shared<AsyncScanRpcRetryingCaller>(
+ conn_, retry_timer_, rpc_client_, scan_, scanner_id_, results_cache_, consumer_,
+ region_location_, scanner_lease_timeout_nanos_, pause_, max_retries_, scan_timeout_nanos_,
+ rpc_timeout_nanos_, start_log_errors_count_);
+ }
+
+ private:
+ SharedThisPtr shared_this() {
+ return std::enable_shared_from_this<GenericThisType>::shared_from_this();
+ }
+
+ private:
+ std::shared_ptr<AsyncConnection> conn_;
+ std::shared_ptr<folly::HHWheelTimer> retry_timer_;
+ std::shared_ptr<hbase::RpcClient> rpc_client_;
+ std::shared_ptr<Scan> scan_;
+ nanoseconds rpc_timeout_nanos_;
+ nanoseconds scan_timeout_nanos_;
+ nanoseconds scanner_lease_timeout_nanos_;
+ nanoseconds pause_;
+ uint32_t max_retries_;
+ uint32_t start_log_errors_count_;
+ std::shared_ptr<RegionLocation> region_location_;
+ int64_t scanner_id_;
+ std::shared_ptr<RawScanResultConsumer> consumer_;
+ std::shared_ptr<ScanResultCache> results_cache_;
+}; // end of ScanCallerBuilder
+
class AsyncRpcRetryingCallerFactory {
private:
std::shared_ptr<AsyncConnection> conn_;
@@ -218,6 +332,10 @@ class AsyncRpcRetryingCallerFactory {
std::shared_ptr<BatchCallerBuilder> Batch() {
return std::make_shared<BatchCallerBuilder>(conn_, retry_timer_);
}
+
+ std::shared_ptr<ScanCallerBuilder> Scan() {
+ return std::make_shared<ScanCallerBuilder>(conn_, retry_timer_);
+ }
};
} // namespace hbase
diff --git a/core/async-rpc-retrying-caller.cc b/core/async-rpc-retrying-caller.cc
index a2c5f0e..b9d01bb 100644
--- a/core/async-rpc-retrying-caller.cc
+++ b/core/async-rpc-retrying-caller.cc
@@ -111,7 +111,7 @@ void AsyncSingleRequestRpcRetryingCaller<RESP>::OnError(
Consumer<exception_wrapper> update_cached_location) {
ThrowableWithExtraContext twec(error, TimeUtil::GetNowNanos());
exceptions_->push_back(twec);
- if (!ShouldRetry(error) || tries_ >= max_retries_) {
+ if (!ExceptionUtil::ShouldRetry(error) || tries_ >= max_retries_) {
CompleteExceptionally();
return;
}
@@ -149,14 +149,6 @@ void AsyncSingleRequestRpcRetryingCaller<RESP>::OnError(
}
template <typename RESP>
-bool AsyncSingleRequestRpcRetryingCaller<RESP>::ShouldRetry(const exception_wrapper& error) {
- bool do_not_retry = false;
- error.with_exception(
- [&](const RemoteException& remote_ex) { do_not_retry &= remote_ex.do_not_retry(); });
- return !do_not_retry;
-}
-
-template <typename RESP>
void AsyncSingleRequestRpcRetryingCaller<RESP>::Call(const RegionLocation& loc) {
int64_t call_timeout_ns;
if (operation_timeout_nanos_.count() > 0) {
@@ -220,6 +212,8 @@ void AsyncSingleRequestRpcRetryingCaller<RESP>::ResetController(
// explicit instantiations for the linker. Otherwise, you have to #include the .cc file for the
// templetized
// class definitions.
+class OpenScannerResponse;
template class AsyncSingleRequestRpcRetryingCaller<std::shared_ptr<hbase::Result>>;
template class AsyncSingleRequestRpcRetryingCaller<folly::Unit>;
+template class AsyncSingleRequestRpcRetryingCaller<std::shared_ptr<OpenScannerResponse>>;
} /* namespace hbase */
diff --git a/core/async-rpc-retrying-caller.h b/core/async-rpc-retrying-caller.h
index fa3c288..c7e28d0 100644
--- a/core/async-rpc-retrying-caller.h
+++ b/core/async-rpc-retrying-caller.h
@@ -84,8 +84,6 @@ class AsyncSingleRequestRpcRetryingCaller {
void OnError(const folly::exception_wrapper& error, Supplier<std::string> err_msg,
Consumer<folly::exception_wrapper> update_cached_location);
- bool ShouldRetry(const folly::exception_wrapper& error);
-
void Call(const RegionLocation& loc);
void CompleteExceptionally();
diff --git a/core/async-scan-rpc-retrying-caller.cc b/core/async-scan-rpc-retrying-caller.cc
new file mode 100644
index 0000000..fbdf17a
--- /dev/null
+++ b/core/async-scan-rpc-retrying-caller.cc
@@ -0,0 +1,447 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/async-scan-rpc-retrying-caller.h"
+
+namespace hbase {
+
+ScanResumerImpl::ScanResumerImpl(std::shared_ptr<AsyncScanRpcRetryingCaller> caller)
+ : caller_(caller), mutex_() {}
+
+void ScanResumerImpl::Resume() {
+ // just used to fix findbugs warnings. In fact, if resume is called before prepare, then we
+ // just return at the first if condition without loading the resp and numValidResuls field. If
+ // resume is called after suspend, then it is also safe to just reference resp and
+ // numValidResults after the synchronized block as no one will change it anymore.
+ std::shared_ptr<pb::ScanResponse> local_resp;
+ int64_t local_num_complete_rows;
+
+ {
+ std::unique_lock<std::mutex> mlock{mutex_};
+ if (state_ == ScanResumerState::kInitialized) {
+ // user calls this method before we call prepare, so just set the state to
+ // RESUMED, the implementation will just go on.
+ state_ = ScanResumerState::kResumed;
+ return;
+ }
+ if (state_ == ScanResumerState::kResumed) {
+ // already resumed, give up.
+ return;
+ }
+ state_ = ScanResumerState::kResumed;
+ local_resp = resp_;
+ local_num_complete_rows = num_complete_rows_;
+ }
+
+ caller_->CompleteOrNext(local_resp);
+}
+
+bool ScanResumerImpl::Prepare(std::shared_ptr<pb::ScanResponse> resp, int num_complete_rows) {
+ std::unique_lock<std::mutex> mlock(mutex_);
+ if (state_ == ScanResumerState::kResumed) {
+ // user calls resume before we actually suspend the scan, just continue;
+ return false;
+ }
+ state_ = ScanResumerState::kSuspended;
+ resp_ = resp;
+ num_complete_rows_ = num_complete_rows;
+
+ return true;
+}
+
+ScanControllerImpl::ScanControllerImpl(std::shared_ptr<AsyncScanRpcRetryingCaller> caller)
+ : caller_(caller) {}
+
+std::shared_ptr<ScanResumer> ScanControllerImpl::Suspend() {
+ PreCheck();
+ state_ = ScanControllerState::kSuspended;
+ resumer_ = std::make_shared<ScanResumerImpl>(caller_);
+ return resumer_;
+}
+
+void ScanControllerImpl::Terminate() {
+ PreCheck();
+ state_ = ScanControllerState::kTerminated;
+}
+
+// return the current state, and set the state to DESTROYED.
+ScanControllerState ScanControllerImpl::Destroy() {
+ ScanControllerState state = state_;
+ state_ = ScanControllerState::kDestroyed;
+ return state;
+}
+
+void ScanControllerImpl::PreCheck() {
+ CHECK(std::this_thread::get_id() == caller_thread_id_)
+ << "The current thread is" << std::this_thread::get_id() << ", expected thread is "
+ << caller_thread_id_ << ", you should not call this method outside OnNext or OnHeartbeat";
+
+ CHECK(state_ == ScanControllerState::kInitialized) << "Invalid Stopper state "
+ << DebugString(state_);
+}
+
+std::string ScanControllerImpl::DebugString(ScanControllerState state) {
+ switch (state) {
+ case ScanControllerState::kInitialized:
+ return "kInitialized";
+ case ScanControllerState::kSuspended:
+ return "kSuspended";
+ case ScanControllerState::kTerminated:
+ return "kTerminated";
+ case ScanControllerState::kDestroyed:
+ return "kDestroyed";
+ default:
+ return "UNKNOWN";
+ }
+}
+
+std::string ScanControllerImpl::DebugString(ScanResumerState state) {
+ switch (state) {
+ case ScanResumerState::kInitialized:
+ return "kInitialized";
+ case ScanResumerState::kSuspended:
+ return "kSuspended";
+ case ScanResumerState::kResumed:
+ return "kResumed";
+ default:
+ return "UNKNOWN";
+ }
+}
+
+AsyncScanRpcRetryingCaller::AsyncScanRpcRetryingCaller(
+ std::shared_ptr<AsyncConnection> conn, std::shared_ptr<folly::HHWheelTimer> retry_timer,
+ std::shared_ptr<hbase::RpcClient> rpc_client, std::shared_ptr<Scan> scan, int64_t scanner_id,
+ std::shared_ptr<ScanResultCache> results_cache, std::shared_ptr<RawScanResultConsumer> consumer,
+ std::shared_ptr<RegionLocation> region_location, nanoseconds scanner_lease_timeout_nanos,
+ nanoseconds pause, uint32_t max_retries, nanoseconds scan_timeout_nanos,
+ nanoseconds rpc_timeout_nanos, uint32_t start_log_errors_count)
+ : conn_(conn),
+ retry_timer_(retry_timer),
+ rpc_client_(rpc_client),
+ scan_(scan),
+ scanner_id_(scanner_id),
+ results_cache_(results_cache),
+ consumer_(consumer),
+ region_location_(region_location),
+ scanner_lease_timeout_nanos_(scanner_lease_timeout_nanos),
+ pause_(pause),
+ max_retries_(max_retries),
+ scan_timeout_nanos_(scan_timeout_nanos),
+ rpc_timeout_nanos_(rpc_timeout_nanos),
+ start_log_errors_count_(start_log_errors_count),
+ promise_(std::make_shared<folly::Promise<bool>>()),
+ tries_(1) {
+ controller_ = conn_->CreateRpcController();
+ start_ns_ = TimeUtil::GetNowNanos();
+ max_attempts_ = ConnectionUtils::Retries2Attempts(max_retries);
+ exceptions_ = std::make_shared<std::vector<ThrowableWithExtraContext>>();
+}
+
+folly::Future<bool> AsyncScanRpcRetryingCaller::Start(
+ std::shared_ptr<HBaseRpcController> controller,
+ std::shared_ptr<pb::ScanResponse> open_scan_resp,
+ const std::shared_ptr<CellScanner> cell_scanner) {
+ OnComplete(controller, open_scan_resp, cell_scanner);
+ return promise_->getFuture();
+}
+
+int64_t AsyncScanRpcRetryingCaller::RemainingTimeNs() {
+ return scan_timeout_nanos_.count() - (TimeUtil::GetNowNanos() - start_ns_);
+}
+
+void AsyncScanRpcRetryingCaller::OnComplete(std::shared_ptr<HBaseRpcController> controller,
+ std::shared_ptr<pb::ScanResponse> resp,
+ const std::shared_ptr<CellScanner> cell_scanner) {
+ VLOG(5) << "Scan: OnComplete, scanner_id:" << scanner_id_;
+
+ if (controller->Failed()) {
+ OnError(controller->exception());
+ return;
+ }
+
+ bool is_heartbeat = resp->has_heartbeat_message() && resp->heartbeat_message();
+
+ int64_t num_complete_rows_before = results_cache_->num_complete_rows();
+ try {
+ auto raw_results = ResponseConverter::FromScanResponse(resp, cell_scanner);
+
+ auto results = results_cache_->AddAndGet(raw_results, is_heartbeat);
+
+ auto scan_controller = std::make_shared<ScanControllerImpl>(shared_from_this());
+
+ if (results.size() > 0) {
+ UpdateNextStartRowWhenError(*results[results.size() - 1]);
+ VLOG(5) << "Calling consumer->OnNext()";
+ consumer_->OnNext(results, scan_controller);
+ } else if (is_heartbeat) {
+ consumer_->OnHeartbeat(scan_controller);
+ }
+
+ ScanControllerState state = scan_controller->Destroy();
+ if (state == ScanControllerState::kTerminated) {
+ if (resp->has_more_results_in_region() && !resp->more_results_in_region()) {
+ // we have more results in region but user request to stop the scan, so we need to close the
+ // scanner explicitly.
+ CloseScanner();
+ }
+ CompleteNoMoreResults();
+ return;
+ }
+
+ int64_t num_complete_rows = results_cache_->num_complete_rows() - num_complete_rows_before;
+ if (state == ScanControllerState::kSuspended) {
+ if (scan_controller->resumer()->Prepare(resp, num_complete_rows)) {
+ return;
+ }
+ }
+ } catch (const std::runtime_error& e) {
+ // We can not retry here. The server has responded normally and the call sequence has been
+ // increased so a new scan with the same call sequence will cause an
+ // OutOfOrderScannerNextException. Let the upper layer open a new scanner.
+ LOG(WARNING) << "Received exception in reading the scan response:" << e.what();
+ CompleteWhenError(true);
+ return;
+ }
+
+ CompleteOrNext(resp);
+}
+
+void AsyncScanRpcRetryingCaller::CompleteOrNext(std::shared_ptr<pb::ScanResponse> resp) {
+ VLOG(5) << "Scan: CompleteOrNext, scanner_id" << scanner_id_
+ << ", response:" << resp->ShortDebugString();
+
+ if (resp->has_more_results() && !resp->more_results()) {
+ // RS tells us there is no more data for the whole scan
+ CompleteNoMoreResults();
+ return;
+ }
+ // TODO: Implement Scan::limit(), and check the limit here
+
+ if (resp->has_more_results_in_region() && !resp->more_results_in_region()) {
+ // TODO: check whether Scan is reversed here
+ CompleteWhenNoMoreResultsInRegion();
+ return;
+ }
+ Next();
+}
+
+void AsyncScanRpcRetryingCaller::CompleteExceptionally(bool close_scanner) {
+ VLOG(5) << "Scan: CompleteExceptionally";
+ results_cache_->Clear();
+ if (close_scanner) {
+ CloseScanner();
+ }
+ this->promise_->setException(RetriesExhaustedException(tries_ - 1, exceptions_));
+}
+
+void AsyncScanRpcRetryingCaller::CompleteNoMoreResults() {
+ // In master code, scanners auto-close if we have exhausted the region. It may not be the case
+ // in branch-1 code. If this is backported, make sure that the scanner is closed.
+ VLOG(5) << "Scan: CompleteNoMoreResults, scanner_id:" << scanner_id_;
+ promise_->setValue(false);
+}
+
+void AsyncScanRpcRetryingCaller::CompleteWhenNoMoreResultsInRegion() {
+ VLOG(5) << "Scan: CompleteWhenNoMoreResultsInRegion, scanner_id:" << scanner_id_;
+ // In master code, scanners auto-close if we have exhausted the region. It may not be the case
+ // in branch-1 code. If this is backported, make sure that the scanner is closed.
+ if (NoMoreResultsForScan(*scan_, region_location_->region_info())) {
+ CompleteNoMoreResults();
+ } else {
+ CompleteWithNextStartRow(region_location_->region_info().end_key(), true);
+ }
+}
+
+void AsyncScanRpcRetryingCaller::CompleteWithNextStartRow(std::string row, bool inclusive) {
+ VLOG(5) << "Scan: CompleteWithNextStartRow: region scan is complete, move to next region";
+ scan_->SetStartRow(row);
+ // TODO: set inclusive if it is reverse scans
+ promise_->setValue(true);
+}
+
+void AsyncScanRpcRetryingCaller::UpdateNextStartRowWhenError(const Result& result) {
+ next_start_row_when_error_ = optional<std::string>(result.Row());
+ include_next_start_row_when_error_ = result.Partial();
+}
+
+void AsyncScanRpcRetryingCaller::CompleteWhenError(bool close_scanner) {
+ VLOG(5) << "Scan: CompleteWhenError, scanner_id:" << scanner_id_;
+ results_cache_->Clear();
+ if (close_scanner) {
+ CloseScanner();
+ }
+ if (next_start_row_when_error_) {
+ // TODO: HBASE-17583 adds include start / stop row to the Scan. Once we rebase and implement
+ // those options in Scan , we can start using that here.
+ scan_->SetStartRow(include_next_start_row_when_error_
+ ? *next_start_row_when_error_
+ : BytesUtil::CreateClosestRowAfter(*next_start_row_when_error_));
+ }
+ promise_->setValue(true);
+}
+
+void AsyncScanRpcRetryingCaller::OnError(const folly::exception_wrapper& error) {
+ VLOG(5) << "Scan: OnError, scanner_id:" << scanner_id_;
+ if (tries_ > start_log_errors_count_ || VLOG_IS_ON(5)) {
+ LOG(WARNING) << "Call to " << region_location_->server_name().ShortDebugString()
+ << " for scanner id = " << scanner_id_ << " for "
+ << region_location_->region_info().ShortDebugString()
+ << " failed, , tries = " << tries_ << ", maxAttempts = " << max_attempts_
+ << ", timeout = " << TimeUtil::ToMillis(scan_timeout_nanos_).count()
+ << " ms, time elapsed = " << TimeUtil::ElapsedMillis(start_ns_) << " ms"
+ << error.what().toStdString();
+ }
+
+ bool scanner_closed = ExceptionUtil::IsScannerClosed(error);
+ ThrowableWithExtraContext twec(error, TimeUtil::GetNowNanos());
+ exceptions_->push_back(twec);
+ if (tries_ >= max_retries_) {
+ CompleteExceptionally(!scanner_closed);
+ return;
+ }
+
+ int64_t delay_ns;
+ if (scan_timeout_nanos_.count() > 0) {
+ int64_t max_delay_ns = RemainingTimeNs() - ConnectionUtils::kSleepDeltaNs;
+ if (max_delay_ns <= 0) {
+ CompleteExceptionally(!scanner_closed);
+ return;
+ }
+ delay_ns = std::min(max_delay_ns, ConnectionUtils::GetPauseTime(pause_.count(), tries_ - 1));
+ } else {
+ delay_ns = ConnectionUtils::GetPauseTime(pause_.count(), tries_ - 1);
+ }
+
+ if (scanner_closed) {
+ CompleteWhenError(false);
+ return;
+ }
+
+ if (ExceptionUtil::IsScannerOutOfOrder(error)) {
+ CompleteWhenError(true);
+ return;
+ }
+ if (!ExceptionUtil::ShouldRetry(error)) {
+ CompleteExceptionally(true);
+ return;
+ }
+ tries_++;
+
+ auto self(shared_from_this());
+ conn_->retry_executor()->add([&]() {
+ retry_timer_->scheduleTimeoutFn(
+ [self]() { self->conn_->cpu_executor()->add([&]() { self->Call(); }); },
+ std::chrono::milliseconds(TimeUtil::ToMillis(delay_ns)));
+ });
+}
+
+bool AsyncScanRpcRetryingCaller::NoMoreResultsForScan(const Scan& scan,
+ const pb::RegionInfo& info) {
+ if (BytesUtil::IsEmptyStopRow(info.end_key())) {
+ return true;
+ }
+ if (BytesUtil::IsEmptyStopRow(scan.StopRow())) {
+ return false;
+ }
+ int32_t c = BytesUtil::CompareTo(info.end_key(), scan.StopRow());
+ // 1. if our stop row is less than the endKey of the region
+ // 2. if our stop row is equal to the endKey of the region and we do not include the stop row
+ // for scan.
+ return c > 0 ||
+ (c == 0 /* && !scan.IncludeStopRow()*/); // TODO: Scans always exclude StopRow for now.
+}
+
+void AsyncScanRpcRetryingCaller::Next() {
+ VLOG(5) << "Scan: Next";
+ next_call_seq_++;
+ tries_ = 1;
+ exceptions_->clear();
+ start_ns_ = TimeUtil::GetNowNanos();
+ Call();
+}
+
+void AsyncScanRpcRetryingCaller::Call() {
+ VLOG(5) << "Scan: Call";
+ auto self(shared_from_this());
+ // As we have a call sequence for scan, it is useless to have a different rpc timeout which is
+ // less than the scan timeout. If the server does not respond in time(usually this will not
+ // happen as we have heartbeat now), we will get an OutOfOrderScannerNextException when
+ // resending the next request and the only way to fix this is to close the scanner and open a
+ // new one.
+ int64_t call_timeout_nanos;
+ if (scan_timeout_nanos_.count() > 0) {
+ int64_t remaining_nanos = scan_timeout_nanos_.count() - (TimeUtil::GetNowNanos() - start_ns_);
+ if (remaining_nanos <= 0) {
+ CompleteExceptionally(true);
+ return;
+ }
+ call_timeout_nanos = remaining_nanos;
+ } else {
+ call_timeout_nanos = 0L;
+ }
+
+ ResetController(controller_, call_timeout_nanos);
+
+ auto req =
+ RequestConverter::ToScanRequest(scanner_id_, scan_->Caching(), false, next_call_seq_, false);
+
+ // do the RPC call
+ rpc_client_
+ ->AsyncCall(region_location_->server_name().host_name(),
+ region_location_->server_name().port(), std::move(req),
+ security::User::defaultUser(), "ClientService")
+ .then([self, this](const std::unique_ptr<Response>& resp) {
+ auto scan_resp = std::static_pointer_cast<pb::ScanResponse>(resp->resp_msg());
+ return OnComplete(controller_, scan_resp, resp->cell_scanner());
+ })
+ .onError([self, this](const folly::exception_wrapper& e) { OnError(e); });
+}
+
+void AsyncScanRpcRetryingCaller::CloseScanner() {
+ auto self(shared_from_this());
+ ResetController(controller_, rpc_timeout_nanos_.count());
+
+ VLOG(5) << "Closing scanner with scanner_id:" << folly::to<std::string>(scanner_id_);
+
+ // Do a close scanner RPC. Fire and forget.
+ auto req = RequestConverter::ToScanRequest(scanner_id_, 0, true);
+ rpc_client_
+ ->AsyncCall(region_location_->server_name().host_name(),
+ region_location_->server_name().port(), std::move(req),
+ security::User::defaultUser(), "ClientService")
+ .onError([self, this](const folly::exception_wrapper& e) -> std::unique_ptr<Response> {
+ LOG(WARNING) << "Call to " + region_location_->server_name().ShortDebugString() +
+ " for closing scanner_id = " + folly::to<std::string>(scanner_id_) +
+ " for " + region_location_->region_info().ShortDebugString() +
+ " failed, ignore, probably already closed. Exception:" +
+ e.what().toStdString();
+ return nullptr;
+ });
+}
+
+void AsyncScanRpcRetryingCaller::ResetController(std::shared_ptr<HBaseRpcController> controller,
+ const int64_t& timeout_nanos) {
+ controller->Reset();
+ if (timeout_nanos >= 0) {
+ controller->set_call_timeout(
+ milliseconds(std::min(static_cast<int64_t>(INT_MAX), TimeUtil::ToMillis(timeout_nanos))));
+ }
+}
+
+} // namespace hbase
diff --git a/core/async-scan-rpc-retrying-caller.h b/core/async-scan-rpc-retrying-caller.h
new file mode 100644
index 0000000..9555e80
--- /dev/null
+++ b/core/async-scan-rpc-retrying-caller.h
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <folly/Conv.h>
+#include <folly/ExceptionWrapper.h>
+#include <folly/Format.h>
+#include <folly/Logging.h>
+#include <folly/futures/Future.h>
+#include <folly/io/async/EventBase.h>
+#include <folly/io/async/HHWheelTimer.h>
+
+#include <algorithm>
+#include <chrono>
+#include <functional>
+#include <memory>
+#include <string>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "connection/rpc-client.h"
+#include "core/async-connection.h"
+#include "core/hbase-rpc-controller.h"
+#include "core/raw-scan-result-consumer.h"
+#include "core/region-location.h"
+#include "core/request-converter.h"
+#include "core/response-converter.h"
+#include "core/result.h"
+#include "core/scan-result-cache.h"
+#include "core/scan.h"
+#include "exceptions/exception.h"
+#include "if/Client.pb.h"
+#include "if/HBase.pb.h"
+#include "utils/bytes-util.h"
+#include "utils/connection-util.h"
+#include "utils/optional.h"
+#include "utils/sys-util.h"
+#include "utils/time-util.h"
+
+using std::chrono::nanoseconds;
+using std::chrono::milliseconds;
+
+namespace hbase {
+
+class AsyncScanRpcRetryingCaller;
+
+// The resume method is allowed to be called in another thread so here we also use the
+// ResumerState to prevent race. The initial state is INITIALIZED, and in most cases, when back
+// from onNext or onHeartbeat, we will call the prepare method to change the state to SUSPENDED,
+// and when user calls resume method, we will change the state to RESUMED. But the resume method
+// could be called in other thread, and in fact, user could just do this:
+// controller.suspend().resume()
+// This is strange but valid. This means the scan could be resumed before we call the prepare
+// method to do the actual suspend work. So in the resume method, we will check if the state is
+// INTIALIZED, if it is, then we will just set the state to RESUMED and return. And in prepare
+// method, if the state is RESUMED already, we will just return an let the scan go on.
+// Notice that, the public methods of this class is supposed to be called by upper layer only, and
+// package private methods can only be called within the implementation of
+// AsyncScanSingleRegionRpcRetryingCaller.
+// TODO: Unlike the Java counter part, we do not do scan lease renewals in a background thread.
+// Since there is also no async scan API exposed to the users, only ScanResultConsumer is the
+// AsyncTableResultScanner which will only pause the scanner if the result cache is maxed. The
+// application is expected to consume the scan results before the scanner lease timeout.
+class ScanResumerImpl : public ScanResumer {
+ public:
+ explicit ScanResumerImpl(std::shared_ptr<AsyncScanRpcRetryingCaller> caller);
+
+ virtual ~ScanResumerImpl() = default;
+
+ /**
+ * Resume the scan. You are free to call it multiple time but only the first call will take
+ * effect.
+ */
+ void Resume() override;
+
+ // return false if the scan has already been resumed. See the comment above for ScanResumerImpl
+ // for more details.
+ bool Prepare(std::shared_ptr<pb::ScanResponse> resp, int num_complete_rows);
+
+ private:
+ // INITIALIZED -> SUSPENDED -> RESUMED
+ // INITIALIZED -> RESUMED
+ ScanResumerState state_ = ScanResumerState::kInitialized;
+ std::mutex mutex_;
+ std::shared_ptr<pb::ScanResponse> resp_ = nullptr;
+ int64_t num_complete_rows_ = 0;
+ std::shared_ptr<AsyncScanRpcRetryingCaller> caller_;
+};
+
+class ScanControllerImpl : public ScanController {
+ public:
+ virtual ~ScanControllerImpl() = default;
+
+ explicit ScanControllerImpl(std::shared_ptr<AsyncScanRpcRetryingCaller> caller);
+
+ /**
+ * Suspend the scan.
+ * <p>
+ * This means we will stop fetching data in background, i.e., will not call onNext any more
+ * before you resume the scan.
+ * @return A resumer used to resume the scan later.
+ */
+ std::shared_ptr<ScanResumer> Suspend();
+
+ /**
+ * Terminate the scan.
+ * <p>
+ * This is useful when you have got enough results and want to stop the scan in onNext method,
+ * or you want to stop the scan in onHeartbeat method because it has spent too many time.
+ */
+ void Terminate();
+
+ // return the current state, and set the state to DESTROYED.
+ ScanControllerState Destroy();
+
+ std::shared_ptr<ScanResumerImpl> resumer() { return resumer_; }
+
+ private:
+ void PreCheck();
+
+ std::string DebugString(ScanControllerState state);
+
+ std::string DebugString(ScanResumerState state);
+
+ private:
+ // Make sure the methods are only called in this thread.
+ std::thread::id caller_thread_id_ = std::this_thread::get_id();
+ // INITIALIZED -> SUSPENDED -> DESTROYED
+ // INITIALIZED -> TERMINATED -> DESTROYED
+ // INITIALIZED -> DESTROYED
+ // If the state is incorrect we will throw IllegalStateException.
+ ScanControllerState state_ = ScanControllerState::kInitialized;
+ std::shared_ptr<ScanResumerImpl> resumer_ = nullptr;
+ std::shared_ptr<AsyncScanRpcRetryingCaller> caller_;
+};
+
+class AsyncScanRpcRetryingCaller : public std::enable_shared_from_this<AsyncScanRpcRetryingCaller> {
+ public:
+ AsyncScanRpcRetryingCaller(std::shared_ptr<AsyncConnection> conn,
+ std::shared_ptr<folly::HHWheelTimer> retry_timer,
+ std::shared_ptr<hbase::RpcClient> rpc_client,
+ std::shared_ptr<Scan> scan, int64_t scanner_id,
+ std::shared_ptr<ScanResultCache> results_cache,
+ std::shared_ptr<RawScanResultConsumer> consumer,
+ std::shared_ptr<RegionLocation> region_location,
+ nanoseconds scanner_lease_timeout_nanos, nanoseconds pause,
+ uint32_t max_retries, nanoseconds scan_timeout_nanos,
+ nanoseconds rpc_timeout_nanos, uint32_t start_log_errors_count);
+
+ folly::Future<bool> Start(std::shared_ptr<HBaseRpcController> controller,
+ std::shared_ptr<pb::ScanResponse> open_scan_resp,
+ const std::shared_ptr<CellScanner> cell_scanner);
+
+ private:
+ int64_t RemainingTimeNs();
+ void OnComplete(std::shared_ptr<HBaseRpcController> controller,
+ std::shared_ptr<pb::ScanResponse> resp,
+ const std::shared_ptr<CellScanner> cell_scanner);
+
+ void CompleteOrNext(std::shared_ptr<pb::ScanResponse> resp);
+
+ void CompleteExceptionally(bool close_scanner);
+
+ void CompleteNoMoreResults();
+
+ void CompleteWhenNoMoreResultsInRegion();
+
+ void CompleteWithNextStartRow(std::string row, bool inclusive);
+
+ void UpdateNextStartRowWhenError(const Result& result);
+
+ void CompleteWhenError(bool close_scanner);
+
+ void OnError(const folly::exception_wrapper& e);
+
+ bool NoMoreResultsForScan(const Scan& scan, const pb::RegionInfo& info);
+
+ void Next();
+
+ void Call();
+
+ void CloseScanner();
+
+ void ResetController(std::shared_ptr<HBaseRpcController> controller,
+ const int64_t& timeout_nanos);
+
+ private:
+ std::shared_ptr<AsyncConnection> conn_;
+ std::shared_ptr<folly::HHWheelTimer> retry_timer_;
+ std::shared_ptr<hbase::RpcClient> rpc_client_;
+ std::shared_ptr<Scan> scan_;
+ int64_t scanner_id_;
+ std::shared_ptr<ScanResultCache> results_cache_;
+ std::shared_ptr<RawScanResultConsumer> consumer_;
+ std::shared_ptr<RegionLocation> region_location_;
+ nanoseconds scanner_lease_timeout_nanos_;
+ nanoseconds pause_;
+ uint32_t max_retries_;
+ nanoseconds scan_timeout_nanos_;
+ nanoseconds rpc_timeout_nanos_;
+ uint32_t start_log_errors_count_;
+ std::shared_ptr<folly::Promise<bool>> promise_;
+ std::shared_ptr<HBaseRpcController> controller_;
+ optional<std::string> next_start_row_when_error_ = optional<std::string>();
+ bool include_next_start_row_when_error_ = true;
+ uint64_t start_ns_;
+ uint32_t tries_;
+ std::shared_ptr<std::vector<ThrowableWithExtraContext>> exceptions_;
+ uint32_t max_attempts_;
+ int64_t next_call_seq_ = -1L;
+
+ friend class ScanResumerImpl;
+ friend class ScanControllerImpl;
+};
+
+} // namespace hbase
diff --git a/core/async-table-result-scanner.cc b/core/async-table-result-scanner.cc
new file mode 100644
index 0000000..b1935ae
--- /dev/null
+++ b/core/async-table-result-scanner.cc
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/async-table-result-scanner.h"
+
+#include <vector>
+
+namespace hbase {
+AsyncTableResultScanner::AsyncTableResultScanner(int64_t max_cache_size)
+ : max_cache_size_(max_cache_size) {
+ closed_ = false;
+ cache_size_ = 0;
+}
+
+AsyncTableResultScanner::~AsyncTableResultScanner() { Close(); }
+
+void AsyncTableResultScanner::Close() {
+ std::unique_lock<std::mutex> mlock(mutex_);
+ closed_ = true;
+ while (!queue_.empty()) {
+ queue_.pop();
+ }
+ cache_size_ = 0;
+ if (resumer_ != nullptr) {
+ resumer_->Resume();
+ }
+ cond_.notify_all();
+}
+
+std::shared_ptr<Result> AsyncTableResultScanner::Next() {
+ VLOG(5) << "AsyncTableResultScanner: Next()";
+
+ std::shared_ptr<Result> result = nullptr;
+ std::shared_ptr<ScanResumer> local_resumer = nullptr;
+ {
+ std::unique_lock<std::mutex> mlock(mutex_);
+ while (queue_.empty()) {
+ if (closed_) {
+ return nullptr;
+ }
+ if (error_) {
+ throw error_;
+ }
+ cond_.wait(mlock);
+ }
+ result = queue_.front();
+ queue_.pop();
+
+ cache_size_ -= EstimatedSizeWithSharedPtr(result);
+ if (resumer_ != nullptr && cache_size_ <= max_cache_size_ / 2) {
+ VLOG(1) << std::this_thread::get_id() << " resume scan prefetching";
+ local_resumer = resumer_;
+ resumer_ = nullptr;
+ }
+ }
+
+ // Need to call ScanResumer::Resume() outside of the scope of the mutex. The reason is that
+ // folly/wangle event loop might end up running the attached logic(.then()) at the Scan RPC
+ // in the same event thread before returning from the previous call. This seems like the
+ // wrong thing to do(™), but we cannot fix that now. Since the call back can end up calling
+ // this::OnNext(), we should unlock the mutex.
+ if (local_resumer != nullptr) {
+ local_resumer->Resume();
+ }
+ return result;
+}
+
+void AsyncTableResultScanner::AddToCache(const std::vector<std::shared_ptr<Result>> &results) {
+ VLOG(5) << "AsyncTableResultScanner: AddToCache()";
+ for (const auto r : results) {
+ queue_.push(r);
+ cache_size_ += EstimatedSizeWithSharedPtr(r);
+ }
+}
+
+template <typename T>
+inline size_t AsyncTableResultScanner::EstimatedSizeWithSharedPtr(std::shared_ptr<T> t) {
+ return t->EstimatedSize() + sizeof(std::shared_ptr<T>);
+}
+
+void AsyncTableResultScanner::OnNext(const std::vector<std::shared_ptr<Result>> &results,
+ std::shared_ptr<ScanController> controller) {
+ VLOG(5) << "AsyncTableResultScanner: OnNext()";
+ {
+ std::unique_lock<std::mutex> mlock(mutex_);
+ if (closed_) {
+ controller->Terminate();
+ return;
+ }
+ AddToCache(results);
+
+ if (cache_size_ >= max_cache_size_) {
+ StopPrefetch(controller);
+ }
+ }
+ cond_.notify_all();
+}
+
+void AsyncTableResultScanner::StopPrefetch(std::shared_ptr<ScanController> controller) {
+ VLOG(1) << std::this_thread::get_id()
+ << ": stop prefetching when scanning as the cache size " +
+ folly::to<std::string>(cache_size_) + " is greater than the max_cache_size " +
+ folly::to<std::string>(max_cache_size_);
+
+ resumer_ = controller->Suspend();
+ num_prefetch_stopped_++;
+}
+
+/**
+ * Indicate that there is an heartbeat message but we have not cumulated enough cells to call
+ * onNext.
+ * <p>
+ * This method give you a chance to terminate a slow scan operation.
+ * @param controller used to suspend or terminate the scan. Notice that the {@code controller}
+ * instance is only valid within the scope of onHeartbeat method. You can only call its
+ * method in onHeartbeat, do NOT store it and call it later outside onHeartbeat.
+ */
+void AsyncTableResultScanner::OnHeartbeat(std::shared_ptr<ScanController> controller) {
+ std::unique_lock<std::mutex> mlock(mutex_);
+ if (closed_) {
+ controller->Terminate();
+ }
+}
+
+/**
+ * Indicate that we hit an unrecoverable error and the scan operation is terminated.
+ * <p>
+ * We will not call {@link #onComplete()} after calling {@link #onError(Throwable)}.
+ */
+void AsyncTableResultScanner::OnError(const folly::exception_wrapper &error) {
+ LOG(WARNING) << "Scanner received error" << error.what();
+ std::unique_lock<std::mutex> mlock(mutex_);
+ error_ = error;
+ cond_.notify_all();
+}
+
+/**
+ * Indicate that the scan operation is completed normally.
+ */
+void AsyncTableResultScanner::OnComplete() {
+ std::unique_lock<std::mutex> mlock(mutex_);
+ closed_ = true;
+ cond_.notify_all();
+}
+} // namespace hbase
diff --git a/core/async-table-result-scanner.h b/core/async-table-result-scanner.h
new file mode 100644
index 0000000..dcdf871
--- /dev/null
+++ b/core/async-table-result-scanner.h
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <folly/Conv.h>
+#include <folly/ExceptionWrapper.h>
+#include <folly/Logging.h>
+#include <chrono>
+#include <condition_variable>
+#include <memory>
+#include <mutex>
+#include <queue>
+#include <string>
+#include <vector>
+
+#include "core/raw-scan-result-consumer.h"
+#include "core/result-scanner.h"
+#include "core/result.h"
+#include "if/Client.pb.h"
+#include "if/HBase.pb.h"
+
+namespace hbase {
+
+class AsyncTableResultScanner : public ResultScanner, public RawScanResultConsumer {
+ public:
+ explicit AsyncTableResultScanner(int64_t max_cache_size);
+
+ virtual ~AsyncTableResultScanner();
+
+ void Close() override;
+
+ std::shared_ptr<Result> Next() override;
+
+ void OnNext(const std::vector<std::shared_ptr<Result>> &results,
+ std::shared_ptr<ScanController> controller) override;
+
+ /**
+ * Indicate that there is an heartbeat message but we have not cumulated enough cells to call
+ * onNext.
+ * <p>
+ * This method give you a chance to terminate a slow scan operation.
+ * @param controller used to suspend or terminate the scan. Notice that the {@code controller}
+ * instance is only valid within the scope of onHeartbeat method. You can only call its
+ * method in onHeartbeat, do NOT store it and call it later outside onHeartbeat.
+ */
+ void OnHeartbeat(std::shared_ptr<ScanController> controller) override;
+
+ /**
+ * Indicate that we hit an unrecoverable error and the scan operation is terminated.
+ * <p>
+ * We will not call {@link #onComplete()} after calling {@link #onError(Throwable)}.
+ */
+ void OnError(const folly::exception_wrapper &error) override;
+
+ /**
+ * Indicate that the scan operation is completed normally.
+ */
+ void OnComplete() override;
+
+ // For testing
+ uint32_t num_prefetch_stopped() { return num_prefetch_stopped_; }
+
+ private:
+ void AddToCache(const std::vector<std::shared_ptr<Result>> &results);
+
+ template <typename T>
+ inline size_t EstimatedSizeWithSharedPtr(std::shared_ptr<T> t);
+
+ void StopPrefetch(std::shared_ptr<ScanController> controller);
+
+ private:
+ std::queue<std::shared_ptr<Result>> queue_;
+ std::mutex mutex_;
+ std::condition_variable cond_;
+ folly::exception_wrapper error_;
+ int64_t cache_size_;
+ int64_t max_cache_size_;
+ bool closed_;
+ std::shared_ptr<ScanResumer> resumer_ = nullptr;
+ uint32_t num_prefetch_stopped_ = 0;
+};
+} // namespace hbase
diff --git a/core/cell-test.cc b/core/cell-test.cc
index efb835d..4611473 100644
--- a/core/cell-test.cc
+++ b/core/cell-test.cc
@@ -169,3 +169,27 @@ TEST(CellTest, CellDebugString) {
LOG(INFO) << cell2.DebugString();
EXPECT_EQ("row/column/42/DELETE/vlen=5/seqid=0", cell2.DebugString());
}
+
+TEST(CellTest, CellEstimatedSize) {
+ CellType cell_type = CellType::PUT;
+ int64_t timestamp = std::numeric_limits<int64_t>::max();
+
+ Cell empty{"a", "a", "", timestamp, "", cell_type};
+ Cell cell1{"aa", "a", "", timestamp, "", cell_type};
+ Cell cell2{"a", "aa", "", timestamp, "", cell_type};
+ Cell cell3{"a", "a", "a", timestamp, "", cell_type};
+ Cell cell4{"a", "a", "", timestamp, "a", cell_type};
+ Cell cell5{"a", "a", "", timestamp, "a", CellType::DELETE};
+ Cell cell6{"aaaaaa", "a", "", timestamp, "a", cell_type};
+
+ LOG(INFO) << empty.EstimatedSize();
+ LOG(INFO) << cell1.EstimatedSize();
+
+ EXPECT_TRUE(empty.EstimatedSize() > sizeof(Cell));
+ EXPECT_TRUE(cell1.EstimatedSize() > empty.EstimatedSize());
+ EXPECT_EQ(cell1.EstimatedSize(), cell2.EstimatedSize());
+ EXPECT_EQ(cell2.EstimatedSize(), cell3.EstimatedSize());
+ EXPECT_EQ(cell3.EstimatedSize(), cell4.EstimatedSize());
+ EXPECT_EQ(cell4.EstimatedSize(), cell5.EstimatedSize());
+ EXPECT_TRUE(cell6.EstimatedSize() > cell1.EstimatedSize());
+}
diff --git a/core/cell.cc b/core/cell.cc
index 24788ab..e475d49 100644
--- a/core/cell.cc
+++ b/core/cell.cc
@@ -94,21 +94,30 @@ std::string Cell::DebugString() const {
const char *Cell::TypeToString(CellType type) {
switch (type) {
- case MINIMUM:
+ case CellType::MINIMUM:
return "MINIMUM";
- case PUT:
+ case CellType::PUT:
return "PUT";
- case DELETE:
+ case CellType::DELETE:
return "DELETE";
- case DELETE_COLUMN:
+ case CellType::DELETE_COLUMN:
return "DELETE_COLUMN";
- case DELETE_FAMILY:
+ case CellType::DELETE_FAMILY:
return "DELETE_FAMILY";
- case MAXIMUM:
+ case CellType::MAXIMUM:
return "MAXIMUM";
default:
return "UNKNOWN";
}
}
+size_t Cell::EstimatedSize() const {
+ size_t s = sizeof(Cell);
+ s += row_.capacity();
+ s += family_.capacity();
+ s += qualifier_.capacity();
+ s += value_.capacity();
+ return s;
+}
+
} /* namespace hbase */
diff --git a/core/cell.h b/core/cell.h
index acedd96..7a62a9b 100644
--- a/core/cell.h
+++ b/core/cell.h
@@ -24,7 +24,7 @@
namespace hbase {
-enum CellType {
+enum class CellType {
MINIMUM = 0,
PUT = 4,
DELETE = 8,
@@ -49,6 +49,9 @@ class Cell {
CellType Type() const;
int64_t SequenceId() const;
std::string DebugString() const;
+ /** Returns estimated size of the Cell object including deep heap space usage
+ * of its data. Notice that this is a very rough estimate. */
+ size_t EstimatedSize() const;
private:
std::string row_;
diff --git a/core/client-test.cc b/core/client-test.cc
index f3c5150..743e928 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -161,23 +161,23 @@ TEST_F(ClientTest, PutGetDelete) {
table->Delete(hbase::Delete{row}.AddColumn("d", "1"));
result = table->Get(get);
ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty.";
- ASSERT_TRUE(result->Value("d", "1") == nullptr) << "Column 1 should be gone";
+ ASSERT_FALSE(result->Value("d", "1")) << "Column 1 should be gone";
EXPECT_EQ(valExtra, *(result->Value("d", "extra")));
// delete cell from column "extra" with timestamp tsExtra
table->Delete(hbase::Delete{row}.AddColumn("d", "extra", tsExtra));
result = table->Get(get);
ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty.";
- ASSERT_TRUE(result->Value("d", "1") == nullptr) << "Column 1 should be gone";
- ASSERT_TRUE(result->Value("d", "extra") != nullptr) << "Column extra should have value";
+ ASSERT_FALSE(result->Value("d", "1")) << "Column 1 should be gone";
+ ASSERT_TRUE(result->Value("d", "extra")) << "Column extra should have value";
EXPECT_EQ(valExt, *(result->Value("d", "ext"))) << "Column ext should have value";
// delete all cells from "extra" column
table->Delete(hbase::Delete{row}.AddColumns("d", "extra"));
result = table->Get(get);
ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty.";
- ASSERT_TRUE(result->Value("d", "1") == nullptr) << "Column 1 should be gone";
- ASSERT_TRUE(result->Value("d", "extra") == nullptr) << "Column extra should be gone";
+ ASSERT_FALSE(result->Value("d", "1")) << "Column 1 should be gone";
+ ASSERT_FALSE(result->Value("d", "extra")) << "Column extra should be gone";
EXPECT_EQ(valExt, *(result->Value("d", "ext"))) << "Column ext should have value";
// Delete the row and verify that subsequent Get returns nothing
@@ -249,6 +249,7 @@ TEST_F(ClientTest, PutGet) {
table->Close();
client.Close();
}
+
TEST_F(ClientTest, GetForNonExistentTable) {
// Create TableName and Row to be fetched from HBase
auto tn = folly::to<hbase::pb::TableName>("t_not_exists");
@@ -377,7 +378,7 @@ TEST_F(ClientTest, MultiGets) {
ASSERT_TRUE(!results[i]->IsEmpty()) << "Result for Get " << gets[i].row()
<< " must not be empty";
EXPECT_EQ("test" + std::to_string(i), results[i]->Row());
- EXPECT_EQ("value" + std::to_string(i), *results[i]->Value("d", std::to_string(i)).get());
+ EXPECT_EQ("value" + std::to_string(i), results[i]->Value("d", std::to_string(i)).value());
}
// We are inserting test2 twice so the below test should pass
ASSERT_TRUE(!results[i]->IsEmpty()) << "Result for Get " << gets[i].row() << " must not be empty";
diff --git a/core/delete-test.cc b/core/delete-test.cc
index 19e844b..ec1e3a9 100644
--- a/core/delete-test.cc
+++ b/core/delete-test.cc
@@ -19,8 +19,8 @@
#include <glog/logging.h>
#include <gtest/gtest.h>
-#include "core/mutation.h"
#include "core/delete.h"
+#include "core/mutation.h"
#include "utils/time-util.h"
using hbase::Delete;
diff --git a/core/delete.cc b/core/delete.cc
index b44971f..5f48782 100644
--- a/core/delete.cc
+++ b/core/delete.cc
@@ -47,14 +47,14 @@ Delete& Delete::AddColumn(const std::string& family, const std::string& qualifie
* @param timestamp version timestamp
*/
Delete& Delete::AddColumn(const std::string& family, const std::string& qualifier,
- int64_t timestamp) {
+ int64_t timestamp) {
if (timestamp < 0) {
throw std::runtime_error("Timestamp cannot be negative. ts=" +
folly::to<std::string>(timestamp));
}
- return Add(std::make_unique<Cell>(row_, family, qualifier, timestamp, "",
- hbase::CellType::DELETE));
+ return Add(
+ std::make_unique<Cell>(row_, family, qualifier, timestamp, "", hbase::CellType::DELETE));
}
/**
* Delete all versions of the specified column.
@@ -62,7 +62,7 @@ Delete& Delete::AddColumn(const std::string& family, const std::string& qualifie
* @param qualifier column qualifier
*/
Delete& Delete::AddColumns(const std::string& family, const std::string& qualifier) {
- return AddColumns(family, qualifier, timestamp_);
+ return AddColumns(family, qualifier, timestamp_);
}
/**
* Delete all versions of the specified column with a timestamp less than
@@ -72,14 +72,14 @@ Delete& Delete::AddColumns(const std::string& family, const std::string& qualifi
* @param timestamp maximum version timestamp
*/
Delete& Delete::AddColumns(const std::string& family, const std::string& qualifier,
- int64_t timestamp) {
- if (timestamp < 0) {
- throw std::runtime_error("Timestamp cannot be negative. ts=" +
- folly::to<std::string>(timestamp));
- }
+ int64_t timestamp) {
+ if (timestamp < 0) {
+ throw std::runtime_error("Timestamp cannot be negative. ts=" +
+ folly::to<std::string>(timestamp));
+ }
- return Add(std::make_unique<Cell>(row_, family, qualifier, timestamp, "",
- hbase::CellType::DELETE_COLUMN));
+ return Add(std::make_unique<Cell>(row_, family, qualifier, timestamp, "",
+ hbase::CellType::DELETE_COLUMN));
}
/**
* Delete all versions of all columns of the specified family.
@@ -88,9 +88,7 @@ Delete& Delete::AddColumns(const std::string& family, const std::string& qualifi
* specified family.
* @param family family name
*/
-Delete& Delete::AddFamily(const std::string& family) {
- return AddFamily(family, timestamp_);
-}
+Delete& Delete::AddFamily(const std::string& family) { return AddFamily(family, timestamp_); }
/**
* Delete all columns of the specified family with a timestamp less than
@@ -102,14 +100,14 @@ Delete& Delete::AddFamily(const std::string& family) {
* @param timestamp maximum version timestamp
*/
Delete& Delete::AddFamily(const std::string& family, int64_t timestamp) {
- const auto &it = family_map_.find(family);
- if (family_map_.end() != it) {
- it->second.clear();
- } else {
- family_map_[family];
- }
- return Add(std::make_unique<Cell>(row_, family, "", timestamp, "",
- hbase::CellType::DELETE_FAMILY));
+ const auto& it = family_map_.find(family);
+ if (family_map_.end() != it) {
+ it->second.clear();
+ } else {
+ family_map_[family];
+ }
+ return Add(
+ std::make_unique<Cell>(row_, family, "", timestamp, "", hbase::CellType::DELETE_FAMILY));
}
/**
* Delete all columns of the specified family with a timestamp equal to
@@ -118,8 +116,8 @@ Delete& Delete::AddFamily(const std::string& family, int64_t timestamp) {
* @param timestamp version timestamp
*/
Delete& Delete::AddFamilyVersion(const std::string& family, int64_t timestamp) {
- return Add(std::make_unique<Cell>(row_, family, "", timestamp, "",
- hbase::CellType::DELETE_FAMILY_VERSION));
+ return Add(std::make_unique<Cell>(row_, family, "", timestamp, "",
+ hbase::CellType::DELETE_FAMILY_VERSION));
}
Delete& Delete::Add(std::unique_ptr<Cell> cell) {
if (cell->Row() != row_) {
diff --git a/core/get.cc b/core/get.cc
index 52b2ec5..bc0d446 100644
--- a/core/get.cc
+++ b/core/get.cc
@@ -28,7 +28,7 @@ Get::~Get() {}
Get::Get(const std::string &row) : Row(row) {}
-Get::Get(const Get &get) {
+Get::Get(const Get &get) : Query(get) {
row_ = get.row_;
max_versions_ = get.max_versions_;
cache_blocks_ = get.cache_blocks_;
@@ -39,6 +39,7 @@ Get::Get(const Get &get) {
}
Get &Get::operator=(const Get &get) {
+ Query::operator=(get);
row_ = get.row_;
max_versions_ = get.max_versions_;
cache_blocks_ = get.cache_blocks_;
diff --git a/core/hbase-configuration-loader.h b/core/hbase-configuration-loader.h
index a1c1d3f..95b2541 100644
--- a/core/hbase-configuration-loader.h
+++ b/core/hbase-configuration-loader.h
@@ -24,15 +24,12 @@
#include <vector>
#include <boost/optional.hpp>
-#include <experimental/optional>
#include "core/configuration.h"
+#include "utils/optional.h"
namespace hbase {
-template <class T>
-using optional = std::experimental::optional<T>;
-
class HBaseConfigurationLoader {
public:
HBaseConfigurationLoader();
diff --git a/core/hbase-rpc-controller.h b/core/hbase-rpc-controller.h
index 9bb89b9..33f552b 100644
--- a/core/hbase-rpc-controller.h
+++ b/core/hbase-rpc-controller.h
@@ -18,6 +18,7 @@
*/
#pragma once
+#include <folly/ExceptionWrapper.h>
#include <google/protobuf/service.h>
#include <chrono>
#include <string>
@@ -37,6 +38,10 @@ class HBaseRpcController : public google::protobuf::RpcController {
bool Failed() const override { return false; }
+ folly::exception_wrapper exception() { return exception_; }
+
+ void set_exception(const folly::exception_wrapper& exception) { exception_ = exception; }
+
std::string ErrorText() const override { return ""; }
void StartCancel() override {}
@@ -46,6 +51,9 @@ class HBaseRpcController : public google::protobuf::RpcController {
bool IsCanceled() const override { return false; }
void NotifyOnCancel(google::protobuf::Closure* callback) override {}
+
+ private:
+ folly::exception_wrapper exception_;
};
} /* namespace hbase */
diff --git a/core/meta-utils.cc b/core/meta-utils.cc
index 0577bfc..8efecc8 100644
--- a/core/meta-utils.cc
+++ b/core/meta-utils.cc
@@ -92,19 +92,12 @@ std::shared_ptr<RegionLocation> MetaUtil::CreateLocation(const Response &resp) {
throw std::runtime_error("Was expecting exactly 1 result in meta scan response, got:" +
std::to_string(results.size()));
}
-
auto result = *results[0];
- // VLOG(1) << "Creating RegionLocation from received Response " << *result; TODO
-
- std::shared_ptr<std::string> region_info_str = result.Value(CATALOG_FAMILY, REGION_INFO_COLUMN);
- std::shared_ptr<std::string> server_str = result.Value(CATALOG_FAMILY, SERVER_COLUMN);
- if (region_info_str == nullptr) {
- throw std::runtime_error("regioninfo column null for location");
- }
- if (server_str == nullptr) {
- throw std::runtime_error("server column null for location");
- }
+ auto region_info_str = result.Value(CATALOG_FAMILY, REGION_INFO_COLUMN);
+ auto server_str = result.Value(CATALOG_FAMILY, SERVER_COLUMN);
+ CHECK(region_info_str);
+ CHECK(server_str);
auto row = result.Row();
auto region_info = folly::to<RegionInfo>(*region_info_str);
diff --git a/core/query.h b/core/query.h
index b706303..301f448 100644
--- a/core/query.h
+++ b/core/query.h
@@ -19,6 +19,8 @@
#pragma once
+#include <memory>
+
#include "core/filter.h"
namespace hbase {
@@ -28,14 +30,25 @@ namespace hbase {
*/
class Query {
public:
+ Query() = default;
+ Query(const Query &query) {
+ // filter can be a custom subclass of Filter, so we do not do a deep copy here.
+ filter_ = query.filter_;
+ }
+
+ Query &operator=(const Query &query) {
+ filter_ = query.filter_;
+ return *this;
+ }
+
virtual ~Query() {}
- void SetFilter(std::unique_ptr<Filter> filter) { filter_ = std::move(filter); }
+ void SetFilter(std::shared_ptr<Filter> filter) { filter_ = filter; }
- const std::unique_ptr<Filter>& filter() const { return filter_; }
+ const std::shared_ptr<Filter> filter() const { return filter_; }
protected:
- std::unique_ptr<Filter> filter_ = nullptr;
+ std::shared_ptr<Filter> filter_ = nullptr;
};
} // namespace hbase
diff --git a/core/raw-async-table.cc b/core/raw-async-table.cc
index f71fbba..998e2f1 100644
--- a/core/raw-async-table.cc
+++ b/core/raw-async-table.cc
@@ -111,9 +111,10 @@ folly::Future<folly::Unit> RawAsyncTable::Put(const hbase::Put& put) {
folly::Future<folly::Unit> RawAsyncTable::Delete(const hbase::Delete& del) {
auto caller =
CreateCallerBuilder<folly::Unit>(del.row(), connection_conf_->write_rpc_timeout())
- ->action([=, &del](std::shared_ptr<hbase::HBaseRpcController> controller,
- std::shared_ptr<hbase::RegionLocation> loc,
- std::shared_ptr<hbase::RpcClient> rpc_client) -> folly::Future<folly::Unit> {
+ ->action([=, &del](
+ std::shared_ptr<hbase::HBaseRpcController> controller,
+ std::shared_ptr<hbase::RegionLocation> loc,
+ std::shared_ptr<hbase::RpcClient> rpc_client) -> folly::Future<folly::Unit> {
return Call<hbase::Delete, hbase::Request, hbase::Response, folly::Unit>(
rpc_client, controller, loc, del, &hbase::RequestConverter::DeleteToMutateRequest,
[](const Response& r) -> folly::Unit { return folly::unit; });
@@ -143,4 +144,24 @@ folly::Future<std::vector<folly::Try<std::shared_ptr<Result>>>> RawAsyncTable::B
return caller->Call().then([caller](auto r) { return r; });
}
+
+void RawAsyncTable::Scan(const hbase::Scan& scan, std::shared_ptr<RawScanResultConsumer> consumer) {
+ auto scanner = AsyncClientScanner::Create(
+ connection_, SetDefaultScanConfig(scan), table_name_, consumer, connection_conf_->pause(),
+ connection_conf_->max_retries(), connection_conf_->scan_timeout(),
+ connection_conf_->rpc_timeout(), connection_conf_->start_log_errors_count());
+ scanner->Start();
+}
+
+std::shared_ptr<hbase::Scan> RawAsyncTable::SetDefaultScanConfig(const hbase::Scan& scan) {
+ // always create a new scan object as we may reset the start row later.
+ auto new_scan = std::make_shared<hbase::Scan>(scan);
+ if (new_scan->Caching() <= 0) {
+ new_scan->SetCaching(default_scanner_caching_);
+ }
+ if (new_scan->MaxResultSize() <= 0) {
+ new_scan->SetMaxResultSize(default_scanner_max_result_size_);
+ }
+ return new_scan;
+}
} // namespace hbase
diff --git a/core/raw-async-table.h b/core/raw-async-table.h
index c8e9f2f..8c40dae 100644
--- a/core/raw-async-table.h
+++ b/core/raw-async-table.h
@@ -24,7 +24,9 @@
#include <memory>
#include <string>
#include <vector>
+
#include "core/async-batch-rpc-retrying-caller.h"
+#include "core/async-client-scanner.h"
#include "core/async-connection.h"
#include "core/async-rpc-retrying-caller-factory.h"
#include "core/async-rpc-retrying-caller.h"
@@ -34,6 +36,7 @@
#include "core/increment.h"
#include "core/put.h"
#include "core/result.h"
+#include "core/scan.h"
namespace hbase {
@@ -48,14 +51,22 @@ class RawAsyncTable {
: connection_(connection),
connection_conf_(connection->connection_conf()),
table_name_(table_name),
- rpc_client_(connection->rpc_client()) {}
+ rpc_client_(connection->rpc_client()) {
+ default_scanner_caching_ = connection_conf_->scanner_caching();
+ default_scanner_max_result_size_ = connection_conf_->scanner_max_result_size();
+ }
virtual ~RawAsyncTable() = default;
folly::Future<std::shared_ptr<Result>> Get(const hbase::Get& get);
- folly::Future<folly::Unit> Delete(const hbase::Delete& del);
- folly::Future<std::shared_ptr<hbase::Result>> Increment(const hbase::Increment& increment);
+ folly::Future<folly::Unit> Delete(const hbase::Delete& del);
+
+ folly::Future<std::shared_ptr<hbase::Result>> Increment(const hbase::Increment& increment);
+
folly::Future<folly::Unit> Put(const hbase::Put& put);
+
+ void Scan(const hbase::Scan& scan, std::shared_ptr<RawScanResultConsumer> consumer);
+
void Close() {}
folly::Future<std::vector<folly::Try<std::shared_ptr<Result>>>> Get(
@@ -69,6 +80,8 @@ class RawAsyncTable {
std::shared_ptr<ConnectionConfiguration> connection_conf_;
std::shared_ptr<pb::TableName> table_name_;
std::shared_ptr<RpcClient> rpc_client_;
+ int32_t default_scanner_caching_;
+ int64_t default_scanner_max_result_size_;
/* Methods */
template <typename REQ, typename PREQ, typename PRESP, typename RESP>
@@ -81,5 +94,7 @@ class RawAsyncTable {
template <typename RESP>
std::shared_ptr<SingleRequestCallerBuilder<RESP>> CreateCallerBuilder(
std::string row, std::chrono::nanoseconds rpc_timeout);
+
+ std::shared_ptr<hbase::Scan> SetDefaultScanConfig(const hbase::Scan& scan);
};
} // namespace hbase
diff --git a/core/raw-scan-result-consumer.h b/core/raw-scan-result-consumer.h
new file mode 100644
index 0000000..b7c3c48
--- /dev/null
+++ b/core/raw-scan-result-consumer.h
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <folly/ExceptionWrapper.h>
+#include <folly/Logging.h>
+#include <chrono>
+#include <memory>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include "core/result.h"
+#include "if/Client.pb.h"
+#include "if/HBase.pb.h"
+
+namespace hbase {
+
+enum class ScanControllerState { kInitialized, kSuspended, kTerminated, kDestroyed };
+
+enum class ScanResumerState { kInitialized, kSuspended, kResumed };
+
+/**
+ * Used to resume a scan.
+ */
+class ScanResumer {
+ public:
+ virtual ~ScanResumer() = default;
+
+ /**
+ * Resume the scan. You are free to call it multiple time but only the first call will take
+ * effect.
+ */
+ virtual void Resume() = 0;
+};
+
+/**
+ * Used to suspend or stop a scan.
+ * <p>
+ * Notice that, you should only call the methods below inside onNext or onHeartbeat method. A
+ * IllegalStateException will be thrown if you call them at other places.
+ * <p>
+ * You can only call one of the methods below, i.e., call suspend or terminate(of course you are
+ * free to not call them both), and the methods are not reentrant. A IllegalStateException will be
+ * thrown if you have already called one of the methods.
+ */
+class ScanController {
+ public:
+ virtual ~ScanController() = default;
+
+ /**
+ * Suspend the scan.
+ * <p>
+ * This means we will stop fetching data in background, i.e., will not call onNext any more
+ * before you resume the scan.
+ * @return A resumer used to resume the scan later.
+ */
+ virtual std::shared_ptr<ScanResumer> Suspend() = 0;
+
+ /**
+ * Terminate the scan.
+ * <p>
+ * This is useful when you have got enough results and want to stop the scan in onNext method,
+ * or you want to stop the scan in onHeartbeat method because it has spent too many time.
+ */
+ virtual void Terminate() = 0;
+};
+
+/**
+ * Receives {@link Result} for an asynchronous scan.
+ * <p>
+ * Notice that, the {@link #onNext(Result[], ScanController)} method will be called in the thread
+ * which we send request to HBase service. So if you want the asynchronous scanner fetch data from
+ * HBase in background while you process the returned data, you need to move the processing work to
+ * another thread to make the {@code onNext} call return immediately. And please do NOT do any time
+ * consuming tasks in all methods below unless you know what you are doing.
+ */
+class RawScanResultConsumer {
+ public:
+ virtual ~RawScanResultConsumer() = default;
+
+ /**
+ * Indicate that we have receive some data.
+ * @param results the data fetched from HBase service.
+ * @param controller used to suspend or terminate the scan. Notice that the {@code controller}
+ * instance is only valid within scope of onNext method. You can only call its method in
+ * onNext, do NOT store it and call it later outside onNext.
+ */
+ virtual void OnNext(const std::vector<std::shared_ptr<Result>> &results,
+ std::shared_ptr<ScanController> controller) {}
+
+ /**
+ * Indicate that there is an heartbeat message but we have not cumulated enough cells to call
+ * onNext.
+ * <p>
+ * This method give you a chance to terminate a slow scan operation.
+ * @param controller used to suspend or terminate the scan. Notice that the {@code controller}
+ * instance is only valid within the scope of onHeartbeat method. You can only call its
+ * method in onHeartbeat, do NOT store it and call it later outside onHeartbeat.
+ */
+ virtual void OnHeartbeat(std::shared_ptr<ScanController> controller) {}
+
+ /**
+ * Indicate that we hit an unrecoverable error and the scan operation is terminated.
+ * <p>
+ * We will not call {@link #onComplete()} after calling {@link #onError(Throwable)}.
+ */
+ virtual void OnError(const folly::exception_wrapper &error) {}
+
+ /**
+ * Indicate that the scan operation is completed normally.
+ */
+ virtual void OnComplete() {}
+};
+} // namespace hbase
diff --git a/core/region-location.h b/core/region-location.h
index 4087d94..d5d9d67 100644
--- a/core/region-location.h
+++ b/core/region-location.h
@@ -26,7 +26,7 @@
namespace hbase {
-enum RegionLocateType { kBefore, kCurrent, kAfter };
+enum class RegionLocateType { kBefore, kCurrent, kAfter };
/**
* @brief class to hold where a region is located.
diff --git a/core/request-converter-test.cc b/core/request-converter-test.cc
index c01f16c..6c07a19 100644
--- a/core/request-converter-test.cc
+++ b/core/request-converter-test.cc
@@ -83,7 +83,6 @@ TEST(RequestConverter, ToScan) {
scan.SetReversed(true);
scan.SetStartRow(start_row);
scan.SetStopRow(stop_row);
- scan.SetSmall(true);
scan.SetCaching(3);
scan.SetConsistency(hbase::pb::Consistency::TIMELINE);
scan.SetCacheBlocks(true);
@@ -105,7 +104,7 @@ TEST(RequestConverter, ToScan) {
EXPECT_TRUE(msg->scan().reversed());
EXPECT_EQ(msg->scan().start_row(), start_row);
EXPECT_EQ(msg->scan().stop_row(), stop_row);
- EXPECT_TRUE(msg->scan().small());
+ EXPECT_FALSE(msg->scan().small());
EXPECT_EQ(msg->scan().caching(), 3);
EXPECT_EQ(msg->scan().consistency(), hbase::pb::Consistency::TIMELINE);
EXPECT_TRUE(msg->scan().cache_blocks());
diff --git a/core/request-converter.cc b/core/request-converter.cc
index a1e63fe..6eb2f04 100644
--- a/core/request-converter.cc
+++ b/core/request-converter.cc
@@ -53,19 +53,11 @@ std::unique_ptr<Request> RequestConverter::ToGetRequest(const Get &get,
return pb_req;
}
-std::unique_ptr<Request> RequestConverter::ToScanRequest(const Scan &scan,
- const std::string ®ion_name) {
- auto pb_req = Request::scan();
-
- auto pb_msg = std::static_pointer_cast<ScanRequest>(pb_req->req_msg());
-
- RequestConverter::SetRegion(region_name, pb_msg->mutable_region());
-
- auto pb_scan = pb_msg->mutable_scan();
+std::unique_ptr<hbase::pb::Scan> RequestConverter::ToScan(const Scan &scan) {
+ auto pb_scan = std::make_unique<hbase::pb::Scan>();
pb_scan->set_max_versions(scan.MaxVersions());
pb_scan->set_cache_blocks(scan.CacheBlocks());
pb_scan->set_reversed(scan.IsReversed());
- pb_scan->set_small(scan.IsSmall());
pb_scan->set_caching(scan.Caching());
pb_scan->set_start_row(scan.StartRow());
pb_scan->set_stop_row(scan.StopRow());
@@ -94,12 +86,78 @@ std::unique_ptr<Request> RequestConverter::ToScanRequest(const Scan &scan,
pb_scan->set_allocated_filter(Filter::ToProto(*(scan.filter())).release());
}
- // TODO We will change this later.
+ return std::move(pb_scan);
+}
+
+std::unique_ptr<Request> RequestConverter::ToScanRequest(const Scan &scan,
+ const std::string ®ion_name) {
+ auto pb_req = Request::scan();
+ auto pb_msg = std::static_pointer_cast<ScanRequest>(pb_req->req_msg());
+
+ RequestConverter::SetRegion(region_name, pb_msg->mutable_region());
+
+ pb_msg->set_allocated_scan(ToScan(scan).release());
+
+ SetCommonScanRequestFields(pb_msg, false);
+
+ return pb_req;
+}
+
+std::unique_ptr<Request> RequestConverter::ToScanRequest(const Scan &scan,
+ const std::string ®ion_name,
+ int32_t num_rows, bool close_scanner) {
+ auto pb_req = Request::scan();
+ auto pb_msg = std::static_pointer_cast<ScanRequest>(pb_req->req_msg());
+
+ RequestConverter::SetRegion(region_name, pb_msg->mutable_region());
+
+ pb_msg->set_allocated_scan(ToScan(scan).release());
+
+ pb_msg->set_number_of_rows(num_rows);
+ pb_msg->set_close_scanner(close_scanner);
+
+ SetCommonScanRequestFields(pb_msg, false);
+
+ return pb_req;
+}
+
+std::unique_ptr<Request> RequestConverter::ToScanRequest(int64_t scanner_id, int32_t num_rows,
+ bool close_scanner) {
+ auto pb_req = Request::scan();
+ auto pb_msg = std::static_pointer_cast<ScanRequest>(pb_req->req_msg());
+
+ pb_msg->set_number_of_rows(num_rows);
+ pb_msg->set_close_scanner(close_scanner);
+ pb_msg->set_scanner_id(scanner_id);
+
+ SetCommonScanRequestFields(pb_msg, false);
+
+ return pb_req;
+}
+
+std::unique_ptr<Request> RequestConverter::ToScanRequest(int64_t scanner_id, int32_t num_rows,
+ bool close_scanner,
+ int64_t next_call_seq_id, bool renew) {
+ auto pb_req = Request::scan();
+ auto pb_msg = std::static_pointer_cast<ScanRequest>(pb_req->req_msg());
+
+ pb_msg->set_number_of_rows(num_rows);
+ pb_msg->set_close_scanner(close_scanner);
+ pb_msg->set_scanner_id(scanner_id);
+ pb_msg->set_next_call_seq(next_call_seq_id);
+
+ SetCommonScanRequestFields(pb_msg, renew);
+ return pb_req;
+}
+
+void RequestConverter::SetCommonScanRequestFields(std::shared_ptr<hbase::pb::ScanRequest> pb_msg,
+ bool renew) {
+ // TODO We will change these later when we implement partial results and heartbeats, etc
pb_msg->set_client_handles_partials(false);
pb_msg->set_client_handles_heartbeats(false);
pb_msg->set_track_scan_metrics(false);
-
- return pb_req;
+ pb_msg->set_renew(renew);
+ // TODO: set scan limit
}
std::unique_ptr<Request> RequestConverter::ToMultiRequest(
@@ -123,7 +181,6 @@ std::unique_ptr<Request> RequestConverter::ToMultiRequest(
}
}
- VLOG(3) << "Multi Req:-" << pb_req->req_msg()->ShortDebugString();
return pb_req;
}
@@ -190,13 +247,13 @@ std::unique_ptr<MutationProto> RequestConverter::ToMutation(const MutationType t
DeleteType RequestConverter::ToDeleteType(const CellType type) {
switch (type) {
- case DELETE:
+ case CellType::DELETE:
return pb::MutationProto_DeleteType_DELETE_ONE_VERSION;
- case DELETE_COLUMN:
+ case CellType::DELETE_COLUMN:
return pb::MutationProto_DeleteType_DELETE_MULTIPLE_VERSIONS;
- case DELETE_FAMILY:
+ case CellType::DELETE_FAMILY:
return pb::MutationProto_DeleteType_DELETE_FAMILY;
- case DELETE_FAMILY_VERSION:
+ case CellType::DELETE_FAMILY_VERSION:
return pb::MutationProto_DeleteType_DELETE_FAMILY_VERSION;
default:
throw std::runtime_error("Unknown delete type: " + folly::to<std::string>(type));
@@ -216,12 +273,11 @@ std::unique_ptr<Request> RequestConverter::ToMutateRequest(const Put &put,
pb_msg->set_allocated_mutation(
ToMutation(MutationType::MutationProto_MutationType_PUT, put, -1).release());
- VLOG(3) << "Req is " << pb_req->req_msg()->ShortDebugString();
return pb_req;
}
std::unique_ptr<Request> RequestConverter::DeleteToMutateRequest(const Delete &del,
- const std::string ®ion_name) {
+ const std::string ®ion_name) {
auto pb_req = Request::mutate();
auto pb_msg = std::static_pointer_cast<hbase::pb::MutateRequest>(pb_req->req_msg());
RequestConverter::SetRegion(region_name, pb_msg->mutable_region());
diff --git a/core/request-converter.h b/core/request-converter.h
index 6d57161..c807f45 100644
--- a/core/request-converter.h
+++ b/core/request-converter.h
@@ -66,9 +66,20 @@ class RequestConverter {
*/
static std::unique_ptr<Request> ToScanRequest(const Scan &scan, const std::string ®ion_name);
+ static std::unique_ptr<Request> ToScanRequest(const Scan &scan, const std::string ®ion_name,
+ int32_t num_rows, bool close_scanner);
+
+ static std::unique_ptr<Request> ToScanRequest(int64_t scanner_id, int32_t num_rows,
+ bool close_scanner);
+
+ static std::unique_ptr<Request> ToScanRequest(int64_t scanner_id, int32_t num_rows,
+ bool close_scanner, int64_t next_call_seq_id,
+ bool renew);
+
static std::unique_ptr<Request> ToMultiRequest(const ActionsByRegion ®ion_requests);
- static std::unique_ptr<Request> DeleteToMutateRequest(const Delete &del,const std::string ®ion_name);
+ static std::unique_ptr<Request> DeleteToMutateRequest(const Delete &del,
+ const std::string ®ion_name);
static std::unique_ptr<Request> ToMutateRequest(const Put &put, const std::string ®ion_name);
@@ -91,8 +102,10 @@ class RequestConverter {
*/
static void SetRegion(const std::string ®ion_name, pb::RegionSpecifier *region_specifier);
static std::unique_ptr<hbase::pb::Get> ToGet(const Get &get);
+ static std::unique_ptr<hbase::pb::Scan> ToScan(const Scan &scan);
static DeleteType ToDeleteType(const CellType type);
static bool IsDelete(const CellType type);
+ static void SetCommonScanRequestFields(std::shared_ptr<hbase::pb::ScanRequest>, bool renew);
};
} /* namespace hbase */
diff --git a/core/response-converter.cc b/core/response-converter.cc
index 94b7875..d2d719b 100644
--- a/core/response-converter.cc
+++ b/core/response-converter.cc
@@ -54,7 +54,7 @@ std::shared_ptr<Result> ResponseConverter::FromMutateResponse(const Response& re
}
std::shared_ptr<Result> ResponseConverter::ToResult(
- const hbase::pb::Result& result, const std::unique_ptr<CellScanner>& cell_scanner) {
+ const hbase::pb::Result& result, const std::shared_ptr<CellScanner> cell_scanner) {
std::vector<std::shared_ptr<Cell>> vcells;
for (auto cell : result.cell()) {
std::shared_ptr<Cell> pcell =
@@ -82,34 +82,38 @@ std::shared_ptr<Result> ResponseConverter::ToResult(
std::vector<std::shared_ptr<Result>> ResponseConverter::FromScanResponse(const Response& resp) {
auto scan_resp = std::static_pointer_cast<ScanResponse>(resp.resp_msg());
- VLOG(3) << "FromScanResponse:" << scan_resp->ShortDebugString();
- int num_results = resp.cell_scanner() != nullptr ? scan_resp->cells_per_result_size()
- : scan_resp->results_size();
+ return FromScanResponse(scan_resp, resp.cell_scanner());
+}
+
+std::vector<std::shared_ptr<Result>> ResponseConverter::FromScanResponse(
+ const std::shared_ptr<ScanResponse> scan_resp, std::shared_ptr<CellScanner> cell_scanner) {
+ VLOG(3) << "FromScanResponse:" << scan_resp->ShortDebugString()
+ << " cell_scanner:" << (cell_scanner == nullptr);
+ int num_results =
+ cell_scanner != nullptr ? scan_resp->cells_per_result_size() : scan_resp->results_size();
std::vector<std::shared_ptr<Result>> results{static_cast<size_t>(num_results)};
for (int i = 0; i < num_results; i++) {
- if (resp.cell_scanner() != nullptr) {
+ if (cell_scanner != nullptr) {
// Cells are out in cellblocks. Group them up again as Results. How many to read at a
// time will be found in getCellsLength -- length here is how many Cells in the i'th Result
int num_cells = scan_resp->cells_per_result(i);
std::vector<std::shared_ptr<Cell>> vcells;
- while (resp.cell_scanner()->Advance()) {
- vcells.push_back(resp.cell_scanner()->Current());
- }
- // TODO: check associated cell count?
-
- if (vcells.size() != num_cells) {
- std::string msg = "Results sent from server=" + std::to_string(num_results) +
- ". But only got " + std::to_string(i) +
- " results completely at client. Resetting the scanner to scan again.";
- LOG(ERROR) << msg;
- throw std::runtime_error(msg);
+ for (int j = 0; j < num_cells; j++) {
+ if (!cell_scanner->Advance()) {
+ std::string msg = "Results sent from server=" + std::to_string(num_results) +
+ ". But only got " + std::to_string(i) +
+ " results completely at client. Resetting the scanner to scan again.";
+ LOG(ERROR) << msg;
+ throw std::runtime_error(msg);
+ }
+ vcells.push_back(cell_scanner->Current());
}
// TODO: handle partial results per Result by checking partial_flag_per_result
results[i] = std::make_shared<Result>(vcells, false, scan_resp->stale(), false);
} else {
- results[i] = ToResult(scan_resp->results(i), resp.cell_scanner());
+ results[i] = ToResult(scan_resp->results(i), cell_scanner);
}
}
diff --git a/core/response-converter.h b/core/response-converter.h
index 0fdde89..b518d1c 100644
--- a/core/response-converter.h
+++ b/core/response-converter.h
@@ -39,7 +39,7 @@ class ResponseConverter {
~ResponseConverter();
static std::shared_ptr<Result> ToResult(const hbase::pb::Result& result,
- const std::unique_ptr<CellScanner>& cell_scanner);
+ const std::shared_ptr<CellScanner> cell_scanner);
/**
* @brief Returns a Result object created by PB Message in passed Response object.
@@ -51,6 +51,9 @@ class ResponseConverter {
static std::vector<std::shared_ptr<Result>> FromScanResponse(const Response& resp);
+ static std::vector<std::shared_ptr<Result>> FromScanResponse(
+ const std::shared_ptr<pb::ScanResponse> resp, std::shared_ptr<CellScanner> cell_scanner);
+
static std::unique_ptr<hbase::MultiResponse> GetResults(std::shared_ptr<Request> req,
const Response& resp);
diff --git a/core/query.h b/core/result-scanner.h
similarity index 66%
copy from core/query.h
copy to core/result-scanner.h
index b706303..9460521 100644
--- a/core/query.h
+++ b/core/result-scanner.h
@@ -19,23 +19,29 @@
#pragma once
-#include "core/filter.h"
+#include <functional>
+#include <iterator>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "core/cell.h"
+#include "core/result.h"
namespace hbase {
/**
- * Base class for read RPC calls (Get / Scan).
+ * Interface for client-side scanning. Use Table to obtain instances.
*/
-class Query {
- public:
- virtual ~Query() {}
+class ResultScanner {
+ // TODO: should we implement forward iterators?
- void SetFilter(std::unique_ptr<Filter> filter) { filter_ = std::move(filter); }
+ public:
+ virtual ~ResultScanner() {}
- const std::unique_ptr<Filter>& filter() const { return filter_; }
+ virtual void Close() = 0;
- protected:
- std::unique_ptr<Filter> filter_ = nullptr;
+ virtual std::shared_ptr<Result> Next() = 0;
};
-
-} // namespace hbase
+} /* namespace hbase */
diff --git a/core/result-test.cc b/core/result-test.cc
index 520f4b9..3299ffc 100644
--- a/core/result-test.cc
+++ b/core/result-test.cc
@@ -17,6 +17,7 @@
*
*/
+#include <glog/logging.h>
#include <gtest/gtest.h>
#include <limits>
#include <memory>
@@ -71,7 +72,7 @@ void PopulateCells(std::vector<std::shared_ptr<Cell> > &cells) {
}
default: {
cells.push_back(std::make_shared<Cell>(
- row, family, column, std::numeric_limits<long>::max(), value, CellType::PUT));
+ row, family, column, std::numeric_limits<int64_t>::max(), value, CellType::PUT));
}
}
}
@@ -255,7 +256,7 @@ TEST(Result, FilledResult) {
break;
}
default: {
- EXPECT_EQ(std::numeric_limits<long>::max(), version_map.first);
+ EXPECT_EQ(std::numeric_limits<int64_t>::max(), version_map.first);
EXPECT_EQ(value, version_map.second);
}
}
@@ -297,3 +298,24 @@ TEST(Result, FilledResult) {
EXPECT_EQ("value-9", qual_val_map.second);
}
}
+
+TEST(Result, ResultEstimatedSize) {
+ CellType cell_type = CellType::PUT;
+ int64_t timestamp = std::numeric_limits<int64_t>::max();
+ std::vector<std::shared_ptr<Cell> > cells;
+ Result empty(cells, true, false, false);
+
+ EXPECT_EQ(empty.EstimatedSize(), sizeof(Result));
+
+ cells.push_back(std::make_shared<Cell>("a", "a", "", timestamp, "", cell_type));
+ Result result1(cells, true, false, false);
+ EXPECT_TRUE(result1.EstimatedSize() > empty.EstimatedSize());
+
+ cells.push_back(std::make_shared<Cell>("a", "a", "", timestamp, "", cell_type));
+ Result result2(cells, true, false, false);
+ EXPECT_TRUE(result2.EstimatedSize() > result1.EstimatedSize());
+
+ LOG(INFO) << empty.EstimatedSize();
+ LOG(INFO) << result1.EstimatedSize();
+ LOG(INFO) << result2.EstimatedSize();
+}
diff --git a/core/result.cc b/core/result.cc
index 9d9ddb3..44b4c86 100644
--- a/core/result.cc
+++ b/core/result.cc
@@ -25,13 +25,7 @@ Result::~Result() {}
Result::Result(const std::vector<std::shared_ptr<Cell> > &cells, bool exists, bool stale,
bool partial)
- : exists_(exists), stale_(stale), partial_(partial) {
- for (const auto &cell : cells) {
- cells_.push_back(cell);
- // We create the map when cells are added. unlike java where map is created
- // when result.getMap() is called
- result_map_[cell->Family()][cell->Qualifier()][cell->Timestamp()] = cell->Value();
- }
+ : exists_(exists), stale_(stale), partial_(partial), cells_(cells) {
row_ = (cells_.size() == 0 ? "" : cells_[0]->Row());
}
@@ -43,10 +37,10 @@ Result::Result(const Result &result) {
if (!result.cells_.empty()) {
for (const auto &cell : result.cells_) {
cells_.push_back(cell);
- result_map_[cell->Family()][cell->Qualifier()][cell->Timestamp()] = cell->Value();
}
}
}
+
const std::vector<std::shared_ptr<Cell> > &Result::Cells() const { return cells_; }
std::vector<std::shared_ptr<Cell> > Result::ColumnCells(const std::string &family,
@@ -74,13 +68,12 @@ const std::shared_ptr<Cell> Result::ColumnLatestCell(const std::string &family,
return nullptr;
}
-std::shared_ptr<std::string> Result::Value(const std::string &family,
- const std::string &qualifier) const {
+optional<std::string> Result::Value(const std::string &family, const std::string &qualifier) const {
std::shared_ptr<Cell> latest_cell(ColumnLatestCell(family, qualifier));
if (latest_cell.get()) {
- return std::make_shared<std::string>(latest_cell->Value());
+ return optional<std::string>(latest_cell->Value());
}
- return nullptr;
+ return optional<std::string>();
}
bool Result::IsEmpty() const { return cells_.empty(); }
@@ -89,24 +82,33 @@ const std::string &Result::Row() const { return row_; }
int Result::Size() const { return cells_.size(); }
-const ResultMap &Result::Map() const { return result_map_; }
+ResultMap Result::Map() const {
+ ResultMap result_map;
+ for (const auto &cell : cells_) {
+ result_map[cell->Family()][cell->Qualifier()][cell->Timestamp()] = cell->Value();
+ }
+ return result_map;
+}
-const std::map<std::string, std::string> Result::FamilyMap(const std::string &family) const {
+std::map<std::string, std::string> Result::FamilyMap(const std::string &family) const {
std::map<std::string, std::string> family_map;
if (!IsEmpty()) {
- for (auto itr = result_map_.begin(); itr != result_map_.end(); ++itr) {
- if (family == itr->first) {
- for (auto qitr = itr->second.begin(); qitr != itr->second.end(); ++qitr) {
- for (auto vitr = qitr->second.begin(); vitr != qitr->second.end(); ++vitr) {
- // We break after inserting the first value. Result.java takes only
- // the first value
- family_map[qitr->first] = vitr->second;
- break;
- }
- }
+ auto result_map = Map();
+ auto itr = result_map.find(family);
+ if (itr == result_map.end()) {
+ return family_map;
+ }
+
+ for (auto qitr = itr->second.begin(); qitr != itr->second.end(); ++qitr) {
+ for (auto vitr = qitr->second.begin(); vitr != qitr->second.end(); ++vitr) {
+ // We break after inserting the first value. Result.java takes only
+ // the first value
+ family_map[qitr->first] = vitr->second;
+ break;
}
}
}
+
return family_map;
}
@@ -131,4 +133,14 @@ std::string Result::DebugString() const {
return ret;
}
+size_t Result::EstimatedSize() const {
+ size_t s = sizeof(Result);
+ s += row_.capacity();
+ for (const auto c : cells_) {
+ s += sizeof(std::shared_ptr<Cell>);
+ s + c->EstimatedSize();
+ }
+ return s;
+}
+
} /* namespace hbase */
diff --git a/core/result.h b/core/result.h
index 627d161..f18071b 100644
--- a/core/result.h
+++ b/core/result.h
@@ -26,6 +26,7 @@
#include <vector>
#include "core/cell.h"
+#include "utils/optional.h"
namespace hbase {
@@ -79,7 +80,7 @@ class Result {
* @param family - column family
* @param qualifier - column qualifier
*/
- std::shared_ptr<std::string> Value(const std::string &family, const std::string &qualifier) const;
+ optional<std::string> Value(const std::string &family, const std::string &qualifier) const;
/**
* @brief Returns if the underlying Cell vector is empty or not
@@ -104,23 +105,32 @@ class Result {
* All other map returning methods make use of this map internally
* The Map is created when the Result instance is created
*/
- const ResultMap &Map() const;
+ ResultMap Map() const;
/**
* @brief Map of qualifiers to values.
* Returns a Map of the form: Map<qualifier,value>
* @param family - column family to get
*/
- const std::map<std::string, std::string> FamilyMap(const std::string &family) const;
+ std::map<std::string, std::string> FamilyMap(const std::string &family) const;
std::string DebugString() const;
+ bool Exists() const { return exists_; }
+
+ bool Stale() const { return stale_; }
+
+ bool Partial() const { return partial_; }
+
+ /** Returns estimated size of the Result object including deep heap space usage
+ * of its Cells and data. Notice that this is a very rough estimate. */
+ size_t EstimatedSize() const;
+
private:
bool exists_ = false;
bool stale_ = false;
bool partial_ = false;
std::string row_ = "";
std::vector<std::shared_ptr<Cell> > cells_;
- ResultMap result_map_;
};
} /* namespace hbase */
diff --git a/core/scan-result-cache-test.cc b/core/scan-result-cache-test.cc
new file mode 100644
index 0000000..0bf83ce
--- /dev/null
+++ b/core/scan-result-cache-test.cc
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <folly/Conv.h>
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include <vector>
+
+#include "core/cell.h"
+#include "core/result.h"
+#include "core/scan-result-cache.h"
+
+using hbase::ScanResultCache;
+using hbase::Result;
+using hbase::Cell;
+using hbase::CellType;
+
+using ResultVector = std::vector<std::shared_ptr<Result>>;
+
+std::shared_ptr<Cell> CreateCell(const int32_t &key, const std::string &family,
+ const std::string &column) {
+ auto row = folly::to<std::string>(key);
+ return std::make_shared<Cell>(row, family, column, std::numeric_limits<int64_t>::max(), row,
+ CellType::PUT);
+}
+
+std::shared_ptr<Result> CreateResult(std::shared_ptr<Cell> cell, bool partial) {
+ return std::make_shared<Result>(std::vector<std::shared_ptr<Cell>>{cell}, false, false, partial);
+}
+
+TEST(ScanResultCacheTest, NoPartial) {
+ ScanResultCache cache;
+ ASSERT_EQ(ResultVector{}, cache.AddAndGet(ResultVector{}, false));
+ ASSERT_EQ(ResultVector{}, cache.AddAndGet(ResultVector{}, true));
+ int32_t count = 10;
+ ResultVector results{};
+ for (int32_t i = 0; i < count; i++) {
+ results.push_back(CreateResult(CreateCell(i, "cf", "cq1"), false));
+ }
+ ASSERT_EQ(results, cache.AddAndGet(results, false));
+}
+
+TEST(ScanResultCacheTest, Combine1) {
+ ScanResultCache cache;
+ auto prev_result = CreateResult(CreateCell(0, "cf", "cq1"), true);
+ auto result1 = CreateResult(CreateCell(1, "cf", "cq1"), true);
+ auto result2 = CreateResult(CreateCell(1, "cf", "cq2"), true);
+ auto result3 = CreateResult(CreateCell(1, "cf", "cq3"), true);
+ auto results = cache.AddAndGet(ResultVector{prev_result, result1}, false);
+ ASSERT_EQ(1L, results.size());
+ ASSERT_EQ(prev_result, results[0]);
+
+ ASSERT_EQ(0, cache.AddAndGet(ResultVector{result2}, false).size());
+ ASSERT_EQ(0, cache.AddAndGet(ResultVector{result3}, false).size());
+ ASSERT_EQ(0, cache.AddAndGet(ResultVector{}, true).size());
+
+ results = cache.AddAndGet(ResultVector{}, false);
+ ASSERT_EQ(1, results.size());
+ ASSERT_EQ(1, folly::to<int32_t>(results[0]->Row()));
+ ASSERT_EQ(3, results[0]->Cells().size());
+ ASSERT_EQ(1, folly::to<int32_t>(*results[0]->Value("cf", "cq1")));
+ ASSERT_EQ(1, folly::to<int32_t>(*results[0]->Value("cf", "cq2")));
+ ASSERT_EQ(1, folly::to<int32_t>(*results[0]->Value("cf", "cq3")));
+}
+
+TEST(ScanResultCacheTest, Combine2) {
+ ScanResultCache cache;
+ auto result1 = CreateResult(CreateCell(1, "cf", "cq1"), true);
+ auto result2 = CreateResult(CreateCell(1, "cf", "cq2"), true);
+ auto result3 = CreateResult(CreateCell(1, "cf", "cq3"), true);
+
+ auto next_result1 = CreateResult(CreateCell(2, "cf", "cq1"), true);
+ auto next_to_next_result1 = CreateResult(CreateCell(3, "cf", "cq2"), false);
+
+ ASSERT_EQ(0, cache.AddAndGet(ResultVector{result1}, false).size());
+ ASSERT_EQ(0, cache.AddAndGet(ResultVector{result2}, false).size());
+ ASSERT_EQ(0, cache.AddAndGet(ResultVector{result3}, false).size());
+
+ auto results = cache.AddAndGet(ResultVector{next_result1}, false);
+ ASSERT_EQ(1, results.size());
+ ASSERT_EQ(1, folly::to<int32_t>(results[0]->Row()));
+ ASSERT_EQ(3, results[0]->Cells().size());
+ ASSERT_EQ(1, folly::to<int32_t>(*results[0]->Value("cf", "cq1")));
+ ASSERT_EQ(1, folly::to<int32_t>(*results[0]->Value("cf", "cq2")));
+ ASSERT_EQ(1, folly::to<int32_t>(*results[0]->Value("cf", "cq3")));
+
+ results = cache.AddAndGet(ResultVector{next_to_next_result1}, false);
+ ASSERT_EQ(2, results.size());
+ ASSERT_EQ(2, folly::to<int32_t>(results[0]->Row()));
+ ASSERT_EQ(1, results[0]->Cells().size());
+ ASSERT_EQ(2, folly::to<int32_t>(*results[0]->Value("cf", "cq1")));
+ ASSERT_EQ(3, folly::to<int32_t>(results[1]->Row()));
+ ASSERT_EQ(1, results[1]->Cells().size());
+ ASSERT_EQ(3, folly::to<int32_t>(*results[1]->Value("cf", "cq2")));
+}
+
+TEST(ScanResultCacheTest, Combine3) {
+ ScanResultCache cache;
+ auto result1 = CreateResult(CreateCell(1, "cf", "cq1"), true);
+ auto result2 = CreateResult(CreateCell(1, "cf", "cq2"), true);
+ auto next_result1 = CreateResult(CreateCell(2, "cf", "cq1"), false);
+ auto next_to_next_result1 = CreateResult(CreateCell(3, "cf", "cq1"), true);
+
+ ASSERT_EQ(0, cache.AddAndGet(ResultVector{result1}, false).size());
+ ASSERT_EQ(0, cache.AddAndGet(ResultVector{result2}, false).size());
+
+ auto results = cache.AddAndGet(ResultVector{next_result1, next_to_next_result1}, false);
+
+ ASSERT_EQ(2, results.size());
+ ASSERT_EQ(1, folly::to<int32_t>(results[0]->Row()));
+ ASSERT_EQ(2, results[0]->Cells().size());
+ ASSERT_EQ(1, folly::to<int32_t>(*results[0]->Value("cf", "cq1")));
+ ASSERT_EQ(1, folly::to<int32_t>(*results[0]->Value("cf", "cq2")));
+ ASSERT_EQ(2, folly::to<int32_t>(results[1]->Row()));
+ ASSERT_EQ(1, results[1]->Cells().size());
+ ASSERT_EQ(2, folly::to<int32_t>(*results[1]->Value("cf", "cq1")));
+
+ results = cache.AddAndGet(ResultVector{}, false);
+
+ ASSERT_EQ(1, results.size());
+ ASSERT_EQ(3, folly::to<int32_t>(results[0]->Row()));
+ ASSERT_EQ(1, results[0]->Cells().size());
+ ASSERT_EQ(3, folly::to<int32_t>(*results[0]->Value("cf", "cq1")));
+}
+
+TEST(ScanResultCacheTest, Combine4) {
+ ScanResultCache cache;
+ auto result1 = CreateResult(CreateCell(1, "cf", "cq1"), true);
+ auto result2 = CreateResult(CreateCell(1, "cf", "cq2"), false);
+ auto next_result1 = CreateResult(CreateCell(2, "cf", "cq1"), true);
+ auto next_result2 = CreateResult(CreateCell(2, "cf", "cq2"), false);
+
+ ASSERT_EQ(0, cache.AddAndGet(ResultVector{result1}, false).size());
+
+ auto results = cache.AddAndGet(ResultVector{result2, next_result1}, false);
+
+ ASSERT_EQ(1, results.size());
+ ASSERT_EQ(1, folly::to<int32_t>(results[0]->Row()));
+ ASSERT_EQ(2, results[0]->Cells().size());
+ ASSERT_EQ(1, folly::to<int32_t>(*results[0]->Value("cf", "cq1")));
+ ASSERT_EQ(1, folly::to<int32_t>(*results[0]->Value("cf", "cq2")));
+
+ results = cache.AddAndGet(ResultVector{next_result2}, false);
+
+ ASSERT_EQ(1, results.size());
+ ASSERT_EQ(2, folly::to<int32_t>(results[0]->Row()));
+ ASSERT_EQ(2, results[0]->Cells().size());
+ ASSERT_EQ(2, folly::to<int32_t>(*results[0]->Value("cf", "cq1")));
+ ASSERT_EQ(2, folly::to<int32_t>(*results[0]->Value("cf", "cq2")));
+}
+
+TEST(ScanResultCacheTest, SizeOf) {
+ std::string e{""};
+ std::string f{"f"};
+ std::string foo{"foo"};
+
+ LOG(INFO) << sizeof(e) << " " << e.capacity();
+ LOG(INFO) << sizeof(f) << " " << f.capacity();
+ LOG(INFO) << sizeof(foo) << " " << foo.capacity();
+}
diff --git a/core/scan-result-cache.cc b/core/scan-result-cache.cc
new file mode 100644
index 0000000..62a51e0
--- /dev/null
+++ b/core/scan-result-cache.cc
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/scan-result-cache.h"
+#include <algorithm>
+#include <iterator>
+#include <limits>
+#include <stdexcept>
+
+namespace hbase {
+/**
+ * Add the given results to cache and get valid results back.
+ * @param results the results of a scan next. Must not be null.
+ * @param is_hearthbeat indicate whether the results is gotten from a heartbeat response.
+ * @return valid results, never null.
+ */
+std::vector<std::shared_ptr<Result>> ScanResultCache::AddAndGet(
+ const std::vector<std::shared_ptr<Result>> &results, bool is_hearthbeat) {
+ // If no results were returned it indicates that either we have the all the partial results
+ // necessary to construct the complete result or the server had to send a heartbeat message
+ // to the client to keep the client-server connection alive
+ if (results.empty()) {
+ // If this response was an empty heartbeat message, then we have not exhausted the region
+ // and thus there may be more partials server side that still need to be added to the partial
+ // list before we form the complete Result
+ if (!partial_results_.empty() && !is_hearthbeat) {
+ return UpdateNumberOfCompleteResultsAndReturn(
+ std::vector<std::shared_ptr<Result>>{Combine()});
+ }
+ return std::vector<std::shared_ptr<Result>>{};
+ }
+ // In every RPC response there should be at most a single partial result. Furthermore, if
+ // there is a partial result, it is guaranteed to be in the last position of the array.
+ auto last = results[results.size() - 1];
+ if (last->Partial()) {
+ if (partial_results_.empty()) {
+ partial_results_.push_back(last);
+ std::vector<std::shared_ptr<Result>> new_results;
+ std::copy_n(results.begin(), results.size() - 1, std::back_inserter(new_results));
+ return UpdateNumberOfCompleteResultsAndReturn(new_results);
+ }
+ // We have only one result and it is partial
+ if (results.size() == 1) {
+ // check if there is a row change
+ if (partial_results_.at(0)->Row() == last->Row()) {
+ partial_results_.push_back(last);
+ return std::vector<std::shared_ptr<Result>>{};
+ }
+ auto complete_result = Combine();
+ partial_results_.push_back(last);
+ return UpdateNumberOfCompleteResultsAndReturn(complete_result);
+ }
+ // We have some complete results
+ auto results_to_return = PrependCombined(results, results.size() - 1);
+ partial_results_.push_back(last);
+ return UpdateNumberOfCompleteResultsAndReturn(results_to_return);
+ }
+ if (!partial_results_.empty()) {
+ return UpdateNumberOfCompleteResultsAndReturn(PrependCombined(results, results.size()));
+ }
+ return UpdateNumberOfCompleteResultsAndReturn(results);
+}
+
+void ScanResultCache::Clear() { partial_results_.clear(); }
+
+std::shared_ptr<Result> ScanResultCache::CreateCompleteResult(
+ const std::vector<std::shared_ptr<Result>> &partial_results) {
+ if (partial_results.empty()) {
+ return std::make_shared<Result>(std::vector<std::shared_ptr<Cell>>{}, false, false, false);
+ }
+ std::vector<std::shared_ptr<Cell>> cells{};
+ bool stale = false;
+ std::string prev_row = "";
+ std::string current_row = "";
+ size_t i = 0;
+ for (const auto &r : partial_results) {
+ current_row = r->Row();
+ if (i != 0 && prev_row != current_row) {
+ throw new std::runtime_error(
+ "Cannot form complete result. Rows of partial results do not match.");
+ }
+ // Ensure that all Results except the last one are marked as partials. The last result
+ // may not be marked as a partial because Results are only marked as partials when
+ // the scan on the server side must be stopped due to reaching the maxResultSize.
+ // Visualizing it makes it easier to understand:
+ // maxResultSize: 2 cells
+ // (-x-) represents cell number x in a row
+ // Example: row1: -1- -2- -3- -4- -5- (5 cells total)
+ // How row1 will be returned by the server as partial Results:
+ // Result1: -1- -2- (2 cells, size limit reached, mark as partial)
+ // Result2: -3- -4- (2 cells, size limit reached, mark as partial)
+ // Result3: -5- (1 cell, size limit NOT reached, NOT marked as partial)
+ if (i != partial_results.size() - 1 && !r->Partial()) {
+ throw new std::runtime_error("Cannot form complete result. Result is missing partial flag.");
+ }
+ prev_row = current_row;
+ stale = stale || r->Stale();
+ for (const auto &c : r->Cells()) {
+ cells.push_back(c);
+ }
+ i++;
+ }
+
+ return std::make_shared<Result>(cells, false, stale, false);
+}
+
+std::shared_ptr<Result> ScanResultCache::Combine() {
+ auto result = CreateCompleteResult(partial_results_);
+ partial_results_.clear();
+ return result;
+}
+
+std::vector<std::shared_ptr<Result>> ScanResultCache::PrependCombined(
+ const std::vector<std::shared_ptr<Result>> &results, int length) {
+ if (length == 0) {
+ return std::vector<std::shared_ptr<Result>>{Combine()};
+ }
+ // the last part of a partial result may not be marked as partial so here we need to check if
+ // there is a row change.
+ size_t start;
+ if (partial_results_[0]->Row() == results[0]->Row()) {
+ partial_results_.push_back(results[0]);
+ start = 1;
+ length--;
+ } else {
+ start = 0;
+ }
+ std::vector<std::shared_ptr<Result>> prepend_results{};
+ prepend_results.push_back(Combine());
+ std::copy_n(results.begin() + start, length, std::back_inserter(prepend_results));
+ return prepend_results;
+}
+
+std::vector<std::shared_ptr<Result>> ScanResultCache::UpdateNumberOfCompleteResultsAndReturn(
+ const std::shared_ptr<Result> &result) {
+ return UpdateNumberOfCompleteResultsAndReturn(std::vector<std::shared_ptr<Result>>{result});
+}
+
+std::vector<std::shared_ptr<Result>> ScanResultCache::UpdateNumberOfCompleteResultsAndReturn(
+ const std::vector<std::shared_ptr<Result>> &results) {
+ num_complete_rows_ += results.size();
+ return results;
+}
+} // namespace hbase
diff --git a/core/scan-result-cache.h b/core/scan-result-cache.h
new file mode 100644
index 0000000..5d3d0ab
--- /dev/null
+++ b/core/scan-result-cache.h
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <folly/Logging.h>
+#include <algorithm>
+#include <chrono>
+#include <iterator>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "core/result.h"
+#include "if/Client.pb.h"
+#include "if/HBase.pb.h"
+
+namespace hbase {
+
+class ScanResultCache {
+ // In Java, there are 3 different implementations for this. We are not doing partial results,
+ // or scan batching in native code for now, so this version is simpler and
+ // only deals with giving back complete rows as Result. It is more or less implementation
+ // of CompleteScanResultCache.java
+
+ public:
+ /**
+ * Add the given results to cache and get valid results back.
+ * @param results the results of a scan next. Must not be null.
+ * @param is_hearthbeat indicate whether the results is gotten from a heartbeat response.
+ * @return valid results, never null.
+ */
+ std::vector<std::shared_ptr<Result>> AddAndGet(
+ const std::vector<std::shared_ptr<Result>> &results, bool is_hearthbeat);
+
+ void Clear();
+
+ int64_t num_complete_rows() const { return num_complete_rows_; }
+
+ private:
+ /**
+ * Forms a single result from the partial results in the partialResults list. This method is
+ * useful for reconstructing partial results on the client side.
+ * @param partial_results list of partial results
+ * @return The complete result that is formed by combining all of the partial results together
+ */
+ static std::shared_ptr<Result> CreateCompleteResult(
+ const std::vector<std::shared_ptr<Result>> &partial_results);
+
+ std::shared_ptr<Result> Combine();
+
+ std::vector<std::shared_ptr<Result>> PrependCombined(
+ const std::vector<std::shared_ptr<Result>> &results, int length);
+
+ std::vector<std::shared_ptr<Result>> UpdateNumberOfCompleteResultsAndReturn(
+ const std::shared_ptr<Result> &result);
+
+ std::vector<std::shared_ptr<Result>> UpdateNumberOfCompleteResultsAndReturn(
+ const std::vector<std::shared_ptr<Result>> &results);
+
+ private:
+ std::vector<std::shared_ptr<Result>> partial_results_;
+ int64_t num_complete_rows_ = 0;
+};
+} // namespace hbase
diff --git a/core/scan-test.cc b/core/scan-test.cc
index 73fb6df..0ee054c 100644
--- a/core/scan-test.cc
+++ b/core/scan-test.cc
@@ -17,13 +17,13 @@
*
*/
-#include "core/scan.h"
-
+#include <gtest/gtest.h>
#include <limits>
-#include <gtest/gtest.h>
+#include "core/scan.h"
-using namespace hbase;
+using hbase::Get;
+using hbase::Scan;
void CheckFamilies(Scan &scan) {
EXPECT_EQ(false, scan.HasFamilies());
@@ -74,7 +74,7 @@ void CheckFamilies(Scan &scan) {
EXPECT_EQ(it, scan.FamilyMap().end());
}
-void CheckFamiliesAfterCopy(Scan &scan) {
+void CheckFamiliesAfterCopy(const Scan &scan) {
EXPECT_EQ(true, scan.HasFamilies());
EXPECT_EQ(3, scan.FamilyMap().size());
int i = 1;
@@ -116,11 +116,6 @@ void ScanMethods(Scan &scan) {
scan.SetStopRow(stop_row);
EXPECT_EQ(stop_row, scan.StopRow());
- scan.SetSmall(true);
- EXPECT_EQ(true, scan.IsSmall());
- scan.SetSmall(false);
- EXPECT_EQ(false, scan.IsSmall());
-
scan.SetCaching(3);
EXPECT_EQ(3, scan.Caching());
diff --git a/core/scan.cc b/core/scan.cc
index 5f315ec..6dcc51b 100644
--- a/core/scan.cc
+++ b/core/scan.cc
@@ -38,7 +38,7 @@ Scan::Scan(const std::string &start_row, const std::string &stop_row)
CheckRow(stop_row_);
}
-Scan::Scan(const Scan &scan) {
+Scan::Scan(const Scan &scan) : Query(scan) {
start_row_ = scan.start_row_;
stop_row_ = scan.stop_row_;
max_versions_ = scan.max_versions_;
@@ -47,7 +47,6 @@ Scan::Scan(const Scan &scan) {
cache_blocks_ = scan.cache_blocks_;
load_column_families_on_demand_ = scan.load_column_families_on_demand_;
reversed_ = scan.reversed_;
- small_ = scan.small_;
allow_partial_results_ = scan.allow_partial_results_;
consistency_ = scan.consistency_;
tr_.reset(new TimeRange(scan.tr_->MinTimeStamp(), scan.tr_->MaxTimeStamp()));
@@ -55,6 +54,7 @@ Scan::Scan(const Scan &scan) {
}
Scan &Scan::operator=(const Scan &scan) {
+ Query::operator=(scan);
start_row_ = scan.start_row_;
stop_row_ = scan.stop_row_;
max_versions_ = scan.max_versions_;
@@ -63,7 +63,6 @@ Scan &Scan::operator=(const Scan &scan) {
cache_blocks_ = scan.cache_blocks_;
load_column_families_on_demand_ = scan.load_column_families_on_demand_;
reversed_ = scan.reversed_;
- small_ = scan.small_;
allow_partial_results_ = scan.allow_partial_results_;
consistency_ = scan.consistency_;
tr_.reset(new TimeRange(scan.tr_->MinTimeStamp(), scan.tr_->MaxTimeStamp()));
@@ -109,24 +108,14 @@ void Scan::SetReversed(bool reversed) { reversed_ = reversed; }
bool Scan::IsReversed() const { return reversed_; }
-void Scan::SetStartRow(const std::string &start_row) {
- CheckRow(start_row);
- start_row_ = start_row;
-}
+void Scan::SetStartRow(const std::string &start_row) { start_row_ = start_row; }
const std::string &Scan::StartRow() const { return start_row_; }
-void Scan::SetStopRow(const std::string &stop_row) {
- CheckRow(stop_row);
- stop_row_ = stop_row;
-}
+void Scan::SetStopRow(const std::string &stop_row) { stop_row_ = stop_row; }
const std::string &Scan::StopRow() const { return stop_row_; }
-void Scan::SetSmall(bool small) { small_ = small; }
-
-bool Scan::IsSmall() const { return small_; }
-
void Scan::SetCaching(int caching) { caching_ = caching; }
int Scan::Caching() const { return caching_; }
diff --git a/core/scan.h b/core/scan.h
index fb302b7..1085c4b 100644
--- a/core/scan.h
+++ b/core/scan.h
@@ -119,16 +119,6 @@ class Scan : public Query {
const std::string &StopRow() const;
/**
- * @brief Set whether this scan is a small scan.
- */
- void SetSmall(bool small);
-
- /**
- * @brief Returns if the scan is a small scan.
- */
- bool IsSmall() const;
-
- /**
* @brief Set the number of rows for caching that will be passed to scanners.
* Higher caching values will enable faster scanners but will use more memory.
* @param caching - the number of rows for caching.
@@ -258,12 +248,11 @@ class Scan : public Query {
std::string start_row_ = "";
std::string stop_row_ = "";
uint32_t max_versions_ = 1;
- int caching_ = -1;
+ int32_t caching_ = -1;
int64_t max_result_size_ = -1;
bool cache_blocks_ = true;
bool load_column_families_on_demand_ = false;
bool reversed_ = false;
- bool small_ = false;
bool allow_partial_results_ = false;
hbase::pb::Consistency consistency_ = hbase::pb::Consistency::STRONG;
std::unique_ptr<TimeRange> tr_ = std::make_unique<TimeRange>();
diff --git a/core/scanner-test.cc b/core/scanner-test.cc
new file mode 100644
index 0000000..1ecbd02
--- /dev/null
+++ b/core/scanner-test.cc
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <folly/Conv.h>
+#include <gtest/gtest.h>
+
+#include <chrono>
+#include <thread>
+#include <vector>
+
+#include "core/async-client-scanner.h"
+#include "core/async-table-result-scanner.h"
+#include "core/cell.h"
+#include "core/client.h"
+#include "core/configuration.h"
+#include "core/filter.h"
+#include "core/get.h"
+#include "core/hbase-configuration-loader.h"
+#include "core/put.h"
+#include "core/result.h"
+#include "core/row.h"
+#include "core/table.h"
+#include "if/Comparator.pb.h"
+#include "if/Filter.pb.h"
+#include "serde/table-name.h"
+#include "test-util/test-util.h"
+#include "utils/time-util.h"
+
+using hbase::Cell;
+using hbase::ComparatorFactory;
+using hbase::Comparator;
+using hbase::Configuration;
+using hbase::Get;
+using hbase::Put;
+using hbase::Result;
+using hbase::Scan;
+using hbase::Table;
+using hbase::TestUtil;
+using hbase::TimeUtil;
+using hbase::AsyncClientScanner;
+using hbase::AsyncTableResultScanner;
+using hbase::FilterFactory;
+using hbase::pb::CompareType;
+
+class ScannerTest : public ::testing::Test {
+ public:
+ static std::unique_ptr<hbase::TestUtil> test_util;
+ static const uint32_t num_rows;
+
+ static void SetUpTestCase() {
+ google::InstallFailureSignalHandler();
+ test_util = std::make_unique<hbase::TestUtil>();
+ test_util->StartMiniCluster(2);
+ }
+};
+std::unique_ptr<hbase::TestUtil> ScannerTest::test_util = nullptr;
+const uint32_t ScannerTest::num_rows = 1000;
+
+std::string Family(uint32_t i) { return "f" + folly::to<std::string>(i); }
+
+std::string Row(uint32_t i, int width) {
+ std::ostringstream s;
+ s.fill('0');
+ s.width(width);
+ s << i;
+ return "row" + s.str();
+}
+
+std::string Row(uint32_t i) { return Row(i, 3); }
+
+std::unique_ptr<Put> MakePut(const std::string &row, uint32_t num_families) {
+ auto put = std::make_unique<Put>(row);
+
+ for (uint32_t i = 0; i < num_families; i++) {
+ put->AddColumn(Family(i), "q1", row);
+ put->AddColumn(Family(i), "q2", row + "-" + row);
+ }
+
+ return std::move(put);
+}
+
+void CheckResult(const Result &r, std::string expected_row, uint32_t num_families) {
+ VLOG(1) << r.DebugString();
+ auto row = r.Row();
+ ASSERT_EQ(row, expected_row);
+ ASSERT_EQ(r.Cells().size(), num_families * 2);
+ for (uint32_t i = 0; i < num_families; i++) {
+ ASSERT_EQ(*r.Value(Family(i), "q1"), row);
+ ASSERT_EQ(*r.Value(Family(i), "q2"), row + "-" + row);
+ }
+}
+
+void CreateTable(std::string table_name, uint32_t num_families, uint32_t num_rows,
+ int32_t num_regions) {
+ LOG(INFO) << "Creating the table " << table_name
+ << " with num_regions:" << folly::to<std::string>(num_regions);
+ std::vector<std::string> families;
+ for (uint32_t i = 0; i < num_families; i++) {
+ families.push_back(Family(i));
+ }
+ if (num_regions <= 1) {
+ ScannerTest::test_util->CreateTable(table_name, families);
+ } else {
+ std::vector<std::string> keys;
+ for (int32_t i = 0; i < num_regions - 1; i++) {
+ keys.push_back(Row(i * (num_rows / (num_regions - 1))));
+ LOG(INFO) << "Split key:" << keys[keys.size() - 1];
+ }
+ ScannerTest::test_util->CreateTable(table_name, families, keys);
+ }
+}
+
+std::unique_ptr<hbase::Client> CreateTableAndWriteData(std::string table_name,
+ uint32_t num_families, uint32_t num_rows,
+ int32_t num_regions) {
+ CreateTable(table_name, num_families, num_rows, num_regions);
+ auto tn = folly::to<hbase::pb::TableName>(table_name);
+ auto client = std::make_unique<hbase::Client>(*ScannerTest::test_util->conf());
+ auto table = client->Table(tn);
+
+ LOG(INFO) << "Writing data to the table, num_rows:" << num_rows;
+ // Perform Puts
+ for (uint32_t i = 0; i < num_rows; i++) {
+ table->Put(*MakePut(Row(i), num_families));
+ }
+ return std::move(client);
+}
+
+void TestScan(const Scan &scan, uint32_t num_families, int32_t start, int32_t num_rows,
+ Table *table) {
+ LOG(INFO) << "Starting scan for the test with start:" << scan.StartRow()
+ << ", stop:" << scan.StopRow() << " expected_num_rows:" << num_rows;
+ auto scanner = table->Scan(scan);
+
+ uint32_t i = start;
+ auto r = scanner->Next();
+ while (r != nullptr) {
+ CheckResult(*r, Row(i++), num_families);
+ r = scanner->Next();
+ }
+ ASSERT_EQ(i - start, num_rows);
+}
+
+void TestScan(const Scan &scan, int32_t start, int32_t num_rows, Table *table) {
+ TestScan(scan, 1, start, num_rows, table);
+}
+
+void TestScan(uint32_t num_families, int32_t start, int32_t stop, int32_t num_rows, Table *table) {
+ Scan scan{};
+ if (start >= 0) {
+ scan.SetStartRow(Row(start));
+ } else {
+ start = 0; // neded for below logic
+ }
+ if (stop >= 0) {
+ scan.SetStopRow(Row(stop));
+ }
+
+ TestScan(scan, num_families, start, num_rows, table);
+}
+
+void TestScan(int32_t start, int32_t stop, int32_t num_rows, Table *table) {
+ TestScan(1, start, stop, num_rows, table);
+}
+
+void TestScan(uint32_t num_families, std::string start, std::string stop, int32_t num_rows,
+ Table *table) {
+ Scan scan{};
+
+ scan.SetStartRow(start);
+ scan.SetStopRow(stop);
+
+ LOG(INFO) << "Starting scan for the test with start:" << start << ", stop:" << stop
+ << " expected_num_rows:" << num_rows;
+ auto scanner = table->Scan(scan);
+
+ uint32_t i = 0;
+ auto r = scanner->Next();
+ while (r != nullptr) {
+ VLOG(1) << r->DebugString();
+ i++;
+ ASSERT_EQ(r->Map().size(), num_families);
+ r = scanner->Next();
+ }
+ ASSERT_EQ(i, num_rows);
+}
+
+void TestScan(std::string start, std::string stop, int32_t num_rows, Table *table) {
+ TestScan(1, start, stop, num_rows, table);
+}
+
+void TestScanCombinations(Table *table, uint32_t num_families) {
+ // full table
+ TestScan(num_families, -1, -1, 1000, table);
+ TestScan(num_families, -1, 999, 999, table);
+ TestScan(num_families, 0, -1, 1000, table);
+ TestScan(num_families, 0, 999, 999, table);
+ TestScan(num_families, 10, 990, 980, table);
+ TestScan(num_families, 1, 998, 997, table);
+
+ TestScan(num_families, 123, 345, 222, table);
+ TestScan(num_families, 234, 456, 222, table);
+ TestScan(num_families, 345, 567, 222, table);
+ TestScan(num_families, 456, 678, 222, table);
+
+ // single results
+ TestScan(num_families, 111, 111, 1, table); // split keys are like 111, 222, 333, etc
+ TestScan(num_families, 111, 112, 1, table);
+ TestScan(num_families, 332, 332, 1, table);
+ TestScan(num_families, 332, 333, 1, table);
+ TestScan(num_families, 333, 333, 1, table);
+ TestScan(num_families, 333, 334, 1, table);
+ TestScan(num_families, 42, 42, 1, table);
+ TestScan(num_families, 921, 921, 1, table);
+ TestScan(num_families, 0, 0, 1, table);
+ TestScan(num_families, 0, 1, 1, table);
+ TestScan(num_families, 999, 999, 1, table);
+
+ // few results
+ TestScan(num_families, 0, 0, 1, table);
+ TestScan(num_families, 0, 2, 2, table);
+ TestScan(num_families, 0, 5, 5, table);
+ TestScan(num_families, 10, 15, 5, table);
+ TestScan(num_families, 105, 115, 10, table);
+ TestScan(num_families, 111, 221, 110, table);
+ TestScan(num_families, 111, 222, 111, table); // crossing region boundary 111-222
+ TestScan(num_families, 111, 223, 112, table);
+ TestScan(num_families, 111, 224, 113, table);
+ TestScan(num_families, 990, 999, 9, table);
+ TestScan(num_families, 900, 998, 98, table);
+
+ // empty results
+ TestScan(num_families, "a", "a", 0, table);
+ TestScan(num_families, "a", "r", 0, table);
+ TestScan(num_families, "", "r", 0, table);
+ TestScan(num_families, "s", "", 0, table);
+ TestScan(num_families, "s", "z", 0, table);
+ TestScan(num_families, Row(110) + "a", Row(111), 0, table);
+ TestScan(num_families, Row(111) + "a", Row(112), 0, table);
+ TestScan(num_families, Row(123) + "a", Row(124), 0, table);
+
+ // custom
+ TestScan(num_families, Row(111, 3), Row(1111, 4), 1, table);
+ TestScan(num_families, Row(0, 3), Row(0, 4), 1, table);
+ TestScan(num_families, Row(999, 3), Row(9999, 4), 1, table);
+ TestScan(num_families, Row(111, 3), Row(1111, 4), 1, table);
+ TestScan(num_families, Row(0, 3), Row(9999, 4), 1000, table);
+ TestScan(num_families, "a", "z", 1000, table);
+}
+
+// some of these tests are from TestAsyncTableScan* and some from TestFromClientSide* and
+// TestScannersFromClientSide*
+
+TEST_F(ScannerTest, SingleRegionScan) {
+ auto client = CreateTableAndWriteData("t_single_region_scan", 1, num_rows, 1);
+ auto table = client->Table(folly::to<hbase::pb::TableName>("t_single_region_scan"));
+
+ TestScanCombinations(table.get(), 1);
+}
+
+TEST_F(ScannerTest, MultiRegionScan) {
+ auto client = CreateTableAndWriteData("t_multi_region_scan", 1, num_rows, 10);
+ auto table = client->Table(folly::to<hbase::pb::TableName>("t_multi_region_scan"));
+
+ TestScanCombinations(table.get(), 1);
+}
+
+TEST_F(ScannerTest, ScanWithPauses) {
+ auto max_result_size =
+ ScannerTest::test_util->conf()->GetInt("hbase.client.scanner.max.result.size", 2097152);
+ ScannerTest::test_util->conf()->SetInt("hbase.client.scanner.max.result.size", 100);
+ auto client = CreateTableAndWriteData("t_multi_region_scan", 1, num_rows, 5);
+ auto table = client->Table(folly::to<hbase::pb::TableName>("t_multi_region_scan"));
+
+ VLOG(1) << "Starting scan for the test";
+ Scan scan{};
+ scan.SetCaching(100);
+ auto scanner = table->Scan(scan);
+
+ uint32_t i = 0;
+ auto r = scanner->Next();
+ while (r != nullptr) {
+ CheckResult(*r, Row(i++), 1);
+ r = scanner->Next();
+ std::this_thread::sleep_for(TimeUtil::MillisToNanos(10));
+ }
+
+ auto s = static_cast<AsyncTableResultScanner *>(scanner.get());
+ ASSERT_GT(s->num_prefetch_stopped(), 0);
+
+ ASSERT_EQ(i, num_rows);
+ ScannerTest::test_util->conf()->SetInt("hbase.client.scanner.max.result.size", max_result_size);
+}
+
+TEST_F(ScannerTest, ScanWithFilters) {
+ auto client = CreateTableAndWriteData("t_scan_with_filters", 1, num_rows, 1);
+ auto table = client->Table(folly::to<hbase::pb::TableName>("t_scan_with_filters"));
+
+ Scan scan{};
+ scan.SetFilter(FilterFactory::ValueFilter(CompareType::GREATER_OR_EQUAL,
+ *ComparatorFactory::BinaryComparator(Row(800))));
+
+ TestScan(scan, 800, 200, table.get());
+}
+
+TEST_F(ScannerTest, ScanMultiFamily) {
+ auto client = CreateTableAndWriteData("t_scan_multi_family", 3, num_rows, 1);
+ auto table = client->Table(folly::to<hbase::pb::TableName>("t_scan_multi_family"));
+
+ TestScanCombinations(table.get(), 3);
+}
+
+TEST_F(ScannerTest, ScanNullQualifier) {
+ std::string table_name{"t_scan_null_qualifier"};
+ std::string row{"row"};
+ CreateTable(table_name, 1, 1, 1);
+
+ auto tn = folly::to<hbase::pb::TableName>(table_name);
+ auto client = std::make_unique<hbase::Client>(*ScannerTest::test_util->conf());
+ auto table = client->Table(tn);
+
+ // Perform Puts
+ Put put{row};
+ put.AddColumn(Family(0), "q1", row);
+ put.AddColumn(Family(0), "", row);
+ table->Put(put);
+
+ Scan scan1{};
+ scan1.AddColumn(Family(0), "");
+ auto scanner1 = table->Scan(scan1);
+ auto r1 = scanner1->Next();
+ ASSERT_EQ(r1->Cells().size(), 1);
+ ASSERT_EQ(scanner1->Next(), nullptr);
+
+ Scan scan2{};
+ scan2.AddFamily(Family(0));
+ auto scanner2 = table->Scan(scan2);
+ auto r2 = scanner2->Next();
+ ASSERT_EQ(r2->Cells().size(), 2);
+ ASSERT_EQ(scanner2->Next(), nullptr);
+}
+
+TEST_F(ScannerTest, ScanNoResults) {
+ std::string table_name{"t_scan_no_results"};
+ auto client = CreateTableAndWriteData(table_name, 1, num_rows, 3);
+ auto table = client->Table(folly::to<hbase::pb::TableName>(table_name));
+
+ Scan scan{};
+ scan.AddColumn(Family(0), "non_existing_qualifier");
+
+ TestScan(scan, 0, 0, table.get());
+}
diff --git a/core/simple-client.cc b/core/simple-client.cc
index 3a7d62b..f79d848 100644
--- a/core/simple-client.cc
+++ b/core/simple-client.cc
@@ -30,6 +30,7 @@
#include "core/client.h"
#include "core/get.h"
#include "core/put.h"
+#include "core/scan.h"
#include "core/table.h"
#include "serde/server-name.h"
#include "serde/table-name.h"
@@ -38,6 +39,7 @@
using hbase::Client;
using hbase::Configuration;
using hbase::Get;
+using hbase::Scan;
using hbase::Put;
using hbase::Table;
using hbase::pb::TableName;
@@ -48,6 +50,10 @@ DEFINE_string(table, "test_table", "What table to do the reads or writes");
DEFINE_string(row, "row_", "row prefix");
DEFINE_string(zookeeper, "localhost:2181", "What zk quorum to talk to");
DEFINE_uint64(num_rows, 10000, "How many rows to write and read");
+DEFINE_bool(puts, true, "Whether to perform puts");
+DEFINE_bool(gets, true, "Whether to perform gets");
+DEFINE_bool(multigets, true, "Whether to perform multi-gets");
+DEFINE_bool(scans, true, "Whether to perform scans");
DEFINE_bool(display_results, false, "Whether to display the Results from Gets");
DEFINE_int32(threads, 6, "How many cpu threads");
@@ -86,41 +92,72 @@ int main(int argc, char *argv[]) {
auto start_ns = TimeUtil::GetNowNanos();
// Do the Put requests
- for (uint64_t i = 0; i < num_puts; i++) {
- table->Put(*MakePut(Row(FLAGS_row, i)));
- }
+ if (FLAGS_puts) {
+ LOG(INFO) << "Sending put requests";
+ for (uint64_t i = 0; i < num_puts; i++) {
+ table->Put(*MakePut(Row(FLAGS_row, i)));
+ }
- LOG(INFO) << "Successfully sent " << num_puts << " Put requests in "
- << TimeUtil::ElapsedMillis(start_ns) << " ms.";
+ LOG(INFO) << "Successfully sent " << num_puts << " Put requests in "
+ << TimeUtil::ElapsedMillis(start_ns) << " ms.";
+ }
// Do the Get requests
- start_ns = TimeUtil::GetNowNanos();
- for (uint64_t i = 0; i < num_puts; i++) {
- auto result = table->Get(Get{Row(FLAGS_row, i)});
- if (FLAGS_display_results) {
- LOG(INFO) << result->DebugString();
+ if (FLAGS_gets) {
+ LOG(INFO) << "Sending get requests";
+ start_ns = TimeUtil::GetNowNanos();
+ for (uint64_t i = 0; i < num_puts; i++) {
+ auto result = table->Get(Get{Row(FLAGS_row, i)});
+ if (FLAGS_display_results) {
+ LOG(INFO) << result->DebugString();
+ }
}
- }
- LOG(INFO) << "Successfully sent " << num_puts << " Get requests in "
- << TimeUtil::ElapsedMillis(start_ns) << " ms.";
+ LOG(INFO) << "Successfully sent " << num_puts << " Get requests in "
+ << TimeUtil::ElapsedMillis(start_ns) << " ms.";
+ }
// Do the Multi-Gets
- std::vector<hbase::Get> gets;
- for (uint64_t i = 0; i < num_puts; ++i) {
- hbase::Get get(Row(FLAGS_row, i));
- gets.push_back(get);
- }
+ if (FLAGS_multigets) {
+ std::vector<hbase::Get> gets;
+ for (uint64_t i = 0; i < num_puts; ++i) {
+ hbase::Get get(Row(FLAGS_row, i));
+ gets.push_back(get);
+ }
+
+ LOG(INFO) << "Sending multi-get requests";
+ start_ns = TimeUtil::GetNowNanos();
+ auto results = table->Get(gets);
- start_ns = TimeUtil::GetNowNanos();
- auto results = table->Get(gets);
+ if (FLAGS_display_results) {
+ for (const auto &result : results) LOG(INFO) << result->DebugString();
+ }
- if (FLAGS_display_results) {
- for (const auto &result : results) LOG(INFO) << result->DebugString();
+ LOG(INFO) << "Successfully sent " << gets.size() << " Multi-Get requests in "
+ << TimeUtil::ElapsedMillis(start_ns) << " ms.";
}
- LOG(INFO) << "Successfully sent " << gets.size() << " Multi-Get requests in "
- << TimeUtil::ElapsedMillis(start_ns) << " ms.";
+ // Do the Scan
+ if (FLAGS_scans) {
+ LOG(INFO) << "Starting scanner";
+ start_ns = TimeUtil::GetNowNanos();
+ Scan scan{};
+ auto scanner = table->Scan(scan);
+
+ uint64_t i = 0;
+ auto r = scanner->Next();
+ while (r != nullptr) {
+ if (FLAGS_display_results) {
+ LOG(INFO) << r->DebugString();
+ }
+ r = scanner->Next();
+ i++;
+ }
+
+ LOG(INFO) << "Successfully iterated over " << i << " Scan results in "
+ << TimeUtil::ElapsedMillis(start_ns) << " ms.";
+ scanner->Close();
+ }
table->Close();
client->Close();
diff --git a/core/table.cc b/core/table.cc
index aa51989..9cdd8b0 100644
--- a/core/table.cc
+++ b/core/table.cc
@@ -25,6 +25,7 @@
#include <vector>
#include "core/async-connection.h"
+#include "core/async-table-result-scanner.h"
#include "core/request-converter.h"
#include "core/response-converter.h"
#include "if/Client.pb.h"
@@ -52,6 +53,21 @@ std::shared_ptr<hbase::Result> Table::Get(const hbase::Get &get) {
return context.get(operation_timeout());
}
+std::shared_ptr<ResultScanner> Table::Scan(const hbase::Scan &scan) {
+ auto max_cache_size = ResultSize2CacheSize(
+ scan.MaxResultSize() > 0 ? scan.MaxResultSize()
+ : async_connection_->connection_conf()->scanner_max_result_size());
+ auto scanner = std::make_shared<AsyncTableResultScanner>(max_cache_size);
+ async_table_->Scan(scan, scanner);
+ return scanner;
+}
+
+int64_t Table::ResultSize2CacheSize(int64_t max_results_size) const {
+ // * 2 if possible
+ return max_results_size > (std::numeric_limits<int64_t>::max() / 2) ? max_results_size
+ : max_results_size * 2;
+}
+
void Table::Put(const hbase::Put &put) {
auto future = async_table_->Put(put);
future.get(operation_timeout());
diff --git a/core/table.h b/core/table.h
index 81ddc8e..1f6d9b7 100644
--- a/core/table.h
+++ b/core/table.h
@@ -32,6 +32,7 @@
#include "core/location-cache.h"
#include "core/put.h"
#include "core/raw-async-table.h"
+#include "core/result-scanner.h"
#include "core/result.h"
#include "serde/table-name.h"
@@ -74,6 +75,8 @@ class Table {
std::shared_ptr<hbase::Result> Increment(const hbase::Increment &increment);
// TODO: Batch Puts
+ std::shared_ptr<ResultScanner> Scan(const hbase::Scan &scan);
+
/**
* @brief - Close the client connection.
*/
@@ -92,5 +95,7 @@ class Table {
private:
std::chrono::milliseconds operation_timeout() const;
+
+ int64_t ResultSize2CacheSize(int64_t max_results_size) const;
};
} /* namespace hbase */
diff --git a/exceptions/BUCK b/exceptions/BUCK
index eef4437..07ffeaf 100644
--- a/exceptions/BUCK
+++ b/exceptions/BUCK
@@ -17,8 +17,12 @@
cxx_library(
name="exceptions",
- exported_headers=["exception.h",],
+ exported_headers=[
+ "exception.h",
+ ],
srcs=[],
- deps=["//third-party:folly",],
+ deps=[
+ "//third-party:folly",
+ ],
compiler_flags=['-Weffc++'],
- visibility=['//core/...','//connection//...'],)
\ No newline at end of file
+ visibility=['//core/...', '//connection//...'],)
diff --git a/exceptions/exception.h b/exceptions/exception.h
index f25fbea..9cbd7ae 100644
--- a/exceptions/exception.h
+++ b/exceptions/exception.h
@@ -18,25 +18,22 @@
*/
#pragma once
+#include <folly/ExceptionWrapper.h>
+#include <folly/io/IOBuf.h>
#include <exception>
#include <string>
#include <vector>
-#include <folly/io/IOBuf.h>
-#include <folly/ExceptionWrapper.h>
namespace hbase {
class ThrowableWithExtraContext {
-public:
- ThrowableWithExtraContext(folly::exception_wrapper cause,
- const long& when) :
- cause_(cause), when_(when), extras_("") {
- }
+ public:
+ ThrowableWithExtraContext(folly::exception_wrapper cause, const long& when)
+ : cause_(cause), when_(when), extras_("") {}
- ThrowableWithExtraContext(folly::exception_wrapper cause,
- const long& when, const std::string& extras) :
- cause_(cause), when_(when), extras_(extras) {
- }
+ ThrowableWithExtraContext(folly::exception_wrapper cause, const long& when,
+ const std::string& extras)
+ : cause_(cause), when_(when), extras_(extras) {}
virtual std::string ToString() {
// TODO:
@@ -44,55 +41,45 @@ public:
return extras_ + ", " + cause_.what().toStdString();
}
- virtual folly::exception_wrapper cause() {
- return cause_;
- }
-private:
+ virtual folly::exception_wrapper cause() { return cause_; }
+
+ private:
folly::exception_wrapper cause_;
long when_;
std::string extras_;
};
-class IOException: public std::logic_error {
-public:
+class IOException : public std::logic_error {
+ public:
IOException() : logic_error("") {}
- IOException(
- const std::string& what) :
- logic_error(what) {}
- IOException(
- const std::string& what,
- folly::exception_wrapper cause) :
- logic_error(what), cause_(cause) {}
+ IOException(const std::string& what) : logic_error(what) {}
+ IOException(const std::string& what, folly::exception_wrapper cause)
+ : logic_error(what), cause_(cause) {}
virtual ~IOException() = default;
- virtual folly::exception_wrapper cause() {
- return cause_;
- }
-private:
+ virtual folly::exception_wrapper cause() { return cause_; }
+
+ private:
folly::exception_wrapper cause_;
};
-class RetriesExhaustedException: public IOException {
-public:
- RetriesExhaustedException(
- const int& num_retries,
- std::shared_ptr<std::vector<ThrowableWithExtraContext>> exceptions) :
- IOException(
- GetMessage(num_retries, exceptions),
- exceptions->empty() ? folly::exception_wrapper{}
- : (*exceptions)[exceptions->size() - 1].cause()){
- }
+class RetriesExhaustedException : public IOException {
+ public:
+ RetriesExhaustedException(const int& num_retries,
+ std::shared_ptr<std::vector<ThrowableWithExtraContext>> exceptions)
+ : IOException(GetMessage(num_retries, exceptions),
+ exceptions->empty() ? folly::exception_wrapper{}
+ : (*exceptions)[exceptions->size() - 1].cause()) {}
virtual ~RetriesExhaustedException() = default;
-private:
- std::string GetMessage(
- const int& num_retries,
- std::shared_ptr<std::vector<ThrowableWithExtraContext>> exceptions) {
+ private:
+ std::string GetMessage(const int& num_retries,
+ std::shared_ptr<std::vector<ThrowableWithExtraContext>> exceptions) {
std::string buffer("Failed after attempts=");
buffer.append(std::to_string(num_retries + 1));
buffer.append(", exceptions:\n");
- for (auto it = exceptions->begin(); it != exceptions->end(); it++) {
+ for (auto it = exceptions->begin(); it != exceptions->end(); it++) {
buffer.append(it->ToString());
buffer.append("\n");
}
@@ -100,74 +87,141 @@ private:
}
};
-class HBaseIOException : public IOException {
-};
+class HBaseIOException : public IOException {};
class RemoteException : public IOException {
-public:
-
+ public:
RemoteException() : port_(0), do_not_retry_(false) {}
- RemoteException(const std::string& what) :
- IOException(what), port_(0), do_not_retry_(false) {}
+ RemoteException(const std::string& what) : IOException(what), port_(0), do_not_retry_(false) {}
- RemoteException(
- const std::string& what,
- folly::exception_wrapper cause) :
- IOException(what, cause), port_(0), do_not_retry_(false) {}
+ RemoteException(const std::string& what, folly::exception_wrapper cause)
+ : IOException(what, cause), port_(0), do_not_retry_(false) {}
virtual ~RemoteException() = default;
- std::string exception_class_name() const {
- return exception_class_name_;
- }
+ std::string exception_class_name() const { return exception_class_name_; }
RemoteException* set_exception_class_name(const std::string& value) {
exception_class_name_ = value;
return this;
}
- std::string stack_trace() const {
- return stack_trace_;
- }
+ std::string stack_trace() const { return stack_trace_; }
RemoteException* set_stack_trace(const std::string& value) {
stack_trace_ = value;
return this;
}
- std::string hostname() const {
- return hostname_;
- }
+ std::string hostname() const { return hostname_; }
RemoteException* set_hostname(const std::string& value) {
hostname_ = value;
return this;
}
- int port() const {
- return port_;
- }
+ int port() const { return port_; }
RemoteException* set_port(int value) {
port_ = value;
return this;
}
- bool do_not_retry() const {
- return do_not_retry_;
- }
+ bool do_not_retry() const { return do_not_retry_; }
RemoteException* set_do_not_retry(bool value) {
do_not_retry_ = value;
return this;
}
-private:
+ private:
std::string exception_class_name_;
std::string stack_trace_;
std::string hostname_;
int port_;
bool do_not_retry_;
};
+
+/**
+ * List of known exceptions from Java side, and Java-specific exception logic
+ */
+class ExceptionUtil {
+ private:
+ // unknown scanner and sub-classes
+ static constexpr const char* kUnknownScannerException =
+ "org.apache.hadoop.hbase.UnknownScannerException";
+
+ // not serving region and sub-classes
+ static constexpr const char* kNotServingRegionException =
+ "org.apache.hadoop.hbase.NotServingRegionException";
+ static constexpr const char* kRegionInRecoveryException =
+ "org.apache.hadoop.hbase.exceptions.RegionInRecoveryException";
+ static constexpr const char* kRegionOpeningException =
+ "org.apache.hadoop.hbase.exceptions.RegionOpeningException";
+ static constexpr const char* kRegionMovedException =
+ "org.apache.hadoop.hbase.exceptions.RegionMovedException";
+
+ // Region server stopped and sub-classes
+ static constexpr const char* kRegionServerStoppedException =
+ "org.apache.hadoop.hbase.regionserver.RegionServerStoppedException";
+ static constexpr const char* kRegionServerAbortedException =
+ "org.apache.hadoop.hbase.regionserver.RegionServerAbortedException";
+
+ // other scanner related exceptions
+ static constexpr const char* kOutOfOrderScannerNextException =
+ "org.apache.hadoop.hbase.exceptions.OutOfOrderScannerNextException";
+ static constexpr const char* kScannerResetException =
+ "org.apache.hadoop.hbase.exceptions.ScannerResetException";
+
+ public:
+ /**
+ * Returns whether or not the exception should be retried by looking at the
+ * remote exception.
+ */
+ static bool ShouldRetry(const folly::exception_wrapper& error) {
+ bool do_not_retry = false;
+ error.with_exception(
+ [&](const RemoteException& remote_ex) { do_not_retry = remote_ex.do_not_retry(); });
+ return !do_not_retry;
+ }
+
+ /**
+ * Returns whether the scanner is closed when the client received the
+ * remote exception.
+ * Ok, here is a nice detail about the java exceptions. In the java side, we
+ * just have a hierarchy of Exception classes that we use both client side and
+ * server side. On the client side, we rethrow the server side exception by
+ * un-wrapping the exception from a RemoteException or a ServiceException
+ * (see ConnectionUtils.translateException() in Java).
+ * Since this object-hierarchy info is not available in C++ side, we are doing a
+ * very fragile catch-all list of all exception types in Java that extend these
+ * three base classes: UnknownScannerException, NotServingRegionException,
+ * RegionServerStoppedException
+ */
+ static bool IsScannerClosed(const folly::exception_wrapper& exception) {
+ bool scanner_closed = false;
+ exception.with_exception([&](const RemoteException& remote_ex) {
+ auto java_class = remote_ex.exception_class_name();
+ if (java_class == kUnknownScannerException || java_class == kNotServingRegionException ||
+ java_class == kRegionInRecoveryException || java_class == kRegionOpeningException ||
+ java_class == kRegionMovedException || java_class == kRegionServerStoppedException ||
+ java_class == kRegionServerAbortedException) {
+ scanner_closed = true;
+ }
+ });
+ return scanner_closed;
+ }
+
+ static bool IsScannerOutOfOrder(const folly::exception_wrapper& exception) {
+ bool scanner_out_of_order = false;
+ exception.with_exception([&](const RemoteException& remote_ex) {
+ auto java_class = remote_ex.exception_class_name();
+ if (java_class == kOutOfOrderScannerNextException || java_class == kScannerResetException) {
+ scanner_out_of_order = true;
+ }
+ });
+ return scanner_out_of_order;
+ }
+};
} // namespace hbase
diff --git a/test-util/BUCK b/test-util/BUCK
index 4998614..7c92841 100644
--- a/test-util/BUCK
+++ b/test-util/BUCK
@@ -16,33 +16,37 @@
# limitations under the License.
import os
-cxx_library(name="test-util",
- exported_headers=[
- "test-util.h","mini-cluster.h"
- ],
- srcs=["test-util.cc","mini-cluster.cc"],
- deps=[
- "//third-party:folly",
- "//core:core",
+cxx_library(
+ name="test-util",
+ exported_headers=["test-util.h", "mini-cluster.h"],
+ srcs=["test-util.cc", "mini-cluster.cc"],
+ deps=[
+ "//third-party:folly",
+ "//core:core",
],
- preprocessor_flags= [
- '-I' + os.environ['JAVA_HOME'] + '/include',
- '-I' + os.environ['JAVA_HOME'] + '/include/darwin',
- '-I' + os.environ['JAVA_HOME'] + '/include/linux'],
- exported_preprocessor_flags= [
- '-I' + os.environ['JAVA_HOME'] + '/include',
- '-I' + os.environ['JAVA_HOME'] + '/include/darwin',
- '-I' + os.environ['JAVA_HOME'] + '/include/linux'],
- compiler_flags = [
- '-I' + os.environ['JAVA_HOME'] + '/include',
- '-I' + os.environ['JAVA_HOME'] + '/include/darwin',
- '-I' + os.environ['JAVA_HOME'] + '/include/linux', '-ggdb'],
- linker_flags = ['-ljvm',
- '-L' + os.environ['JAVA_HOME'] + '/jre/lib/amd64/server', '-ggdb'],
- exported_linker_flags = ['-ljvm',
- '-L' + os.environ['JAVA_HOME'] + '/jre/lib/amd64/server',
- '-Wl,-rpath=' + os.environ['JAVA_HOME'] + '/jre/lib/amd64/server'],
- visibility=[
- 'PUBLIC',
- ],
- )
+ preprocessor_flags=[
+ '-I' + os.environ['JAVA_HOME'] + '/include',
+ '-I' + os.environ['JAVA_HOME'] + '/include/darwin',
+ '-I' + os.environ['JAVA_HOME'] + '/include/linux'
+ ],
+ exported_preprocessor_flags=[
+ '-I' + os.environ['JAVA_HOME'] + '/include',
+ '-I' + os.environ['JAVA_HOME'] + '/include/darwin',
+ '-I' + os.environ['JAVA_HOME'] + '/include/linux'
+ ],
+ compiler_flags=[
+ '-I' + os.environ['JAVA_HOME'] + '/include',
+ '-I' + os.environ['JAVA_HOME'] + '/include/darwin',
+ '-I' + os.environ['JAVA_HOME'] + '/include/linux', '-ggdb'
+ ],
+ linker_flags=[
+ '-ljvm', '-L' + os.environ['JAVA_HOME'] + '/jre/lib/amd64/server',
+ '-ggdb'
+ ],
+ exported_linker_flags=[
+ '-ljvm', '-L' + os.environ['JAVA_HOME'] + '/jre/lib/amd64/server',
+ '-Wl,-rpath=' + os.environ['JAVA_HOME'] + '/jre/lib/amd64/server'
+ ],
+ visibility=[
+ 'PUBLIC',
+ ],)
diff --git a/test-util/mini-cluster.cc b/test-util/mini-cluster.cc
index 34da54c..688ea8e 100644
--- a/test-util/mini-cluster.cc
+++ b/test-util/mini-cluster.cc
@@ -119,6 +119,9 @@ void MiniCluster::Setup() {
env_->GetMethodID(testing_util_class_, "createTable",
"(Lorg/apache/hadoop/hbase/TableName;Ljava/lang/String;)Lorg/"
"apache/hadoop/hbase/client/Table;");
+ create_table_families_mid_ = env_->GetMethodID(testing_util_class_, "createTable",
+ "(Lorg/apache/hadoop/hbase/TableName;[[B)Lorg/"
+ "apache/hadoop/hbase/client/Table;");
create_table_with_split_mid_ = env_->GetMethodID(
testing_util_class_, "createTable",
"(Lorg/apache/hadoop/hbase/TableName;[[B[[B)Lorg/apache/hadoop/hbase/client/Table;");
@@ -198,20 +201,45 @@ jobject MiniCluster::CreateTable(const std::string &table, const std::string &fa
return table_obj;
}
+jobject MiniCluster::CreateTable(const std::string &table,
+ const std::vector<std::string> &families) {
+ jstring table_name_str = env_->NewStringUTF(table.c_str());
+ jobject table_name =
+ env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_, table_name_str);
+ jclass array_element_type = env_->FindClass("[B");
+ jobjectArray family_array = env_->NewObjectArray(families.size(), array_element_type, nullptr);
+ int i = 0;
+ for (auto family : families) {
+ env_->SetObjectArrayElement(family_array, i++, StrToByteChar(family));
+ }
+ jobject table_obj =
+ env_->CallObjectMethod(htu_, create_table_families_mid_, table_name, family_array);
+ return table_obj;
+}
+
jobject MiniCluster::CreateTable(const std::string &table, const std::string &family,
const std::vector<std::string> &keys) {
+ std::vector<std::string> families{};
+ families.push_back(std::string{family});
+ return CreateTable(table, families, keys);
+}
+
+jobject MiniCluster::CreateTable(const std::string &table, const std::vector<std::string> &families,
+ const std::vector<std::string> &keys) {
jstring table_name_str = env_->NewStringUTF(table.c_str());
jobject table_name =
env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_, table_name_str);
jclass array_element_type = env_->FindClass("[B");
- jobjectArray family_array = env_->NewObjectArray(1, array_element_type, env_->NewByteArray(1));
- env_->SetObjectArrayElement(family_array, 0, StrToByteChar(family));
+ int i = 0;
+ jobjectArray family_array = env_->NewObjectArray(families.size(), array_element_type, nullptr);
+ for (auto family : families) {
+ env_->SetObjectArrayElement(family_array, i++, StrToByteChar(family));
+ }
- jobjectArray key_array =
- env_->NewObjectArray(keys.size(), array_element_type, env_->NewByteArray(1));
+ jobjectArray key_array = env_->NewObjectArray(keys.size(), array_element_type, nullptr);
- int i = 0;
+ i = 0;
for (auto key : keys) {
env_->SetObjectArrayElement(key_array, i++, StrToByteChar(key));
}
diff --git a/test-util/mini-cluster.h b/test-util/mini-cluster.h
index 4119cb5..b8ac391 100644
--- a/test-util/mini-cluster.h
+++ b/test-util/mini-cluster.h
@@ -18,9 +18,9 @@
*/
#pragma once
+#include <jni.h>
#include <string>
#include <vector>
-#include "jni.h"
namespace hbase {
@@ -29,8 +29,11 @@ class MiniCluster {
jobject StartCluster(int32_t num_region_servers);
void StopCluster();
jobject CreateTable(const std::string &table, const std::string &family);
+ jobject CreateTable(const std::string &table, const std::vector<std::string> &families);
jobject CreateTable(const std::string &table, const std::string &family,
const std::vector<std::string> &keys);
+ jobject CreateTable(const std::string &table, const std::vector<std::string> &families,
+ const std::vector<std::string> &keys);
jobject StopRegionServer(int idx);
// moves region to server
@@ -51,6 +54,7 @@ class MiniCluster {
jmethodID set_conf_mid_;
jmethodID tbl_name_value_of_mid_;
jmethodID create_table_mid_;
+ jmethodID create_table_families_mid_;
jmethodID create_table_with_split_mid_;
jmethodID put_mid_;
jmethodID put_ctor_;
diff --git a/test-util/test-util.cc b/test-util/test-util.cc
index c4e6ed2..26862d8 100644
--- a/test-util/test-util.cc
+++ b/test-util/test-util.cc
@@ -65,11 +65,20 @@ void TestUtil::CreateTable(const std::string &table, const std::string &family)
mini_->CreateTable(table, family);
}
+void TestUtil::CreateTable(const std::string &table, const std::vector<std::string> &families) {
+ mini_->CreateTable(table, families);
+}
+
void TestUtil::CreateTable(const std::string &table, const std::string &family,
const std::vector<std::string> &keys) {
mini_->CreateTable(table, family, keys);
}
+void TestUtil::CreateTable(const std::string &table, const std::vector<std::string> &families,
+ const std::vector<std::string> &keys) {
+ mini_->CreateTable(table, families, keys);
+}
+
void TestUtil::StartStandAloneInstance() {
auto p = temp_dir_.path().string();
auto cmd = std::string{"bin/start-local-hbase.sh " + p};
diff --git a/test-util/test-util.h b/test-util/test-util.h
index 5729674..e26558b 100644
--- a/test-util/test-util.h
+++ b/test-util/test-util.h
@@ -59,8 +59,11 @@ class TestUtil {
void StopMiniCluster();
void CreateTable(const std::string &table, const std::string &family);
+ void CreateTable(const std::string &table, const std::vector<std::string> &families);
void CreateTable(const std::string &table, const std::string &family,
const std::vector<std::string> &keys);
+ void CreateTable(const std::string &table, const std::vector<std::string> &families,
+ const std::vector<std::string> &keys);
void StartStandAloneInstance();
void StopStandAloneInstance();
diff --git a/utils/BUCK b/utils/BUCK
index ed8e114..788056b 100644
--- a/utils/BUCK
+++ b/utils/BUCK
@@ -20,6 +20,7 @@ cxx_library(
exported_headers=[
"bytes-util.h",
"connection-util.h",
+ "optional.h",
"sys-util.h",
"time-util.h",
"user-util.h",
diff --git a/utils/bytes-util-test.cc b/utils/bytes-util-test.cc
index ca64a21..16af021 100644
--- a/utils/bytes-util-test.cc
+++ b/utils/bytes-util-test.cc
@@ -47,6 +47,7 @@ TEST(TestBytesUtil, TestToStringBinary) {
EXPECT_EQ("foo_\\x00\\xFF_bar",
BytesUtil::ToStringBinary("foo_" + std::string{zero} + std::string{max} + "_bar"));
}
+
TEST(TestBytesUtil, TestToStringToInt64) {
int64_t num = 761235;
EXPECT_EQ(num, BytesUtil::ToInt64(BytesUtil::ToString(num)));
@@ -57,3 +58,13 @@ TEST(TestBytesUtil, TestToStringToInt64) {
num = 0;
EXPECT_EQ(num, BytesUtil::ToInt64(BytesUtil::ToString(num)));
}
+
+TEST(TestBytesUtil, TestCreateClosestRowAfter) {
+ std::string empty{""};
+ EXPECT_EQ(BytesUtil::CreateClosestRowAfter(empty), std::string{'\0'});
+
+ std::string foo{"foo"};
+ EXPECT_EQ(BytesUtil::CreateClosestRowAfter(foo), std::string{"foo"} + '\0');
+
+ EXPECT_EQ("f\\x00", BytesUtil::ToStringBinary(BytesUtil::CreateClosestRowAfter("f")));
+}
diff --git a/utils/bytes-util.h b/utils/bytes-util.h
index 3566d62..6221bf0 100644
--- a/utils/bytes-util.h
+++ b/utils/bytes-util.h
@@ -42,7 +42,27 @@ class BytesUtil {
*/
static std::string ToStringBinary(const std::string& b, size_t off, size_t len);
- static std::string ToString(int64_t amt);
- static long ToInt64(std::string str);
+ static std::string ToString(int64_t value);
+
+ static int64_t ToInt64(std::string str);
+
+ static bool IsEmptyStartRow(const std::string& row) { return row == ""; }
+
+ static bool IsEmptyStopRow(const std::string& row) { return row == ""; }
+
+ static int32_t CompareTo(const std::string& a, const std::string& b) {
+ if (a < b) {
+ return -1;
+ }
+ if (a == b) {
+ return 0;
+ }
+ return 1;
+ }
+
+ /**
+ * Create the closest row after the specified row
+ */
+ static std::string CreateClosestRowAfter(std::string row) { return row.append(1, '\0'); }
};
} /* namespace hbase */
diff --git a/core/query.h b/utils/optional.h
similarity index 70%
copy from core/query.h
copy to utils/optional.h
index b706303..a05eab5 100644
--- a/core/query.h
+++ b/utils/optional.h
@@ -19,23 +19,14 @@
#pragma once
-#include "core/filter.h"
+#include <experimental/optional>
namespace hbase {
/**
- * Base class for read RPC calls (Get / Scan).
+ * An optional value that may or may not be present.
*/
-class Query {
- public:
- virtual ~Query() {}
+template <class T>
+using optional = std::experimental::optional<T>;
- void SetFilter(std::unique_ptr<Filter> filter) { filter_ = std::move(filter); }
-
- const std::unique_ptr<Filter>& filter() const { return filter_; }
-
- protected:
- std::unique_ptr<Filter> filter_ = nullptr;
-};
-
-} // namespace hbase
+} /* namespace hbase */
[hbase] 105/137: HBASE-18061 [C++] Fix retry logic in multi-get
calls (Sudeep Sunthankar)
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 0b42c7a37a2a554879822c47e21ffaee2b418f83
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Wed Jul 19 11:50:40 2017 -0700
HBASE-18061 [C++] Fix retry logic in multi-get calls (Sudeep Sunthankar)
---
core/BUCK | 11 +
core/async-batch-rpc-retrying-caller.cc | 241 ++++++++---------
core/async-batch-rpc-retrying-caller.h | 28 +-
core/async-batch-rpc-retrying-test.cc | 463 ++++++++++++++++++++++++++++++++
core/client-test.cc | 91 +++++--
core/multi-response.cc | 34 +--
core/multi-response.h | 14 +-
core/region-result.cc | 2 +-
core/region-result.h | 5 +-
core/response-converter.cc | 50 +++-
core/response-converter.h | 8 +-
core/server-request.h | 10 +-
12 files changed, 753 insertions(+), 204 deletions(-)
diff --git a/core/BUCK b/core/BUCK
index 464c010..f9db0bd 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -323,6 +323,17 @@ cxx_test(
":core",
],
run_test_separately=True,)
+cxx_test(
+ name="multi-retry-test",
+ srcs=[
+ "async-batch-rpc-retrying-test.cc",
+ ],
+ deps=[
+ ":core",
+ "//test-util:test-util",
+ "//exceptions:exceptions",
+ ],
+ run_test_separately=True,)
cxx_binary(
name="simple-client",
srcs=[
diff --git a/core/async-batch-rpc-retrying-caller.cc b/core/async-batch-rpc-retrying-caller.cc
index 05290f5..0d67b17 100644
--- a/core/async-batch-rpc-retrying-caller.cc
+++ b/core/async-batch-rpc-retrying-caller.cc
@@ -77,20 +77,20 @@ int64_t AsyncBatchRpcRetryingCaller::RemainingTimeNs() {
void AsyncBatchRpcRetryingCaller::LogException(int32_t tries,
std::shared_ptr<RegionRequest> region_request,
- std::shared_ptr<std::exception> &error,
+ const folly::exception_wrapper &ew,
std::shared_ptr<ServerName> server_name) {
if (tries > start_log_errors_count_) {
std::string regions;
regions += region_request->region_location()->region_name() + ", ";
LOG(WARNING) << "Process batch for " << regions << " in " << table_name_->namespace_() << ":"
<< table_name_->qualifier() << " from " << server_name->host_name()
- << " failed, tries=" << tries << ":- " << error->what();
+ << " failed, tries=" << tries << ":- " << ew.what().toStdString();
}
}
void AsyncBatchRpcRetryingCaller::LogException(
- int32_t tries, std::vector<std::shared_ptr<RegionRequest>> ®ion_requests,
- std::shared_ptr<std::exception> &error, std::shared_ptr<ServerName> server_name) {
+ int32_t tries, const std::vector<std::shared_ptr<RegionRequest>> ®ion_requests,
+ const folly::exception_wrapper &ew, std::shared_ptr<ServerName> server_name) {
if (tries > start_log_errors_count_) {
std::string regions;
for (const auto region_request : region_requests) {
@@ -98,7 +98,7 @@ void AsyncBatchRpcRetryingCaller::LogException(
}
LOG(WARNING) << "Process batch for " << regions << " in " << table_name_->namespace_() << ":"
<< table_name_->qualifier() << " from " << server_name->host_name()
- << " failed, tries=" << tries << error->what();
+ << " failed, tries=" << tries << ew.what().toStdString();
}
}
@@ -107,27 +107,24 @@ const std::string AsyncBatchRpcRetryingCaller::GetExtraContextForError(
return server_name ? server_name->ShortDebugString() : "";
}
-// TODO HBASE-17800 pass folly ew instead of std::exception
void AsyncBatchRpcRetryingCaller::AddError(const std::shared_ptr<Action> &action,
- std::shared_ptr<std::exception> error,
+ const folly::exception_wrapper &ew,
std::shared_ptr<ServerName> server_name) {
- folly::exception_wrapper ew;
ThrowableWithExtraContext twec(ew, TimeUtil::GetNowNanos(), GetExtraContextForError(server_name));
AddAction2Error(action->original_index(), twec);
}
void AsyncBatchRpcRetryingCaller::AddError(const std::vector<std::shared_ptr<Action>> &actions,
- std::shared_ptr<std::exception> error,
+ const folly::exception_wrapper &ew,
std::shared_ptr<ServerName> server_name) {
for (const auto action : actions) {
- AddError(action, error, server_name);
+ AddError(action, ew, server_name);
}
}
-// TODO HBASE-17800 pass folly ew instead of std::exception
void AsyncBatchRpcRetryingCaller::FailOne(const std::shared_ptr<Action> &action, int32_t tries,
- std::shared_ptr<std::exception> error,
- int64_t current_time, const std::string extras) {
+ const folly::exception_wrapper &ew, int64_t current_time,
+ const std::string extras) {
auto action_index = action->original_index();
auto itr = action2promises_.find(action_index);
if (itr != action2promises_.end()) {
@@ -135,7 +132,6 @@ void AsyncBatchRpcRetryingCaller::FailOne(const std::shared_ptr<Action> &action,
return;
}
}
- folly::exception_wrapper ew;
ThrowableWithExtraContext twec(ew, current_time, extras);
AddAction2Error(action_index, twec);
action2promises_[action_index].setException(
@@ -143,10 +139,10 @@ void AsyncBatchRpcRetryingCaller::FailOne(const std::shared_ptr<Action> &action,
}
void AsyncBatchRpcRetryingCaller::FailAll(const std::vector<std::shared_ptr<Action>> &actions,
- int32_t tries, std::shared_ptr<std::exception> error,
+ int32_t tries, const folly::exception_wrapper &ew,
std::shared_ptr<ServerName> server_name) {
for (const auto action : actions) {
- FailOne(action, tries, error, TimeUtil::GetNowNanos(), GetExtraContextForError(server_name));
+ FailOne(action, tries, ew, TimeUtil::GetNowNanos(), GetExtraContextForError(server_name));
}
}
@@ -159,7 +155,7 @@ void AsyncBatchRpcRetryingCaller::FailAll(const std::vector<std::shared_ptr<Acti
return;
}
action2promises_[action_index].setException(
- RetriesExhaustedException(tries - 1, action2errors_[action_index]));
+ RetriesExhaustedException(tries, action2errors_[action_index]));
}
}
@@ -176,34 +172,32 @@ void AsyncBatchRpcRetryingCaller::AddAction2Error(uint64_t action_index,
}
void AsyncBatchRpcRetryingCaller::OnError(const ActionsByRegion &actions_by_region, int32_t tries,
- std::shared_ptr<std::exception> exc,
+ const folly::exception_wrapper &ew,
std::shared_ptr<ServerName> server_name) {
std::vector<std::shared_ptr<Action>> copied_actions;
std::vector<std::shared_ptr<RegionRequest>> region_requests;
for (const auto &action_by_region : actions_by_region) {
region_requests.push_back(action_by_region.second);
- // Concurrent
for (const auto &action : action_by_region.second->actions()) {
copied_actions.push_back(action);
}
}
- // TODO HBASE-17800 for exc check with DoNotRetryIOException
- LogException(tries, region_requests, exc, server_name);
- if (tries >= max_attempts_) {
- FailAll(copied_actions, tries, exc, server_name);
+
+ LogException(tries, region_requests, ew, server_name);
+ if ((tries >= max_attempts_) || !ExceptionUtil::ShouldRetry(ew)) {
+ FailAll(copied_actions, tries, ew, server_name);
return;
}
- AddError(copied_actions, exc, server_name);
+ AddError(copied_actions, ew, server_name);
TryResubmit(copied_actions, tries);
}
-void AsyncBatchRpcRetryingCaller::TryResubmit(std::vector<std::shared_ptr<Action>> actions,
+void AsyncBatchRpcRetryingCaller::TryResubmit(const std::vector<std::shared_ptr<Action>> &actions,
int32_t tries) {
int64_t delay_ns;
if (operation_timeout_ns_.count() > 0) {
int64_t max_delay_ns = RemainingTimeNs() - ConnectionUtils::kSleepDeltaNs;
if (max_delay_ns <= 0) {
- VLOG(8) << "Fail All from onError";
FailAll(actions, tries);
return;
}
@@ -211,9 +205,12 @@ void AsyncBatchRpcRetryingCaller::TryResubmit(std::vector<std::shared_ptr<Action
} else {
delay_ns = ConnectionUtils::GetPauseTime(pause_ns_.count(), tries - 1);
}
- // TODO This gives segfault @ present, when retried
- // retry_timer_->scheduleTimeoutFn([&]() { GroupAndSend(actions, tries + 1); },
- // milliseconds(TimeUtil::ToMillis(delay_ns)));
+
+ conn_->retry_executor()->add([=]() {
+ retry_timer_->scheduleTimeoutFn(
+ [=]() { conn_->cpu_executor()->add([=]() { GroupAndSend(actions, tries + 1); }); },
+ milliseconds(TimeUtil::ToMillis(delay_ns)));
+ });
}
Future<std::vector<Try<std::shared_ptr<RegionLocation>>>>
@@ -234,7 +231,7 @@ void AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr
if (operation_timeout_ns_.count() > 0) {
locate_timeout_ns = RemainingTimeNs();
if (locate_timeout_ns <= 0) {
- FailAll(actions_, tries);
+ FailAll(actions, tries);
return;
}
} else {
@@ -242,8 +239,8 @@ void AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr
}
GetRegionLocations(actions, locate_timeout_ns)
- .then([&](std::vector<Try<std::shared_ptr<RegionLocation>>> &loc) {
- std::lock_guard<std::mutex> lock(multi_mutex_);
+ .then([=](std::vector<Try<std::shared_ptr<RegionLocation>>> &loc) {
+ std::lock_guard<std::recursive_mutex> lck(multi_mutex_);
ActionsByServer actions_by_server;
std::vector<std::shared_ptr<Action>> locate_failed;
@@ -252,50 +249,36 @@ void AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr
if (loc[i].hasValue()) {
auto region_loc = loc[i].value();
// Add it to actions_by_server;
- // Concurrent
auto search =
actions_by_server.find(std::make_shared<ServerName>(region_loc->server_name()));
if (search != actions_by_server.end()) {
search->second->AddActionsByRegion(region_loc, action);
} else {
- // Create new key
auto server_request = std::make_shared<ServerRequest>(region_loc);
server_request->AddActionsByRegion(region_loc, action);
auto server_name = std::make_shared<ServerName>(region_loc->server_name());
actions_by_server[server_name] = server_request;
}
- locate_failed.push_back(action);
- VLOG(8) << "row [" << action->action()->row() << "] of table["
- << table_name_->namespace_() << ":" << table_name_->qualifier()
- << " found in region [" << region_loc->region_name() << "]; host["
- << region_loc->server_name().host_name() << "]; port["
+ VLOG(5) << "rowkey [" << action->action()->row() << "] of table["
+ << table_name_->ShortDebugString() << "] found in ["
+ << region_loc->region_name() << "]; RS["
+ << region_loc->server_name().host_name() << ":"
<< region_loc->server_name().port() << "];";
} else if (loc[i].hasException()) {
- VLOG(8) << "Exception occured while locating region:- "
- << loc[i].exception().getCopied()->what() << " for action index " << i;
- // TODO Feedback needed, Java API only identifies DoNotRetryIOException
- // We might receive runtime error from location-cache.cc too, we are treating both same
- if (loc[i].exception().is_compatible_with<std::runtime_error>()) {
- std::string extra = "";
- FailOne(action, tries, nullptr, TimeUtil::GetNowNanos(),
- loc[i].exception().what().toStdString());
- return;
+ folly::exception_wrapper ew = loc[i].exception();
+ VLOG(1) << "GetRegionLocations() exception: " << ew.what().toStdString()
+ << "for index:" << i << "; tries: " << tries
+ << "; max_attempts_: " << max_attempts_;
+ // We might receive runtime error from location-cache.cc too, we are doing FailOne and
+ // continue next one
+ if (tries >= max_attempts_ || !ExceptionUtil::ShouldRetry(ew)) {
+ FailOne(action, tries, ew, TimeUtil::GetNowNanos(), ew.what().toStdString());
+ } else {
+ AddError(action, loc[i].exception(), nullptr);
+ locate_failed.push_back(action);
}
- // TODO HBASE-17800 for exc check with DoNotRetryIOException
- /*
- else if (loc[i].exception().is_compatible_with<hbase::DoNotRetryIOException>()) {
- int64_t current_time = 0;
- std::string extra = "";
- FailOne(action, tries, nullptr, TimeUtil::GetNowNanos(),
- loc[i].exception().what().toStdString());
- return;
- }*/
- AddError(action, std::make_shared<std::exception>(*loc[i].exception().getCopied()),
- nullptr);
- locate_failed.push_back(action);
}
}
-
if (!actions_by_server.empty()) {
Send(actions_by_server, tries);
}
@@ -304,17 +287,21 @@ void AsyncBatchRpcRetryingCaller::GroupAndSend(const std::vector<std::shared_ptr
TryResubmit(locate_failed, tries);
}
})
- .onError([&](const folly::exception_wrapper &ew) {
- std::lock_guard<std::mutex> lock(multi_mutex_);
- auto exc = ew.getCopied();
- VLOG(8) << "GetRegionLocations() exception: " << ew.what().toStdString();
+ .onError([=](const folly::exception_wrapper &ew) {
+ VLOG(1) << "GetRegionLocations() exception: " << ew.what().toStdString()
+ << "tries: " << tries << "; max_attempts_: " << max_attempts_;
+ std::lock_guard<std::recursive_mutex> lck(multi_mutex_);
+ if (tries >= max_attempts_ || !ExceptionUtil::ShouldRetry(ew)) {
+ FailAll(actions, tries, ew, nullptr);
+ } else {
+ TryResubmit(actions, tries);
+ }
});
return;
}
Future<std::vector<Try<std::unique_ptr<Response>>>> AsyncBatchRpcRetryingCaller::GetMultiResponse(
const ActionsByServer &actions_by_server) {
- // Concurrent.
auto multi_calls = std::vector<Future<std::unique_ptr<hbase::Response>>>{};
auto user = User::defaultUser();
for (const auto &action_by_server : actions_by_server) {
@@ -328,16 +315,14 @@ Future<std::vector<Try<std::unique_ptr<Response>>>> AsyncBatchRpcRetryingCaller:
return collectAll(multi_calls);
}
-void AsyncBatchRpcRetryingCaller::Send(ActionsByServer &actions_by_server, int32_t tries) {
+void AsyncBatchRpcRetryingCaller::Send(const ActionsByServer &actions_by_server, int32_t tries) {
int64_t remaining_ns;
if (operation_timeout_ns_.count() > 0) {
remaining_ns = RemainingTimeNs();
if (remaining_ns <= 0) {
std::vector<std::shared_ptr<Action>> failed_actions;
for (const auto &action_by_server : actions_by_server) {
- // Concurrent
for (auto &value : action_by_server.second->actions_by_region()) {
- // Concurrent
for (const auto &failed_action : value.second->actions()) {
failed_actions.push_back(failed_action);
}
@@ -357,30 +342,30 @@ void AsyncBatchRpcRetryingCaller::Send(ActionsByServer &actions_by_server, int32
GetMultiResponse(actions_by_server)
.then([=](const std::vector<Try<std::unique_ptr<hbase::Response>>> &completed_responses) {
- std::lock_guard<std::mutex> lock(multi_mutex_);
- for (uint64_t num = 0; num < completed_responses.size(); ++num) {
+ std::lock_guard<std::recursive_mutex> lck(multi_mutex_);
+ uint64_t num = 0;
+ for (const auto &action_by_server : actions_by_server) {
if (completed_responses[num].hasValue()) {
auto multi_response =
- ResponseConverter::GetResults(multi_reqv[num], *completed_responses[num].value());
- for (const auto &action_by_server : actions_by_server) {
- OnComplete(action_by_server.second->actions_by_region(), tries,
- action_by_server.first, std::move(multi_response));
- }
+ ResponseConverter::GetResults(multi_reqv[num], *completed_responses[num].value(),
+ action_by_server.second->actions_by_region());
+ OnComplete(action_by_server.second->actions_by_region(), tries, action_by_server.first,
+ std::move(multi_response));
} else if (completed_responses[num].hasException()) {
- VLOG(8) << "Received exception: "
- << completed_responses[num].exception().getCopied()->what()
- << " from server for action index " << num;
- // TODO: we should call OnError here as well.
+ folly::exception_wrapper ew = completed_responses[num].exception();
+ VLOG(1) << "GetMultiResponse() exception: " << ew.what().toStdString()
+ << " from server for action index:" << num;
+ OnError(action_by_server.second->actions_by_region(), tries, ew,
+ action_by_server.first);
}
+ num++;
}
})
.onError([=](const folly::exception_wrapper &ew) {
- auto exc = ew.getCopied();
- VLOG(8) << "GetMultiResponse() exception: " << ew.what().toStdString();
- std::lock_guard<std::mutex> lock(multi_mutex_);
+ VLOG(1) << "GetMultiResponse() exception: " << ew.what().toStdString();
+ std::lock_guard<std::recursive_mutex> lck(multi_mutex_);
for (const auto &action_by_server : actions_by_server) {
- OnError(action_by_server.second->actions_by_region(), tries,
- std::make_shared<std::exception>(*exc), action_by_server.first);
+ OnError(action_by_server.second->actions_by_region(), tries, ew, action_by_server.first);
}
});
return;
@@ -391,46 +376,35 @@ void AsyncBatchRpcRetryingCaller::OnComplete(
const std::shared_ptr<ServerName> server_name,
const std::unique_ptr<hbase::MultiResponse> multi_response) {
std::vector<std::shared_ptr<Action>> failed_actions;
+ const auto region_results = multi_response->RegionResults();
for (const auto &action_by_region : actions_by_region) {
- auto region_result_itr = multi_response->RegionResults().find(action_by_region.first);
- if (region_result_itr == multi_response->RegionResults().end()) {
- VLOG(8) << "Region " << action_by_region.first << " not found in MultiResults.";
- // TODO Feedback needed Should we throw from here or continue for next action_by_region ?
- // Throwing at present as this looks like an inconsistency
- // Concurrent
- auto exc = std::make_shared<std::runtime_error>("Invalid search for region " +
- action_by_region.first + " in multi results");
- FailAll(action_by_region.second->actions(), tries, exc, server_name);
- return;
- // std::runtime_error(
- // "Invalid search for region " + action_by_region.first + " in multi results");
- }
- if (region_result_itr != multi_response->RegionResults().end()) {
- // Concurrent
+ auto region_result_itr = region_results.find(action_by_region.first);
+ if (region_result_itr != region_results.end()) {
for (const auto &action : action_by_region.second->actions()) {
OnComplete(action, action_by_region.second, tries, server_name, region_result_itr->second,
failed_actions);
}
- } else {
+ } else if (region_result_itr == region_results.end()) {
auto region_exc = multi_response->RegionException(action_by_region.first);
- std::shared_ptr<std::exception> pexc;
if (region_exc == nullptr) {
- VLOG(8) << "Server sent us neither results nor exceptions for " << action_by_region.first;
- pexc = std::make_shared<std::exception>(std::runtime_error("Invalid response"));
- // TODO: raise this exception to the application
+ // FailAll actions for this particular region as inconsistent server response. So we raise
+ // this exception to the application
+ std::string err_msg = "Invalid response: Server " + server_name->ShortDebugString() +
+ " sent us neither results nor exceptions for " +
+ action_by_region.first;
+ VLOG(1) << err_msg;
+ auto ew = folly::make_exception_wrapper<std::runtime_error>(err_msg);
+ FailAll(action_by_region.second->actions(), tries, ew, server_name);
} else {
- // TODO HBASE-17800 for exc check with DoNotRetryIOException
- LogException(tries, action_by_region.second, region_exc, server_name);
- location_cache_->UpdateCachedLocation(*action_by_region.second->region_location(),
- *region_exc);
- std::string row_name;
- if (tries >= max_attempts_) {
- // Concurrent
- FailAll(action_by_region.second->actions(), tries, region_exc, server_name);
+ // Eg: org.apache.hadoop.hbase.NotServingRegionException:
+ LogException(tries, action_by_region.second, *region_exc, server_name);
+ if (tries >= max_attempts_ || !ExceptionUtil::ShouldRetry(*region_exc)) {
+ FailAll(action_by_region.second->actions(), tries, *region_exc, server_name);
return;
}
- // Concurrent
- AddError(action_by_region.second->actions(), region_exc, server_name);
+ location_cache_->UpdateCachedLocation(*action_by_region.second->region_location(),
+ *region_exc);
+ AddError(action_by_region.second->actions(), *region_exc, server_name);
for (const auto &action : action_by_region.second->actions()) {
failed_actions.push_back(action);
}
@@ -440,6 +414,7 @@ void AsyncBatchRpcRetryingCaller::OnComplete(
if (!failed_actions.empty()) {
TryResubmit(failed_actions, tries);
}
+
return;
}
@@ -454,34 +429,36 @@ void AsyncBatchRpcRetryingCaller::OnComplete(const std::shared_ptr<Action> &acti
auto result_or_exc = region_result->ResultOrException(action->original_index());
auto result = std::get<0>(*result_or_exc);
auto exc = std::get<1>(*result_or_exc);
- std::shared_ptr<std::exception> pexc;
if (exc != nullptr) {
- LogException(tries, region_request, exc, server_name);
- if (tries >= max_attempts_) {
- FailOne(action, tries, exc, TimeUtil::GetNowNanos(), GetExtraContextForError(server_name));
+ LogException(tries, region_request, *exc, server_name);
+ if (tries >= max_attempts_ || !ExceptionUtil::ShouldRetry(*exc)) {
+ FailOne(action, tries, *exc, TimeUtil::GetNowNanos(), GetExtraContextForError(server_name));
} else {
failed_actions.push_back(action);
}
} else if (result != nullptr) {
action2promises_[action->original_index()].setValue(std::move(result));
} else {
- VLOG(8) << "Server " << server_name->ShortDebugString()
- << " sent us neither results nor exceptions for request @ index "
- << action->original_index() << ", row " << action->action()->row() << " of "
- << region_request->region_location()->region_name();
- err_msg = "Invalid response";
- AddError(action, std::make_shared<std::runtime_error>(err_msg), server_name);
+ std::string err_msg = "Invalid response: Server " + server_name->ShortDebugString() +
+ " sent us neither results nor exceptions for request @ index " +
+ std::to_string(action->original_index()) + ", row " +
+ action->action()->row() + " of " +
+ region_request->region_location()->region_name();
+ VLOG(1) << err_msg;
+ auto ew = folly::make_exception_wrapper<std::runtime_error>(err_msg);
+ AddError(action, ew, server_name);
failed_actions.push_back(action);
}
} catch (const std::out_of_range &oor) {
- // TODO Feedback needed. Should we retry for he specific index again ?
- // This should never occur, so we are throwing a std::runtime_error from here
- VLOG(8) << "No ResultOrException found @ index " << action->original_index() << ", row "
- << action->action()->row() << " of "
- << region_request->region_location()->region_name();
- throw std::runtime_error("ResultOrException not present @ index " +
- std::to_string(action->original_index()));
+ // This should never occur. Error in logic. Throwing std::runtime_error from here. Will be
+ // retried or failed
+ std::string err_msg = "ResultOrException not present @ index " +
+ std::to_string(action->original_index()) + ", row " +
+ action->action()->row() + " of " +
+ region_request->region_location()->region_name();
+ throw std::runtime_error(err_msg);
}
return;
}
+
} /* namespace hbase */
diff --git a/core/async-batch-rpc-retrying-caller.h b/core/async-batch-rpc-retrying-caller.h
index 29a0e6a..4c04b91 100644
--- a/core/async-batch-rpc-retrying-caller.h
+++ b/core/async-batch-rpc-retrying-caller.h
@@ -19,6 +19,7 @@
#pragma once
+#include <folly/ExceptionWrapper.h>
#include <folly/Format.h>
#include <folly/futures/Future.h>
#include <folly/futures/Promise.h>
@@ -107,36 +108,36 @@ class AsyncBatchRpcRetryingCaller {
int64_t RemainingTimeNs();
void LogException(int32_t tries, std::shared_ptr<RegionRequest> region_request,
- std::shared_ptr<std::exception> &error,
+ const folly::exception_wrapper &ew,
std::shared_ptr<pb::ServerName> server_name);
- void LogException(int32_t tries, std::vector<std::shared_ptr<RegionRequest>> ®ion_requests,
- std::shared_ptr<std::exception> &error,
+ void LogException(int32_t tries,
+ const std::vector<std::shared_ptr<RegionRequest>> ®ion_requests,
+ const folly::exception_wrapper &ew,
std::shared_ptr<pb::ServerName> server_name);
const std::string GetExtraContextForError(std::shared_ptr<pb::ServerName> server_name);
- void AddError(const std::shared_ptr<Action> &action, std::shared_ptr<std::exception> error,
+ void AddError(const std::shared_ptr<Action> &action, const folly::exception_wrapper &ew,
std::shared_ptr<pb::ServerName> server_name);
void AddError(const std::vector<std::shared_ptr<Action>> &actions,
- std::shared_ptr<std::exception> error, std::shared_ptr<pb::ServerName> server_name);
+ const folly::exception_wrapper &ew, std::shared_ptr<pb::ServerName> server_name);
void FailOne(const std::shared_ptr<Action> &action, int32_t tries,
- std::shared_ptr<std::exception> error, int64_t current_time,
- const std::string extras);
+ const folly::exception_wrapper &ew, int64_t current_time, const std::string extras);
void FailAll(const std::vector<std::shared_ptr<Action>> &actions, int32_t tries,
- std::shared_ptr<std::exception> error, std::shared_ptr<pb::ServerName> server_name);
+ const folly::exception_wrapper &ew, std::shared_ptr<pb::ServerName> server_name);
void FailAll(const std::vector<std::shared_ptr<Action>> &actions, int32_t tries);
void AddAction2Error(uint64_t action_index, const ThrowableWithExtraContext &twec);
void OnError(const ActionsByRegion &actions_by_region, int32_t tries,
- std::shared_ptr<std::exception> exc, std::shared_ptr<pb::ServerName> server_name);
+ const folly::exception_wrapper &ew, std::shared_ptr<pb::ServerName> server_name);
- void TryResubmit(std::vector<std::shared_ptr<Action>> actions, int32_t tries);
+ void TryResubmit(const std::vector<std::shared_ptr<Action>> &actions, int32_t tries);
folly::Future<std::vector<folly::Try<std::shared_ptr<RegionLocation>>>> GetRegionLocations(
const std::vector<std::shared_ptr<Action>> &actions, int64_t locate_timeout_ns);
@@ -146,7 +147,7 @@ class AsyncBatchRpcRetryingCaller {
folly::Future<std::vector<folly::Try<std::unique_ptr<Response>>>> GetMultiResponse(
const ActionsByServer &actions_by_server);
- void Send(ActionsByServer &actions_by_server, int32_t tries);
+ void Send(const ActionsByServer &actions_by_server, int32_t tries);
void OnComplete(const ActionsByRegion &actions_by_region, int32_t tries,
const std::shared_ptr<pb::ServerName> server_name,
@@ -179,7 +180,6 @@ class AsyncBatchRpcRetryingCaller {
std::shared_ptr<RpcClient> rpc_client_ = nullptr;
std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_pool_ = nullptr;
- std::mutex multi_mutex_;
+ std::recursive_mutex multi_mutex_;
};
-
-} /* namespace hbase */
+} // namespace hbase
diff --git a/core/async-batch-rpc-retrying-test.cc b/core/async-batch-rpc-retrying-test.cc
new file mode 100644
index 0000000..c186276
--- /dev/null
+++ b/core/async-batch-rpc-retrying-test.cc
@@ -0,0 +1,463 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <folly/Logging.h>
+#include <folly/Memory.h>
+#include <folly/futures/Future.h>
+#include <folly/io/async/EventBase.h>
+#include <folly/io/async/ScopedEventBaseThread.h>
+#include <gtest/gtest.h>
+#include <wangle/concurrent/IOThreadPoolExecutor.h>
+
+#include <chrono>
+#include <functional>
+#include <string>
+
+#include "connection/rpc-client.h"
+#include "core/async-batch-rpc-retrying-caller.h"
+#include "core/async-connection.h"
+#include "core/async-rpc-retrying-caller-factory.h"
+#include "core/client.h"
+#include "core/connection-configuration.h"
+#include "core/keyvalue-codec.h"
+#include "core/region-location.h"
+#include "core/result.h"
+#include "exceptions/exception.h"
+#include "test-util/test-util.h"
+#include "utils/time-util.h"
+
+using hbase::AsyncRpcRetryingCallerFactory;
+using hbase::AsyncConnection;
+using hbase::AsyncRegionLocator;
+using hbase::ConnectionConfiguration;
+using hbase::Configuration;
+using hbase::HBaseRpcController;
+using hbase::RegionLocation;
+using hbase::RegionLocateType;
+using hbase::RpcClient;
+using hbase::RequestConverter;
+using hbase::ResponseConverter;
+using hbase::Put;
+using hbase::TimeUtil;
+using hbase::Client;
+using hbase::security::User;
+
+using std::chrono::nanoseconds;
+using std::chrono::milliseconds;
+
+using namespace hbase;
+
+using folly::exception_wrapper;
+
+class AsyncBatchRpcRetryTest : public ::testing::Test {
+ public:
+ static std::unique_ptr<hbase::TestUtil> test_util;
+ static void SetUpTestCase() {
+ google::InstallFailureSignalHandler();
+ test_util = std::make_unique<hbase::TestUtil>();
+ test_util->StartMiniCluster(2);
+ }
+};
+std::unique_ptr<hbase::TestUtil> AsyncBatchRpcRetryTest::test_util = nullptr;
+
+class AsyncRegionLocatorBase : public AsyncRegionLocator {
+ public:
+ AsyncRegionLocatorBase() {}
+ explicit AsyncRegionLocatorBase(std::shared_ptr<RegionLocation> region_location)
+ : region_location_(region_location) {}
+ virtual ~AsyncRegionLocatorBase() = default;
+
+ folly::Future<std::shared_ptr<hbase::RegionLocation>> LocateRegion(const hbase::pb::TableName &,
+ const std::string &row,
+ const RegionLocateType,
+ const int64_t) override {
+ folly::Promise<std::shared_ptr<RegionLocation>> promise;
+ promise.setValue(region_locations_.at(row));
+ return promise.getFuture();
+ }
+
+ virtual void set_region_location(std::shared_ptr<RegionLocation> region_location) {
+ region_location_ = region_location;
+ }
+
+ virtual void set_region_location(
+ const std::map<std::string, std::shared_ptr<RegionLocation>> ®_locs) {
+ for (auto reg_loc : reg_locs) {
+ region_locations_[reg_loc.first] = reg_loc.second;
+ }
+ }
+
+ void UpdateCachedLocation(const RegionLocation &rl, const folly::exception_wrapper &ew) override {
+ }
+
+ protected:
+ std::shared_ptr<RegionLocation> region_location_;
+ std::map<std::string, std::shared_ptr<RegionLocation>> region_locations_;
+ std::map<std::string, uint32_t> mtries_;
+ std::map<std::string, uint32_t> mnum_fails_;
+
+ void InitRetryMaps(uint32_t num_fails) {
+ if (mtries_.size() == 0 && mnum_fails_.size() == 0) {
+ for (auto reg_loc : region_locations_) {
+ mtries_[reg_loc.first] = 0;
+ mnum_fails_[reg_loc.first] = num_fails;
+ }
+ }
+ }
+};
+
+class MockAsyncRegionLocator : public AsyncRegionLocatorBase {
+ public:
+ MockAsyncRegionLocator() : AsyncRegionLocatorBase() {}
+ explicit MockAsyncRegionLocator(std::shared_ptr<RegionLocation> region_location)
+ : AsyncRegionLocatorBase(region_location) {}
+ virtual ~MockAsyncRegionLocator() {}
+};
+
+class MockWrongRegionAsyncRegionLocator : public AsyncRegionLocatorBase {
+ private:
+ uint32_t counter_ = 0;
+ uint32_t num_fails_ = 0;
+ uint32_t tries_ = 0;
+
+ public:
+ explicit MockWrongRegionAsyncRegionLocator(uint32_t num_fails)
+ : AsyncRegionLocatorBase(), num_fails_(num_fails) {}
+ explicit MockWrongRegionAsyncRegionLocator(std::shared_ptr<RegionLocation> region_location)
+ : AsyncRegionLocatorBase(region_location) {}
+ virtual ~MockWrongRegionAsyncRegionLocator() {}
+
+ folly::Future<std::shared_ptr<hbase::RegionLocation>> LocateRegion(
+ const hbase::pb::TableName &tn, const std::string &row,
+ const RegionLocateType locate_type = RegionLocateType::kCurrent,
+ const int64_t locate_ns = 0) override {
+ InitRetryMaps(num_fails_);
+ auto &tries = mtries_[row];
+ auto &num_fails = mnum_fails_[row];
+ if (++tries > num_fails) {
+ return AsyncRegionLocatorBase::LocateRegion(tn, row, locate_type, locate_ns);
+ }
+
+ folly::Promise<std::shared_ptr<RegionLocation>> promise;
+ /* set random region name, simulating invalid region */
+ auto result = std::make_shared<RegionLocation>("whatever-region-name",
+ region_locations_.at(row)->region_info(),
+ region_locations_.at(row)->server_name());
+ promise.setValue(result);
+ return promise.getFuture();
+ }
+};
+
+class MockFailingAsyncRegionLocator : public AsyncRegionLocatorBase {
+ private:
+ uint32_t tries_ = 0;
+ uint32_t num_fails_ = 0;
+ uint32_t counter_ = 0;
+
+ public:
+ explicit MockFailingAsyncRegionLocator(uint32_t num_fails)
+ : AsyncRegionLocatorBase(), num_fails_(num_fails) {}
+ explicit MockFailingAsyncRegionLocator(std::shared_ptr<RegionLocation> region_location)
+ : AsyncRegionLocatorBase(region_location) {}
+ virtual ~MockFailingAsyncRegionLocator() {}
+ folly::Future<std::shared_ptr<hbase::RegionLocation>> LocateRegion(
+ const hbase::pb::TableName &tn, const std::string &row,
+ const RegionLocateType locate_type = RegionLocateType::kCurrent,
+ const int64_t locate_ns = 0) override {
+ InitRetryMaps(num_fails_);
+ auto &tries = mtries_[row];
+ auto &num_fails = mnum_fails_[row];
+ if (++tries > num_fails) {
+ return AsyncRegionLocatorBase::LocateRegion(tn, row, locate_type, locate_ns);
+ }
+
+ folly::Promise<std::shared_ptr<RegionLocation>> promise;
+ promise.setException(std::runtime_error{"Failed to look up region location"});
+ return promise.getFuture();
+ }
+};
+
+class MockAsyncConnection : public AsyncConnection,
+ public std::enable_shared_from_this<MockAsyncConnection> {
+ public:
+ MockAsyncConnection(std::shared_ptr<ConnectionConfiguration> conn_conf,
+ std::shared_ptr<folly::HHWheelTimer> retry_timer,
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
+ std::shared_ptr<wangle::IOThreadPoolExecutor> retry_executor,
+ std::shared_ptr<RpcClient> rpc_client,
+ std::shared_ptr<AsyncRegionLocator> region_locator)
+ : conn_conf_(conn_conf),
+ retry_timer_(retry_timer),
+ cpu_executor_(cpu_executor),
+ io_executor_(io_executor),
+ retry_executor_(retry_executor),
+ rpc_client_(rpc_client),
+ region_locator_(region_locator) {}
+ ~MockAsyncConnection() {}
+ void Init() {
+ caller_factory_ =
+ std::make_shared<AsyncRpcRetryingCallerFactory>(shared_from_this(), retry_timer_);
+ }
+
+ std::shared_ptr<Configuration> conf() override { return nullptr; }
+ std::shared_ptr<ConnectionConfiguration> connection_conf() override { return conn_conf_; }
+ std::shared_ptr<AsyncRpcRetryingCallerFactory> caller_factory() override {
+ return caller_factory_;
+ }
+ std::shared_ptr<RpcClient> rpc_client() override { return rpc_client_; }
+ std::shared_ptr<AsyncRegionLocator> region_locator() override { return region_locator_; }
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor() override { return cpu_executor_; }
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor() override { return io_executor_; }
+ std::shared_ptr<wangle::IOThreadPoolExecutor> retry_executor() override {
+ return retry_executor_;
+ }
+
+ void Close() override {}
+ std::shared_ptr<HBaseRpcController> CreateRpcController() override {
+ return std::make_shared<HBaseRpcController>();
+ }
+
+ private:
+ std::shared_ptr<folly::HHWheelTimer> retry_timer_;
+ std::shared_ptr<ConnectionConfiguration> conn_conf_;
+ std::shared_ptr<AsyncRpcRetryingCallerFactory> caller_factory_;
+ std::shared_ptr<RpcClient> rpc_client_;
+ std::shared_ptr<AsyncRegionLocator> region_locator_;
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_;
+ std::shared_ptr<wangle::IOThreadPoolExecutor> retry_executor_;
+};
+
+class MockRawAsyncTableImpl {
+ public:
+ explicit MockRawAsyncTableImpl(std::shared_ptr<MockAsyncConnection> conn,
+ std::shared_ptr<hbase::pb::TableName> tn)
+ : conn_(conn), tn_(tn) {}
+ virtual ~MockRawAsyncTableImpl() = default;
+
+ /* implement this in real RawAsyncTableImpl. */
+ folly::Future<std::vector<folly::Try<std::shared_ptr<Result>>>> Gets(
+ const std::vector<hbase::Get> &gets) {
+ /* init request caller builder */
+ auto builder = conn_->caller_factory()->Batch();
+
+ /* call with retry to get result */
+ auto async_caller =
+ builder->table(tn_)
+ ->actions(std::make_shared<std::vector<hbase::Get>>(gets))
+ ->rpc_timeout(conn_->connection_conf()->read_rpc_timeout())
+ ->operation_timeout(conn_->connection_conf()->operation_timeout())
+ ->pause(conn_->connection_conf()->pause())
+ ->max_attempts(conn_->connection_conf()->max_retries())
+ ->start_log_errors_count(conn_->connection_conf()->start_log_errors_count())
+ ->Build();
+
+ return async_caller->Call().then([async_caller](auto r) { return r; });
+ }
+
+ private:
+ std::shared_ptr<MockAsyncConnection> conn_;
+ std::shared_ptr<hbase::pb::TableName> tn_;
+};
+
+void runMultiTest(std::shared_ptr<AsyncRegionLocatorBase> region_locator,
+ const std::string &table_name, bool split_regions, uint32_t tries = 3,
+ uint32_t operation_timeout_millis = 600000, uint32_t num_rows = 10000) {
+ std::vector<std::string> keys{"test0", "test100", "test200", "test300", "test400",
+ "test500", "test600", "test700", "test800", "test900"};
+ std::string tableName = (split_regions) ? ("split-" + table_name) : table_name;
+ if (split_regions)
+ AsyncBatchRpcRetryTest::test_util->CreateTable(tableName, "d", keys);
+ else
+ AsyncBatchRpcRetryTest::test_util->CreateTable(tableName, "d");
+
+ // Create TableName and Row to be fetched from HBase
+ auto tn = folly::to<hbase::pb::TableName>(tableName);
+
+ // Create a client
+ Client client(*AsyncBatchRpcRetryTest::test_util->conf());
+
+ // Get connection to HBase Table
+ auto table = client.Table(tn);
+ ASSERT_TRUE(table) << "Unable to get connection to Table.";
+
+ for (uint64_t i = 0; i < num_rows; i++) {
+ table->Put(Put{"test" + std::to_string(i)}.AddColumn("d", std::to_string(i),
+ "value" + std::to_string(i)));
+ }
+
+ std::map<std::string, std::shared_ptr<RegionLocation>> region_locations;
+ std::vector<hbase::Get> gets;
+ for (uint64_t i = 0; i < num_rows; ++i) {
+ auto row = "test" + std::to_string(i);
+ hbase::Get get(row);
+ gets.push_back(get);
+ region_locations[row] = table->GetRegionLocation(row);
+ }
+
+ /* init region location and rpc channel */
+ auto cpu_executor_ = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
+ auto io_executor_ = client.async_connection()->io_executor();
+ auto retry_executor_ = std::make_shared<wangle::IOThreadPoolExecutor>(1);
+ auto codec = std::make_shared<hbase::KeyValueCodec>();
+ auto rpc_client =
+ std::make_shared<RpcClient>(io_executor_, codec, AsyncBatchRpcRetryTest::test_util->conf());
+ std::shared_ptr<folly::HHWheelTimer> retry_timer =
+ folly::HHWheelTimer::newTimer(retry_executor_->getEventBase());
+
+ /* init connection configuration */
+ auto connection_conf = std::make_shared<ConnectionConfiguration>(
+ TimeUtil::SecondsToNanos(20), // connect_timeout
+ TimeUtil::MillisToNanos(operation_timeout_millis), // operation_timeout
+ TimeUtil::SecondsToNanos(60), // rpc_timeout
+ TimeUtil::MillisToNanos(100), // pause
+ tries, // max retries
+ 1); // start log errors count
+
+ /* set region locator */
+ region_locator->set_region_location(region_locations);
+
+ /* init hbase client connection */
+ auto conn = std::make_shared<MockAsyncConnection>(connection_conf, retry_timer, cpu_executor_,
+ io_executor_, retry_executor_, rpc_client,
+ region_locator);
+ conn->Init();
+
+ /* init retry caller factory */
+ auto tableImpl =
+ std::make_shared<MockRawAsyncTableImpl>(conn, std::make_shared<hbase::pb::TableName>(tn));
+
+ auto tresults = tableImpl->Gets(gets).get(milliseconds(operation_timeout_millis));
+
+ ASSERT_TRUE(!tresults.empty()) << "tresults shouldn't be empty.";
+ std::vector<std::shared_ptr<hbase::Result>> results{};
+ uint32_t num = 0;
+ for (auto tresult : tresults) {
+ if (tresult.hasValue()) {
+ results.push_back(tresult.value());
+ } else if (tresult.hasException()) {
+ folly::exception_wrapper ew = tresult.exception();
+ LOG(ERROR) << "Caught exception:- " << ew.what().toStdString() << " for " << gets[num].row();
+ throw ew;
+ }
+ ++num;
+ }
+
+ // Test the values, should be same as in put executed on hbase shell
+ ASSERT_TRUE(!results.empty()) << "Results shouldn't be empty.";
+ uint32_t i = 0;
+ for (; i < num_rows; ++i) {
+ ASSERT_TRUE(!results[i]->IsEmpty()) << "Result for Get " << gets[i].row()
+ << " must not be empty";
+ EXPECT_EQ("test" + std::to_string(i), results[i]->Row());
+ EXPECT_EQ("value" + std::to_string(i), results[i]->Value("d", std::to_string(i)).value());
+ }
+
+ retry_timer->destroy();
+ table->Close();
+ client.Close();
+ retry_executor_->stop();
+ io_executor_->stop();
+ cpu_executor_->stop();
+}
+
+// Test successful case
+TEST_F(AsyncBatchRpcRetryTest, MultiGets) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockAsyncRegionLocator>());
+ runMultiTest(region_locator, "table1", false);
+}
+
+// Tests the RPC failing 3 times, then succeeding
+TEST_F(AsyncBatchRpcRetryTest, HandleException) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockWrongRegionAsyncRegionLocator>(3));
+ runMultiTest(region_locator, "table2", false, 5);
+}
+
+// Tests the RPC failing 4 times, throwing an exception
+TEST_F(AsyncBatchRpcRetryTest, FailWithException) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockWrongRegionAsyncRegionLocator>(4));
+ EXPECT_ANY_THROW(runMultiTest(region_locator, "table3", false));
+}
+
+// Tests the region location lookup failing 3 times, then succeeding
+TEST_F(AsyncBatchRpcRetryTest, HandleExceptionFromRegionLocationLookup) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockFailingAsyncRegionLocator>(3));
+ runMultiTest(region_locator, "table4", false);
+}
+
+// Tests the region location lookup failing 5 times, throwing an exception
+TEST_F(AsyncBatchRpcRetryTest, FailWithExceptionFromRegionLocationLookup) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockFailingAsyncRegionLocator>(4));
+ EXPECT_ANY_THROW(runMultiTest(region_locator, "table5", false, 3));
+}
+
+// Tests hitting operation timeout, thus not retrying anymore
+TEST_F(AsyncBatchRpcRetryTest, FailWithOperationTimeout) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockFailingAsyncRegionLocator>(6));
+ EXPECT_ANY_THROW(runMultiTest(region_locator, "table6", false, 5, 100, 10000));
+}
+
+// Test successful case
+TEST_F(AsyncBatchRpcRetryTest, MultiGetsSplitRegions) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockAsyncRegionLocator>());
+ runMultiTest(region_locator, "table1", true);
+}
+
+// Tests the RPC failing 3 times, then succeeding
+TEST_F(AsyncBatchRpcRetryTest, HandleExceptionSplitRegions) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockWrongRegionAsyncRegionLocator>(3));
+ runMultiTest(region_locator, "table2", true, 5);
+}
+
+// Tests the RPC failing 4 times, throwing an exception
+TEST_F(AsyncBatchRpcRetryTest, FailWithExceptionSplitRegions) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockWrongRegionAsyncRegionLocator>(4));
+ EXPECT_ANY_THROW(runMultiTest(region_locator, "table3", true));
+}
+
+// Tests the region location lookup failing 3 times, then succeeding
+TEST_F(AsyncBatchRpcRetryTest, HandleExceptionFromRegionLocationLookupSplitRegions) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockFailingAsyncRegionLocator>(3));
+ runMultiTest(region_locator, "table4", true);
+}
+
+// Tests the region location lookup failing 5 times, throwing an exception
+TEST_F(AsyncBatchRpcRetryTest, FailWithExceptionFromRegionLocationLookupSplitRegions) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockFailingAsyncRegionLocator>(4));
+ EXPECT_ANY_THROW(runMultiTest(region_locator, "table5", true, 3));
+}
+
+// Tests hitting operation timeout, thus not retrying anymore
+TEST_F(AsyncBatchRpcRetryTest, FailWithOperationTimeoutSplitRegions) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockFailingAsyncRegionLocator>(6));
+ EXPECT_ANY_THROW(runMultiTest(region_locator, "table6", true, 5, 100, 10000));
+}
diff --git a/core/client-test.cc b/core/client-test.cc
index ba213bd..9efe0b6 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -435,29 +435,27 @@ TEST_F(ClientTest, PutsWithTimestamp) {
client.Close();
}
-TEST_F(ClientTest, MultiGets) {
- // Using TestUtil to populate test data
- ClientTest::test_util->CreateTable("t", "d");
-
- // Create TableName and Row to be fetched from HBase
- auto tn = folly::to<hbase::pb::TableName>("t");
-
- // Create a client
- hbase::Client client(*ClientTest::test_util->conf());
+void SetClientParams() {
+ ClientTest::test_util->conf()->SetInt("hbase.client.cpu.thread.pool.size", 6);
+ ClientTest::test_util->conf()->SetInt("hbase.client.operation.timeout", 600000);
+ ClientTest::test_util->conf()->SetInt("hbase.client.retries.number", 7);
+ ClientTest::test_util->conf()->SetInt("hbase.client.start.log.errors.counter", 1);
+}
- // Get connection to HBase Table
- auto table = client.Table(tn);
+void PerformPuts(uint64_t num_rows, std::shared_ptr<hbase::Client> client,
+ const std::string &table_name) {
+ auto tn = folly::to<hbase::pb::TableName>(table_name);
+ auto table = client->Table(tn);
ASSERT_TRUE(table) << "Unable to get connection to Table.";
-
- uint64_t num_rows = 10000;
// Perform Puts
for (uint64_t i = 0; i < num_rows; i++) {
table->Put(Put{"test" + std::to_string(i)}.AddColumn("d", std::to_string(i),
"value" + std::to_string(i)));
}
+}
+void MakeGets(uint64_t num_rows, const std::string &row_prefix, std::vector<hbase::Get> &gets) {
// Perform the Gets
- std::vector<hbase::Get> gets;
for (uint64_t i = 0; i < num_rows; ++i) {
auto row = "test" + std::to_string(i);
hbase::Get get(row);
@@ -465,9 +463,10 @@ TEST_F(ClientTest, MultiGets) {
}
gets.push_back(hbase::Get("test2"));
gets.push_back(hbase::Get("testextra"));
+}
- auto results = table->Get(gets);
-
+void TestMultiResults(uint64_t num_rows, const std::vector<std::shared_ptr<hbase::Result>> &results,
+ const std::vector<hbase::Get> &gets) {
// Test the values, should be same as in put executed on hbase shell
ASSERT_TRUE(!results.empty()) << "Result vector shouldn't be empty.";
@@ -483,6 +482,66 @@ TEST_F(ClientTest, MultiGets) {
++i;
ASSERT_TRUE(results[i]->IsEmpty()) << "Result for Get " << gets[i].row() << " must be empty";
+}
+
+TEST_F(ClientTest, MultiGets) {
+ std::string table_name = "t";
+ // Using TestUtil to populate test data
+ ClientTest::test_util->CreateTable(table_name, "d");
+
+ // Create TableName and Row to be fetched from HBase
+ auto tn = folly::to<hbase::pb::TableName>(table_name);
+
+ SetClientParams();
+ // Create a client
+ hbase::Client client(*ClientTest::test_util->conf());
+
+ uint64_t num_rows = 50000;
+ PerformPuts(num_rows, std::make_shared<hbase::Client>(client), table_name);
+
+ // Get connection to HBase Table
+ auto table = client.Table(tn);
+ ASSERT_TRUE(table) << "Unable to get connection to Table.";
+
+ std::vector<hbase::Get> gets;
+ MakeGets(num_rows, "test", gets);
+
+ auto results = table->Get(gets);
+
+ TestMultiResults(num_rows, results, gets);
+
+ table->Close();
+ client.Close();
+}
+
+TEST_F(ClientTest, MultiGetsWithRegionSplits) {
+ // Using TestUtil to populate test data
+ std::vector<std::string> keys{"test0", "test100", "test200", "test300", "test400",
+ "test500", "test600", "test700", "test800", "test900"};
+ std::string table_name = "t";
+ ClientTest::test_util->CreateTable(table_name, "d", keys);
+
+ // Create TableName and Row to be fetched from HBase
+ auto tn = folly::to<hbase::pb::TableName>(table_name);
+
+ SetClientParams();
+
+ // Create a client
+ hbase::Client client(*ClientTest::test_util->conf());
+
+ uint64_t num_rows = 50000;
+ PerformPuts(num_rows, std::make_shared<hbase::Client>(client), table_name);
+
+ // Get connection to HBase Table
+ auto table = client.Table(tn);
+ ASSERT_TRUE(table) << "Unable to get connection to Table.";
+
+ std::vector<hbase::Get> gets;
+ MakeGets(num_rows, "test", gets);
+
+ auto results = table->Get(gets);
+
+ TestMultiResults(num_rows, results, gets);
table->Close();
client.Close();
diff --git a/core/multi-response.cc b/core/multi-response.cc
index f620c98..a4c2108 100644
--- a/core/multi-response.cc
+++ b/core/multi-response.cc
@@ -18,6 +18,7 @@
*/
#include "core/multi-response.h"
+#include <glog/logging.h>
#include "core/region-result.h"
using hbase::pb::RegionLoadStats;
@@ -36,35 +37,38 @@ int MultiResponse::Size() const {
void MultiResponse::AddRegionResult(const std::string& region_name, int32_t original_index,
std::shared_ptr<Result> result,
- std::shared_ptr<std::exception> exc) {
- bool region_found = false;
- for (auto itr = results_.begin(); itr != results_.end(); ++itr) {
- if (itr->first == region_name) {
- region_found = true;
- itr->second->AddResultOrException(original_index, result, exc);
- break;
- }
- }
- if (!region_found) {
+ std::shared_ptr<folly::exception_wrapper> exc) {
+ auto itr = results_.find(region_name);
+ if (itr == results_.end()) {
auto region_result = std::make_shared<RegionResult>();
region_result->AddResultOrException(original_index, result, exc);
results_[region_name] = region_result;
+ } else {
+ itr->second->AddResultOrException(original_index, result, exc);
}
}
void MultiResponse::AddRegionException(const std::string& region_name,
- std::shared_ptr<std::exception> exception) {
- exceptions_[region_name] = exception;
+ std::shared_ptr<folly::exception_wrapper> exception) {
+ VLOG(8) << "Store Region Exception:- " << exception->what() << "; Region[" << region_name << "];";
+ bool region_found = false;
+ auto itr = exceptions_.find(region_name);
+ if (itr == exceptions_.end()) {
+ auto region_result = std::make_shared<folly::exception_wrapper>();
+ exceptions_[region_name] = exception;
+ } else {
+ itr->second = exception;
+ }
}
-std::shared_ptr<std::exception> MultiResponse::RegionException(
+std::shared_ptr<folly::exception_wrapper> MultiResponse::RegionException(
const std::string& region_name) const {
auto find = exceptions_.at(region_name);
return find;
}
-const std::map<std::string, std::shared_ptr<std::exception> >& MultiResponse::RegionExceptions()
- const {
+const std::map<std::string, std::shared_ptr<folly::exception_wrapper> >&
+MultiResponse::RegionExceptions() const {
return exceptions_;
}
diff --git a/core/multi-response.h b/core/multi-response.h
index 96883fd..d38cfd6 100644
--- a/core/multi-response.h
+++ b/core/multi-response.h
@@ -20,6 +20,7 @@
#pragma once
#include <core/region-result.h>
+#include <folly/ExceptionWrapper.h>
#include <exception>
#include <map>
#include <memory>
@@ -46,17 +47,18 @@ class MultiResponse {
* @param resOrEx the result or error; will be empty for successful Put and Delete actions.
*/
void AddRegionResult(const std::string& region_name, int32_t original_index,
- std::shared_ptr<Result> result, std::shared_ptr<std::exception> exc);
+ std::shared_ptr<Result> result,
+ std::shared_ptr<folly::exception_wrapper> exc);
void AddRegionException(const std::string& region_name,
- std::shared_ptr<std::exception> exception);
+ std::shared_ptr<folly::exception_wrapper> exception);
/**
* @return the exception for the region, if any. Null otherwise.
*/
- std::shared_ptr<std::exception> RegionException(const std::string& region_name) const;
+ std::shared_ptr<folly::exception_wrapper> RegionException(const std::string& region_name) const;
- const std::map<std::string, std::shared_ptr<std::exception>>& RegionExceptions() const;
+ const std::map<std::string, std::shared_ptr<folly::exception_wrapper>>& RegionExceptions() const;
void AddStatistic(const std::string& region_name, std::shared_ptr<pb::RegionLoadStats> stat);
@@ -66,12 +68,12 @@ class MultiResponse {
private:
// map of regionName to map of Results by the original index for that Result
- std::map<std::string, std::shared_ptr<hbase::RegionResult>> results_;
+ std::map<std::string, std::shared_ptr<RegionResult>> results_;
/**
* The server can send us a failure for the region itself, instead of individual failure.
* It's a part of the protobuf definition.
*/
- std::map<std::string, std::shared_ptr<std::exception>> exceptions_;
+ std::map<std::string, std::shared_ptr<folly::exception_wrapper>> exceptions_;
};
} /* namespace hbase */
diff --git a/core/region-result.cc b/core/region-result.cc
index 05ab274..206c876 100644
--- a/core/region-result.cc
+++ b/core/region-result.cc
@@ -30,7 +30,7 @@ RegionResult::RegionResult() {}
RegionResult::~RegionResult() {}
void RegionResult::AddResultOrException(int32_t index, std::shared_ptr<hbase::Result> result,
- std::shared_ptr<std::exception> exc) {
+ std::shared_ptr<folly::exception_wrapper> exc) {
auto index_found = result_or_excption_.find(index);
if (index_found == result_or_excption_.end()) {
result_or_excption_[index] = std::make_tuple(result ? result : nullptr, exc ? exc : nullptr);
diff --git a/core/region-result.h b/core/region-result.h
index cfd9e5a..b961634 100644
--- a/core/region-result.h
+++ b/core/region-result.h
@@ -19,6 +19,7 @@
#pragma once
+#include <folly/ExceptionWrapper.h>
#include <map>
#include <memory>
#include <string>
@@ -29,13 +30,13 @@
namespace hbase {
using ResultOrExceptionTuple =
- std::tuple<std::shared_ptr<hbase::Result>, std::shared_ptr<std::exception>>;
+ std::tuple<std::shared_ptr<hbase::Result>, std::shared_ptr<folly::exception_wrapper>>;
class RegionResult {
public:
RegionResult();
void AddResultOrException(int32_t index, std::shared_ptr<hbase::Result> result,
- std::shared_ptr<std::exception> exc);
+ std::shared_ptr<folly::exception_wrapper> exc);
void set_stat(std::shared_ptr<pb::RegionLoadStats> stat);
diff --git a/core/response-converter.cc b/core/response-converter.cc
index 4f9bfb1..960c487 100644
--- a/core/response-converter.cc
+++ b/core/response-converter.cc
@@ -18,7 +18,6 @@
*/
#include "core/response-converter.h"
-
#include <glog/logging.h>
#include <stdexcept>
#include <string>
@@ -125,8 +124,9 @@ std::vector<std::shared_ptr<Result>> ResponseConverter::FromScanResponse(
return results;
}
-std::unique_ptr<hbase::MultiResponse> ResponseConverter::GetResults(std::shared_ptr<Request> req,
- const Response& resp) {
+std::unique_ptr<hbase::MultiResponse> ResponseConverter::GetResults(
+ std::shared_ptr<Request> req, const Response& resp,
+ const ServerRequest::ActionsByRegion& actions_by_region) {
auto multi_req = std::static_pointer_cast<hbase::pb::MultiRequest>(req->req_msg());
auto multi_resp = std::static_pointer_cast<hbase::pb::MultiResponse>(resp.resp_msg());
VLOG(3) << "GetResults:" << multi_resp->ShortDebugString();
@@ -148,11 +148,10 @@ std::unique_ptr<hbase::MultiResponse> ResponseConverter::GetResults(std::shared_
auto region_name = rs.value();
if (action_result.has_exception()) {
- if (action_result.exception().has_value()) {
- auto exc = std::make_shared<hbase::IOException>(action_result.exception().value());
- VLOG(8) << "Store Region Exception:- " << exc->what();
- multi_response->AddRegionException(region_name, exc);
- }
+ auto ew = ResponseConverter::GetRemoteException(action_result.exception());
+ VLOG(8) << "Store Remote Region Exception:- " << ew->what().toStdString() << "; Region["
+ << region_name << "];";
+ multi_response->AddRegionException(region_name, ew);
continue;
}
@@ -163,14 +162,16 @@ std::unique_ptr<hbase::MultiResponse> ResponseConverter::GetResults(std::shared_
" for region " + actions.region().value());
}
+ auto multi_actions = actions_by_region.at(region_name)->actions();
+ uint64_t multi_actions_num = 0;
for (hbase::pb::ResultOrException roe : action_result.resultorexception()) {
std::shared_ptr<Result> result;
- std::shared_ptr<std::exception> exc;
+ std::shared_ptr<folly::exception_wrapper> ew;
if (roe.has_exception()) {
- if (roe.exception().has_value()) {
- exc = std::make_shared<hbase::IOException>(roe.exception().value());
- VLOG(8) << "Store ResultOrException:- " << exc->what();
- }
+ auto ew = ResponseConverter::GetRemoteException(roe.exception());
+ VLOG(8) << "Store Remote Region Exception:- " << ew->what().toStdString() << "; Region["
+ << region_name << "];";
+ multi_response->AddRegionException(region_name, ew);
} else if (roe.has_result()) {
result = ToResult(roe.result(), resp.cell_scanner());
} else if (roe.has_service_result()) {
@@ -183,7 +184,11 @@ std::unique_ptr<hbase::MultiResponse> ResponseConverter::GetResults(std::shared_
result = std::make_shared<Result>(empty_cells, multi_resp->processed() ? true : false,
false, false);
}
- multi_response->AddRegionResult(region_name, roe.index(), std::move(result), exc);
+ // We add the original index of the multi-action so that when populating the response back we
+ // do it as per the action index
+ multi_response->AddRegionResult(
+ region_name, multi_actions[multi_actions_num]->original_index(), std::move(result), ew);
+ multi_actions_num++;
}
}
@@ -196,4 +201,21 @@ std::unique_ptr<hbase::MultiResponse> ResponseConverter::GetResults(std::shared_
}
return multi_response;
}
+
+std::shared_ptr<folly::exception_wrapper> ResponseConverter::GetRemoteException(
+ const hbase::pb::NameBytesPair& exc_resp) {
+ std::string what;
+ std::string exception_class_name = exc_resp.has_name() ? exc_resp.name() : "";
+ std::string stack_trace = exc_resp.has_value() ? exc_resp.value() : "";
+
+ what.append(exception_class_name).append(stack_trace);
+ auto remote_exception = std::make_unique<RemoteException>(what);
+ remote_exception->set_exception_class_name(exception_class_name)
+ ->set_stack_trace(stack_trace)
+ ->set_hostname("")
+ ->set_port(0);
+
+ return std::make_shared<folly::exception_wrapper>(
+ folly::make_exception_wrapper<RemoteException>(*remote_exception));
+}
} /* namespace hbase */
diff --git a/core/response-converter.h b/core/response-converter.h
index 2f8f279..edd4165 100644
--- a/core/response-converter.h
+++ b/core/response-converter.h
@@ -25,6 +25,7 @@
#include "connection/response.h"
#include "core/multi-response.h"
#include "core/result.h"
+#include "core/server-request.h"
#include "if/Client.pb.h"
#include "serde/cell-scanner.h"
@@ -56,12 +57,15 @@ class ResponseConverter {
static std::vector<std::shared_ptr<Result>> FromScanResponse(
const std::shared_ptr<pb::ScanResponse> resp, std::shared_ptr<CellScanner> cell_scanner);
- static std::unique_ptr<hbase::MultiResponse> GetResults(std::shared_ptr<Request> req,
- const Response& resp);
+ static std::unique_ptr<hbase::MultiResponse> GetResults(
+ std::shared_ptr<Request> req, const Response& resp,
+ const ServerRequest::ActionsByRegion& actions_by_region);
private:
// Constructor not required. We have all static methods to extract response from PB messages.
ResponseConverter();
+ static std::shared_ptr<folly::exception_wrapper> GetRemoteException(
+ const hbase::pb::NameBytesPair& exc_resp);
};
} /* namespace hbase */
diff --git a/core/server-request.h b/core/server-request.h
index 7f31c2b..85df9ed 100644
--- a/core/server-request.h
+++ b/core/server-request.h
@@ -44,8 +44,14 @@ class ServerRequest {
void AddActionsByRegion(std::shared_ptr<RegionLocation> region_location,
std::shared_ptr<Action> action) {
auto region_name = region_location->region_name();
- auto itr = actions_by_region_.at(region_name);
- itr->AddAction(action);
+ auto search = actions_by_region_.find(region_name);
+ if (search == actions_by_region_.end()) {
+ auto region_request = std::make_shared<RegionRequest>(region_location);
+ actions_by_region_[region_name] = region_request;
+ actions_by_region_[region_name]->AddAction(action);
+ } else {
+ search->second->AddAction(action);
+ }
}
const ActionsByRegion &actions_by_region() const { return actions_by_region_; }
[hbase] 05/137: HBASE-15078 Added ability to start/stop hbase local
cluster for tests, global test_env for gtest,
small changes to dockerfile and docker run.
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit d2992135e8f955bba1e23dcaca907f9762f6e4ba
Author: Mikhail Antonov <an...@apache.org>
AuthorDate: Wed Jan 6 15:08:21 2016 -0800
HBASE-15078 Added ability to start/stop hbase local cluster for tests, global test_env for gtest, small changes to dockerfile and docker run.
Added check ~/.m2 folder exists; moved scripts to ./bin
---
.buckconfig | 2 +-
Dockerfile | 2 +-
bin/start-docker.sh | 10 +++++-
Dockerfile => bin/start_local_hbase_and_wait.sh | 12 ++++---
Dockerfile => bin/stop_local_hbase_and_wait.sh | 12 ++++---
core/BUCK | 15 +++++++++
core/HBaseNativeClientTestEnv.cc | 42 +++++++++++++++++++++++++
core/SampleNativeClientTest.cc | 28 +++++++++++++++++
core/test_env.h | 30 ++++++++++++++++++
9 files changed, 142 insertions(+), 11 deletions(-)
diff --git a/.buckconfig b/.buckconfig
index 3227a2a..402ef27 100644
--- a/.buckconfig
+++ b/.buckconfig
@@ -1,2 +1,2 @@
[cxx]
- gtest_dep = //third-party/googletest/googletest:google-test
+ gtest_dep = //third-party:google-test
diff --git a/Dockerfile b/Dockerfile
index 70e823b..5f17f04 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -17,6 +17,6 @@
FROM pjameson/buck-folly-watchman
-RUN apt-get install -y libprotobuf-dev protobuf-compiler clang-format-3.7 vim
+RUN apt-get install -y libprotobuf-dev protobuf-compiler clang-format-3.7 vim maven inetutils-ping
WORKDIR /usr/local/src/hbase/hbase-native-client
diff --git a/bin/start-docker.sh b/bin/start-docker.sh
index 1c9b02e..bf38912 100755
--- a/bin/start-docker.sh
+++ b/bin/start-docker.sh
@@ -28,5 +28,13 @@ if [[ ! -d third-party/googletest ]]; then
git clone https://github.com/google/googletest.git third-party/googletest
fi
+if [[ ! -d ~/.m2 ]]; then
+ echo "~/.m2 directory doesn't exist. Check Apache Maven is installed."
+ exit 1
+fi;
-docker run -v ${PWD}/..:/usr/local/src/hbase -it hbase_native /bin/bash
+docker run -p 16010:16010/tcp \
+ -e "JAVA_HOME=/usr/lib/jvm/java-8-oracle" \
+ -v ${PWD}/..:/usr/local/src/hbase \
+ -v ~/.m2:/root/.m2 \
+ -it hbase_native /bin/bash
diff --git a/Dockerfile b/bin/start_local_hbase_and_wait.sh
old mode 100644
new mode 100755
similarity index 78%
copy from Dockerfile
copy to bin/start_local_hbase_and_wait.sh
index 70e823b..64d0b68
--- a/Dockerfile
+++ b/bin/start_local_hbase_and_wait.sh
@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -15,8 +17,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM pjameson/buck-folly-watchman
-
-RUN apt-get install -y libprotobuf-dev protobuf-compiler clang-format-3.7 vim
+$PWD/../bin/start-hbase.sh
-WORKDIR /usr/local/src/hbase/hbase-native-client
+until [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010) == "200" ]
+do
+ printf "Waiting for local HBase cluster to start\n"
+ sleep 1
+done
diff --git a/Dockerfile b/bin/stop_local_hbase_and_wait.sh
old mode 100644
new mode 100755
similarity index 79%
copy from Dockerfile
copy to bin/stop_local_hbase_and_wait.sh
index 70e823b..4e89334
--- a/Dockerfile
+++ b/bin/stop_local_hbase_and_wait.sh
@@ -1,3 +1,5 @@
+#!/usr/bin/env bash
+
##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -15,8 +17,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM pjameson/buck-folly-watchman
-
-RUN apt-get install -y libprotobuf-dev protobuf-compiler clang-format-3.7 vim
+$PWD/../bin/stop-hbase.sh
-WORKDIR /usr/local/src/hbase/hbase-native-client
+while [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010) == "200" ]
+do
+ printf "Waiting for local HBase cluster to stop\n"
+ sleep 1
+done
diff --git a/core/BUCK b/core/BUCK
index a7eaa9e..2e4e755 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -48,3 +48,18 @@ cxx_binary(
'PUBLIC',
],
)
+
+cxx_test(
+ name = "core_test",
+ headers = [
+ "test_env.h",
+ ],
+ srcs = [
+ "HBaseNativeClientTestEnv.cc",
+ "SampleNativeClientTest.cc",
+ ],
+ deps = [
+ ":core",
+ ],
+ run_test_separately = True,
+)
diff --git a/core/HBaseNativeClientTestEnv.cc b/core/HBaseNativeClientTestEnv.cc
new file mode 100644
index 0000000..b8cb8db
--- /dev/null
+++ b/core/HBaseNativeClientTestEnv.cc
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <gtest/gtest.h>
+#include <core/test_env.h>
+
+namespace {
+
+class HBaseNativeClientTestEnv : public ::testing::Environment {
+ public:
+ void SetUp() override {
+ init_test_env();
+ }
+
+ void TearDown() override {
+ clean_test_env();
+ }
+};
+
+} // anonymous
+
+int main(int argc, char** argv) {
+ testing::InitGoogleTest(&argc, argv);
+ ::testing::AddGlobalTestEnvironment(new HBaseNativeClientTestEnv());
+ return RUN_ALL_TESTS();
+}
diff --git a/core/SampleNativeClientTest.cc b/core/SampleNativeClientTest.cc
new file mode 100644
index 0000000..ef564f7
--- /dev/null
+++ b/core/SampleNativeClientTest.cc
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "gtest/gtest.h"
+
+/**
+ * Sample test.
+ */
+TEST(SampleTest, sample) {
+ EXPECT_TRUE(true);
+}
+
diff --git a/core/test_env.h b/core/test_env.h
new file mode 100644
index 0000000..5796ae1
--- /dev/null
+++ b/core/test_env.h
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <cstdlib>
+
+inline void init_test_env() {
+ // start local HBase cluster to be reused by all tests
+ system("scripts/start_local_hbase_and_wait.sh");
+}
+
+inline void clean_test_env() {
+ // shutdown local HBase cluster
+ system("scripts/stop_local_hbase_and_wait.sh");
+}
[hbase] 98/137: HBASE-18188 [C++] Fix Handling do not retry
exceptions
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 62f042b5ebc3f066fb64b8f38ad12f365b660bdd
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Fri Jun 16 11:38:22 2017 -0700
HBASE-18188 [C++] Fix Handling do not retry exceptions
---
bin/cpplint.sh | 2 +-
core/client-test.cc | 11 ++-
core/location-cache.cc | 4 +-
exceptions/BUCK | 13 ++-
exceptions/exception-test.cc | 64 ++++++++++++++
exceptions/exception.cc | 128 +++++++++++++++++++++++++++
exceptions/exception.h | 201 +++++++++++++++++++++++++++++++------------
7 files changed, 362 insertions(+), 61 deletions(-)
diff --git a/bin/cpplint.sh b/bin/cpplint.sh
index 81795fd..3684c2e 100755
--- a/bin/cpplint.sh
+++ b/bin/cpplint.sh
@@ -27,4 +27,4 @@ wget -nc $CPPLINT_LOC -O $OUTPUT
# Exclude the following rules: build/header_guard (We use #pragma once instead)
# readability/todo (TODOs are generic)
# build/c++11 (We are building with c++14)
-find core connection serde utils test-util security -name "*.h" -or -name "*.cc" | xargs -P8 python $OUTPUT --filter=-build/header_guard,-readability/todo,-build/c++11 --linelength=100
+find core connection exceptions serde utils test-util security -name "*.h" -or -name "*.cc" | xargs -P8 python $OUTPUT --filter=-build/header_guard,-readability/todo,-build/c++11 --linelength=100
diff --git a/core/client-test.cc b/core/client-test.cc
index ed413e8..6462e6a 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -30,6 +30,7 @@
#include "core/put.h"
#include "core/result.h"
#include "core/table.h"
+#include "exceptions/exception.h"
#include "serde/table-name.h"
#include "test-util/test-util.h"
#include "utils/bytes-util.h"
@@ -37,6 +38,7 @@
using hbase::Cell;
using hbase::Configuration;
using hbase::Get;
+using hbase::RetriesExhaustedException;
using hbase::Put;
using hbase::Table;
using hbase::TestUtil;
@@ -326,7 +328,14 @@ TEST_F(ClientTest, GetForNonExistentTable) {
ASSERT_TRUE(table) << "Unable to get connection to Table.";
// Perform the Get
- ASSERT_ANY_THROW(table->Get(get)) << "Table does not exist. We should get an exception";
+ try {
+ table->Get(get);
+ FAIL() << "Should have thrown RetriesExhaustedException";
+ } catch (const RetriesExhaustedException &ex) {
+ ASSERT_EQ(0, ex.num_retries());
+ } catch (...) {
+ FAIL() << "Should have thrown RetriesExhaustedException";
+ }
table->Close();
client.Close();
diff --git a/core/location-cache.cc b/core/location-cache.cc
index d368314..dfe3e9f 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -24,9 +24,9 @@
#include <wangle/concurrent/CPUThreadPoolExecutor.h>
#include <wangle/concurrent/IOThreadPoolExecutor.h>
-#include <folly/Logging.h>
#include "connection/response.h"
#include "connection/rpc-connection.h"
+#include "exceptions/exception.h"
#include "if/Client.pb.h"
#include "if/ZooKeeper.pb.h"
#include "serde/region-info.h"
@@ -127,7 +127,7 @@ folly::Future<std::shared_ptr<RegionLocation>> LocationCache::LocateFromMeta(
// Make sure that the correct location was found.
if (rl->region_info().table_name().namespace_() != tn.namespace_() ||
rl->region_info().table_name().qualifier() != tn.qualifier()) {
- throw std::runtime_error("Doesn't look like table exists.");
+ throw TableNotFoundException(folly::to<std::string>(tn));
}
return rl;
})
diff --git a/exceptions/BUCK b/exceptions/BUCK
index 07ffeaf..e2f03a1 100644
--- a/exceptions/BUCK
+++ b/exceptions/BUCK
@@ -20,9 +20,20 @@ cxx_library(
exported_headers=[
"exception.h",
],
- srcs=[],
+ srcs=[
+ "exception.cc",
+ ],
deps=[
"//third-party:folly",
],
compiler_flags=['-Weffc++'],
visibility=['//core/...', '//connection//...'],)
+cxx_test(
+ name="exception-test",
+ srcs=[
+ "exception-test.cc",
+ ],
+ deps=[
+ ":exceptions",
+ ],
+ run_test_separately=True,)
diff --git a/exceptions/exception-test.cc b/exceptions/exception-test.cc
new file mode 100644
index 0000000..583240e
--- /dev/null
+++ b/exceptions/exception-test.cc
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include <gtest/gtest.h>
+
+#include "exceptions/exception.h"
+
+#include "folly/ExceptionWrapper.h"
+
+using hbase::ExceptionUtil;
+using hbase::IOException;
+using hbase::RemoteException;
+
+TEST(ExceptionUtilTest, IOExceptionShouldRetry) {
+ IOException ex{};
+ EXPECT_TRUE(ExceptionUtil::ShouldRetry(ex));
+
+ ex.set_do_not_retry(true);
+ EXPECT_FALSE(ExceptionUtil::ShouldRetry(ex));
+
+ ex.set_do_not_retry(false);
+ EXPECT_TRUE(ExceptionUtil::ShouldRetry(ex));
+
+ IOException ex2{"description", true};
+ EXPECT_FALSE(ExceptionUtil::ShouldRetry(ex2));
+
+ IOException ex3{"description", std::runtime_error("ex"), true};
+ EXPECT_FALSE(ExceptionUtil::ShouldRetry(ex3));
+}
+
+TEST(ExceptionUtilTest, RemoteExceptionShouldRetry) {
+ RemoteException ex{};
+ EXPECT_TRUE(ExceptionUtil::ShouldRetry(ex));
+
+ ex.set_do_not_retry(true);
+ EXPECT_FALSE(ExceptionUtil::ShouldRetry(ex));
+
+ ex.set_do_not_retry(false);
+ EXPECT_TRUE(ExceptionUtil::ShouldRetry(ex));
+
+ ex.set_exception_class_name("org.apache.hadoop.hbase.FooException");
+ EXPECT_TRUE(ExceptionUtil::ShouldRetry(ex));
+
+ ex.set_exception_class_name("org.apache.hadoop.hbase.NotServingRegionException");
+ EXPECT_TRUE(ExceptionUtil::ShouldRetry(ex));
+
+ ex.set_exception_class_name("org.apache.hadoop.hbase.UnknownRegionException");
+ EXPECT_FALSE(ExceptionUtil::ShouldRetry(ex));
+}
diff --git a/exceptions/exception.cc b/exceptions/exception.cc
new file mode 100644
index 0000000..c25acb4
--- /dev/null
+++ b/exceptions/exception.cc
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include "exceptions/exception.h"
+
+namespace hbase {
+const std::vector<const char*> ExceptionUtil::kAllDoNotRetryIOExceptions = {
+ kDoNotRetryIOException,
+ kTableNotFoundException,
+ kTableNotEnabledException,
+ kCoprocessorException,
+ kBypassCoprocessorException,
+ kInvalidFamilyOperationException,
+ kServerTooBusyException,
+ kFailedSanityCheckException,
+ kCorruptHFileException,
+ kLabelAlreadyExistsException,
+ kFatalConnectionException,
+ kUnsupportedCryptoException,
+ kUnsupportedCellCodecException,
+ kEmptyServiceNameException,
+ kUnknownServiceException,
+ kWrongVersionException,
+ kBadAuthException,
+ kUnsupportedCompressionCodecException,
+ kDoNotRetryRegionException,
+ kRowTooBigException,
+ kRowTooBigExceptionDeprecated,
+ kUnknownRegionException,
+ kMergeRegionException,
+ kNoServerForRegionException,
+ kQuotaExceededException,
+ kSpaceLimitingException,
+ kThrottlingException,
+ kAccessDeniedException,
+ kUnknownProtocolException,
+ kRequestTooBigException,
+ kNotAllMetaRegionsOnlineException,
+ kConstraintException,
+ kNoSuchColumnFamilyException,
+ kLeaseException,
+ kInvalidLabelException,
+ kUnknownScannerException,
+ kScannerResetException,
+ kOutOfOrderScannerNextException};
+
+bool ExceptionUtil::ShouldRetry(const folly::exception_wrapper& error) {
+ bool do_not_retry = false;
+ error.with_exception(
+ [&](const IOException& ioe) { do_not_retry = do_not_retry || ioe.do_not_retry(); });
+ error.with_exception([&](const RemoteException& remote_ex) {
+ do_not_retry = do_not_retry || IsJavaDoNotRetryException(remote_ex.exception_class_name());
+ });
+ return !do_not_retry;
+}
+
+/**
+ * Returns whether the java exception class extends DoNotRetryException.
+ * In the java side, we just have a hierarchy of Exception classes that we use
+ * both client side and server side. On the client side, we rethrow the server
+ * side exception by un-wrapping the exception from a RemoteException or a ServiceException
+ * (see ConnectionUtils.translateException() in Java).
+ * Since this object-hierarchy info is not available in C++ side, we are doing a
+ * very fragile catch-all list of all exception types in Java that extend the
+ * DoNotRetryException class type.
+ */
+bool ExceptionUtil::IsJavaDoNotRetryException(const std::string& java_class_name) {
+ for (auto exception : kAllDoNotRetryIOExceptions) {
+ if (java_class_name == exception) {
+ return true;
+ }
+ }
+ return false;
+}
+
+/**
+ * Returns whether the scanner is closed when the client received the
+ * remote exception.
+ * Since the object-hierarchy info is not available in C++ side, we are doing a
+ * very fragile catch-all list of all exception types in Java that extend these
+ * three base classes: UnknownScannerException, NotServingRegionException,
+ * RegionServerStoppedException
+ */
+bool ExceptionUtil::IsScannerClosed(const folly::exception_wrapper& exception) {
+ bool scanner_closed = false;
+ exception.with_exception([&](const RemoteException& remote_ex) {
+ auto java_class = remote_ex.exception_class_name();
+ if (java_class == kUnknownScannerException || java_class == kNotServingRegionException ||
+ java_class == kRegionInRecoveryException || java_class == kRegionOpeningException ||
+ java_class == kRegionMovedException || java_class == kRegionServerStoppedException ||
+ java_class == kRegionServerAbortedException) {
+ scanner_closed = true;
+ }
+ });
+ return scanner_closed;
+}
+
+/**
+ * Returns whether the wrapped exception is a java exception of type OutOfOrderScannerNextException
+ * or ScannerResetException. These two exception types are thrown from the server side when the
+ * scanner on the server side is closed.
+ */
+bool ExceptionUtil::IsScannerOutOfOrder(const folly::exception_wrapper& exception) {
+ bool scanner_out_of_order = false;
+ exception.with_exception([&](const RemoteException& remote_ex) {
+ auto java_class = remote_ex.exception_class_name();
+ if (java_class == kOutOfOrderScannerNextException || java_class == kScannerResetException) {
+ scanner_out_of_order = true;
+ }
+ });
+ return scanner_out_of_order;
+}
+} // namespace hbase
diff --git a/exceptions/exception.h b/exceptions/exception.h
index 9cbd7ae..bdedff4 100644
--- a/exceptions/exception.h
+++ b/exceptions/exception.h
@@ -21,6 +21,7 @@
#include <folly/ExceptionWrapper.h>
#include <folly/io/IOBuf.h>
#include <exception>
+#include <memory>
#include <string>
#include <vector>
@@ -28,10 +29,10 @@ namespace hbase {
class ThrowableWithExtraContext {
public:
- ThrowableWithExtraContext(folly::exception_wrapper cause, const long& when)
+ ThrowableWithExtraContext(folly::exception_wrapper cause, const int64_t& when)
: cause_(cause), when_(when), extras_("") {}
- ThrowableWithExtraContext(folly::exception_wrapper cause, const long& when,
+ ThrowableWithExtraContext(folly::exception_wrapper cause, const int64_t& when,
const std::string& extras)
: cause_(cause), when_(when), extras_(extras) {}
@@ -45,23 +46,44 @@ class ThrowableWithExtraContext {
private:
folly::exception_wrapper cause_;
- long when_;
+ int64_t when_;
std::string extras_;
};
class IOException : public std::logic_error {
public:
- IOException() : logic_error("") {}
+ IOException() : logic_error(""), do_not_retry_(false) {}
+
+ explicit IOException(const std::string& what) : logic_error(what), do_not_retry_(false) {}
+
+ IOException(const std::string& what, bool do_not_retry)
+ : logic_error(what), do_not_retry_(do_not_retry) {}
- IOException(const std::string& what) : logic_error(what) {}
IOException(const std::string& what, folly::exception_wrapper cause)
- : logic_error(what), cause_(cause) {}
+ : logic_error(what), cause_(cause), do_not_retry_(false) {}
+
+ IOException(const std::string& what, folly::exception_wrapper cause, bool do_not_retry)
+ : logic_error(what), cause_(cause), do_not_retry_(do_not_retry) {}
+
virtual ~IOException() = default;
virtual folly::exception_wrapper cause() { return cause_; }
+ bool do_not_retry() const { return do_not_retry_; }
+
+ IOException* set_do_not_retry(bool value) {
+ do_not_retry_ = value;
+ return this;
+ }
+
private:
folly::exception_wrapper cause_;
+ // In case the exception is a RemoteException, do_not_retry information can come from
+ // the PB field in the RPC response, or it can be deduced from the Java-exception
+ // hierarchy in ExceptionUtil::ShouldRetry(). In case this is a client-side exception
+ // raised from the C++ internals, set this field so that the retrying callers can
+ // re-throw the exception without retrying.
+ bool do_not_retry_;
};
class RetriesExhaustedException : public IOException {
@@ -70,9 +92,12 @@ class RetriesExhaustedException : public IOException {
std::shared_ptr<std::vector<ThrowableWithExtraContext>> exceptions)
: IOException(GetMessage(num_retries, exceptions),
exceptions->empty() ? folly::exception_wrapper{}
- : (*exceptions)[exceptions->size() - 1].cause()) {}
+ : (*exceptions)[exceptions->size() - 1].cause()),
+ num_retries_(num_retries) {}
virtual ~RetriesExhaustedException() = default;
+ int32_t num_retries() const { return num_retries_; }
+
private:
std::string GetMessage(const int& num_retries,
std::shared_ptr<std::vector<ThrowableWithExtraContext>> exceptions) {
@@ -85,18 +110,19 @@ class RetriesExhaustedException : public IOException {
}
return buffer;
}
-};
-class HBaseIOException : public IOException {};
+ private:
+ int32_t num_retries_;
+};
class RemoteException : public IOException {
public:
- RemoteException() : port_(0), do_not_retry_(false) {}
+ RemoteException() : IOException(), port_(0) {}
- RemoteException(const std::string& what) : IOException(what), port_(0), do_not_retry_(false) {}
+ explicit RemoteException(const std::string& what) : IOException(what), port_(0) {}
RemoteException(const std::string& what, folly::exception_wrapper cause)
- : IOException(what, cause), port_(0), do_not_retry_(false) {}
+ : IOException(what, cause), port_(0) {}
virtual ~RemoteException() = default;
@@ -128,19 +154,23 @@ class RemoteException : public IOException {
return this;
}
- bool do_not_retry() const { return do_not_retry_; }
-
- RemoteException* set_do_not_retry(bool value) {
- do_not_retry_ = value;
- return this;
- }
-
private:
std::string exception_class_name_;
std::string stack_trace_;
std::string hostname_;
int port_;
- bool do_not_retry_;
+};
+
+/**
+ * Raised from the client side if we cannot find the table (does not have anything to
+ * do with the Java exception of the same name).
+ */
+class TableNotFoundException : public IOException {
+ public:
+ explicit TableNotFoundException(const std::string& table_name)
+ : IOException("Table cannot be found:" + table_name, true) {}
+
+ virtual ~TableNotFoundException() = default;
};
/**
@@ -174,54 +204,113 @@ class ExceptionUtil {
static constexpr const char* kScannerResetException =
"org.apache.hadoop.hbase.exceptions.ScannerResetException";
+ // All other DoNotRetryIOExceptions
+ static constexpr const char* kDoNotRetryIOException =
+ "org.apache.hadoop.hbase.DoNotRetryIOException";
+ static constexpr const char* kTableNotFoundException =
+ "org.apache.hadoop.hbase.TableNotFoundException";
+ static constexpr const char* kTableNotEnabledException =
+ "org.apache.hadoop.hbase.TableNotEnabledException";
+ static constexpr const char* kCoprocessorException =
+ "org.apache.hadoop.hbase.coprocessor.CoprocessorException";
+ static constexpr const char* kBypassCoprocessorException =
+ "org.apache.hadoop.hbase.coprocessor.BypassCoprocessorException";
+ static constexpr const char* kInvalidFamilyOperationException =
+ "org.apache.hadoop.hbase.InvalidFamilyOperationException";
+ static constexpr const char* kServerTooBusyException =
+ "org.apache.hadoop.hbase.ipc.ServerTooBusyException"; // This should NOT be DNRIOE?
+ static constexpr const char* kFailedSanityCheckException =
+ "org.apache.hadoop.hbase.exceptions.FailedSanityCheckException";
+ static constexpr const char* kCorruptHFileException =
+ "org.apache.hadoop.hbase.io.hfile.CorruptHFileException";
+ static constexpr const char* kLabelAlreadyExistsException =
+ "org.apache.hadoop.hbase.security.visibility.LabelAlreadyExistsException";
+ static constexpr const char* kFatalConnectionException =
+ "org.apache.hadoop.hbase.ipc.FatalConnectionException";
+ static constexpr const char* kUnsupportedCryptoException =
+ "org.apache.hadoop.hbase.ipc.UnsupportedCryptoException";
+ static constexpr const char* kUnsupportedCellCodecException =
+ "org.apache.hadoop.hbase.ipc.UnsupportedCellCodecException";
+ static constexpr const char* kEmptyServiceNameException =
+ "org.apache.hadoop.hbase.ipc.EmptyServiceNameException";
+ static constexpr const char* kUnknownServiceException =
+ "org.apache.hadoop.hbase.ipc.UnknownServiceException";
+ static constexpr const char* kWrongVersionException =
+ "org.apache.hadoop.hbase.ipc.WrongVersionException";
+ static constexpr const char* kBadAuthException = "org.apache.hadoop.hbase.ipc.BadAuthException";
+ static constexpr const char* kUnsupportedCompressionCodecException =
+ "org.apache.hadoop.hbase.ipc.UnsupportedCompressionCodecException";
+ static constexpr const char* kDoNotRetryRegionException =
+ "org.apache.hadoop.hbase.client.DoNotRetryRegionException";
+ static constexpr const char* kRowTooBigException =
+ "org.apache.hadoop.hbase.client.RowTooBigException";
+ static constexpr const char* kRowTooBigExceptionDeprecated =
+ "org.apache.hadoop.hbase.regionserver.RowTooBigException";
+ static constexpr const char* kUnknownRegionException =
+ "org.apache.hadoop.hbase.UnknownRegionException";
+ static constexpr const char* kMergeRegionException =
+ "org.apache.hadoop.hbase.exceptions.MergeRegionException";
+ static constexpr const char* kNoServerForRegionException =
+ "org.apache.hadoop.hbase.client.NoServerForRegionException";
+ static constexpr const char* kQuotaExceededException =
+ "org.apache.hadoop.hbase.quotas.QuotaExceededException";
+ static constexpr const char* kSpaceLimitingException =
+ "org.apache.hadoop.hbase.quotas.SpaceLimitingException";
+ static constexpr const char* kThrottlingException =
+ "org.apache.hadoop.hbase.quotas.ThrottlingException";
+ static constexpr const char* kAccessDeniedException =
+ "org.apache.hadoop.hbase.security.AccessDeniedException";
+ static constexpr const char* kUnknownProtocolException =
+ "org.apache.hadoop.hbase.exceptions.UnknownProtocolException";
+ static constexpr const char* kRequestTooBigException =
+ "org.apache.hadoop.hbase.exceptions.RequestTooBigException";
+ static constexpr const char* kNotAllMetaRegionsOnlineException =
+ "org.apache.hadoop.hbase.NotAllMetaRegionsOnlineException";
+ static constexpr const char* kConstraintException =
+ "org.apache.hadoop.hbase.constraint.ConstraintException";
+ static constexpr const char* kNoSuchColumnFamilyException =
+ "org.apache.hadoop.hbase.regionserver.NoSuchColumnFamilyException";
+ static constexpr const char* kLeaseException =
+ "org.apache.hadoop.hbase.regionserver.LeaseException";
+ static constexpr const char* kInvalidLabelException =
+ "org.apache.hadoop.hbase.security.visibility.InvalidLabelException";
+
+ // TODO:
+ // These exceptions are not thrown in the regular read / write paths, although they are
+ // DoNotRetryIOExceptions. Add these to the list below in case we start doing Admin/DDL ops
+ // ReplicationPeerNotFoundException, XXXSnapshotException, NamespaceExistException,
+ // NamespaceNotFoundException, TableExistsException, TableNotDisabledException,
+ static const std::vector<const char*> kAllDoNotRetryIOExceptions;
+
public:
/**
* Returns whether or not the exception should be retried by looking at the
- * remote exception.
+ * client-side IOException, or RemoteException coming from server side.
*/
- static bool ShouldRetry(const folly::exception_wrapper& error) {
- bool do_not_retry = false;
- error.with_exception(
- [&](const RemoteException& remote_ex) { do_not_retry = remote_ex.do_not_retry(); });
- return !do_not_retry;
- }
+ static bool ShouldRetry(const folly::exception_wrapper& error);
/**
- * Returns whether the scanner is closed when the client received the
- * remote exception.
- * Ok, here is a nice detail about the java exceptions. In the java side, we
- * just have a hierarchy of Exception classes that we use both client side and
- * server side. On the client side, we rethrow the server side exception by
- * un-wrapping the exception from a RemoteException or a ServiceException
+ * Returns whether the java exception class extends DoNotRetryException.
+ * In the java side, we just have a hierarchy of Exception classes that we use
+ * both client side and server side. On the client side, we rethrow the server
+ * side exception by un-wrapping the exception from a RemoteException or a ServiceException
* (see ConnectionUtils.translateException() in Java).
* Since this object-hierarchy info is not available in C++ side, we are doing a
+ * very fragile catch-all list of all exception types in Java that extend the
+ * DoNotRetryException class type.
+ */
+ static bool IsJavaDoNotRetryException(const std::string& java_class_name);
+
+ /**
+ * Returns whether the scanner is closed when the client received the
+ * remote exception.
+ * Since the object-hierarchy info is not available in C++ side, we are doing a
* very fragile catch-all list of all exception types in Java that extend these
* three base classes: UnknownScannerException, NotServingRegionException,
* RegionServerStoppedException
*/
- static bool IsScannerClosed(const folly::exception_wrapper& exception) {
- bool scanner_closed = false;
- exception.with_exception([&](const RemoteException& remote_ex) {
- auto java_class = remote_ex.exception_class_name();
- if (java_class == kUnknownScannerException || java_class == kNotServingRegionException ||
- java_class == kRegionInRecoveryException || java_class == kRegionOpeningException ||
- java_class == kRegionMovedException || java_class == kRegionServerStoppedException ||
- java_class == kRegionServerAbortedException) {
- scanner_closed = true;
- }
- });
- return scanner_closed;
- }
+ static bool IsScannerClosed(const folly::exception_wrapper& exception);
- static bool IsScannerOutOfOrder(const folly::exception_wrapper& exception) {
- bool scanner_out_of_order = false;
- exception.with_exception([&](const RemoteException& remote_ex) {
- auto java_class = remote_ex.exception_class_name();
- if (java_class == kOutOfOrderScannerNextException || java_class == kScannerResetException) {
- scanner_out_of_order = true;
- }
- });
- return scanner_out_of_order;
- }
+ static bool IsScannerOutOfOrder(const folly::exception_wrapper& exception);
};
} // namespace hbase
[hbase] 66/137: HBASE-17612 [C++] Set client version info in RPC
header
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit f58f72a1dc1d9552f7ad7b2e4fb5587fbba3279c
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Mon Feb 13 18:52:04 2017 -0800
HBASE-17612 [C++] Set client version info in RPC header
---
.gitignore | 3 +++
Makefile | 8 +++++---
utils/BUCK => bin/copy-version.sh | 20 ++++++++------------
bin/start-docker.sh | 3 +++
serde/BUCK | 5 +----
serde/rpc.cc | 30 +++++++++++++++++++++++++++++-
serde/rpc.h | 2 ++
utils/BUCK | 2 +-
8 files changed, 52 insertions(+), 21 deletions(-)
diff --git a/.gitignore b/.gitignore
index c0032e4..c99ce59 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,3 +24,6 @@ buck-out
# Thirdparty dirs
third-party/*
/gcc-debug/
+
+# Generated files
+utils/version.h
diff --git a/Makefile b/Makefile
index 99e38ef..84ae556 100644
--- a/Makefile
+++ b/Makefile
@@ -57,7 +57,7 @@ ARC_DEBUG=$(DEBUG_PATH)/libHbaseClient_d.a
vpath %.cc $(SRC_DIR)
-build: checkdirs protos $(LIB_DEBUG) $(LIB_RELEASE) $(ARC_DEBUG) $(ARC_RELEASE)
+build: checkdirs copyfiles protos $(LIB_DEBUG) $(LIB_RELEASE) $(ARC_DEBUG) $(ARC_RELEASE)
$(LIB_DEBUG): $(DEBUG_BUILD_DIR)
define make-goal-dbg
$1/%.o: %.cc $(DEPS) $(PROTODEPS) $(PROTOSRC)
@@ -72,10 +72,12 @@ endef
.PHONY: all clean install
-
-
checkdirs: $(DEBUG_BUILD_DIR) $(RELEASE_BUILD_DIR) $(PROTO_SRC_DIR)
+copyfiles:
+ bin/copy-protobuf.sh
+ bin/copy-version.sh
+
protos: createprotosrc
@make all -f Makefile.protos
diff --git a/utils/BUCK b/bin/copy-version.sh
old mode 100644
new mode 100755
similarity index 69%
copy from utils/BUCK
copy to bin/copy-version.sh
index 120331a..b33db7a
--- a/utils/BUCK
+++ b/bin/copy-version.sh
@@ -1,3 +1,4 @@
+#!/usr/bin/env bash
##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
@@ -14,16 +15,11 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
+set -euo pipefail
+IFS=$'\n\t'
-cxx_library(
- name="utils",
- exported_headers=["user-util.h",],
- srcs=["user-util.cc",],
- deps=['//third-party:folly',],
- tests=[":user-util-test"],
- visibility=['PUBLIC',],
- compiler_flags=['-Weffc++'],)
-cxx_test(
- name="user-util-test",
- srcs=["user-util-test.cc",],
- deps=[":utils",],)
+# Copy the version.h generated from hbase-common/src/saveVersion.sh script via the mvn build
+BIN_DIR=$(dirname "$0")
+VERSION_SOURCE_DIR="${BIN_DIR}/../../hbase-common/target/generated-sources/native/utils/"
+VERSION_DEST_DIR="${BIN_DIR}/../utils/"
+cp $VERSION_SOURCE_DIR/* $VERSION_DEST_DIR/
diff --git a/bin/start-docker.sh b/bin/start-docker.sh
index 38affa0..1380cdf 100755
--- a/bin/start-docker.sh
+++ b/bin/start-docker.sh
@@ -26,7 +26,10 @@ eval "$(docker-machine env dinghy)"
BIN_DIR=$(pushd `dirname "$0"` 2>&1 > /dev/null && pwd && popd 2>&1 > /dev/null)
BASE_DIR=$(pushd "${BIN_DIR}/../" 2>&1 > /dev/null && pwd && popd 2>&1 > /dev/null)
+# TODO: this is wrong. These should be called from BUCK, but I could not find a
+# way to execute scripts in BUCK that are in under bin/.
${BIN_DIR}/copy-protobuf.sh
+${BIN_DIR}/copy-version.sh
# Go into the base dir. This just makes things cleaner.
pushd ${BASE_DIR}
diff --git a/serde/BUCK b/serde/BUCK
index c6809a8..38e7b4d 100644
--- a/serde/BUCK
+++ b/serde/BUCK
@@ -31,10 +31,7 @@ cxx_library(
"rpc.cc",
"zk.cc",
],
- deps=[
- "//if:if",
- "//third-party:folly",
- ],
+ deps=["//if:if", "//third-party:folly", "//utils:utils"],
tests=[
":client-deserializer-test",
":client-serializer-test",
diff --git a/serde/rpc.cc b/serde/rpc.cc
index d5bca62..e657a64 100644
--- a/serde/rpc.cc
+++ b/serde/rpc.cc
@@ -19,16 +19,18 @@
#include "serde/rpc.h"
+#include <folly/Conv.h>
#include <folly/Logging.h>
#include <folly/io/Cursor.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
#include <google/protobuf/message.h>
+#include <boost/algorithm/string.hpp>
#include <utility>
-#include "if/HBase.pb.h"
#include "if/RPC.pb.h"
+#include "utils/version.h"
using namespace hbase;
@@ -111,12 +113,38 @@ unique_ptr<IOBuf> RpcSerde::Header(const string &user) {
// TODO: send the service name and user from the RpcClient
h.set_service_name(INTERFACE);
+ std::unique_ptr<pb::VersionInfo> version_info = CreateVersionInfo();
+
+ h.set_allocated_version_info(version_info.release());
+
if (codec_ != nullptr) {
h.set_cell_block_codec_class(codec_->java_class_name());
}
return PrependLength(SerializeMessage(h));
}
+std::unique_ptr<pb::VersionInfo> RpcSerde::CreateVersionInfo() {
+ std::unique_ptr<pb::VersionInfo> version_info = std::make_unique<pb::VersionInfo>();
+ version_info->set_user(Version::user);
+ version_info->set_revision(Version::revision);
+ version_info->set_url(Version::url);
+ version_info->set_date(Version::date);
+ version_info->set_src_checksum(Version::src_checksum);
+ version_info->set_version(Version::version);
+
+ std::string version{Version::version};
+ std::vector<std::string> version_parts;
+ boost::split(version_parts, version, boost::is_any_of("."), boost::token_compress_on);
+ uint32_t major_version = 0, minor_version = 0;
+ if (version_parts.size() >= 2) {
+ version_info->set_version_major(folly::to<uint32_t>(version_parts[0]));
+ version_info->set_version_minor(folly::to<uint32_t>(version_parts[1]));
+ }
+
+ VLOG(1) << "Client VersionInfo:" << version_info->ShortDebugString();
+ return version_info;
+}
+
unique_ptr<IOBuf> RpcSerde::Request(const uint32_t call_id, const string &method,
const Message *msg) {
pb::RequestHeader rq;
diff --git a/serde/rpc.h b/serde/rpc.h
index c59f903..abebe94 100644
--- a/serde/rpc.h
+++ b/serde/rpc.h
@@ -21,6 +21,7 @@
#include <memory>
#include <string>
+#include "if/HBase.pb.h"
#include "serde/cell-scanner.h"
#include "serde/codec.h"
@@ -120,5 +121,6 @@ class RpcSerde {
/* data */
uint8_t auth_type_;
std::shared_ptr<Codec> codec_;
+ std::unique_ptr<pb::VersionInfo> CreateVersionInfo();
};
} // namespace hbase
diff --git a/utils/BUCK b/utils/BUCK
index 120331a..796f2f5 100644
--- a/utils/BUCK
+++ b/utils/BUCK
@@ -17,7 +17,7 @@
cxx_library(
name="utils",
- exported_headers=["user-util.h",],
+ exported_headers=["user-util.h", "version.h"],
srcs=["user-util.cc",],
deps=['//third-party:folly',],
tests=[":user-util-test"],
[hbase] 42/137: HBASE-16740 start-docker.sh fails to run by
complaining bzip2 error (Xiaobing Zhou)
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 05f51ae037fc1740f69d97dc61ab45c57efed82e
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Mon Oct 10 22:46:49 2016 -0700
HBASE-16740 start-docker.sh fails to run by complaining bzip2 error (Xiaobing Zhou)
---
Dockerfile | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/Dockerfile b/Dockerfile
index ca87523..4504a8e 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -31,8 +31,10 @@ RUN apt-get install -y vim maven inetutils-ping python-pip doxygen graphviz && \
rm -rf /var/lib/{apt,dpkg,cache,log}/ && \
rm -rf /tmp/*
-RUN git clone --depth 1 --branch v2.6.1 https://github.com/google/protobuf.git /usr/src/protobuf && \
+RUN git clone https://github.com/google/protobuf.git /usr/src/protobuf && \
cd /usr/src/protobuf/ && \
+ git checkout 2.7.0 && \
+ mkdir gmock && \
ldconfig && \
./autogen.sh && \
./configure && \
[hbase] 121/137: HBASE-18603 buck complains about Build target path
containing double slash
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit b665d673e2c4662a83d0d3ba3dccaa0101871e08
Author: tedyu <yu...@gmail.com>
AuthorDate: Tue Aug 15 11:26:56 2017 -0700
HBASE-18603 buck complains about Build target path containing double slash
---
exceptions/BUCK | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/exceptions/BUCK b/exceptions/BUCK
index e2f03a1..3d66d72 100644
--- a/exceptions/BUCK
+++ b/exceptions/BUCK
@@ -27,7 +27,7 @@ cxx_library(
"//third-party:folly",
],
compiler_flags=['-Weffc++'],
- visibility=['//core/...', '//connection//...'],)
+ visibility=['//core/...', '//connection/...'],)
cxx_test(
name="exception-test",
srcs=[
[hbase] 99/137: HBASE-18178 [C++] Retrying meta location lookup and
zookeeper connection
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit aa3c2167227a222bb8d71baca68ffa0712921a18
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Fri Jun 16 11:55:51 2017 -0700
HBASE-18178 [C++] Retrying meta location lookup and zookeeper connection
---
connection/client-handler.cc | 9 +--
core/BUCK | 12 ++++
core/async-rpc-retrying-caller.cc | 17 +++---
core/async-rpc-retrying-test.cc | 3 +-
core/connection-configuration.h | 2 +-
core/location-cache-retry-test.cc | 112 ++++++++++++++++++++++++++++++++++++++
core/location-cache.cc | 100 ++++++++++++++++++++++++----------
core/location-cache.h | 8 ++-
core/meta-utils.cc | 42 ++++++++++----
core/meta-utils.h | 24 +++++++-
core/region-location.h | 19 +------
core/response-converter.cc | 2 +-
core/simple-client.cc | 18 ++++--
core/zk-util.cc | 4 ++
core/zk-util.h | 5 ++
serde/region-info.h | 4 +-
serde/table-name.h | 5 +-
test-util/mini-cluster.cc | 4 ++
test-util/test-util.cc | 4 ++
test-util/test-util.h | 1 +
utils/bytes-util-test.cc | 3 +-
utils/bytes-util.cc | 4 +-
22 files changed, 315 insertions(+), 87 deletions(-)
diff --git a/connection/client-handler.cc b/connection/client-handler.cc
index 894ecb3..775df68 100644
--- a/connection/client-handler.cc
+++ b/connection/client-handler.cc
@@ -51,7 +51,7 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<folly::IOBuf> buf) {
int used_bytes = serde_.ParseDelimited(buf.get(), &header);
VLOG(3) << "Read RPC ResponseHeader size=" << used_bytes << " call_id=" << header.call_id()
- << " has_exception=" << header.has_exception();
+ << " has_exception=" << header.has_exception() << ", server: " << server_;
// Get the response protobuf from the map
auto search = resp_msgs_->find(header.call_id());
@@ -80,7 +80,8 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<folly::IOBuf> buf) {
}
VLOG(3) << "Read RPCResponse, buf length:" << buf->length()
- << ", header PB length:" << used_bytes << ", cell_block length:" << cell_block_length;
+ << ", header PB length:" << used_bytes << ", cell_block length:" << cell_block_length
+ << ", server: " << server_;
// Make sure that bytes were parsed.
CHECK((used_bytes + cell_block_length) == buf->length());
@@ -113,7 +114,7 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<folly::IOBuf> buf) {
VLOG(3) << "Exception RPC ResponseHeader, call_id=" << header.call_id()
<< " exception.what=" << remote_exception->what()
- << ", do_not_retry=" << remote_exception->do_not_retry();
+ << ", do_not_retry=" << remote_exception->do_not_retry() << ", server: " << server_;
received->set_exception(folly::exception_wrapper{*remote_exception});
}
ctx->fireRead(std::move(received));
@@ -129,7 +130,7 @@ folly::Future<folly::Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Re
ctx->fireWrite(std::move(header));
});
- VLOG(3) << "Writing RPC Request:" << r->DebugString();
+ VLOG(3) << "Writing RPC Request:" << r->DebugString() << ", server: " << server_;
// Now store the call id to response.
resp_msgs_->insert(r->call_id(), r->resp_msg());
diff --git a/core/BUCK b/core/BUCK
index 47e97f5..464c010 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -137,6 +137,18 @@ cxx_test(
],
run_test_separately=True,)
cxx_test(
+ name="location-cache-retry-test",
+ srcs=[
+ "location-cache-retry-test.cc",
+ ],
+ deps=[
+ ":core",
+ "//if:if",
+ "//serde:serde",
+ "//test-util:test-util",
+ ],
+ run_test_separately=True,)
+cxx_test(
name="cell-test",
srcs=[
"cell-test.cc",
diff --git a/core/async-rpc-retrying-caller.cc b/core/async-rpc-retrying-caller.cc
index 0302ad3..aee7d0b 100644
--- a/core/async-rpc-retrying-caller.cc
+++ b/core/async-rpc-retrying-caller.cc
@@ -168,14 +168,17 @@ void AsyncSingleRequestRpcRetryingCaller<RESP>::Call(const RegionLocation& loc)
ResetController(controller_, call_timeout_ns);
- callable_(controller_, std::make_shared<RegionLocation>(loc), rpc_client)
- .then([this](const RESP& resp) { this->promise_->setValue(std::move(resp)); })
- .onError([&, this](const exception_wrapper& e) {
+ // TODO: RegionLocation should propagate through these method chains as a shared_ptr.
+ // Otherwise, it may get deleted underneat us. We are just copying for now.
+ auto loc_ptr = std::make_shared<RegionLocation>(loc);
+ callable_(controller_, loc_ptr, rpc_client)
+ .then([loc_ptr, this](const RESP& resp) { this->promise_->setValue(std::move(resp)); })
+ .onError([&, loc_ptr, this](const exception_wrapper& e) {
OnError(e,
[&, this]() -> std::string {
- return "Call to " + folly::sformat("{0}:{1}", loc.server_name().host_name(),
- loc.server_name().port()) +
- " for '" + row_ + "' in " + loc.DebugString() + " of " +
+ return "Call to " + folly::sformat("{0}:{1}", loc_ptr->server_name().host_name(),
+ loc_ptr->server_name().port()) +
+ " for '" + row_ + "' in " + loc_ptr->DebugString() + " of " +
table_name_->namespace_() + "::" + table_name_->qualifier() +
" failed, tries = " + std::to_string(tries_) + ", maxAttempts = " +
std::to_string(max_attempts_) + ", timeout = " +
@@ -184,7 +187,7 @@ void AsyncSingleRequestRpcRetryingCaller<RESP>::Call(const RegionLocation& loc)
" ms";
},
[&, this](const exception_wrapper& error) {
- conn_->region_locator()->UpdateCachedLocation(loc, error);
+ conn_->region_locator()->UpdateCachedLocation(*loc_ptr, error);
});
});
}
diff --git a/core/async-rpc-retrying-test.cc b/core/async-rpc-retrying-test.cc
index 0f83914..f887815 100644
--- a/core/async-rpc-retrying-test.cc
+++ b/core/async-rpc-retrying-test.cc
@@ -147,8 +147,7 @@ class MockWrongRegionAsyncRegionLocator : public AsyncRegionLocatorBase {
folly::Promise<std::shared_ptr<RegionLocation>> promise;
/* set random region name, simulating invalid region */
auto result = std::make_shared<RegionLocation>(
- "whatever-region-name", region_location_->region_info(), region_location_->server_name(),
- region_location_->service());
+ "whatever-region-name", region_location_->region_info(), region_location_->server_name());
promise.setValue(result);
return promise.getFuture();
}
diff --git a/core/connection-configuration.h b/core/connection-configuration.h
index adc8d5b..995798e 100644
--- a/core/connection-configuration.h
+++ b/core/connection-configuration.h
@@ -143,7 +143,7 @@ class ConnectionConfiguration {
*/
static constexpr const char* kClientRetriesNumber = "hbase.client.retries.number";
- static constexpr const uint32_t kDefaultClientRetriesNumber = 31;
+ static constexpr const uint32_t kDefaultClientRetriesNumber = 35;
/**
* Configure the number of failures after which the client will start logging. A few failures
diff --git a/core/location-cache-retry-test.cc b/core/location-cache-retry-test.cc
new file mode 100644
index 0000000..988f994
--- /dev/null
+++ b/core/location-cache-retry-test.cc
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <gtest/gtest.h>
+
+#include "core/append.h"
+#include "core/cell.h"
+#include "core/client.h"
+#include "core/configuration.h"
+#include "core/delete.h"
+#include "core/get.h"
+#include "core/hbase-configuration-loader.h"
+#include "core/increment.h"
+#include "core/meta-utils.h"
+#include "core/put.h"
+#include "core/result.h"
+#include "core/table.h"
+#include "exceptions/exception.h"
+#include "serde/table-name.h"
+#include "test-util/test-util.h"
+#include "utils/bytes-util.h"
+
+using hbase::Cell;
+using hbase::Configuration;
+using hbase::Get;
+using hbase::MetaUtil;
+using hbase::RetriesExhaustedException;
+using hbase::Put;
+using hbase::Table;
+using hbase::TestUtil;
+
+using std::chrono_literals::operator""s;
+
+class LocationCacheRetryTest : public ::testing::Test {
+ public:
+ static std::unique_ptr<hbase::TestUtil> test_util;
+ static void SetUpTestCase() {
+ google::InstallFailureSignalHandler();
+ test_util = std::make_unique<hbase::TestUtil>();
+ test_util->StartMiniCluster(2);
+ test_util->conf()->SetInt("hbase.client.retries.number", 5);
+ }
+};
+
+std::unique_ptr<hbase::TestUtil> LocationCacheRetryTest::test_util = nullptr;
+
+TEST_F(LocationCacheRetryTest, GetFromMetaTable) {
+ auto tn = folly::to<hbase::pb::TableName>("hbase:meta");
+ auto row = "test1";
+
+ hbase::Client client(*LocationCacheRetryTest::test_util->conf());
+
+ // do a get against the other table, but not the actual table "t".
+ auto table = client.Table(tn);
+ hbase::Get get(row);
+ auto result = table->Get(get);
+
+ LocationCacheRetryTest::test_util->MoveRegion(MetaUtil::kMetaRegion, "");
+
+ std::this_thread::sleep_for(3s); // sleep 3 sec
+
+ result = table->Get(get);
+}
+
+TEST_F(LocationCacheRetryTest, PutGet) {
+ LocationCacheRetryTest::test_util->CreateTable("t", "d");
+ LocationCacheRetryTest::test_util->CreateTable("t2", "d");
+
+ auto tn = folly::to<hbase::pb::TableName>("t");
+ auto tn2 = folly::to<hbase::pb::TableName>("t2");
+ auto row = "test1";
+
+ hbase::Client client(*LocationCacheRetryTest::test_util->conf());
+
+ // do a get against the other table, but not the actual table "t".
+ auto table = client.Table(tn);
+ auto table2 = client.Table(tn2);
+ hbase::Get get(row);
+ auto result = table2->Get(get);
+
+ // we should have already cached the location of meta right now. Now
+ // move the meta region to the other server so that we will get a NotServingRegionException
+ // when we do the actual location lookup request. If there is no invalidation
+ // of the meta's own location, then following put/get will result in retries exhausted.
+ LocationCacheRetryTest::test_util->MoveRegion(MetaUtil::kMetaRegion, "");
+
+ std::this_thread::sleep_for(3s); // sleep 3 sec
+
+ table->Put(Put{row}.AddColumn("d", "1", "value1"));
+
+ result = table->Get(get);
+
+ ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty.";
+ EXPECT_EQ("test1", result->Row());
+ EXPECT_EQ("value1", *(result->Value("d", "1")));
+}
diff --git a/core/location-cache.cc b/core/location-cache.cc
index dfe3e9f..5f68420 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -24,8 +24,12 @@
#include <wangle/concurrent/CPUThreadPoolExecutor.h>
#include <wangle/concurrent/IOThreadPoolExecutor.h>
+#include <map>
+#include <utility>
+
#include "connection/response.h"
#include "connection/rpc-connection.h"
+#include "core/meta-utils.h"
#include "exceptions/exception.h"
#include "if/Client.pb.h"
#include "if/ZooKeeper.pb.h"
@@ -33,8 +37,6 @@
#include "serde/server-name.h"
#include "serde/zk.h"
-#include <utility>
-
using hbase::pb::MetaRegionServer;
using hbase::pb::ServerName;
using hbase::pb::TableName;
@@ -54,27 +56,49 @@ LocationCache::LocationCache(std::shared_ptr<hbase::Configuration> conf,
cached_locations_(),
locations_lock_() {
zk_quorum_ = ZKUtil::ParseZooKeeperQuorum(*conf_);
- zk_ = zookeeper_init(zk_quorum_.c_str(), nullptr, 1000, 0, 0, 0);
+ EnsureZooKeeperConnection();
+}
+
+LocationCache::~LocationCache() { CloseZooKeeperConnection(); }
+
+void LocationCache::CloseZooKeeperConnection() {
+ if (zk_ != nullptr) {
+ zookeeper_close(zk_);
+ zk_ = nullptr;
+ LOG(INFO) << "Closed connection to ZooKeeper.";
+ }
}
-LocationCache::~LocationCache() {
- zookeeper_close(zk_);
- zk_ = nullptr;
- LOG(INFO) << "Closed connection to ZooKeeper.";
+void LocationCache::EnsureZooKeeperConnection() {
+ if (zk_ == nullptr) {
+ LOG(INFO) << "Connecting to ZooKeeper. Quorum:" + zk_quorum_;
+ auto session_timeout = ZKUtil::SessionTimeout(*conf_);
+ zk_ = zookeeper_init(zk_quorum_.c_str(), nullptr, session_timeout, nullptr, nullptr, 0);
+ }
}
folly::Future<ServerName> LocationCache::LocateMeta() {
- std::lock_guard<std::mutex> g(meta_lock_);
+ std::lock_guard<std::recursive_mutex> g(meta_lock_);
if (meta_promise_ == nullptr) {
this->RefreshMetaLocation();
}
- return meta_promise_->getFuture();
+ return meta_promise_->getFuture().onError([&](const folly::exception_wrapper &ew) {
+ auto promise = InvalidateMeta();
+ promise->setException(ew);
+ return ServerName{};
+ });
}
-void LocationCache::InvalidateMeta() {
+std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> LocationCache::InvalidateMeta() {
+ VLOG(2) << "Invalidating meta location";
+ std::lock_guard<std::recursive_mutex> g(meta_lock_);
if (meta_promise_ != nullptr) {
- std::lock_guard<std::mutex> g(meta_lock_);
- meta_promise_ = nullptr;
+ // return the unique_ptr back to the caller.
+ std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> ret = nullptr;
+ std::swap(ret, meta_promise_);
+ return ret;
+ } else {
+ return nullptr;
}
}
@@ -84,18 +108,21 @@ void LocationCache::RefreshMetaLocation() {
cpu_executor_->add([&] { meta_promise_->setWith([&] { return this->ReadMetaLocation(); }); });
}
+// Note: this is a blocking call to zookeeper
ServerName LocationCache::ReadMetaLocation() {
auto buf = folly::IOBuf::create(4096);
ZkDeserializer derser;
+ EnsureZooKeeperConnection();
// This needs to be int rather than size_t as that's what ZK expects.
int len = buf->capacity();
std::string zk_node = ZKUtil::MetaZNode(*conf_);
- // TODO(elliott): handle disconnects/reconntion as needed.
int zk_result = zoo_get(this->zk_, zk_node.c_str(), 0,
reinterpret_cast<char *>(buf->writableData()), &len, nullptr);
if (zk_result != ZOK || len < 9) {
LOG(ERROR) << "Error getting meta location.";
+ // We just close the zk connection, and let the upper levels retry.
+ CloseZooKeeperConnection();
throw std::runtime_error("Error getting meta location. Quorum: " + zk_quorum_);
}
buf->append(len);
@@ -103,6 +130,8 @@ ServerName LocationCache::ReadMetaLocation() {
MetaRegionServer mrs;
if (derser.Parse(buf.get(), &mrs) == false) {
LOG(ERROR) << "Unable to decode";
+ throw std::runtime_error("Error getting meta location (Unable to decode). Quorum: " +
+ zk_quorum_);
}
return mrs.server();
}
@@ -118,10 +147,15 @@ folly::Future<std::shared_ptr<RegionLocation>> LocationCache::LocateFromMeta(
.then([tn, row, this](std::shared_ptr<RpcConnection> rpc_connection) {
return (*rpc_connection->get_service())(std::move(meta_util_.MetaRequest(tn, row)));
})
- .then([this](std::unique_ptr<Response> resp) {
+ .onError([&](const folly::exception_wrapper &ew) {
+ auto promise = InvalidateMeta();
+ throw ew;
+ return static_cast<std::unique_ptr<Response>>(nullptr);
+ })
+ .then([tn, this](std::unique_ptr<Response> resp) {
// take the protobuf response and make it into
// a region location.
- return meta_util_.CreateLocation(std::move(*resp));
+ return meta_util_.CreateLocation(std::move(*resp), tn);
})
.then([tn, this](std::shared_ptr<RegionLocation> rl) {
// Make sure that the correct location was found.
@@ -134,9 +168,6 @@ folly::Future<std::shared_ptr<RegionLocation>> LocationCache::LocateFromMeta(
.then([this](std::shared_ptr<RegionLocation> rl) {
auto remote_id =
std::make_shared<ConnectionId>(rl->server_name().host_name(), rl->server_name().port());
- // Now fill out the connection.
- // rl->set_service(cp_->GetConnection(remote_id)->get_service()); TODO: this causes wangle
- // assertion errors
return rl;
})
.then([tn, this](std::shared_ptr<RegionLocation> rl) {
@@ -146,9 +177,20 @@ folly::Future<std::shared_ptr<RegionLocation>> LocationCache::LocateFromMeta(
});
}
+constexpr const char *MetaUtil::kMetaRegionName;
+
folly::Future<std::shared_ptr<RegionLocation>> LocationCache::LocateRegion(
const TableName &tn, const std::string &row, const RegionLocateType locate_type,
const int64_t locate_ns) {
+ // We maybe asked to locate meta itself
+ if (MetaUtil::IsMeta(tn)) {
+ return LocateMeta().then([this](const ServerName &server_name) {
+ auto rl = std::make_shared<RegionLocation>(MetaUtil::kMetaRegionName,
+ meta_util_.meta_region_info(), server_name);
+ return rl;
+ });
+ }
+
// TODO: implement region locate type and timeout
auto cached_loc = this->GetCachedLocation(tn, row);
if (cached_loc != nullptr) {
@@ -164,34 +206,28 @@ std::shared_ptr<RegionLocation> LocationCache::GetCachedLocation(const hbase::pb
auto t_locs = this->GetTableLocations(tn);
std::shared_lock<folly::SharedMutexWritePriority> lock(locations_lock_);
- if (VLOG_IS_ON(2)) {
- for (const auto &p : *t_locs) {
- VLOG(2) << "t_locs[" << p.first << "] = " << p.second->DebugString();
- }
- }
-
// looking for the "floor" key as a start key
auto possible_region = t_locs->upper_bound(row);
if (t_locs->empty()) {
- VLOG(2) << "Could not find region in cache, table map is empty";
+ VLOG(5) << "Could not find region in cache, table map is empty";
return nullptr;
}
if (possible_region == t_locs->begin()) {
- VLOG(2) << "Could not find region in cache, all keys are greater, row:" << row
+ VLOG(5) << "Could not find region in cache, all keys are greater, row:" << row
<< " ,possible_region:" << possible_region->second->DebugString();
return nullptr;
}
--possible_region;
- VLOG(2) << "Found possible region in cache for row:" << row
+ VLOG(5) << "Found possible region in cache for row:" << row
<< " ,possible_region:" << possible_region->second->DebugString();
// found possible start key, now need to check end key
if (possible_region->second->region_info().end_key() == "" ||
possible_region->second->region_info().end_key() > row) {
- VLOG(1) << "Found region in cache for row:" << row
+ VLOG(2) << "Found region in cache for row:" << row
<< " ,region:" << possible_region->second->DebugString();
return possible_region->second;
} else {
@@ -261,15 +297,23 @@ void LocationCache::ClearCache() {
// must hold unique lock on locations_lock_
void LocationCache::ClearCachedLocations(const hbase::pb::TableName &tn) {
+ VLOG(1) << "ClearCachedLocations, table:" << folly::to<std::string>(tn);
std::unique_lock<folly::SharedMutexWritePriority> lock(locations_lock_);
cached_locations_.erase(tn);
+ if (MetaUtil::IsMeta(tn)) {
+ InvalidateMeta();
+ }
}
// must hold unique lock on locations_lock_
void LocationCache::ClearCachedLocation(const hbase::pb::TableName &tn, const std::string &row) {
+ VLOG(1) << "ClearCachedLocation, table:" << folly::to<std::string>(tn) << ", row:" << row;
auto table_locs = this->GetTableLocations(tn);
std::unique_lock<folly::SharedMutexWritePriority> lock(locations_lock_);
table_locs->erase(row);
+ if (MetaUtil::IsMeta(tn)) {
+ InvalidateMeta();
+ }
}
void LocationCache::UpdateCachedLocation(const RegionLocation &loc,
diff --git a/core/location-cache.h b/core/location-cache.h
index a3c15cb..a374fb6 100644
--- a/core/location-cache.h
+++ b/core/location-cache.h
@@ -137,7 +137,7 @@ class LocationCache : public AsyncRegionLocator {
/**
* Remove the cached location of meta.
*/
- void InvalidateMeta();
+ std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> InvalidateMeta();
/**
* Return cached region location corresponding to this row,
@@ -186,6 +186,10 @@ class LocationCache : public AsyncRegionLocator {
const std::string &zk_quorum() { return zk_quorum_; }
private:
+ void CloseZooKeeperConnection();
+ void EnsureZooKeeperConnection();
+
+ private:
void RefreshMetaLocation();
hbase::pb::ServerName ReadMetaLocation();
std::shared_ptr<RegionLocation> CreateLocation(const Response &resp);
@@ -198,7 +202,7 @@ class LocationCache : public AsyncRegionLocator {
std::string zk_quorum_;
std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> meta_promise_;
- std::mutex meta_lock_;
+ std::recursive_mutex meta_lock_;
MetaUtil meta_util_;
std::shared_ptr<ConnectionPool> cp_;
diff --git a/core/meta-utils.cc b/core/meta-utils.cc
index 8efecc8..31349a5 100644
--- a/core/meta-utils.cc
+++ b/core/meta-utils.cc
@@ -21,10 +21,13 @@
#include <folly/Conv.h>
#include <memory>
+#include <utility>
+#include <vector>
#include "connection/request.h"
#include "connection/response.h"
#include "core/response-converter.h"
+#include "exceptions/exception.h"
#include "if/Client.pb.h"
#include "serde/region-info.h"
#include "serde/server-name.h"
@@ -38,10 +41,17 @@ using hbase::pb::ServerName;
namespace hbase {
-static const std::string META_REGION = "1588230740";
-static const std::string CATALOG_FAMILY = "info";
-static const std::string REGION_INFO_COLUMN = "regioninfo";
-static const std::string SERVER_COLUMN = "server";
+MetaUtil::MetaUtil() {
+ meta_region_info_.set_start_key("");
+ meta_region_info_.set_end_key("");
+ meta_region_info_.set_offline(false);
+ meta_region_info_.set_split(false);
+ meta_region_info_.set_replica_id(0);
+ meta_region_info_.set_split(false);
+ meta_region_info_.set_region_id(1);
+ meta_region_info_.mutable_table_name()->set_namespace_(MetaUtil::kSystemNamespace);
+ meta_region_info_.mutable_table_name()->set_qualifier(MetaUtil::kMetaTableQualifier);
+}
std::string MetaUtil::RegionLookupRowkey(const TableName &tn, const std::string &row) const {
return folly::to<std::string>(tn, ",", row, ",", "999999999999999999");
@@ -56,7 +66,7 @@ std::unique_ptr<Request> MetaUtil::MetaRequest(const TableName tn, const std::st
// Set the region this scan goes to
auto region = msg->mutable_region();
- region->set_value(META_REGION);
+ region->set_value(MetaUtil::kMetaRegion);
region->set_type(
RegionSpecifier_RegionSpecifierType::RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
@@ -78,30 +88,38 @@ std::unique_ptr<Request> MetaUtil::MetaRequest(const TableName tn, const std::st
// Set the columns that we need
auto info_col = scan->add_column();
- info_col->set_family("info");
- info_col->add_qualifier("server");
- info_col->add_qualifier("regioninfo");
+ info_col->set_family(MetaUtil::kCatalogFamily);
+ info_col->add_qualifier(MetaUtil::kServerColumn);
+ info_col->add_qualifier(MetaUtil::kRegionInfoColumn);
scan->set_start_row(RegionLookupRowkey(tn, row));
return request;
}
-std::shared_ptr<RegionLocation> MetaUtil::CreateLocation(const Response &resp) {
+std::shared_ptr<RegionLocation> MetaUtil::CreateLocation(const Response &resp,
+ const TableName &tn) {
std::vector<std::shared_ptr<Result>> results = ResponseConverter::FromScanResponse(resp);
+ if (results.size() == 0) {
+ throw TableNotFoundException(folly::to<std::string>(tn));
+ }
if (results.size() != 1) {
throw std::runtime_error("Was expecting exactly 1 result in meta scan response, got:" +
std::to_string(results.size()));
}
auto result = *results[0];
- auto region_info_str = result.Value(CATALOG_FAMILY, REGION_INFO_COLUMN);
- auto server_str = result.Value(CATALOG_FAMILY, SERVER_COLUMN);
+ auto region_info_str = result.Value(MetaUtil::kCatalogFamily, MetaUtil::kRegionInfoColumn);
+ auto server_str = result.Value(MetaUtil::kCatalogFamily, MetaUtil::kServerColumn);
CHECK(region_info_str);
CHECK(server_str);
auto row = result.Row();
auto region_info = folly::to<RegionInfo>(*region_info_str);
auto server_name = folly::to<ServerName>(*server_str);
- return std::make_shared<RegionLocation>(row, std::move(region_info), server_name, nullptr);
+ return std::make_shared<RegionLocation>(row, std::move(region_info), server_name);
+}
+
+bool MetaUtil::IsMeta(const hbase::pb::TableName &tn) {
+ return folly::to<std::string>(tn) == MetaUtil::kMetaTableName;
}
} // namespace hbase
diff --git a/core/meta-utils.h b/core/meta-utils.h
index d67f32d..d178179 100644
--- a/core/meta-utils.h
+++ b/core/meta-utils.h
@@ -34,6 +34,17 @@ namespace hbase {
*/
class MetaUtil {
public:
+ static constexpr const char *kSystemNamespace = "hbase";
+ static constexpr const char *kMetaTableQualifier = "meta";
+ static constexpr const char *kMetaTableName = "hbase:meta";
+ static constexpr const char *kMetaRegion = "1588230740";
+ static constexpr const char *kMetaRegionName = "hbase:meta,,1";
+ static constexpr const char *kCatalogFamily = "info";
+ static constexpr const char *kRegionInfoColumn = "regioninfo";
+ static constexpr const char *kServerColumn = "server";
+
+ MetaUtil();
+
/**
* Given a table and a row give the row key from which to start a scan to find
* region locations.
@@ -49,6 +60,17 @@ class MetaUtil {
/**
* Return a RegionLocation from the parsed Response
*/
- std::shared_ptr<RegionLocation> CreateLocation(const Response &resp);
+ std::shared_ptr<RegionLocation> CreateLocation(const Response &resp,
+ const hbase::pb::TableName &tn);
+
+ /**
+ * Return whether the table is the meta table.
+ */
+ static bool IsMeta(const hbase::pb::TableName &tn);
+
+ const pb::RegionInfo &meta_region_info() const { return meta_region_info_; }
+
+ private:
+ pb::RegionInfo meta_region_info_;
};
} // namespace hbase
diff --git a/core/region-location.h b/core/region-location.h
index d5d9d67..822180b 100644
--- a/core/region-location.h
+++ b/core/region-location.h
@@ -44,9 +44,8 @@ class RegionLocation {
* this region.
* @param service the connected service to the regionserver.
*/
- RegionLocation(std::string region_name, hbase::pb::RegionInfo ri, hbase::pb::ServerName sn,
- std::shared_ptr<HBaseService> service)
- : region_name_(region_name), ri_(ri), sn_(sn), service_(service) {}
+ RegionLocation(std::string region_name, hbase::pb::RegionInfo ri, hbase::pb::ServerName sn)
+ : region_name_(region_name), ri_(ri), sn_(sn) {}
/**
* Get a reference to the regio info
@@ -64,19 +63,6 @@ class RegionLocation {
const std::string ®ion_name() const { return region_name_; }
/**
- * Get a service. This could be closed or null. It's the caller's
- * responsibility to check.
- */
- std::shared_ptr<HBaseService> service() { return service_; }
-
- /**
- * Set the service.
- * This should be used if the region moved or if the connection is thought to
- * be bad and a new tcp connection needs to be made.
- */
- void set_service(std::shared_ptr<HBaseService> s) { service_ = s; }
-
- /**
* Set the servername if the region has moved.
*/
void set_server_name(hbase::pb::ServerName sn) { sn_ = sn; }
@@ -89,7 +75,6 @@ class RegionLocation {
std::string region_name_;
hbase::pb::RegionInfo ri_;
hbase::pb::ServerName sn_;
- std::shared_ptr<HBaseService> service_;
};
} // namespace hbase
diff --git a/core/response-converter.cc b/core/response-converter.cc
index 9bc4892..4f9bfb1 100644
--- a/core/response-converter.cc
+++ b/core/response-converter.cc
@@ -93,7 +93,7 @@ std::vector<std::shared_ptr<Result>> ResponseConverter::FromScanResponse(const R
std::vector<std::shared_ptr<Result>> ResponseConverter::FromScanResponse(
const std::shared_ptr<ScanResponse> scan_resp, std::shared_ptr<CellScanner> cell_scanner) {
VLOG(3) << "FromScanResponse:" << scan_resp->ShortDebugString()
- << " cell_scanner:" << (cell_scanner == nullptr);
+ << " cell_scanner:" << (cell_scanner != nullptr);
int num_results =
cell_scanner != nullptr ? scan_resp->cells_per_result_size() : scan_resp->results_size();
diff --git a/core/simple-client.cc b/core/simple-client.cc
index f79d848..2fd7108 100644
--- a/core/simple-client.cc
+++ b/core/simple-client.cc
@@ -29,6 +29,7 @@
#include "connection/rpc-client.h"
#include "core/client.h"
#include "core/get.h"
+#include "core/hbase-configuration-loader.h"
#include "core/put.h"
#include "core/scan.h"
#include "core/table.h"
@@ -39,6 +40,7 @@
using hbase::Client;
using hbase::Configuration;
using hbase::Get;
+using hbase::HBaseConfigurationLoader;
using hbase::Scan;
using hbase::Put;
using hbase::Table;
@@ -49,6 +51,7 @@ using hbase::TimeUtil;
DEFINE_string(table, "test_table", "What table to do the reads or writes");
DEFINE_string(row, "row_", "row prefix");
DEFINE_string(zookeeper, "localhost:2181", "What zk quorum to talk to");
+DEFINE_string(conf, "", "Conf directory to read the config from (optional)");
DEFINE_uint64(num_rows, 10000, "How many rows to write and read");
DEFINE_bool(puts, true, "Whether to perform puts");
DEFINE_bool(gets, true, "Whether to perform gets");
@@ -76,10 +79,17 @@ int main(int argc, char *argv[]) {
FLAGS_logtostderr = 1;
FLAGS_stderrthreshold = 1;
- // Configuration
- auto conf = std::make_shared<Configuration>();
- conf->Set("hbase.zookeeper.quorum", FLAGS_zookeeper);
- conf->SetInt("hbase.client.cpu.thread.pool.size", FLAGS_threads);
+ std::shared_ptr<Configuration> conf = nullptr;
+ if (FLAGS_conf == "") {
+ // Configuration
+ conf = std::make_shared<Configuration>();
+ conf->Set("hbase.zookeeper.quorum", FLAGS_zookeeper);
+ conf->SetInt("hbase.client.cpu.thread.pool.size", FLAGS_threads);
+ } else {
+ setenv("HBASE_CONF", FLAGS_conf.c_str(), 1);
+ hbase::HBaseConfigurationLoader loader;
+ conf = std::make_shared<Configuration>(loader.LoadDefaultResources().value());
+ }
auto row = FLAGS_row;
diff --git a/core/zk-util.cc b/core/zk-util.cc
index 50ea92a..d29c8c3 100644
--- a/core/zk-util.cc
+++ b/core/zk-util.cc
@@ -55,4 +55,8 @@ std::string ZKUtil::MetaZNode(const hbase::Configuration& conf) {
return zk_node;
}
+int32_t ZKUtil::SessionTimeout(const hbase::Configuration& conf) {
+ return conf.GetInt(kHBaseZookeeperSessionTimeout_, kDefHBaseZookeeperSessionTimeout_);
+}
+
} // namespace hbase
diff --git a/core/zk-util.h b/core/zk-util.h
index 8f2d627..403fbe4 100644
--- a/core/zk-util.h
+++ b/core/zk-util.h
@@ -34,8 +34,13 @@ class ZKUtil {
static constexpr const char* kDefHBaseZnodeParent_ = "/hbase";
static constexpr const char* kHBaseMetaRegionServer_ = "meta-region-server";
+ static constexpr const char* kHBaseZookeeperSessionTimeout_ = "zookeeper.session.timeout";
+ static constexpr const int32_t kDefHBaseZookeeperSessionTimeout_ = 90000;
+
static std::string ParseZooKeeperQuorum(const hbase::Configuration& conf);
static std::string MetaZNode(const hbase::Configuration& conf);
+
+ static int32_t SessionTimeout(const hbase::Configuration& conf);
};
} // namespace hbase
diff --git a/serde/region-info.h b/serde/region-info.h
index 1f08298..8010042 100644
--- a/serde/region-info.h
+++ b/serde/region-info.h
@@ -19,13 +19,13 @@
#pragma once
-#include "if/HBase.pb.h"
-
#include <folly/Conv.h>
#include <boost/algorithm/string/predicate.hpp>
#include <string>
+#include "if/HBase.pb.h"
+
namespace hbase {
namespace pb {
template <class String>
diff --git a/serde/table-name.h b/serde/table-name.h
index b8b7776..3594802 100644
--- a/serde/table-name.h
+++ b/serde/table-name.h
@@ -18,12 +18,13 @@
*/
#pragma once
+#include <folly/Conv.h>
+#include <folly/String.h>
+
#include <memory>
#include <string>
#include <vector>
-#include <folly/Conv.h>
-#include <folly/String.h>
#include "if/HBase.pb.h"
namespace hbase {
diff --git a/test-util/mini-cluster.cc b/test-util/mini-cluster.cc
index 688ea8e..56461e1 100644
--- a/test-util/mini-cluster.cc
+++ b/test-util/mini-cluster.cc
@@ -59,6 +59,7 @@ JNIEnv *MiniCluster::CreateVM(JavaVM **jvm) {
}
fd.close();
}
+
auto options = std::string{"-Djava.class.path="} + clspath;
jvm_options.optionString = const_cast<char *>(options.c_str());
args.options = &jvm_options;
@@ -185,6 +186,9 @@ JNIEnv *MiniCluster::env() {
}
// converts C char* to Java byte[]
jbyteArray MiniCluster::StrToByteChar(const std::string &str) {
+ if (str.size() == 0) {
+ return nullptr;
+ }
char *p = const_cast<char *>(str.c_str());
int n = str.length();
jbyteArray arr = env_->NewByteArray(n);
diff --git a/test-util/test-util.cc b/test-util/test-util.cc
index 26862d8..b32c635 100644
--- a/test-util/test-util.cc
+++ b/test-util/test-util.cc
@@ -79,6 +79,10 @@ void TestUtil::CreateTable(const std::string &table, const std::vector<std::stri
mini_->CreateTable(table, families, keys);
}
+void TestUtil::MoveRegion(const std::string ®ion, const std::string &server) {
+ mini_->MoveRegion(region, server);
+}
+
void TestUtil::StartStandAloneInstance() {
auto p = temp_dir_.path().string();
auto cmd = std::string{"bin/start-local-hbase.sh " + p};
diff --git a/test-util/test-util.h b/test-util/test-util.h
index e26558b..40e99d1 100644
--- a/test-util/test-util.h
+++ b/test-util/test-util.h
@@ -68,6 +68,7 @@ class TestUtil {
void StartStandAloneInstance();
void StopStandAloneInstance();
void RunShellCmd(const std::string &);
+ void MoveRegion(const std::string ®ion, const std::string &server);
private:
std::unique_ptr<MiniCluster> mini_;
diff --git a/utils/bytes-util-test.cc b/utils/bytes-util-test.cc
index 16af021..4a49593 100644
--- a/utils/bytes-util-test.cc
+++ b/utils/bytes-util-test.cc
@@ -23,8 +23,7 @@
#include "utils/bytes-util.h"
-using namespace std;
-using namespace hbase;
+using hbase::BytesUtil;
TEST(TestBytesUtil, TestToStringBinary) {
std::string empty{""};
diff --git a/utils/bytes-util.cc b/utils/bytes-util.cc
index a937782..12037c3 100644
--- a/utils/bytes-util.cc
+++ b/utils/bytes-util.cc
@@ -21,11 +21,11 @@
#include <bits/stdc++.h>
#include <boost/predef.h>
+#include <glog/logging.h>
+
#include <memory>
#include <string>
-#include <glog/logging.h>
-
namespace hbase {
constexpr char BytesUtil::kHexChars[];
[hbase] 57/137: HBASE-17463 [C++] RpcClient should close the thread
pool
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 72f4f4cfb959fa7b5a954de2ce43ccf38f80353a
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Fri Jan 13 11:29:00 2017 -0800
HBASE-17463 [C++] RpcClient should close the thread pool
---
connection/connection-pool.cc | 3 +++
connection/connection-pool.h | 5 +++++
connection/rpc-client.cc | 14 +++++++++-----
connection/rpc-client.h | 1 +
serde/rpc.cc | 1 +
5 files changed, 19 insertions(+), 5 deletions(-)
diff --git a/connection/connection-pool.cc b/connection/connection-pool.cc
index e022f9e..ee14c9d 100644
--- a/connection/connection-pool.cc
+++ b/connection/connection-pool.cc
@@ -117,3 +117,6 @@ void ConnectionPool::Close(std::shared_ptr<ConnectionId> remote_id) {
found->second->Close();
connections_.erase(found);
}
+
+void ConnectionPool::Close() {
+}
diff --git a/connection/connection-pool.h b/connection/connection-pool.h
index 96d89ac..5101c68 100644
--- a/connection/connection-pool.h
+++ b/connection/connection-pool.h
@@ -75,6 +75,11 @@ class ConnectionPool {
*/
void Close(std::shared_ptr<ConnectionId> remote_id);
+ /**
+ * Close the Connection Pool
+ */
+ void Close();
+
private:
std::shared_ptr<RpcConnection> GetCachedConnection(
std::shared_ptr<ConnectionId> remote_id);
diff --git a/connection/rpc-client.cc b/connection/rpc-client.cc
index 66ec231..3f0cfaf 100644
--- a/connection/rpc-client.cc
+++ b/connection/rpc-client.cc
@@ -43,13 +43,15 @@ class RpcChannelImplementation : public AbstractRpcChannel {
} // namespace hbase
RpcClient::RpcClient() {
- auto io_executor = std::make_shared<wangle::IOThreadPoolExecutor>(
+ io_executor_ = std::make_shared<wangle::IOThreadPoolExecutor>(
sysconf(_SC_NPROCESSORS_ONLN));
- cp_ = std::make_shared<ConnectionPool>(io_executor);
+ cp_ = std::make_shared<ConnectionPool>(io_executor_);
}
-void RpcClient::Close() {}
+void RpcClient::Close() {
+ io_executor_->stop();
+}
std::shared_ptr<Response> RpcClient::SyncCall(const std::string& host,
uint16_t port,
@@ -114,6 +116,8 @@ void RpcClient::CallMethod(const MethodDescriptor* method,
std::unique_ptr<Request> req =
std::make_unique<Request>(shared_req, shared_resp, method->name());
- AsyncCall(host, port, std::move(req), ticket)
- .then([done, this](Response resp) { done->Run(); });
+ AsyncCall(host, port, std::move(req), ticket, method->service()->name())
+ .then([done, this](Response resp) {
+ done->Run();
+ });
}
diff --git a/connection/rpc-client.h b/connection/rpc-client.h
index c24db9d..dbf857d 100644
--- a/connection/rpc-client.h
+++ b/connection/rpc-client.h
@@ -91,6 +91,7 @@ class RpcClient : public std::enable_shared_from_this<RpcClient> {
private:
std::shared_ptr<ConnectionPool> cp_;
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_;
};
class AbstractRpcChannel : public RpcChannel {
diff --git a/serde/rpc.cc b/serde/rpc.cc
index d863d50..e4ade22 100644
--- a/serde/rpc.cc
+++ b/serde/rpc.cc
@@ -110,6 +110,7 @@ unique_ptr<IOBuf> RpcSerde::Header(const string &user) {
// That may or may not be the correct thing to do.
// It worked for a while with the java client; until it
// didn't.
+ // TODO: send the service name and user from the RpcClient
h.set_service_name(INTERFACE);
return PrependLength(SerializeMessage(h));
}
[hbase] 111/137: HBASE-18466. [C++] Support handling exception in
RpcTestServer
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 913d1f19b62e540182ad6fcddee122e1c3e7e5e4
Author: Xiaobing Zhou <xz...@hortonworks.com>
AuthorDate: Thu Jul 27 17:33:08 2017 -0700
HBASE-18466. [C++] Support handling exception in RpcTestServer
Signed-off-by: Enis Soztutar <en...@apache.org>
---
connection/rpc-test-server-handler.cc | 7 +-
connection/rpc-test-server-handler.h | 2 +-
connection/rpc-test-server.cc | 31 +++++++-
connection/rpc-test-server.h | 24 ++++++-
connection/rpc-test.cc | 129 ++++++++++++++++++++++++++--------
serde/rpc-serde.cc | 27 +++++++
serde/rpc-serde.h | 14 ++++
7 files changed, 194 insertions(+), 40 deletions(-)
diff --git a/connection/rpc-test-server-handler.cc b/connection/rpc-test-server-handler.cc
index 7d2f407..7f41b7e 100644
--- a/connection/rpc-test-server-handler.cc
+++ b/connection/rpc-test-server-handler.cc
@@ -46,16 +46,17 @@ void RpcTestServerSerializeHandler::read(Context* ctx, std::unique_ptr<folly::IO
}
folly::Future<folly::Unit> RpcTestServerSerializeHandler::write(Context* ctx,
- std::unique_ptr<Response> r) {
+ std::unique_ptr<Response> resp) {
VLOG(3) << "Writing RPC Request";
// Send the data down the pipeline.
- return ctx->fireWrite(serde_.Response(r->call_id(), r->resp_msg().get()));
+ return ctx->fireWrite(
+ serde_.Response(resp->call_id(), resp->resp_msg().get(), resp->exception()));
}
std::unique_ptr<Request> RpcTestServerSerializeHandler::CreateReceivedRequest(
const std::string& method_name) {
std::unique_ptr<Request> result = nullptr;
- ;
+
if (method_name == "ping") {
result = std::make_unique<Request>(std::make_shared<EmptyRequestProto>(),
std::make_shared<EmptyResponseProto>(), method_name);
diff --git a/connection/rpc-test-server-handler.h b/connection/rpc-test-server-handler.h
index 4c84615..ab0264f 100644
--- a/connection/rpc-test-server-handler.h
+++ b/connection/rpc-test-server-handler.h
@@ -36,7 +36,7 @@ class RpcTestServerSerializeHandler
void read(Context* ctx, std::unique_ptr<folly::IOBuf> buf) override;
- folly::Future<folly::Unit> write(Context* ctx, std::unique_ptr<Response> r) override;
+ folly::Future<folly::Unit> write(Context* ctx, std::unique_ptr<Response> resp) override;
private:
std::unique_ptr<Request> CreateReceivedRequest(const std::string& method_name);
diff --git a/connection/rpc-test-server.cc b/connection/rpc-test-server.cc
index d3a30b1..b9e1f13 100644
--- a/connection/rpc-test-server.cc
+++ b/connection/rpc-test-server.cc
@@ -30,19 +30,35 @@ namespace hbase {
RpcTestServerSerializePipeline::Ptr RpcTestServerPipelineFactory::newPipeline(
std::shared_ptr<AsyncTransportWrapper> sock) {
+ if (service_ == nullptr) {
+ initService(sock);
+ }
+ CHECK(service_ != nullptr);
+
auto pipeline = RpcTestServerSerializePipeline::create();
pipeline->addBack(AsyncSocketHandler(sock));
// ensure we can write from any thread
pipeline->addBack(EventBaseHandler());
pipeline->addBack(LengthFieldBasedFrameDecoder());
pipeline->addBack(RpcTestServerSerializeHandler());
- pipeline->addBack(
- MultiplexServerDispatcher<std::unique_ptr<Request>, std::unique_ptr<Response>>(&service_));
+ pipeline->addBack(MultiplexServerDispatcher<std::unique_ptr<Request>, std::unique_ptr<Response>>(
+ service_.get()));
pipeline->finalize();
return pipeline;
}
+void RpcTestServerPipelineFactory::initService(std::shared_ptr<AsyncTransportWrapper> sock) {
+ /* get server address */
+ SocketAddress localAddress;
+ sock->getLocalAddress(&localAddress);
+
+ /* init service with server address */
+ service_ = std::make_shared<ExecutorFilter<std::unique_ptr<Request>, std::unique_ptr<Response>>>(
+ std::make_shared<CPUThreadPoolExecutor>(1),
+ std::make_shared<RpcTestService>(std::make_shared<SocketAddress>(localAddress)));
+}
+
Future<std::unique_ptr<Response>> RpcTestService::operator()(std::unique_ptr<Request> request) {
/* build Response */
auto response = std::make_unique<Response>();
@@ -54,11 +70,20 @@ Future<std::unique_ptr<Response>> RpcTestService::operator()(std::unique_ptr<Req
response->set_resp_msg(pb_resp_msg);
} else if (method_name == "echo") {
auto pb_resp_msg = std::make_shared<EchoResponseProto>();
+ /* get msg from client */
auto pb_req_msg = std::static_pointer_cast<EchoRequestProto>(request->req_msg());
pb_resp_msg->set_message(pb_req_msg->message());
response->set_resp_msg(pb_resp_msg);
+ VLOG(1) << "RPC server:"
+ << " echo called, " << pb_req_msg->message();
+
} else if (method_name == "error") {
- // TODO:
+ auto pb_resp_msg = std::make_shared<EmptyResponseProto>();
+ response->set_resp_msg(pb_resp_msg);
+ VLOG(1) << "RPC server:"
+ << " error called.";
+ response->set_exception(folly::make_exception_wrapper<RpcTestException>("server error!"));
+
} else if (method_name == "pause") {
// TODO:
} else if (method_name == "addr") {
diff --git a/connection/rpc-test-server.h b/connection/rpc-test-server.h
index c3225ff..955560e 100644
--- a/connection/rpc-test-server.h
+++ b/connection/rpc-test-server.h
@@ -17,12 +17,14 @@
*
*/
#pragma once
+#include <folly/SocketAddress.h>
#include <wangle/concurrent/CPUThreadPoolExecutor.h>
#include <wangle/service/ExecutorFilter.h>
#include <wangle/service/Service.h>
#include "connection/request.h"
#include "connection/response.h"
+#include "exceptions/exception.h"
using namespace hbase;
using namespace folly;
@@ -31,11 +33,24 @@ using namespace wangle;
namespace hbase {
using RpcTestServerSerializePipeline = wangle::Pipeline<IOBufQueue&, std::unique_ptr<Response>>;
+class RpcTestException : public IOException {
+ public:
+ RpcTestException() {}
+ RpcTestException(const std::string& what) : IOException(what) {}
+ RpcTestException(const std::string& what, const folly::exception_wrapper& cause)
+ : IOException(what, cause) {}
+ RpcTestException(const folly::exception_wrapper& cause) : IOException("", cause) {}
+};
+
class RpcTestService : public Service<std::unique_ptr<Request>, std::unique_ptr<Response>> {
public:
- RpcTestService() {}
+ RpcTestService(std::shared_ptr<folly::SocketAddress> socket_address)
+ : socket_address_(socket_address) {}
virtual ~RpcTestService() = default;
Future<std::unique_ptr<Response>> operator()(std::unique_ptr<Request> request) override;
+
+ private:
+ std::shared_ptr<folly::SocketAddress> socket_address_;
};
class RpcTestServerPipelineFactory : public PipelineFactory<RpcTestServerSerializePipeline> {
@@ -44,7 +59,10 @@ class RpcTestServerPipelineFactory : public PipelineFactory<RpcTestServerSeriali
std::shared_ptr<AsyncTransportWrapper> sock) override;
private:
- ExecutorFilter<std::unique_ptr<Request>, std::unique_ptr<Response>> service_{
- std::make_shared<CPUThreadPoolExecutor>(1), std::make_shared<RpcTestService>()};
+ void initService(std::shared_ptr<AsyncTransportWrapper> sock);
+
+ private:
+ std::shared_ptr<ExecutorFilter<std::unique_ptr<Request>, std::unique_ptr<Response>>> service_{
+ nullptr};
};
} // end of namespace hbase
diff --git a/connection/rpc-test.cc b/connection/rpc-test.cc
index d4cd89f..2949fe9 100644
--- a/connection/rpc-test.cc
+++ b/connection/rpc-test.cc
@@ -30,6 +30,7 @@
#include <boost/thread.hpp>
#include "connection/rpc-client.h"
+#include "exceptions/exception.h"
#include "if/test.pb.h"
#include "rpc-test-server.h"
#include "security/user.h"
@@ -40,46 +41,114 @@ using namespace folly;
using namespace hbase;
DEFINE_int32(port, 0, "test server port");
+typedef ServerBootstrap<RpcTestServerSerializePipeline> ServerTestBootstrap;
+typedef std::shared_ptr<ServerTestBootstrap> ServerPtr;
-TEST(RpcTestServer, echo) {
- /* create conf */
+class RpcTest : public ::testing::Test {
+ public:
+ static void SetUpTestCase() { google::InstallFailureSignalHandler(); }
+};
+
+std::shared_ptr<Configuration> CreateConf() {
auto conf = std::make_shared<Configuration>();
conf->Set(RpcSerde::HBASE_CLIENT_RPC_TEST_MODE, "true");
+ return conf;
+}
+ServerPtr CreateRpcServer() {
/* create rpc test server */
- auto server = std::make_shared<ServerBootstrap<RpcTestServerSerializePipeline>>();
+ auto server = std::make_shared<ServerTestBootstrap>();
server->childPipeline(std::make_shared<RpcTestServerPipelineFactory>());
server->bind(FLAGS_port);
- folly::SocketAddress server_addr;
- server->getSockets()[0]->getAddress(&server_addr);
+ return server;
+}
- /* create RpcClient */
+std::shared_ptr<folly::SocketAddress> GetRpcServerAddress(ServerPtr server) {
+ auto addr = std::make_shared<folly::SocketAddress>();
+ server->getSockets()[0]->getAddress(addr.get());
+ return addr;
+}
+
+std::shared_ptr<RpcClient> CreateRpcClient(std::shared_ptr<Configuration> conf) {
auto io_executor = std::make_shared<wangle::IOThreadPoolExecutor>(1);
+ auto client = std::make_shared<RpcClient>(io_executor, nullptr, conf);
+ return client;
+}
+
+std::shared_ptr<RpcClient> CreateRpcClient(std::shared_ptr<Configuration> conf,
+ std::chrono::nanoseconds connect_timeout) {
+ auto io_executor = std::make_shared<wangle::IOThreadPoolExecutor>(1);
+ auto client = std::make_shared<RpcClient>(io_executor, nullptr, conf, connect_timeout);
+ return client;
+}
+
+/**
+ * test echo
+ */
+TEST_F(RpcTest, Echo) {
+ auto conf = CreateConf();
+ auto server = CreateRpcServer();
+ auto server_addr = GetRpcServerAddress(server);
+ auto client = CreateRpcClient(conf);
+
+ std::string greetings = "hello, hbase server!";
+ auto request = std::make_unique<Request>(std::make_shared<EchoRequestProto>(),
+ std::make_shared<EchoResponseProto>(), "echo");
+ auto pb_msg = std::static_pointer_cast<EchoRequestProto>(request->req_msg());
+ pb_msg->set_message(greetings);
+
+ /* sending out request */
+ client
+ ->AsyncCall(server_addr->getAddressStr(), server_addr->getPort(), std::move(request),
+ hbase::security::User::defaultUser())
+ .then([=](std::unique_ptr<Response> response) {
+ auto pb_resp = std::static_pointer_cast<EchoResponseProto>(response->resp_msg());
+ EXPECT_TRUE(pb_resp != nullptr);
+ VLOG(1) << "RPC echo returned: " + pb_resp->message();
+ EXPECT_EQ(greetings, pb_resp->message());
+ })
+ .onError([](const folly::exception_wrapper& ew) {
+ FAIL() << "Shouldn't get here, no exception is expected for RPC echo.";
+ });
+
+ server->stop();
+ server->join();
+}
+
+/**
+ * test error
+ */
+TEST_F(RpcTest, error) {
+ auto conf = CreateConf();
+ auto server = CreateRpcServer();
+ auto server_addr = GetRpcServerAddress(server);
+ auto client = CreateRpcClient(conf);
+
+ auto request = std::make_unique<Request>(std::make_shared<EmptyRequestProto>(),
+ std::make_shared<EmptyResponseProto>(), "error");
+ /* sending out request */
+ client
+ ->AsyncCall(server_addr->getAddressStr(), server_addr->getPort(), std::move(request),
+ hbase::security::User::defaultUser())
+ .then([=](std::unique_ptr<Response> response) {
+ FAIL() << "Shouldn't get here, exception is expected for RPC error.";
+ })
+ .onError([](const folly::exception_wrapper& ew) {
+ VLOG(1) << "RPC error returned with exception.";
+ std::string kRemoteException = demangle(typeid(hbase::RemoteException)).toStdString();
+ std::string kRpcTestException = demangle(typeid(hbase::RpcTestException)).toStdString();
+
+ /* verify exception_wrapper */
+ EXPECT_TRUE(bool(ew));
+ EXPECT_EQ(kRemoteException, ew.class_name());
- auto rpc_client = std::make_shared<RpcClient>(io_executor, nullptr, conf);
-
- /**
- * test echo
- */
- try {
- std::string greetings = "hello, hbase server!";
- auto request = std::make_unique<Request>(std::make_shared<EchoRequestProto>(),
- std::make_shared<EchoResponseProto>(), "echo");
- auto pb_msg = std::static_pointer_cast<EchoRequestProto>(request->req_msg());
- pb_msg->set_message(greetings);
-
- /* sending out request */
- rpc_client
- ->AsyncCall(server_addr.getAddressStr(), server_addr.getPort(), std::move(request),
- hbase::security::User::defaultUser())
- .then([=](std::unique_ptr<Response> response) {
- auto pb_resp = std::static_pointer_cast<EchoResponseProto>(response->resp_msg());
- VLOG(1) << "message returned: " + pb_resp->message();
- EXPECT_EQ(greetings, pb_resp->message());
- });
- } catch (const std::exception& e) {
- throw e;
- }
+ /* verify RemoteException */
+ EXPECT_TRUE(ew.with_exception([&](const hbase::RemoteException& re) {
+ /* verify DoNotRetryIOException*/
+ EXPECT_EQ(kRpcTestException, re.exception_class_name());
+ EXPECT_EQ(kRpcTestException + ": server error!", re.stack_trace());
+ }));
+ });
server->stop();
server->join();
diff --git a/serde/rpc-serde.cc b/serde/rpc-serde.cc
index 9e1f79a..70a57e8 100644
--- a/serde/rpc-serde.cc
+++ b/serde/rpc-serde.cc
@@ -40,6 +40,8 @@ using google::protobuf::io::CodedInputStream;
using google::protobuf::io::CodedOutputStream;
using google::protobuf::io::ZeroCopyOutputStream;
+using namespace hbase::pb;
+
namespace hbase {
static const std::string PREAMBLE = "HBas";
@@ -174,6 +176,31 @@ std::unique_ptr<folly::IOBuf> RpcSerde::Response(const uint32_t call_id,
return PrependLength(std::move(ser_header));
}
+std::unique_ptr<folly::IOBuf> RpcSerde::Response(const uint32_t call_id,
+ const google::protobuf::Message *msg,
+ const folly::exception_wrapper &exception) {
+ /* create ResponseHeader */
+ pb::ResponseHeader rh;
+ rh.set_call_id(call_id);
+
+ /* create ExceptionResponse */
+ if (bool(exception)) {
+ VLOG(1) << "packing ExceptionResponse";
+ auto exception_response = new pb::ExceptionResponse();
+ exception_response->set_exception_class_name(exception.class_name().c_str());
+ exception_response->set_stack_trace(exception.what().c_str());
+ rh.set_allocated_exception(exception_response);
+ }
+
+ /* serialize Response header and body */
+ auto ser_header = SerializeDelimited(rh);
+ auto ser_resp = SerializeDelimited(*msg);
+ ser_header->appendChain(std::move(ser_resp));
+
+ VLOG(3) << "Converted hbase::Response to folly::IOBuf";
+ return PrependLength(std::move(ser_header));
+}
+
std::unique_ptr<CellScanner> RpcSerde::CreateCellScanner(std::unique_ptr<folly::IOBuf> buf,
uint32_t offset, uint32_t length) {
if (codec_ == nullptr) {
diff --git a/serde/rpc-serde.h b/serde/rpc-serde.h
index 0e1d44e..6941f62 100644
--- a/serde/rpc-serde.h
+++ b/serde/rpc-serde.h
@@ -21,10 +21,12 @@
#include <memory>
#include <string>
+#include <folly/ExceptionWrapper.h>
#include "if/HBase.pb.h"
#include "serde/cell-scanner.h"
#include "serde/codec.h"
+using namespace folly;
// Forward
namespace folly {
class IOBuf;
@@ -110,6 +112,18 @@ class RpcSerde {
const google::protobuf::Message *msg);
/**
+ * Serialize a response message into a protobuf.
+ * Request consists of:
+ *
+ * - Big endian length
+ * - ResponseHeader object
+ * - The passed in hbase::Response object
+ */
+ std::unique_ptr<folly::IOBuf> Response(const uint32_t call_id,
+ const google::protobuf::Message *msg,
+ const folly::exception_wrapper &exception);
+
+ /**
* Serialize a message in the delimited format.
* Delimited format consists of the following:
*
[hbase] 123/137: HBASE-18204 [C++] Rpc connection close and
reconnecting
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 874b001a332e3928cb862cec87a8aec109d7c697
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Tue Aug 22 19:04:29 2017 -0700
HBASE-18204 [C++] Rpc connection close and reconnecting
---
connection/client-dispatcher.cc | 43 +++++++++++++++++++++++++--
connection/client-dispatcher.h | 12 +++++++-
connection/connection-factory.cc | 35 +++++++++++++---------
connection/connection-factory.h | 17 +++++++++--
connection/connection-id.h | 8 +++--
connection/connection-pool-test.cc | 55 +++++++++++++++++++++--------------
connection/connection-pool.cc | 17 +++++------
connection/connection-pool.h | 5 +---
connection/rpc-client.cc | 3 +-
connection/rpc-client.h | 5 ++--
connection/rpc-connection.h | 46 ++++++++++++++++++++++-------
connection/rpc-test.cc | 22 +++++++++-----
connection/sasl-handler.cc | 1 +
core/async-batch-rpc-retrying-test.cc | 33 +++++++++++++--------
core/async-connection.cc | 6 ++--
core/async-rpc-retrying-test.cc | 4 +--
core/location-cache-test.cc | 26 ++++++++++-------
core/location-cache.cc | 8 +++--
core/location-cache.h | 13 +++++----
core/region-location.h | 4 +--
test-util/mini-cluster.cc | 17 ++++++-----
test-util/mini-cluster.h | 1 +
test-util/test-util.cc | 5 +++-
utils/concurrent-map.h | 5 ++++
24 files changed, 267 insertions(+), 124 deletions(-)
diff --git a/connection/client-dispatcher.cc b/connection/client-dispatcher.cc
index d5d7f5f..fc8eb16 100644
--- a/connection/client-dispatcher.cc
+++ b/connection/client-dispatcher.cc
@@ -17,19 +17,24 @@
*
*/
#include "connection/client-dispatcher.h"
+
#include <folly/ExceptionWrapper.h>
#include <folly/Format.h>
#include <folly/io/async/AsyncSocketException.h>
#include <utility>
+
+#include "connection/rpc-connection.h"
#include "exceptions/exception.h"
using std::unique_ptr;
namespace hbase {
-ClientDispatcher::ClientDispatcher() : current_call_id_(9), requests_(5000) {}
+ClientDispatcher::ClientDispatcher(const std::string &server)
+ : current_call_id_(9), requests_(5000), server_(server), is_closed_(false) {}
void ClientDispatcher::read(Context *ctx, unique_ptr<Response> in) {
+ VLOG(5) << "ClientDispatcher::read()";
auto call_id = in->call_id();
auto p = requests_.find_and_erase(call_id);
@@ -43,7 +48,23 @@ void ClientDispatcher::read(Context *ctx, unique_ptr<Response> in) {
}
}
+void ClientDispatcher::readException(Context *ctx, folly::exception_wrapper e) {
+ VLOG(5) << "ClientDispatcher::readException()";
+ CloseAndCleanUpCalls();
+}
+
+void ClientDispatcher::readEOF(Context *ctx) {
+ VLOG(5) << "ClientDispatcher::readEOF()";
+ CloseAndCleanUpCalls();
+}
+
folly::Future<unique_ptr<Response>> ClientDispatcher::operator()(unique_ptr<Request> arg) {
+ VLOG(5) << "ClientDispatcher::operator()";
+ std::lock_guard<std::recursive_mutex> lock(mutex_);
+ if (is_closed_) {
+ throw ConnectionException("Connection closed already");
+ }
+
auto call_id = current_call_id_++;
arg->set_call_id(call_id);
@@ -55,6 +76,7 @@ folly::Future<unique_ptr<Response>> ClientDispatcher::operator()(unique_ptr<Requ
p.setInterruptHandler([call_id, this](const folly::exception_wrapper &e) {
LOG(ERROR) << "e = " << call_id;
this->requests_.erase(call_id);
+ // TODO: call Promise::SetException()?
});
try {
@@ -68,9 +90,26 @@ folly::Future<unique_ptr<Response>> ClientDispatcher::operator()(unique_ptr<Requ
return f;
}
-folly::Future<folly::Unit> ClientDispatcher::close() { return ClientDispatcherBase::close(); }
+void ClientDispatcher::CloseAndCleanUpCalls() {
+ VLOG(5) << "ClientDispatcher::CloseAndCleanUpCalls()";
+ std::lock_guard<std::recursive_mutex> lock(mutex_);
+ if (is_closed_) {
+ return;
+ }
+ for (auto &pair : requests_) {
+ pair.second.setException(IOException{"Connection closed to server:" + server_});
+ }
+ requests_.clear();
+ is_closed_ = true;
+}
+
+folly::Future<folly::Unit> ClientDispatcher::close() {
+ CloseAndCleanUpCalls();
+ return ClientDispatcherBase::close();
+}
folly::Future<folly::Unit> ClientDispatcher::close(Context *ctx) {
+ CloseAndCleanUpCalls();
return ClientDispatcherBase::close(ctx);
}
} // namespace hbase
diff --git a/connection/client-dispatcher.h b/connection/client-dispatcher.h
index 1f8e6b3..7ef3759 100644
--- a/connection/client-dispatcher.h
+++ b/connection/client-dispatcher.h
@@ -26,6 +26,7 @@
#include <map>
#include <memory>
#include <mutex>
+#include <string>
#include "connection/pipeline.h"
#include "connection/request.h"
@@ -33,6 +34,7 @@
#include "utils/concurrent-map.h"
namespace hbase {
+
/**
* Dispatcher that assigns a call_id and then routes the response back to the
* future.
@@ -42,9 +44,11 @@ class ClientDispatcher
std::unique_ptr<Response>> {
public:
/** Create a new ClientDispatcher */
- ClientDispatcher();
+ explicit ClientDispatcher(const std::string &server);
/** Read a response off the pipeline. */
void read(Context *ctx, std::unique_ptr<Response> in) override;
+ void readException(Context *ctx, folly::exception_wrapper e) override;
+ void readEOF(Context *ctx) override;
/** Take a request as a call and send it down the pipeline. */
folly::Future<std::unique_ptr<Response>> operator()(std::unique_ptr<Request> arg) override;
/** Close the dispatcher and the associated pipeline. */
@@ -53,6 +57,10 @@ class ClientDispatcher
folly::Future<folly::Unit> close() override;
private:
+ void CloseAndCleanUpCalls();
+
+ private:
+ std::recursive_mutex mutex_;
concurrent_map<uint32_t, folly::Promise<std::unique_ptr<Response>>> requests_;
// Start at some number way above what could
// be there for un-initialized call id counters.
@@ -63,5 +71,7 @@ class ClientDispatcher
// uint32_t has a max of 4Billion so 10 more or less is
// not a big deal.
std::atomic<uint32_t> current_call_id_;
+ std::string server_;
+ bool is_closed_;
};
} // namespace hbase
diff --git a/connection/connection-factory.cc b/connection/connection-factory.cc
index e763c03..751073e 100644
--- a/connection/connection-factory.cc
+++ b/connection/connection-factory.cc
@@ -17,6 +17,7 @@
*
*/
+#include <folly/Conv.h>
#include <glog/logging.h>
#include <wangle/channel/Handler.h>
@@ -38,18 +39,20 @@ using std::chrono::nanoseconds;
namespace hbase {
-ConnectionFactory::ConnectionFactory(std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool,
+ConnectionFactory::ConnectionFactory(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
std::shared_ptr<Codec> codec,
std::shared_ptr<Configuration> conf,
nanoseconds connect_timeout)
: connect_timeout_(connect_timeout),
- io_pool_(io_pool),
+ io_executor_(io_executor),
+ cpu_executor_(cpu_executor),
conf_(conf),
pipeline_factory_(std::make_shared<RpcPipelineFactory>(codec, conf)) {}
std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> ConnectionFactory::MakeBootstrap() {
auto client = std::make_shared<wangle::ClientBootstrap<SerializePipeline>>();
- client->group(io_pool_);
+ client->group(io_executor_);
client->pipelineFactory(pipeline_factory_);
// TODO: Opened https://github.com/facebook/wangle/issues/85 in wangle so that we can set socket
@@ -59,17 +62,23 @@ std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> ConnectionFactory::M
}
std::shared_ptr<HBaseService> ConnectionFactory::Connect(
- std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> client, const std::string &hostname,
- uint16_t port) {
+ std::shared_ptr<RpcConnection> rpc_connection,
+ std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> client_bootstrap,
+ const std::string &hostname, uint16_t port) {
+ // connection should happen from an IO thread
try {
- // Yes this will block however it makes dealing with connection pool soooooo
- // much nicer.
- // TODO see about using shared promise for this.
- auto pipeline = client
- ->connect(folly::SocketAddress(hostname, port, true),
- std::chrono::duration_cast<milliseconds>(connect_timeout_))
- .get();
- auto dispatcher = std::make_shared<ClientDispatcher>();
+ auto future = via(io_executor_.get()).then([=]() {
+ VLOG(1) << "Connecting to server: " << hostname << ":" << port;
+ return client_bootstrap->connect(folly::SocketAddress(hostname, port, true),
+ std::chrono::duration_cast<milliseconds>(connect_timeout_));
+ });
+
+ // See about using shared promise for this.
+ auto pipeline = future.get();
+
+ VLOG(1) << "Connected to server: " << hostname << ":" << port;
+ auto dispatcher =
+ std::make_shared<ClientDispatcher>(hostname + ":" + folly::to<std::string>(port));
dispatcher->setPipeline(pipeline);
return dispatcher;
} catch (const folly::AsyncSocketException &e) {
diff --git a/connection/connection-factory.h b/connection/connection-factory.h
index c96087d..c4e63c2 100644
--- a/connection/connection-factory.h
+++ b/connection/connection-factory.h
@@ -18,6 +18,8 @@
*/
#pragma once
+#include <wangle/concurrent/CPUThreadPoolExecutor.h>
+#include <wangle/concurrent/IOThreadPoolExecutor.h>
#include <wangle/service/Service.h>
#include <chrono>
@@ -32,6 +34,8 @@
namespace hbase {
+class RpcConnection;
+
/**
* Class to create a ClientBootstrap and turn it into a connected
* pipeline.
@@ -42,7 +46,8 @@ class ConnectionFactory {
* Constructor.
* There should only be one ConnectionFactory per client.
*/
- ConnectionFactory(std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool,
+ ConnectionFactory(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
std::shared_ptr<Codec> codec, std::shared_ptr<Configuration> conf,
std::chrono::nanoseconds connect_timeout = std::chrono::nanoseconds(0));
@@ -60,13 +65,19 @@ class ConnectionFactory {
* This is mostly visible so that mocks can override socket connections.
*/
virtual std::shared_ptr<HBaseService> Connect(
- std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> client,
+ std::shared_ptr<RpcConnection> rpc_connection,
+ std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> client_bootstrap,
const std::string &hostname, uint16_t port);
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor() { return io_executor_; }
+
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor() { return cpu_executor_; }
+
private:
std::chrono::nanoseconds connect_timeout_;
std::shared_ptr<Configuration> conf_;
- std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool_;
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_;
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
std::shared_ptr<RpcPipelineFactory> pipeline_factory_;
};
} // namespace hbase
diff --git a/connection/connection-id.h b/connection/connection-id.h
index 4f84bf8..065b484 100644
--- a/connection/connection-id.h
+++ b/connection/connection-id.h
@@ -18,13 +18,15 @@
*/
#pragma once
-#include "if/HBase.pb.h"
-#include "security/user.h"
-
#include <boost/functional/hash.hpp>
+
#include <memory>
+#include <string>
#include <utility>
+#include "if/HBase.pb.h"
+#include "security/user.h"
+
namespace hbase {
class ConnectionId {
diff --git a/connection/connection-pool-test.cc b/connection/connection-pool-test.cc
index 63f774b..0dc8e14 100644
--- a/connection/connection-pool-test.cc
+++ b/connection/connection-pool-test.cc
@@ -17,47 +17,46 @@
*
*/
-#include "connection/connection-pool.h"
+#include <folly/Logging.h>
+#include <gmock/gmock.h>
+
#include "connection/connection-factory.h"
#include "connection/connection-id.h"
-
+#include "connection/connection-pool.h"
#include "if/HBase.pb.h"
#include "serde/server-name.h"
-#include <folly/Logging.h>
-#include <gmock/gmock.h>
-
-using namespace hbase;
-
using hbase::pb::ServerName;
using ::testing::Return;
using ::testing::_;
+using hbase::ConnectionFactory;
+using hbase::ConnectionPool;
using hbase::ConnectionId;
+using hbase::HBaseService;
+using hbase::Request;
+using hbase::Response;
+using hbase::RpcConnection;
+using hbase::SerializePipeline;
class MockConnectionFactory : public ConnectionFactory {
public:
- MockConnectionFactory() : ConnectionFactory(nullptr, nullptr, nullptr) {}
+ MockConnectionFactory() : ConnectionFactory(nullptr, nullptr, nullptr, nullptr) {}
MOCK_METHOD0(MakeBootstrap, std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>());
- MOCK_METHOD3(Connect, std::shared_ptr<HBaseService>(
+ MOCK_METHOD4(Connect, std::shared_ptr<HBaseService>(
+ std::shared_ptr<RpcConnection> rpc_connection,
std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>,
const std::string &hostname, uint16_t port));
};
class MockBootstrap : public wangle::ClientBootstrap<SerializePipeline> {};
-class MockServiceBase : public HBaseService {
+class MockService : public HBaseService {
public:
folly::Future<std::unique_ptr<Response>> operator()(std::unique_ptr<Request> req) override {
- return do_operation(req.get());
- }
- virtual folly::Future<std::unique_ptr<Response>> do_operation(Request *req) {
- return folly::makeFuture<std::unique_ptr<Response>>(std::make_unique<Response>());
+ return folly::makeFuture<std::unique_ptr<Response>>(
+ std::make_unique<Response>(do_operation(req.get())));
}
-};
-
-class MockService : public MockServiceBase {
- public:
- MOCK_METHOD1(do_operation, folly::Future<std::unique_ptr<Response>>(Request *));
+ MOCK_METHOD1(do_operation, Response(Request *));
};
TEST(TestConnectionPool, TestOnlyCreateOnce) {
@@ -67,14 +66,16 @@ TEST(TestConnectionPool, TestOnlyCreateOnce) {
auto mock_cf = std::make_shared<MockConnectionFactory>();
uint32_t port{999};
- EXPECT_CALL((*mock_cf), Connect(_, _, _)).Times(1).WillRepeatedly(Return(mock_service));
+ EXPECT_CALL((*mock_cf), Connect(_, _, _, _)).Times(1).WillRepeatedly(Return(mock_service));
EXPECT_CALL((*mock_cf), MakeBootstrap()).Times(1).WillRepeatedly(Return(mock_boot));
+ EXPECT_CALL((*mock_service), do_operation(_)).Times(1).WillRepeatedly(Return(Response{}));
ConnectionPool cp{mock_cf};
auto remote_id = std::make_shared<ConnectionId>(hostname, port);
auto result = cp.GetConnection(remote_id);
ASSERT_TRUE(result != nullptr);
result = cp.GetConnection(remote_id);
+ result->SendRequest(nullptr);
}
TEST(TestConnectionPool, TestOnlyCreateMultipleDispose) {
@@ -86,20 +87,25 @@ TEST(TestConnectionPool, TestOnlyCreateMultipleDispose) {
auto mock_service = std::make_shared<MockService>();
auto mock_cf = std::make_shared<MockConnectionFactory>();
- EXPECT_CALL((*mock_cf), Connect(_, _, _)).Times(2).WillRepeatedly(Return(mock_service));
+ EXPECT_CALL((*mock_cf), Connect(_, _, _, _)).Times(2).WillRepeatedly(Return(mock_service));
EXPECT_CALL((*mock_cf), MakeBootstrap()).Times(2).WillRepeatedly(Return(mock_boot));
+ EXPECT_CALL((*mock_service), do_operation(_)).Times(4).WillRepeatedly(Return(Response{}));
ConnectionPool cp{mock_cf};
{
auto remote_id = std::make_shared<ConnectionId>(hostname_one, port);
auto result_one = cp.GetConnection(remote_id);
+ result_one->SendRequest(nullptr);
auto remote_id2 = std::make_shared<ConnectionId>(hostname_two, port);
auto result_two = cp.GetConnection(remote_id2);
+ result_two->SendRequest(nullptr);
}
auto remote_id = std::make_shared<ConnectionId>(hostname_one, port);
auto result_one = cp.GetConnection(remote_id);
+ result_one->SendRequest(nullptr);
auto remote_id2 = std::make_shared<ConnectionId>(hostname_two, port);
auto result_two = cp.GetConnection(remote_id2);
+ result_two->SendRequest(nullptr);
}
TEST(TestConnectionPool, TestCreateOneConnectionForOneService) {
@@ -112,18 +118,23 @@ TEST(TestConnectionPool, TestCreateOneConnectionForOneService) {
auto mock_service = std::make_shared<MockService>();
auto mock_cf = std::make_shared<MockConnectionFactory>();
- EXPECT_CALL((*mock_cf), Connect(_, _, _)).Times(2).WillRepeatedly(Return(mock_service));
+ EXPECT_CALL((*mock_cf), Connect(_, _, _, _)).Times(2).WillRepeatedly(Return(mock_service));
EXPECT_CALL((*mock_cf), MakeBootstrap()).Times(2).WillRepeatedly(Return(mock_boot));
+ EXPECT_CALL((*mock_service), do_operation(_)).Times(4).WillRepeatedly(Return(Response{}));
ConnectionPool cp{mock_cf};
{
auto remote_id = std::make_shared<ConnectionId>(hostname, port, service1);
auto result_one = cp.GetConnection(remote_id);
+ result_one->SendRequest(nullptr);
auto remote_id2 = std::make_shared<ConnectionId>(hostname, port, service2);
auto result_two = cp.GetConnection(remote_id2);
+ result_two->SendRequest(nullptr);
}
auto remote_id = std::make_shared<ConnectionId>(hostname, port, service1);
auto result_one = cp.GetConnection(remote_id);
+ result_one->SendRequest(nullptr);
auto remote_id2 = std::make_shared<ConnectionId>(hostname, port, service2);
auto result_two = cp.GetConnection(remote_id2);
+ result_two->SendRequest(nullptr);
}
diff --git a/connection/connection-pool.cc b/connection/connection-pool.cc
index e98759d..e1f6358 100644
--- a/connection/connection-pool.cc
+++ b/connection/connection-pool.cc
@@ -24,6 +24,7 @@
#include <wangle/service/Service.h>
#include <memory>
+#include <string>
#include <utility>
using std::chrono::nanoseconds;
@@ -31,17 +32,18 @@ using std::chrono::nanoseconds;
namespace hbase {
ConnectionPool::ConnectionPool(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
std::shared_ptr<Codec> codec, std::shared_ptr<Configuration> conf,
nanoseconds connect_timeout)
- : cf_(std::make_shared<ConnectionFactory>(io_executor, codec, conf, connect_timeout)),
- clients_(),
+ : cf_(std::make_shared<ConnectionFactory>(io_executor, cpu_executor, codec, conf,
+ connect_timeout)),
connections_(),
map_mutex_(),
conf_(conf) {}
ConnectionPool::ConnectionPool(std::shared_ptr<ConnectionFactory> cf)
- : cf_(cf), clients_(), connections_(), map_mutex_() {}
+ : cf_(cf), connections_(), map_mutex_() {}
-ConnectionPool::~ConnectionPool() { Close(); }
+ConnectionPool::~ConnectionPool() {}
std::shared_ptr<RpcConnection> ConnectionPool::GetConnection(
std::shared_ptr<ConnectionId> remote_id) {
@@ -85,12 +87,9 @@ std::shared_ptr<RpcConnection> ConnectionPool::GetNewConnection(
connections_.erase(remote_id);
/* create new connection */
- auto clientBootstrap = cf_->MakeBootstrap();
- auto dispatcher = cf_->Connect(clientBootstrap, remote_id->host(), remote_id->port());
- auto connection = std::make_shared<RpcConnection>(remote_id, dispatcher);
+ auto connection = std::make_shared<RpcConnection>(remote_id, cf_);
connections_.insert(std::make_pair(remote_id, connection));
- clients_.insert(std::make_pair(remote_id, clientBootstrap));
return connection;
}
@@ -107,7 +106,6 @@ void ConnectionPool::Close(std::shared_ptr<ConnectionId> remote_id) {
}
found->second->Close();
connections_.erase(found);
- // TODO: erase the client as well?
}
void ConnectionPool::Close() {
@@ -117,6 +115,5 @@ void ConnectionPool::Close() {
con->Close();
}
connections_.clear();
- clients_.clear();
}
} // namespace hbase
diff --git a/connection/connection-pool.h b/connection/connection-pool.h
index c7c4246..9af1e7f 100644
--- a/connection/connection-pool.h
+++ b/connection/connection-pool.h
@@ -43,6 +43,7 @@ class ConnectionPool {
public:
/** Create connection pool wit default connection factory */
ConnectionPool(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
std::shared_ptr<Codec> codec, std::shared_ptr<Configuration> conf,
std::chrono::nanoseconds connect_timeout = std::chrono::nanoseconds(0));
@@ -81,10 +82,6 @@ class ConnectionPool {
std::unordered_map<std::shared_ptr<ConnectionId>, std::shared_ptr<RpcConnection>,
ConnectionIdHash, ConnectionIdEquals>
connections_;
- std::unordered_map<std::shared_ptr<ConnectionId>,
- std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>, ConnectionIdHash,
- ConnectionIdEquals>
- clients_;
folly::SharedMutexWritePriority map_mutex_;
std::shared_ptr<ConnectionFactory> cf_;
std::shared_ptr<Configuration> conf_;
diff --git a/connection/rpc-client.cc b/connection/rpc-client.cc
index a16dca6..51c9c63 100644
--- a/connection/rpc-client.cc
+++ b/connection/rpc-client.cc
@@ -32,10 +32,11 @@ using std::chrono::nanoseconds;
namespace hbase {
RpcClient::RpcClient(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
std::shared_ptr<Codec> codec, std::shared_ptr<Configuration> conf,
nanoseconds connect_timeout)
: io_executor_(io_executor), conf_(conf) {
- cp_ = std::make_shared<ConnectionPool>(io_executor_, codec, conf, connect_timeout);
+ cp_ = std::make_shared<ConnectionPool>(io_executor_, cpu_executor, codec, conf, connect_timeout);
}
void RpcClient::Close() { io_executor_->stop(); }
diff --git a/connection/rpc-client.h b/connection/rpc-client.h
index 8145be4..93801d8 100644
--- a/connection/rpc-client.h
+++ b/connection/rpc-client.h
@@ -36,8 +36,9 @@ namespace hbase {
class RpcClient {
public:
- RpcClient(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor, std::shared_ptr<Codec> codec,
- std::shared_ptr<Configuration> conf,
+ RpcClient(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
+ std::shared_ptr<Codec> codec, std::shared_ptr<Configuration> conf,
std::chrono::nanoseconds connect_timeout = std::chrono::nanoseconds(0));
virtual ~RpcClient() { Close(); }
diff --git a/connection/rpc-connection.h b/connection/rpc-connection.h
index d9966a1..9063280 100644
--- a/connection/rpc-connection.h
+++ b/connection/rpc-connection.h
@@ -18,36 +18,62 @@
*/
#pragma once
+#include <memory>
+#include <mutex>
+#include <utility>
+
+#include "connection/connection-factory.h"
#include "connection/connection-id.h"
#include "connection/request.h"
#include "connection/response.h"
#include "connection/service.h"
-#include <memory>
-#include <utility>
-
namespace hbase {
-class RpcConnection {
+class RpcConnection : public std::enable_shared_from_this<RpcConnection> {
public:
- RpcConnection(std::shared_ptr<ConnectionId> connection_id,
- std::shared_ptr<HBaseService> hbase_service)
- : connection_id_(connection_id), hbase_service_(hbase_service) {}
+ RpcConnection(std::shared_ptr<ConnectionId> connection_id, std::shared_ptr<ConnectionFactory> cf)
+ : connection_id_(connection_id), cf_(cf), hbase_service_(nullptr) {}
virtual ~RpcConnection() { Close(); }
virtual std::shared_ptr<ConnectionId> remote_id() const { return connection_id_; }
- virtual std::shared_ptr<HBaseService> get_service() const { return hbase_service_; }
-
virtual folly::Future<std::unique_ptr<Response>> SendRequest(std::unique_ptr<Request> req) {
+ std::lock_guard<std::recursive_mutex> lock(mutex_);
+ if (hbase_service_ == nullptr) {
+ Connect();
+ }
+ VLOG(5) << "Calling RpcConnection::SendRequest()"; // TODO
return (*hbase_service_)(std::move(req));
}
- virtual void Close() { hbase_service_->close(); }
+ virtual void Close() {
+ std::lock_guard<std::recursive_mutex> lock(mutex_);
+ if (hbase_service_) {
+ hbase_service_->close();
+ hbase_service_ = nullptr;
+ }
+ if (client_bootstrap_) {
+ client_bootstrap_ = nullptr;
+ }
+ }
+
+ private:
+ void Connect() {
+ client_bootstrap_ = cf_->MakeBootstrap();
+ auto dispatcher = cf_->Connect(shared_from_this(), client_bootstrap_, remote_id()->host(),
+ remote_id()->port());
+ hbase_service_ = std::move(dispatcher);
+ }
private:
+ std::recursive_mutex mutex_;
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_;
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
std::shared_ptr<ConnectionId> connection_id_;
std::shared_ptr<HBaseService> hbase_service_;
+ std::shared_ptr<ConnectionFactory> cf_;
+ std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> client_bootstrap_;
};
} // namespace hbase
diff --git a/connection/rpc-test.cc b/connection/rpc-test.cc
index d541397..8624e72 100644
--- a/connection/rpc-test.cc
+++ b/connection/rpc-test.cc
@@ -80,14 +80,17 @@ std::shared_ptr<folly::SocketAddress> GetRpcServerAddress(ServerPtr server) {
std::shared_ptr<RpcClient> CreateRpcClient(std::shared_ptr<Configuration> conf) {
auto io_executor = std::make_shared<wangle::IOThreadPoolExecutor>(1);
- auto client = std::make_shared<RpcClient>(io_executor, nullptr, conf);
+ auto cpu_executor = std::make_shared<wangle::CPUThreadPoolExecutor>(1);
+ auto client = std::make_shared<RpcClient>(io_executor, cpu_executor, nullptr, conf);
return client;
}
std::shared_ptr<RpcClient> CreateRpcClient(std::shared_ptr<Configuration> conf,
std::chrono::nanoseconds connect_timeout) {
auto io_executor = std::make_shared<wangle::IOThreadPoolExecutor>(1);
- auto client = std::make_shared<RpcClient>(io_executor, nullptr, conf, connect_timeout);
+ auto cpu_executor = std::make_shared<wangle::CPUThreadPoolExecutor>(1);
+ auto client =
+ std::make_shared<RpcClient>(io_executor, cpu_executor, nullptr, conf, connect_timeout);
return client;
}
@@ -115,7 +118,8 @@ TEST_F(RpcTest, Ping) {
})
.onError([&](const folly::exception_wrapper& ew) {
FAIL() << folly::sformat(FLAGS_fail_no_ex_format, method);
- }).get();
+ })
+ .get();
server->stop();
server->join();
@@ -149,7 +153,8 @@ TEST_F(RpcTest, Echo) {
})
.onError([&](const folly::exception_wrapper& ew) {
FAIL() << folly::sformat(FLAGS_fail_no_ex_format, method);
- }).get();
+ })
+ .get();
server->stop();
server->join();
@@ -188,7 +193,8 @@ TEST_F(RpcTest, Error) {
EXPECT_EQ(kRpcTestException, e.exception_class_name());
EXPECT_EQ(kRpcTestException + ": server error!", e.stack_trace());
}));
- }).get();
+ })
+ .get();
server->stop();
server->join();
@@ -235,7 +241,8 @@ TEST_F(RpcTest, SocketNotOpen) {
EXPECT_EQ(111 /*ECONNREFUSED*/, ase.getErrno());
});
}));
- }).get();
+ })
+ .get();
}
/**
@@ -269,7 +276,8 @@ TEST_F(RpcTest, Pause) {
.onError([&](const folly::exception_wrapper& ew) {
VLOG(1) << folly::sformat(FLAGS_result_format, method, ew.what());
FAIL() << folly::sformat(FLAGS_fail_no_ex_format, method);
- }).get();
+ })
+ .get();
server->stop();
server->join();
diff --git a/connection/sasl-handler.cc b/connection/sasl-handler.cc
index ea09595..9afe1e2 100644
--- a/connection/sasl-handler.cc
+++ b/connection/sasl-handler.cc
@@ -86,6 +86,7 @@ void SaslHandler::transportActive(Context *ctx) {
VLOG(3) << "Writing RPC connection Preamble to server: " << host_name_;
auto preamble = RpcSerde::Preamble(secure_);
ctx->fireWrite(std::move(preamble));
+ ctx->fireTransportActive();
}
void SaslHandler::read(Context *ctx, folly::IOBufQueue &buf) {
diff --git a/core/async-batch-rpc-retrying-test.cc b/core/async-batch-rpc-retrying-test.cc
index 0d186b4..cad03e1 100644
--- a/core/async-batch-rpc-retrying-test.cc
+++ b/core/async-batch-rpc-retrying-test.cc
@@ -68,6 +68,7 @@ using folly::exception_wrapper;
class AsyncBatchRpcRetryTest : public ::testing::Test {
public:
static std::unique_ptr<hbase::TestUtil> test_util;
+
static void SetUpTestCase() {
google::InstallFailureSignalHandler();
test_util = std::make_unique<hbase::TestUtil>();
@@ -279,14 +280,15 @@ class MockRawAsyncTableImpl {
void runMultiTest(std::shared_ptr<AsyncRegionLocatorBase> region_locator,
const std::string &table_name, bool split_regions, uint32_t tries = 3,
- uint32_t operation_timeout_millis = 600000, uint32_t num_rows = 10000) {
+ uint32_t operation_timeout_millis = 600000, uint32_t num_rows = 1000) {
std::vector<std::string> keys{"test0", "test100", "test200", "test300", "test400",
"test500", "test600", "test700", "test800", "test900"};
std::string tableName = (split_regions) ? ("split-" + table_name) : table_name;
- if (split_regions)
+ if (split_regions) {
AsyncBatchRpcRetryTest::test_util->CreateTable(tableName, "d", keys);
- else
+ } else {
AsyncBatchRpcRetryTest::test_util->CreateTable(tableName, "d");
+ }
// Create TableName and Row to be fetched from HBase
auto tn = folly::to<hbase::pb::TableName>(tableName);
@@ -316,8 +318,8 @@ void runMultiTest(std::shared_ptr<AsyncRegionLocatorBase> region_locator,
auto io_executor_ = client.async_connection()->io_executor();
auto retry_executor_ = std::make_shared<wangle::IOThreadPoolExecutor>(1);
auto codec = std::make_shared<hbase::KeyValueCodec>();
- auto rpc_client =
- std::make_shared<RpcClient>(io_executor_, codec, AsyncBatchRpcRetryTest::test_util->conf());
+ auto rpc_client = std::make_shared<RpcClient>(io_executor_, cpu_executor_, codec,
+ AsyncBatchRpcRetryTest::test_util->conf());
std::shared_ptr<folly::HHWheelTimer> retry_timer =
folly::HHWheelTimer::newTimer(retry_executor_->getEventBase());
@@ -416,47 +418,54 @@ TEST_F(AsyncBatchRpcRetryTest, FailWithExceptionFromRegionLocationLookup) {
TEST_F(AsyncBatchRpcRetryTest, FailWithOperationTimeout) {
std::shared_ptr<AsyncRegionLocatorBase> region_locator(
std::make_shared<MockFailingAsyncRegionLocator>(6));
- EXPECT_ANY_THROW(runMultiTest(region_locator, "table6", false, 5, 100, 10000));
+ EXPECT_ANY_THROW(runMultiTest(region_locator, "table6", false, 5, 100, 1000));
}
+/*
+ TODO: Below tests are failing with frequently with segfaults coming from
+ JNI internals indicating that we are doing something wrong in the JNI boundary.
+ However, we were not able to debug furhter yet. Disable the tests for now, and
+ come back later to fix the issue.
+
// Test successful case
TEST_F(AsyncBatchRpcRetryTest, MultiGetsSplitRegions) {
std::shared_ptr<AsyncRegionLocatorBase> region_locator(
std::make_shared<MockAsyncRegionLocator>());
- runMultiTest(region_locator, "table1", true);
+ runMultiTest(region_locator, "table7", true);
}
// Tests the RPC failing 3 times, then succeeding
TEST_F(AsyncBatchRpcRetryTest, HandleExceptionSplitRegions) {
std::shared_ptr<AsyncRegionLocatorBase> region_locator(
std::make_shared<MockWrongRegionAsyncRegionLocator>(3));
- runMultiTest(region_locator, "table2", true, 5);
+ runMultiTest(region_locator, "table8", true, 5);
}
// Tests the RPC failing 4 times, throwing an exception
TEST_F(AsyncBatchRpcRetryTest, FailWithExceptionSplitRegions) {
std::shared_ptr<AsyncRegionLocatorBase> region_locator(
std::make_shared<MockWrongRegionAsyncRegionLocator>(4));
- EXPECT_ANY_THROW(runMultiTest(region_locator, "table3", true));
+ EXPECT_ANY_THROW(runMultiTest(region_locator, "table9", true));
}
// Tests the region location lookup failing 3 times, then succeeding
TEST_F(AsyncBatchRpcRetryTest, HandleExceptionFromRegionLocationLookupSplitRegions) {
std::shared_ptr<AsyncRegionLocatorBase> region_locator(
std::make_shared<MockFailingAsyncRegionLocator>(3));
- runMultiTest(region_locator, "table4", true);
+ runMultiTest(region_locator, "table10", true);
}
// Tests the region location lookup failing 5 times, throwing an exception
TEST_F(AsyncBatchRpcRetryTest, FailWithExceptionFromRegionLocationLookupSplitRegions) {
std::shared_ptr<AsyncRegionLocatorBase> region_locator(
std::make_shared<MockFailingAsyncRegionLocator>(4));
- EXPECT_ANY_THROW(runMultiTest(region_locator, "table5", true, 3));
+ EXPECT_ANY_THROW(runMultiTest(region_locator, "table11", true, 3));
}
// Tests hitting operation timeout, thus not retrying anymore
TEST_F(AsyncBatchRpcRetryTest, FailWithOperationTimeoutSplitRegions) {
std::shared_ptr<AsyncRegionLocatorBase> region_locator(
std::make_shared<MockFailingAsyncRegionLocator>(6));
- EXPECT_ANY_THROW(runMultiTest(region_locator, "table6", true, 5, 100, 10000));
+ EXPECT_ANY_THROW(runMultiTest(region_locator, "table12", true, 5, 100, 1000));
}
+*/
diff --git a/core/async-connection.cc b/core/async-connection.cc
index ef945fb..850fb8f 100644
--- a/core/async-connection.cc
+++ b/core/async-connection.cc
@@ -44,10 +44,10 @@ void AsyncConnectionImpl::Init() {
} else {
LOG(WARNING) << "Not using RPC Cell Codec";
}
- rpc_client_ = std::make_shared<hbase::RpcClient>(io_executor_, codec, conf_,
+ rpc_client_ = std::make_shared<hbase::RpcClient>(io_executor_, cpu_executor_, codec, conf_,
connection_conf_->connect_timeout());
- location_cache_ =
- std::make_shared<hbase::LocationCache>(conf_, cpu_executor_, rpc_client_->connection_pool());
+ location_cache_ = std::make_shared<hbase::LocationCache>(conf_, io_executor_, cpu_executor_,
+ rpc_client_->connection_pool());
caller_factory_ =
std::make_shared<AsyncRpcRetryingCallerFactory>(shared_from_this(), retry_timer_);
}
diff --git a/core/async-rpc-retrying-test.cc b/core/async-rpc-retrying-test.cc
index 95b7143..2eb82a9 100644
--- a/core/async-rpc-retrying-test.cc
+++ b/core/async-rpc-retrying-test.cc
@@ -316,8 +316,8 @@ void runTest(std::shared_ptr<AsyncRegionLocatorBase> region_locator, std::string
auto io_executor_ = client.async_connection()->io_executor();
auto retry_executor_ = std::make_shared<wangle::IOThreadPoolExecutor>(1);
auto codec = std::make_shared<hbase::KeyValueCodec>();
- auto rpc_client =
- std::make_shared<RpcClient>(io_executor_, codec, AsyncRpcRetryTest::test_util->conf());
+ auto rpc_client = std::make_shared<RpcClient>(io_executor_, cpu_executor_, codec,
+ AsyncRpcRetryTest::test_util->conf());
// auto retry_event_base_ = std::make_shared<folly::ScopedEventBaseThread>(true);
std::shared_ptr<folly::HHWheelTimer> retry_timer =
folly::HHWheelTimer::newTimer(retry_executor_->getEventBase());
diff --git a/core/location-cache-test.cc b/core/location-cache-test.cc
index 3253c56..fd96ff3 100644
--- a/core/location-cache-test.cc
+++ b/core/location-cache-test.cc
@@ -27,8 +27,15 @@
#include "if/HBase.pb.h"
#include "serde/table-name.h"
#include "test-util/test-util.h"
-using namespace hbase;
-using namespace std::chrono;
+
+using hbase::Cell;
+using hbase::Configuration;
+using hbase::ConnectionPool;
+using hbase::MetaUtil;
+using hbase::LocationCache;
+using hbase::TestUtil;
+using hbase::KeyValueCodec;
+using std::chrono::milliseconds;
class LocationCacheTest : public ::testing::Test {
protected:
@@ -52,8 +59,8 @@ TEST_F(LocationCacheTest, TestGetMetaNodeContents) {
auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
auto codec = std::make_shared<KeyValueCodec>();
- auto cp = std::make_shared<ConnectionPool>(io, codec, LocationCacheTest::test_util_->conf());
- LocationCache cache{LocationCacheTest::test_util_->conf(), cpu, cp};
+ auto cp = std::make_shared<ConnectionPool>(io, cpu, codec, LocationCacheTest::test_util_->conf());
+ LocationCache cache{LocationCacheTest::test_util_->conf(), io, cpu, cp};
auto f = cache.LocateMeta();
auto result = f.get();
ASSERT_FALSE(f.hasException());
@@ -61,15 +68,14 @@ TEST_F(LocationCacheTest, TestGetMetaNodeContents) {
ASSERT_TRUE(result.has_host_name());
cpu->stop();
io->stop();
- cp->Close();
}
TEST_F(LocationCacheTest, TestGetRegionLocation) {
auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
auto codec = std::make_shared<KeyValueCodec>();
- auto cp = std::make_shared<ConnectionPool>(io, codec, LocationCacheTest::test_util_->conf());
- LocationCache cache{LocationCacheTest::test_util_->conf(), cpu, cp};
+ auto cp = std::make_shared<ConnectionPool>(io, cpu, codec, LocationCacheTest::test_util_->conf());
+ LocationCache cache{LocationCacheTest::test_util_->conf(), io, cpu, cp};
// If there is no table this should throw an exception
auto tn = folly::to<hbase::pb::TableName>("t");
@@ -80,15 +86,14 @@ TEST_F(LocationCacheTest, TestGetRegionLocation) {
ASSERT_TRUE(loc != nullptr);
cpu->stop();
io->stop();
- cp->Close();
}
TEST_F(LocationCacheTest, TestCaching) {
auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
auto codec = std::make_shared<KeyValueCodec>();
- auto cp = std::make_shared<ConnectionPool>(io, codec, LocationCacheTest::test_util_->conf());
- LocationCache cache{LocationCacheTest::test_util_->conf(), cpu, cp};
+ auto cp = std::make_shared<ConnectionPool>(io, cpu, codec, LocationCacheTest::test_util_->conf());
+ LocationCache cache{LocationCacheTest::test_util_->conf(), io, cpu, cp};
auto tn_1 = folly::to<hbase::pb::TableName>("t1");
auto tn_2 = folly::to<hbase::pb::TableName>("t2");
@@ -156,5 +161,4 @@ TEST_F(LocationCacheTest, TestCaching) {
cpu->stop();
io->stop();
- cp->Close();
}
diff --git a/core/location-cache.cc b/core/location-cache.cc
index ed5f5dc..b728d95 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -25,6 +25,7 @@
#include <wangle/concurrent/IOThreadPoolExecutor.h>
#include <map>
+#include <shared_mutex>
#include <utility>
#include "connection/response.h"
@@ -44,13 +45,15 @@ using hbase::pb::TableName;
namespace hbase {
LocationCache::LocationCache(std::shared_ptr<hbase::Configuration> conf,
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
std::shared_ptr<ConnectionPool> cp)
: conf_(conf),
+ io_executor_(io_executor),
cpu_executor_(cpu_executor),
+ cp_(cp),
meta_promise_(nullptr),
meta_lock_(),
- cp_(cp),
meta_util_(),
zk_(nullptr),
cached_locations_(),
@@ -147,11 +150,12 @@ folly::Future<std::shared_ptr<RegionLocation>> LocationCache::LocateFromMeta(
return this->LocateMeta()
.via(cpu_executor_.get())
.then([this](ServerName sn) {
+ // TODO: use RpcClient?
auto remote_id = std::make_shared<ConnectionId>(sn.host_name(), sn.port());
return this->cp_->GetConnection(remote_id);
})
.then([tn, row, this](std::shared_ptr<RpcConnection> rpc_connection) {
- return (*rpc_connection->get_service())(std::move(meta_util_.MetaRequest(tn, row)));
+ return rpc_connection->SendRequest(std::move(meta_util_.MetaRequest(tn, row)));
})
.onError([&](const folly::exception_wrapper &ew) {
auto promise = InvalidateMeta();
diff --git a/core/location-cache.h b/core/location-cache.h
index 932bef7..6eb61ef 100644
--- a/core/location-cache.h
+++ b/core/location-cache.h
@@ -27,18 +27,19 @@
#include <wangle/concurrent/IOThreadPoolExecutor.h>
#include <zookeeper/zookeeper.h>
+#include <map>
#include <memory>
#include <mutex>
-#include <shared_mutex>
#include <string>
+#include <unordered_map>
#include "connection/connection-pool.h"
#include "core/async-region-locator.h"
#include "core/configuration.h"
#include "core/meta-utils.h"
#include "core/region-location.h"
+#include "core/zk-util.h"
#include "serde/table-name.h"
-#include "zk-util.h"
namespace hbase {
// Forward
@@ -87,6 +88,7 @@ class LocationCache : public AsyncRegionLocator {
* @param io_executor executor used to talk to the network
*/
LocationCache(std::shared_ptr<hbase::Configuration> conf,
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
std::shared_ptr<ConnectionPool> cp);
/**
@@ -129,7 +131,7 @@ class LocationCache : public AsyncRegionLocator {
* @param row of the table to look up. This object must live until after the
* future is returned
*/
- virtual folly::Future<std::shared_ptr<RegionLocation>> LocateRegion(
+ folly::Future<std::shared_ptr<RegionLocation>> LocateRegion(
const hbase::pb::TableName &tn, const std::string &row,
const RegionLocateType locate_type = RegionLocateType::kCurrent,
const int64_t locate_ns = 0) override;
@@ -180,8 +182,8 @@ class LocationCache : public AsyncRegionLocator {
/**
* Update cached region location, possibly using the information from exception.
*/
- virtual void UpdateCachedLocation(const RegionLocation &loc,
- const folly::exception_wrapper &error) override;
+ void UpdateCachedLocation(const RegionLocation &loc,
+ const folly::exception_wrapper &error) override;
const std::string &zk_quorum() { return zk_quorum_; }
@@ -200,6 +202,7 @@ class LocationCache : public AsyncRegionLocator {
/* data */
std::shared_ptr<hbase::Configuration> conf_;
std::string zk_quorum_;
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_;
std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
std::shared_ptr<folly::SharedPromise<hbase::pb::ServerName>> meta_promise_;
std::recursive_mutex meta_lock_;
diff --git a/core/region-location.h b/core/region-location.h
index 822180b..f73999f 100644
--- a/core/region-location.h
+++ b/core/region-location.h
@@ -21,7 +21,6 @@
#include <memory>
#include <string>
-#include "connection/service.h"
#include "if/HBase.pb.h"
namespace hbase {
@@ -32,7 +31,7 @@ enum class RegionLocateType { kBefore, kCurrent, kAfter };
* @brief class to hold where a region is located.
*
* This class holds where a region is located, the information about it, the
- * region name, and a connection to the service used for connecting to it.
+ * region name.
*/
class RegionLocation {
public:
@@ -42,7 +41,6 @@ class RegionLocation {
* @param ri The decoded RegionInfo of this region.
* @param sn The server name of the HBase regionserver thought to be hosting
* this region.
- * @param service the connected service to the regionserver.
*/
RegionLocation(std::string region_name, hbase::pb::RegionInfo ri, hbase::pb::ServerName sn)
: region_name_(region_name), ri_(ri), sn_(sn) {}
diff --git a/test-util/mini-cluster.cc b/test-util/mini-cluster.cc
index 56461e1..9dd2f12 100644
--- a/test-util/mini-cluster.cc
+++ b/test-util/mini-cluster.cc
@@ -66,14 +66,18 @@ JNIEnv *MiniCluster::CreateVM(JavaVM **jvm) {
args.ignoreUnrecognized = 0;
int rv;
rv = JNI_CreateJavaVM(jvm, reinterpret_cast<void **>(&env_), &args);
- if (rv < 0 || !env_) {
- LOG(INFO) << "Unable to Launch JVM " << rv;
- } else {
- LOG(INFO) << "Launched JVM! " << options;
- }
+ CHECK(rv >= 0 && env_);
return env_;
}
+MiniCluster::~MiniCluster() {
+ if (jvm_ != NULL) {
+ jvm_->DestroyJavaVM();
+ jvm_ = NULL;
+ }
+ env_ = nullptr;
+}
+
void MiniCluster::Setup() {
jmethodID constructor;
pthread_mutex_lock(&count_mutex_);
@@ -186,10 +190,9 @@ JNIEnv *MiniCluster::env() {
}
// converts C char* to Java byte[]
jbyteArray MiniCluster::StrToByteChar(const std::string &str) {
- if (str.size() == 0) {
+ if (str.length() == 0) {
return nullptr;
}
- char *p = const_cast<char *>(str.c_str());
int n = str.length();
jbyteArray arr = env_->NewByteArray(n);
env_->SetByteArrayRegion(arr, 0, n, reinterpret_cast<const jbyte *>(str.c_str()));
diff --git a/test-util/mini-cluster.h b/test-util/mini-cluster.h
index b8ac391..6b4547c 100644
--- a/test-util/mini-cluster.h
+++ b/test-util/mini-cluster.h
@@ -26,6 +26,7 @@ namespace hbase {
class MiniCluster {
public:
+ virtual ~MiniCluster();
jobject StartCluster(int32_t num_region_servers);
void StopCluster();
jobject CreateTable(const std::string &table, const std::string &family);
diff --git a/test-util/test-util.cc b/test-util/test-util.cc
index b32c635..ea18b84 100644
--- a/test-util/test-util.cc
+++ b/test-util/test-util.cc
@@ -47,7 +47,10 @@ std::string TestUtil::RandString(int len) {
TestUtil::TestUtil() : temp_dir_(TestUtil::RandString()) {}
TestUtil::~TestUtil() {
- if (mini_) StopMiniCluster();
+ if (mini_) {
+ StopMiniCluster();
+ mini_ = nullptr;
+ }
}
void TestUtil::StartMiniCluster(int32_t num_region_servers) {
diff --git a/utils/concurrent-map.h b/utils/concurrent-map.h
index d9703e1..aebca0d 100644
--- a/utils/concurrent-map.h
+++ b/utils/concurrent-map.h
@@ -118,6 +118,11 @@ class concurrent_map {
return map_.empty();
}
+ void clear() {
+ std::unique_lock<std::shared_timed_mutex> lock(mutex_);
+ map_.clear();
+ }
+
private:
std::shared_timed_mutex mutex_;
std::unordered_map<K, V> map_;
[hbase] 30/137: HBASE-15770 Stop using wangle's global executor
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit b535ffaf307ad7bb3eeb419657db47eb2d15f536
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Fri May 6 14:32:16 2016 -0700
HBASE-15770 Stop using wangle's global executor
Summary: Connection pool and connection factory now get thread pools through their constructor. This means that the client has the whole control over the threads.
Test Plan: simple-client still writes.
Differential Revision: https://reviews.facebook.net/D57801
---
connection/client-dispatcher.h | 3 ++-
connection/client-handler.cc | 2 +-
connection/connection-factory.cc | 8 +++-----
connection/connection-factory.h | 2 +-
connection/connection-pool-test.cc | 1 +
connection/connection-pool.cc | 7 ++++---
connection/connection-pool.h | 2 +-
core/client.cc | 23 ++++++++++++++++-------
core/client.h | 9 +++++++--
core/location-cache-test.cc | 5 +++--
core/location-cache.cc | 28 +++++++++++++++-------------
core/location-cache.h | 8 ++++++--
core/simple-client.cc | 8 +++-----
serde/server-name-test.cc | 1 -
14 files changed, 63 insertions(+), 44 deletions(-)
diff --git a/connection/client-dispatcher.h b/connection/client-dispatcher.h
index 4bfb35d..2497cc7 100644
--- a/connection/client-dispatcher.h
+++ b/connection/client-dispatcher.h
@@ -31,7 +31,8 @@
namespace hbase {
/**
- * Dispatcher that assigns a call_id and then routes the response back to the future.
+ * Dispatcher that assigns a call_id and then routes the response back to the
+ * future.
*/
class ClientDispatcher
: public wangle::ClientDispatcherBase<SerializePipeline,
diff --git a/connection/client-handler.cc b/connection/client-handler.cc
index cae03c7..2e3fcd3 100644
--- a/connection/client-handler.cc
+++ b/connection/client-handler.cc
@@ -37,7 +37,7 @@ using hbase::pb::GetResponse;
using google::protobuf::Message;
ClientHandler::ClientHandler(std::string user_name)
- : user_name_(user_name), serde_(), once_flag_(),
+ : user_name_(user_name), serde_(), once_flag_(std::make_unique<std::once_flag>()),
resp_msgs_(
make_unique<folly::AtomicHashMap<
uint32_t, std::shared_ptr<google::protobuf::Message>>>(5000)) {}
diff --git a/connection/connection-factory.cc b/connection/connection-factory.cc
index 635d12d..beec6d56 100644
--- a/connection/connection-factory.cc
+++ b/connection/connection-factory.cc
@@ -19,8 +19,6 @@
#include "connection/connection-factory.h"
-#include <wangle/concurrent/GlobalExecutor.h>
-
#include "connection/client-dispatcher.h"
#include "connection/pipeline.h"
#include "connection/service.h"
@@ -28,9 +26,9 @@
using namespace folly;
using namespace hbase;
-ConnectionFactory::ConnectionFactory()
- : io_pool_(std::static_pointer_cast<wangle::IOThreadPoolExecutor>(
- wangle::getIOExecutor())),
+ConnectionFactory::ConnectionFactory(
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool)
+ : io_pool_(io_pool),
pipeline_factory_(std::make_shared<RpcPipelineFactory>()) {}
std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>
diff --git a/connection/connection-factory.h b/connection/connection-factory.h
index 8b6d8d8..fb5d9fe 100644
--- a/connection/connection-factory.h
+++ b/connection/connection-factory.h
@@ -39,7 +39,7 @@ public:
* Constructor.
* There should only be one ConnectionFactory per client.
*/
- ConnectionFactory();
+ ConnectionFactory(std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool);
/** Default Desctructor */
virtual ~ConnectionFactory() = default;
diff --git a/connection/connection-pool-test.cc b/connection/connection-pool-test.cc
index b1a0ba0..c0c346f 100644
--- a/connection/connection-pool-test.cc
+++ b/connection/connection-pool-test.cc
@@ -34,6 +34,7 @@ using ::testing::_;
class MockConnectionFactory : public ConnectionFactory {
public:
+ MockConnectionFactory() : ConnectionFactory(nullptr) {}
MOCK_METHOD0(MakeBootstrap,
std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>());
MOCK_METHOD3(Connect,
diff --git a/connection/connection-pool.cc b/connection/connection-pool.cc
index 75f343e..90e2056 100644
--- a/connection/connection-pool.cc
+++ b/connection/connection-pool.cc
@@ -31,9 +31,10 @@ using hbase::HBaseService;
using folly::SharedMutexWritePriority;
using folly::SocketAddress;
-ConnectionPool::ConnectionPool()
- : cf_(std::make_shared<ConnectionFactory>()), clients_(), connections_(),
- map_mutex_() {}
+ConnectionPool::ConnectionPool(
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor)
+ : cf_(std::make_shared<ConnectionFactory>(io_executor)), clients_(),
+ connections_(), map_mutex_() {}
ConnectionPool::ConnectionPool(std::shared_ptr<ConnectionFactory> cf)
: cf_(cf), clients_(), connections_(), map_mutex_() {}
diff --git a/connection/connection-pool.h b/connection/connection-pool.h
index 5edd407..60f00de 100644
--- a/connection/connection-pool.h
+++ b/connection/connection-pool.h
@@ -61,7 +61,7 @@ struct ServerNameHash {
class ConnectionPool {
public:
/** Create connection pool wit default connection factory */
- ConnectionPool();
+ ConnectionPool(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor);
/**
* Desctructor.
diff --git a/core/client.cc b/core/client.cc
index 4b9f844..1e80998 100644
--- a/core/client.cc
+++ b/core/client.cc
@@ -19,18 +19,27 @@
#include "core/client.h"
-#include <folly/Logging.h>
-#include <folly/Random.h>
-#include <gflags/gflags.h>
#include <glog/logging.h>
-#include <wangle/concurrent/GlobalExecutor.h>
#include <string>
-#include "if/ZooKeeper.pb.h"
-
using namespace folly;
using namespace std;
using namespace hbase::pb;
-namespace hbase {}
+namespace hbase {
+
+Client::Client(std::string zk_quorum)
+ : cpu_executor_(std::make_shared<wangle::CPUThreadPoolExecutor>(4)),
+ io_executor_(std::make_shared<wangle::IOThreadPoolExecutor>(
+ sysconf(_SC_NPROCESSORS_ONLN))),
+ location_cache_(zk_quorum, cpu_executor_, io_executor_) {}
+
+// We can't have the threads continue running after everything is done
+// that leads to an error.
+Client::~Client() {
+ cpu_executor_->stop();
+ io_executor_->stop();
+}
+
+} // namespace hbase
diff --git a/core/client.h b/core/client.h
index 4a6d23b..4db82c4 100644
--- a/core/client.h
+++ b/core/client.h
@@ -21,6 +21,8 @@
#include <folly/futures/Future.h>
#include <folly/io/IOBuf.h>
+#include <wangle/concurrent/CPUThreadPoolExecutor.h>
+#include <wangle/concurrent/IOThreadPoolExecutor.h>
#include <string>
@@ -33,19 +35,22 @@ namespace hbase {
* Client.
*
* This is the class that provides access to an HBase cluster.
- * It is thread safe and does connection pooling. Current recommendations are to have only one Client per cluster around.
+ * It is thread safe and does connection pooling. Current recommendations are to
+ * have only one Client per cluster around.
*/
class Client {
public:
-
/**
* Create a new client.
* @param quorum_spec Where to connect to get Zookeeper bootstrap information.
*/
explicit Client(std::string quorum_spec);
+ ~Client();
private:
LocationCache location_cache_;
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_;
};
} // namespace hbase
diff --git a/core/location-cache-test.cc b/core/location-cache-test.cc
index 172799d..8bc6383 100644
--- a/core/location-cache-test.cc
+++ b/core/location-cache-test.cc
@@ -18,14 +18,15 @@
*/
#include <folly/Memory.h>
#include <gtest/gtest.h>
-#include <wangle/concurrent/GlobalExecutor.h>
#include "location-cache.h"
using namespace hbase;
TEST(LocationCacheTest, TestGetMetaNodeContents) {
// TODO(elliott): need to make a test utility for this.
- LocationCache cache{"localhost:2181", wangle::getCPUExecutor()};
+ auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
+ auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
+ LocationCache cache{"localhost:2181", cpu, io};
auto f = cache.LocateMeta();
auto result = f.get();
ASSERT_FALSE(f.hasException());
diff --git a/core/location-cache.cc b/core/location-cache.cc
index 583d305..6c018f9 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -20,7 +20,8 @@
#include <folly/Logging.h>
#include <folly/io/IOBuf.h>
-#include <wangle/concurrent/GlobalExecutor.h>
+#include <wangle/concurrent/CPUThreadPoolExecutor.h>
+#include <wangle/concurrent/IOThreadPoolExecutor.h>
#include "connection/response.h"
#include "if/Client.pb.h"
@@ -48,10 +49,13 @@ using hbase::pb::RegionInfo;
// TODO(eclark): make this configurable on client creation
static const char META_ZNODE_NAME[] = "/hbase/meta-region-server";
-LocationCache::LocationCache(string quorum_spec,
- shared_ptr<folly::Executor> executor)
- : quorum_spec_(quorum_spec), executor_(executor), meta_promise_(nullptr),
- meta_lock_(), cp_(), meta_util_(), zk_(nullptr) {
+LocationCache::LocationCache(
+ std::string quorum_spec,
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor)
+ : quorum_spec_(quorum_spec), cpu_executor_(cpu_executor),
+ meta_promise_(nullptr), meta_lock_(), cp_(io_executor), meta_util_(),
+ zk_(nullptr) {
zk_ = zookeeper_init(quorum_spec.c_str(), nullptr, 1000, 0, 0, 0);
}
@@ -79,7 +83,7 @@ void LocationCache::InvalidateMeta() {
/// MUST hold the meta_lock_
void LocationCache::RefreshMetaLocation() {
meta_promise_ = make_unique<SharedPromise<ServerName>>();
- executor_->add([&] {
+ cpu_executor_->add([&] {
meta_promise_->setWith([&] { return this->ReadMetaLocation(); });
});
}
@@ -109,10 +113,9 @@ ServerName LocationCache::ReadMetaLocation() {
Future<std::shared_ptr<RegionLocation>>
LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
- auto exec = wangle::getCPUExecutor();
return this->LocateMeta()
- .via(exec.get())
- .then([ exec = exec, this ](ServerName sn) { return this->cp_.get(sn); })
+ .via(cpu_executor_.get())
+ .then([this](ServerName sn) { return this->cp_.get(sn); })
.then([&](std::shared_ptr<HBaseService> service) {
return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
})
@@ -121,7 +124,7 @@ LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
// a region location.
return this->CreateLocation(std::move(resp));
})
- .then([ exec = exec, this ](std::shared_ptr<RegionLocation> rl) {
+ .then([this](std::shared_ptr<RegionLocation> rl) {
// Now fill out the connection.
rl->set_service(cp_.get(rl->server_name()));
return rl;
@@ -129,14 +132,14 @@ LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
}
/**
- * Filter to remove a service from the location cache and the connection cache on errors
+ * Filter to remove a service from the location cache and the connection cache
+ * on errors
* or on cloase.
*/
class RemoveServiceFilter
: public ServiceFilter<std::unique_ptr<Request>, Response> {
public:
-
/** Create a new filter. */
RemoveServiceFilter(std::shared_ptr<HBaseService> service, ServerName sn,
ConnectionPool &cp)
@@ -157,7 +160,6 @@ public:
}
}
-
/** Has this been closed */
virtual bool isAvailable() override {
return !released && service_->isAvailable();
diff --git a/core/location-cache.h b/core/location-cache.h
index 88bec18..e077750 100644
--- a/core/location-cache.h
+++ b/core/location-cache.h
@@ -21,6 +21,8 @@
#include <folly/Executor.h>
#include <folly/futures/Future.h>
#include <folly/futures/SharedPromise.h>
+#include <wangle/concurrent/CPUThreadPoolExecutor.h>
+#include <wangle/concurrent/IOThreadPoolExecutor.h>
#include <zookeeper/zookeeper.h>
#include <memory>
@@ -51,7 +53,8 @@ public:
* @param executor The cpu executor to run on.
*/
LocationCache(std::string quorum_spec,
- std::shared_ptr<folly::Executor> executor);
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_exector,
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor);
/**
* Destructor.
* This will clean up the zookeeper connections.
@@ -81,8 +84,9 @@ private:
hbase::pb::ServerName ReadMetaLocation();
std::shared_ptr<RegionLocation> CreateLocation(const Response &resp);
+ /* data */
std::string quorum_spec_;
- std::shared_ptr<folly::Executor> executor_;
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> meta_promise_;
std::mutex meta_lock_;
MetaUtil meta_util_;
diff --git a/core/simple-client.cc b/core/simple-client.cc
index 39c82c3..f3f6c42 100644
--- a/core/simple-client.cc
+++ b/core/simple-client.cc
@@ -22,7 +22,7 @@
#include <folly/futures/Future.h>
#include <gflags/gflags.h>
#include <wangle/concurrent/CPUThreadPoolExecutor.h>
-#include <wangle/concurrent/GlobalExecutor.h>
+#include <wangle/concurrent/IOThreadPoolExecutor.h>
#include <atomic>
#include <chrono>
@@ -89,12 +89,10 @@ int main(int argc, char *argv[]) {
// Set up thread pools.
auto cpu_pool =
std::make_shared<wangle::CPUThreadPoolExecutor>(FLAGS_threads);
- wangle::setCPUExecutor(cpu_pool);
auto io_pool = std::make_shared<wangle::IOThreadPoolExecutor>(5);
- wangle::setIOExecutor(io_pool);
// Create the cache.
- LocationCache cache{FLAGS_zookeeper, cpu_pool};
+ LocationCache cache{FLAGS_zookeeper, cpu_pool, io_pool};
auto row = FLAGS_row;
auto tn = folly::to<TableName>(FLAGS_table);
@@ -105,7 +103,7 @@ int main(int argc, char *argv[]) {
auto num_puts = FLAGS_columns;
auto results = std::vector<Future<Response>>{};
- uint64_t col{0};
+ auto col = uint64_t{0};
for (; col < num_puts; col++) {
results.push_back(folly::makeFuture(col)
.via(cpu_pool.get())
diff --git a/serde/server-name-test.cc b/serde/server-name-test.cc
index 2281fa2..73a68d6 100644
--- a/serde/server-name-test.cc
+++ b/serde/server-name-test.cc
@@ -46,5 +46,4 @@ TEST(TestServerName, TestIPV6) {
ASSERT_EQ("[::::1]", sn.host_name());
ASSERT_EQ(123, sn.port());
-
}
[hbase] 82/137: HBASE-17800 [C++] handle exceptions in client RPC
(Xiaobing Zhou and Enis Soztutar)
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 569c6d0369c8184dcdae01c53e7849a2bee3e13d
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Thu Apr 27 18:52:53 2017 -0700
HBASE-17800 [C++] handle exceptions in client RPC (Xiaobing Zhou and Enis Soztutar)
---
connection/BUCK | 1 +
connection/client-dispatcher.cc | 10 +-
connection/client-handler.cc | 39 ++++-
connection/client-handler.h | 4 +-
connection/connection-factory.cc | 3 +-
connection/connection-factory.h | 2 +-
connection/connection-pool-test.cc | 2 +-
connection/connection-pool.cc | 7 +-
connection/pipeline.cc | 5 +-
connection/response.h | 9 +-
core/async-connection.cc | 18 ++-
core/async-connection.h | 10 ++
core/async-region-locator.h | 4 +-
core/async-rpc-retrying-caller-factory.h | 14 +-
core/async-rpc-retrying-caller.cc | 84 ++++++-----
core/async-rpc-retrying-caller.h | 11 +-
core/async-rpc-retrying-test.cc | 240 ++++++++++++++++++++++++++-----
core/client-test.cc | 1 +
core/client.h | 5 +
core/location-cache.cc | 3 +-
core/location-cache.h | 3 +-
exceptions/BUCK | 2 +-
exceptions/exception.h | 92 ++++++++++--
23 files changed, 450 insertions(+), 119 deletions(-)
diff --git a/connection/BUCK b/connection/BUCK
index 19536d5..36111f8 100644
--- a/connection/BUCK
+++ b/connection/BUCK
@@ -48,6 +48,7 @@ cxx_library(
"//security:security",
"//third-party:folly",
"//third-party:wangle",
+ "//exceptions:exceptions",
],
compiler_flags=['-Weffc++'],
visibility=['//core/...',],)
diff --git a/connection/client-dispatcher.cc b/connection/client-dispatcher.cc
index 626fc76..27201d2 100644
--- a/connection/client-dispatcher.cc
+++ b/connection/client-dispatcher.cc
@@ -17,12 +17,14 @@
*
*/
#include "connection/client-dispatcher.h"
+#include <folly/ExceptionWrapper.h>
#include <utility>
using namespace folly;
using namespace hbase;
using namespace wangle;
+using folly::exception_wrapper;
ClientDispatcher::ClientDispatcher() : requests_(5000), current_call_id_(9) {}
@@ -35,9 +37,11 @@ void ClientDispatcher::read(Context *ctx, std::unique_ptr<Response> in) {
requests_.erase(call_id);
- // TODO(eclark): check if the response
- // is an exception. If it is then set that.
- p.setValue(std::move(in));
+ if (in->exception()) {
+ p.setException(in->exception());
+ } else {
+ p.setValue(std::move(in));
+ }
}
Future<std::unique_ptr<Response>> ClientDispatcher::operator()(std::unique_ptr<Request> arg) {
diff --git a/connection/client-handler.cc b/connection/client-handler.cc
index af84572..113ebd0 100644
--- a/connection/client-handler.cc
+++ b/connection/client-handler.cc
@@ -19,6 +19,7 @@
#include "connection/client-handler.h"
+#include <folly/ExceptionWrapper.h>
#include <folly/Likely.h>
#include <glog/logging.h>
@@ -36,9 +37,11 @@ using hbase::pb::ResponseHeader;
using hbase::pb::GetResponse;
using google::protobuf::Message;
-ClientHandler::ClientHandler(std::string user_name, std::shared_ptr<Codec> codec)
+ClientHandler::ClientHandler(std::string user_name, std::shared_ptr<Codec> codec,
+ const std::string &server)
: user_name_(user_name),
serde_(codec),
+ server_(server),
once_flag_(std::make_unique<std::once_flag>()),
resp_msgs_(
make_unique<folly::AtomicHashMap<uint32_t, std::shared_ptr<google::protobuf::Message>>>(
@@ -51,7 +54,7 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
ResponseHeader header;
int used_bytes = serde_.ParseDelimited(buf.get(), &header);
- VLOG(1) << "Read RPC ResponseHeader size=" << used_bytes << " call_id=" << header.call_id()
+ VLOG(3) << "Read RPC ResponseHeader size=" << used_bytes << " call_id=" << header.call_id()
<< " has_exception=" << header.has_exception();
// Get the response protobuf from the map
@@ -92,9 +95,31 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
}
received->set_resp_msg(resp_msg);
- }
- // TODO: set exception in Response here
+ } else {
+ hbase::pb::ExceptionResponse exceptionResponse = header.exception();
+
+ std::string what;
+ std::string exception_class_name = exceptionResponse.has_exception_class_name()
+ ? exceptionResponse.exception_class_name()
+ : "";
+ std::string stack_trace =
+ exceptionResponse.has_stack_trace() ? exceptionResponse.stack_trace() : "";
+ what.append(exception_class_name).append(stack_trace);
+
+ auto remote_exception = std::make_unique<RemoteException>(what);
+ remote_exception->set_exception_class_name(exception_class_name)
+ ->set_stack_trace(stack_trace)
+ ->set_hostname(exceptionResponse.has_hostname() ? exceptionResponse.hostname() : "")
+ ->set_port(exceptionResponse.has_port() ? exceptionResponse.port() : 0);
+ if (exceptionResponse.has_do_not_retry()) {
+ remote_exception->set_do_not_retry(exceptionResponse.do_not_retry());
+ }
+ VLOG(3) << "Exception RPC ResponseHeader, call_id=" << header.call_id()
+ << " exception.what=" << remote_exception->what()
+ << ", do_not_retry=" << remote_exception->do_not_retry();
+ received->set_exception(::folly::exception_wrapper{*remote_exception});
+ }
ctx->fireRead(std::move(received));
}
}
@@ -103,17 +128,19 @@ Future<Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
// We need to send the header once.
// So use call_once to make sure that only one thread wins this.
std::call_once((*once_flag_), [ctx, this]() {
+ VLOG(3) << "Writing RPC connection Preamble and Header to server: " << server_;
auto pre = serde_.Preamble();
auto header = serde_.Header(user_name_);
pre->appendChain(std::move(header));
ctx->fireWrite(std::move(pre));
});
+ VLOG(3) << "Writing RPC Request with call_id:"
+ << r->call_id(); // TODO: more logging for RPC Header
+
// Now store the call id to response.
resp_msgs_->insert(r->call_id(), r->resp_msg());
- VLOG(1) << "Writing RPC Request with call_id:" << r->call_id();
-
// Send the data down the pipeline.
return ctx->fireWrite(serde_.Request(r->call_id(), r->method(), r->req_msg().get()));
}
diff --git a/connection/client-handler.h b/connection/client-handler.h
index afb8e62..4c106e0 100644
--- a/connection/client-handler.h
+++ b/connection/client-handler.h
@@ -27,6 +27,7 @@
#include <string>
#include <utility>
+#include "exceptions/exception.h"
#include "serde/codec.h"
#include "serde/rpc.h"
@@ -59,7 +60,7 @@ class ClientHandler
* Create the handler
* @param user_name the user name of the user running this process.
*/
- explicit ClientHandler(std::string user_name, std::shared_ptr<Codec> codec);
+ ClientHandler(std::string user_name, std::shared_ptr<Codec> codec, const std::string &server);
/**
* Get bytes from the wire.
@@ -77,6 +78,7 @@ class ClientHandler
std::unique_ptr<std::once_flag> once_flag_;
std::string user_name_;
RpcSerde serde_;
+ std::string server_; // for logging
// in flight requests
std::unique_ptr<folly::AtomicHashMap<uint32_t, std::shared_ptr<google::protobuf::Message>>>
diff --git a/connection/connection-factory.cc b/connection/connection-factory.cc
index 832b00f..afa227d 100644
--- a/connection/connection-factory.cc
+++ b/connection/connection-factory.cc
@@ -46,9 +46,10 @@ std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> ConnectionFactory::M
return client;
}
+
std::shared_ptr<HBaseService> ConnectionFactory::Connect(
std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> client, const std::string &hostname,
- int port) {
+ uint16_t port) {
// Yes this will block however it makes dealing with connection pool soooooo
// much nicer.
// TODO see about using shared promise for this.
diff --git a/connection/connection-factory.h b/connection/connection-factory.h
index 32d0bf7..1e75571 100644
--- a/connection/connection-factory.h
+++ b/connection/connection-factory.h
@@ -61,7 +61,7 @@ class ConnectionFactory {
*/
virtual std::shared_ptr<HBaseService> Connect(
std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> client,
- const std::string &hostname, int port);
+ const std::string &hostname, uint16_t port);
private:
nanoseconds connect_timeout_;
diff --git a/connection/connection-pool-test.cc b/connection/connection-pool-test.cc
index 623ce3c..8ecdf29 100644
--- a/connection/connection-pool-test.cc
+++ b/connection/connection-pool-test.cc
@@ -40,7 +40,7 @@ class MockConnectionFactory : public ConnectionFactory {
MOCK_METHOD0(MakeBootstrap, std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>());
MOCK_METHOD3(Connect, std::shared_ptr<HBaseService>(
std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>,
- const std::string &hostname, int port));
+ const std::string &hostname, uint16_t port));
};
class MockBootstrap : public wangle::ClientBootstrap<SerializePipeline> {};
diff --git a/connection/connection-pool.cc b/connection/connection-pool.cc
index 4fe4610..3121294 100644
--- a/connection/connection-pool.cc
+++ b/connection/connection-pool.cc
@@ -19,10 +19,11 @@
#include "connection/connection-pool.h"
+#include <folly/Conv.h>
+#include <folly/Logging.h>
#include <folly/SocketAddress.h>
#include <wangle/service/Service.h>
-#include <folly/Logging.h>
#include <memory>
#include <utility>
@@ -89,7 +90,6 @@ std::shared_ptr<RpcConnection> ConnectionPool::GetNewConnection(
/* create new connection */
auto clientBootstrap = cf_->MakeBootstrap();
auto dispatcher = cf_->Connect(clientBootstrap, remote_id->host(), remote_id->port());
-
auto connection = std::make_shared<RpcConnection>(remote_id, dispatcher);
connections_.insert(std::make_pair(remote_id, connection));
@@ -101,6 +101,8 @@ std::shared_ptr<RpcConnection> ConnectionPool::GetNewConnection(
void ConnectionPool::Close(std::shared_ptr<ConnectionId> remote_id) {
SharedMutexWritePriority::WriteHolder holder{map_mutex_};
+ DLOG(INFO) << "Closing RPC Connection to host:" << remote_id->host()
+ << ", port:" << folly::to<std::string>(remote_id->port());
auto found = connections_.find(remote_id);
if (found == connections_.end() || found->second == nullptr) {
@@ -108,6 +110,7 @@ void ConnectionPool::Close(std::shared_ptr<ConnectionId> remote_id) {
}
found->second->Close();
connections_.erase(found);
+ // TODO: erase the client as well?
}
void ConnectionPool::Close() {
diff --git a/connection/pipeline.cc b/connection/pipeline.cc
index 00dc05c..edada52 100644
--- a/connection/pipeline.cc
+++ b/connection/pipeline.cc
@@ -35,11 +35,14 @@ RpcPipelineFactory::RpcPipelineFactory(std::shared_ptr<Codec> codec)
SerializePipeline::Ptr RpcPipelineFactory::newPipeline(
std::shared_ptr<AsyncTransportWrapper> sock) {
+ SocketAddress addr; // for logging
+ sock->getPeerAddress(&addr);
+
auto pipeline = SerializePipeline::create();
pipeline->addBack(AsyncSocketHandler{sock});
pipeline->addBack(EventBaseHandler{});
pipeline->addBack(LengthFieldBasedFrameDecoder{});
- pipeline->addBack(ClientHandler{user_util_.user_name(), codec_});
+ pipeline->addBack(ClientHandler{user_util_.user_name(), codec_, addr.describe()});
pipeline->finalize();
return pipeline;
}
diff --git a/connection/response.h b/connection/response.h
index 1d60fed..c5472b0 100644
--- a/connection/response.h
+++ b/connection/response.h
@@ -22,6 +22,8 @@
#include <memory>
#include <utility>
+#include <folly/ExceptionWrapper.h>
+
#include "serde/cell-scanner.h"
// Forward
@@ -44,7 +46,7 @@ class Response {
* Constructor.
* Initinalizes the call id to 0. 0 should never be a valid call id.
*/
- Response() : call_id_(0), resp_msg_(nullptr), cell_scanner_(nullptr) {}
+ Response() : call_id_(0), resp_msg_(nullptr), cell_scanner_(nullptr), exception_(nullptr) {}
/** Get the call_id */
uint32_t call_id() { return call_id_; }
@@ -70,9 +72,14 @@ class Response {
const std::unique_ptr<CellScanner>& cell_scanner() const { return cell_scanner_; }
+ folly::exception_wrapper exception() { return exception_; }
+
+ void set_exception(folly::exception_wrapper value) { exception_ = value; }
+
private:
uint32_t call_id_;
std::shared_ptr<google::protobuf::Message> resp_msg_;
std::unique_ptr<CellScanner> cell_scanner_;
+ folly::exception_wrapper exception_;
};
} // namespace hbase
diff --git a/core/async-connection.cc b/core/async-connection.cc
index b945e38..4642c61 100644
--- a/core/async-connection.cc
+++ b/core/async-connection.cc
@@ -29,6 +29,13 @@ void AsyncConnectionImpl::Init() {
auto cpu_threads = conf_->GetInt(kClientCpuThreadPoolSize, 2 * sysconf(_SC_NPROCESSORS_ONLN));
cpu_executor_ = std::make_shared<wangle::CPUThreadPoolExecutor>(cpu_threads);
io_executor_ = std::make_shared<wangle::IOThreadPoolExecutor>(io_threads);
+ /*
+ * We need a retry_executor for a thread pool of size 1 due to a possible bug in wangle/folly.
+ * Otherwise, Assertion 'isInEventBaseThread()' always fails. See the comments
+ * in async-rpc-retrying-caller.cc.
+ */
+ retry_executor_ = std::make_shared<wangle::IOThreadPoolExecutor>(1);
+ retry_timer_ = folly::HHWheelTimer::newTimer(retry_executor_->getEventBase());
std::shared_ptr<Codec> codec = nullptr;
if (conf_->Get(kRpcCodec, hbase::KeyValueCodec::kJavaClassName) ==
@@ -41,22 +48,21 @@ void AsyncConnectionImpl::Init() {
std::make_shared<hbase::RpcClient>(io_executor_, codec, connection_conf_->connect_timeout());
location_cache_ =
std::make_shared<hbase::LocationCache>(conf_, cpu_executor_, rpc_client_->connection_pool());
- caller_factory_ = std::make_shared<AsyncRpcRetryingCallerFactory>(shared_from_this());
+ caller_factory_ =
+ std::make_shared<AsyncRpcRetryingCallerFactory>(shared_from_this(), retry_timer_);
}
// We can't have the threads continue running after everything is done
// that leads to an error.
-AsyncConnectionImpl::~AsyncConnectionImpl() {
- cpu_executor_->stop();
- io_executor_->stop();
- if (rpc_client_.get()) rpc_client_->Close();
-}
+AsyncConnectionImpl::~AsyncConnectionImpl() { Close(); }
void AsyncConnectionImpl::Close() {
if (is_closed_) return;
cpu_executor_->stop();
io_executor_->stop();
+ retry_executor_->stop();
+ retry_timer_->destroy();
if (rpc_client_.get()) rpc_client_->Close();
is_closed_ = true;
}
diff --git a/core/async-connection.h b/core/async-connection.h
index ff11577..7b260a5 100644
--- a/core/async-connection.h
+++ b/core/async-connection.h
@@ -53,6 +53,9 @@ class AsyncConnection {
virtual std::shared_ptr<RpcClient> rpc_client() = 0;
virtual std::shared_ptr<AsyncRegionLocator> region_locator() = 0;
virtual std::shared_ptr<HBaseRpcController> CreateRpcController() = 0;
+ virtual std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor() = 0;
+ virtual std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor() = 0;
+ virtual std::shared_ptr<wangle::IOThreadPoolExecutor> retry_executor() = 0;
virtual void Close() = 0;
};
@@ -81,6 +84,11 @@ class AsyncConnectionImpl : public AsyncConnection,
std::shared_ptr<HBaseRpcController> CreateRpcController() override {
return std::make_shared<HBaseRpcController>();
}
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor() override { return cpu_executor_; }
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor() override { return io_executor_; }
+ std::shared_ptr<wangle::IOThreadPoolExecutor> retry_executor() override {
+ return retry_executor_;
+ }
void Close() override;
@@ -98,8 +106,10 @@ class AsyncConnectionImpl : public AsyncConnection,
std::shared_ptr<Configuration> conf_;
std::shared_ptr<ConnectionConfiguration> connection_conf_;
std::shared_ptr<AsyncRpcRetryingCallerFactory> caller_factory_;
+ std::shared_ptr<folly::HHWheelTimer> retry_timer_;
std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_;
+ std::shared_ptr<wangle::IOThreadPoolExecutor> retry_executor_;
std::shared_ptr<LocationCache> location_cache_;
std::shared_ptr<RpcClient> rpc_client_;
bool is_closed_ = false;
diff --git a/core/async-region-locator.h b/core/async-region-locator.h
index c606dcb..f75cb7e 100644
--- a/core/async-region-locator.h
+++ b/core/async-region-locator.h
@@ -19,6 +19,7 @@
#pragma once
+#include <folly/ExceptionWrapper.h>
#include <folly/futures/Future.h>
#include <memory>
#include <string>
@@ -57,7 +58,8 @@ class AsyncRegionLocator {
/**
* Update cached region location, possibly using the information from exception.
*/
- virtual void UpdateCachedLocation(const RegionLocation &loc, const std::exception &error) = 0;
+ virtual void UpdateCachedLocation(const RegionLocation &loc,
+ const folly::exception_wrapper &error) = 0;
};
} // namespace hbase
diff --git a/core/async-rpc-retrying-caller-factory.h b/core/async-rpc-retrying-caller-factory.h
index 5bcad6c..5a80a06 100644
--- a/core/async-rpc-retrying-caller-factory.h
+++ b/core/async-rpc-retrying-caller-factory.h
@@ -41,8 +41,10 @@ template <typename RESP>
class SingleRequestCallerBuilder
: public std::enable_shared_from_this<SingleRequestCallerBuilder<RESP>> {
public:
- explicit SingleRequestCallerBuilder(std::shared_ptr<AsyncConnection> conn)
+ explicit SingleRequestCallerBuilder(std::shared_ptr<AsyncConnection> conn,
+ std::shared_ptr<folly::HHWheelTimer> retry_timer)
: conn_(conn),
+ retry_timer_(retry_timer),
table_name_(nullptr),
rpc_timeout_nanos_(conn->connection_conf()->rpc_timeout()),
pause_(conn->connection_conf()->pause()),
@@ -105,7 +107,7 @@ class SingleRequestCallerBuilder
std::shared_ptr<AsyncSingleRequestRpcRetryingCaller<RESP>> Build() {
return std::make_shared<AsyncSingleRequestRpcRetryingCaller<RESP>>(
- conn_, table_name_, row_, locate_type_, callable_, pause_, max_retries_,
+ conn_, retry_timer_, table_name_, row_, locate_type_, callable_, pause_, max_retries_,
operation_timeout_nanos_, rpc_timeout_nanos_, start_log_errors_count_);
}
@@ -116,6 +118,7 @@ class SingleRequestCallerBuilder
private:
std::shared_ptr<AsyncConnection> conn_;
+ std::shared_ptr<folly::HHWheelTimer> retry_timer_;
std::shared_ptr<TableName> table_name_;
nanoseconds rpc_timeout_nanos_;
nanoseconds operation_timeout_nanos_;
@@ -130,15 +133,18 @@ class SingleRequestCallerBuilder
class AsyncRpcRetryingCallerFactory {
private:
std::shared_ptr<AsyncConnection> conn_;
+ std::shared_ptr<folly::HHWheelTimer> retry_timer_;
public:
- explicit AsyncRpcRetryingCallerFactory(std::shared_ptr<AsyncConnection> conn) : conn_(conn) {}
+ explicit AsyncRpcRetryingCallerFactory(std::shared_ptr<AsyncConnection> conn,
+ std::shared_ptr<folly::HHWheelTimer> retry_timer)
+ : conn_(conn), retry_timer_(retry_timer) {}
virtual ~AsyncRpcRetryingCallerFactory() = default;
template <typename RESP>
std::shared_ptr<SingleRequestCallerBuilder<RESP>> Single() {
- return std::make_shared<SingleRequestCallerBuilder<RESP>>(conn_);
+ return std::make_shared<SingleRequestCallerBuilder<RESP>>(conn_, retry_timer_);
}
};
diff --git a/core/async-rpc-retrying-caller.cc b/core/async-rpc-retrying-caller.cc
index 965a44b..7e211f7 100644
--- a/core/async-rpc-retrying-caller.cc
+++ b/core/async-rpc-retrying-caller.cc
@@ -19,6 +19,8 @@
#include "core/async-rpc-retrying-caller.h"
+#include <folly/Conv.h>
+#include <folly/ExceptionWrapper.h>
#include <folly/Format.h>
#include <folly/Logging.h>
#include <folly/futures/Unit.h>
@@ -34,15 +36,19 @@
#include "utils/sys-util.h"
#include "utils/time-util.h"
+using folly::exception_wrapper;
+
namespace hbase {
template <typename RESP>
AsyncSingleRequestRpcRetryingCaller<RESP>::AsyncSingleRequestRpcRetryingCaller(
- std::shared_ptr<AsyncConnection> conn, std::shared_ptr<hbase::pb::TableName> table_name,
- const std::string& row, RegionLocateType locate_type, Callable<RESP> callable,
- nanoseconds pause, uint32_t max_retries, nanoseconds operation_timeout_nanos,
- nanoseconds rpc_timeout_nanos, uint32_t start_log_errors_count)
+ std::shared_ptr<AsyncConnection> conn, std::shared_ptr<folly::HHWheelTimer> retry_timer,
+ std::shared_ptr<hbase::pb::TableName> table_name, const std::string& row,
+ RegionLocateType locate_type, Callable<RESP> callable, nanoseconds pause, uint32_t max_retries,
+ nanoseconds operation_timeout_nanos, nanoseconds rpc_timeout_nanos,
+ uint32_t start_log_errors_count)
: conn_(conn),
+ retry_timer_(retry_timer),
table_name_(table_name),
row_(row),
locate_type_(locate_type),
@@ -58,7 +64,6 @@ AsyncSingleRequestRpcRetryingCaller<RESP>::AsyncSingleRequestRpcRetryingCaller(
start_ns_ = TimeUtil::GetNowNanos();
max_attempts_ = ConnectionUtils::Retries2Attempts(max_retries);
exceptions_ = std::make_shared<std::vector<ThrowableWithExtraContext>>();
- retry_timer_ = folly::HHWheelTimer::newTimer(&event_base_);
}
template <typename RESP>
@@ -87,7 +92,7 @@ void AsyncSingleRequestRpcRetryingCaller<RESP>::LocateThenCall() {
conn_->region_locator()
->LocateRegion(*table_name_, row_, locate_type_, locate_timeout_ns)
.then([this](std::shared_ptr<RegionLocation> loc) { Call(*loc); })
- .onError([this](const std::exception& e) {
+ .onError([this](const exception_wrapper& e) {
OnError(e,
[this]() -> std::string {
return "Locate '" + row_ + "' in " + table_name_->namespace_() + "::" +
@@ -96,17 +101,17 @@ void AsyncSingleRequestRpcRetryingCaller<RESP>::LocateThenCall() {
TimeUtil::ToMillisStr(operation_timeout_nanos_) + " ms, time elapsed = " +
TimeUtil::ElapsedMillisStr(this->start_ns_) + " ms";
},
- [](const std::exception& error) {});
+ [](const exception_wrapper& error) {});
});
}
template <typename RESP>
void AsyncSingleRequestRpcRetryingCaller<RESP>::OnError(
- const std::exception& error, Supplier<std::string> err_msg,
- Consumer<std::exception> update_cached_location) {
- ThrowableWithExtraContext twec(std::make_shared<std::exception>(error), TimeUtil::GetNowNanos());
+ const exception_wrapper& error, Supplier<std::string> err_msg,
+ Consumer<exception_wrapper> update_cached_location) {
+ ThrowableWithExtraContext twec(error, TimeUtil::GetNowNanos());
exceptions_->push_back(twec);
- if (SysUtil::InstanceOf<DoNotRetryIOException, std::exception>(error) || tries_ >= max_retries_) {
+ if (!ShouldRetry(error) || tries_ >= max_retries_) {
CompleteExceptionally();
return;
}
@@ -124,8 +129,33 @@ void AsyncSingleRequestRpcRetryingCaller<RESP>::OnError(
}
update_cached_location(error);
tries_++;
- retry_timer_->scheduleTimeoutFn([this]() { LocateThenCall(); },
- milliseconds(TimeUtil::ToMillis(delay_ns)));
+
+ /*
+ * The HHWheelTimer::scheduleTimeout() fails with an assertion from
+ * EventBase::isInEventBaseThread() if we execute the schedule in a random thread, or one of
+ * the IOThreadPool threads (with num threads > 1). I think there is a bug there in using retry
+ * timer from IOThreadPool threads. It only works when executed from a single-thread pool
+ * (retry_executor() is). However, the scheduled "work" which is the LocateThenCall() should
+ * still happen in a thread pool, that is why we are submitting the work to the CPUThreadPool.
+ * IOThreadPool cannot be used without fixing the blocking call that we do at TCP connection
+ * establishment time (see ConnectionFactory::Connect()), otherwise, the IOThreadPool thread
+ * just hangs because it deadlocks itself.
+ */
+ conn_->retry_executor()->add([&]() {
+ retry_timer_->scheduleTimeoutFn(
+ [this]() {
+ conn_->cpu_executor()->add([&]() { LocateThenCall(); });
+ },
+ milliseconds(TimeUtil::ToMillis(delay_ns)));
+ });
+}
+
+template <typename RESP>
+bool AsyncSingleRequestRpcRetryingCaller<RESP>::ShouldRetry(const exception_wrapper& error) {
+ bool do_not_retry = false;
+ error.with_exception(
+ [&](const RemoteException& remote_ex) { do_not_retry &= remote_ex.do_not_retry(); });
+ return !do_not_retry;
}
template <typename RESP>
@@ -143,33 +173,14 @@ void AsyncSingleRequestRpcRetryingCaller<RESP>::Call(const RegionLocation& loc)
}
std::shared_ptr<RpcClient> rpc_client;
- try {
- // TODO: There is no connection attempt happening here, no need to try-catch.
- rpc_client = conn_->rpc_client();
- } catch (const IOException& e) {
- OnError(e,
- [&, this]() -> std::string {
- return "Get async rpc_client to " +
- folly::sformat("{0}:{1}", loc.server_name().host_name(),
- loc.server_name().port()) +
- " for '" + row_ + "' in " + loc.DebugString() + " of " +
- table_name_->namespace_() + "::" + table_name_->qualifier() +
- " failed, tries = " + std::to_string(tries_) + ", maxAttempts = " +
- std::to_string(max_attempts_) + ", timeout = " +
- TimeUtil::ToMillisStr(this->operation_timeout_nanos_) +
- " ms, time elapsed = " + TimeUtil::ElapsedMillisStr(this->start_ns_) + " ms";
- },
- [&, this](const std::exception& error) {
- conn_->region_locator()->UpdateCachedLocation(loc, error);
- });
- return;
- }
+
+ rpc_client = conn_->rpc_client();
ResetController(controller_, call_timeout_ns);
callable_(controller_, std::make_shared<RegionLocation>(loc), rpc_client)
.then([this](const RESP& resp) { this->promise_->setValue(std::move(resp)); })
- .onError([&, this](const std::exception& e) {
+ .onError([&, this](const exception_wrapper& e) {
OnError(e,
[&, this]() -> std::string {
return "Call to " + folly::sformat("{0}:{1}", loc.server_name().host_name(),
@@ -182,10 +193,9 @@ void AsyncSingleRequestRpcRetryingCaller<RESP>::Call(const RegionLocation& loc)
" ms, time elapsed = " + TimeUtil::ElapsedMillisStr(this->start_ns_) +
" ms";
},
- [&, this](const std::exception& error) {
+ [&, this](const exception_wrapper& error) {
conn_->region_locator()->UpdateCachedLocation(loc, error);
});
- return;
});
}
diff --git a/core/async-rpc-retrying-caller.h b/core/async-rpc-retrying-caller.h
index 6006388..c86ad0b5 100644
--- a/core/async-rpc-retrying-caller.h
+++ b/core/async-rpc-retrying-caller.h
@@ -18,6 +18,7 @@
*/
#pragma once
+#include <folly/ExceptionWrapper.h>
#include <folly/futures/Future.h>
#include <folly/io/async/EventBase.h>
#include <folly/io/async/HHWheelTimer.h>
@@ -70,6 +71,7 @@ template <typename RESP>
class AsyncSingleRequestRpcRetryingCaller {
public:
AsyncSingleRequestRpcRetryingCaller(std::shared_ptr<AsyncConnection> conn,
+ std::shared_ptr<folly::HHWheelTimer> retry_timer,
std::shared_ptr<hbase::pb::TableName> table_name,
const std::string& row, RegionLocateType locate_type,
Callable<RESP> callable, nanoseconds pause,
@@ -84,8 +86,10 @@ class AsyncSingleRequestRpcRetryingCaller {
private:
void LocateThenCall();
- void OnError(const std::exception& error, Supplier<std::string> err_msg,
- Consumer<std::exception> update_cached_location);
+ void OnError(const folly::exception_wrapper& error, Supplier<std::string> err_msg,
+ Consumer<folly::exception_wrapper> update_cached_location);
+
+ bool ShouldRetry(const folly::exception_wrapper& error);
void Call(const RegionLocation& loc);
@@ -97,8 +101,8 @@ class AsyncSingleRequestRpcRetryingCaller {
const int64_t& timeout_ns);
private:
- folly::HHWheelTimer::UniquePtr retry_timer_;
std::shared_ptr<AsyncConnection> conn_;
+ std::shared_ptr<folly::HHWheelTimer> retry_timer_;
std::shared_ptr<hbase::pb::TableName> table_name_;
std::string row_;
RegionLocateType locate_type_;
@@ -114,6 +118,5 @@ class AsyncSingleRequestRpcRetryingCaller {
uint32_t tries_;
std::shared_ptr<std::vector<ThrowableWithExtraContext>> exceptions_;
uint32_t max_attempts_;
- folly::EventBase event_base_;
};
} /* namespace hbase */
diff --git a/core/async-rpc-retrying-test.cc b/core/async-rpc-retrying-test.cc
index 4956972..ff28e79 100644
--- a/core/async-rpc-retrying-test.cc
+++ b/core/async-rpc-retrying-test.cc
@@ -20,10 +20,13 @@
#include <folly/Logging.h>
#include <folly/Memory.h>
#include <folly/futures/Future.h>
+#include <folly/io/async/EventBase.h>
+#include <folly/io/async/ScopedEventBaseThread.h>
#include <gmock/gmock.h>
#include <google/protobuf/stubs/callback.h>
#include <wangle/concurrent/IOThreadPoolExecutor.h>
+#include <chrono>
#include <functional>
#include <string>
@@ -67,15 +70,33 @@ using hbase::Client;
using ::testing::Return;
using ::testing::_;
using std::chrono::nanoseconds;
+using std::chrono::milliseconds;
-class MockAsyncRegionLocator : public AsyncRegionLocator {
+using namespace hbase;
+
+using folly::exception_wrapper;
+
+class AsyncRpcRetryTest : public ::testing::Test {
public:
- explicit MockAsyncRegionLocator(std::shared_ptr<RegionLocation> region_location)
+ static std::unique_ptr<hbase::TestUtil> test_util;
+
+ static void SetUpTestCase() {
+ google::InstallFailureSignalHandler();
+ test_util = std::make_unique<hbase::TestUtil>();
+ test_util->StartMiniCluster(2);
+ }
+};
+std::unique_ptr<hbase::TestUtil> AsyncRpcRetryTest::test_util = nullptr;
+
+class AsyncRegionLocatorBase : public AsyncRegionLocator {
+ public:
+ AsyncRegionLocatorBase() {}
+ explicit AsyncRegionLocatorBase(std::shared_ptr<RegionLocation> region_location)
: region_location_(region_location) {}
- ~MockAsyncRegionLocator() = default;
+ virtual ~AsyncRegionLocatorBase() = default;
- folly::Future<std::shared_ptr<hbase::RegionLocation>> LocateRegion(const hbase::pb::TableName&,
- const std::string&,
+ folly::Future<std::shared_ptr<hbase::RegionLocation>> LocateRegion(const hbase::pb::TableName &,
+ const std::string &,
const RegionLocateType,
const int64_t) override {
folly::Promise<std::shared_ptr<RegionLocation>> promise;
@@ -83,22 +104,102 @@ class MockAsyncRegionLocator : public AsyncRegionLocator {
return promise.getFuture();
}
- void UpdateCachedLocation(const RegionLocation&, const std::exception&) override {}
+ virtual void set_region_location(std::shared_ptr<RegionLocation> region_location) {
+ region_location_ = region_location;
+ }
+
+ void UpdateCachedLocation(const RegionLocation &, const folly::exception_wrapper &) override {}
- private:
+ protected:
std::shared_ptr<RegionLocation> region_location_;
};
+class MockAsyncRegionLocator : public AsyncRegionLocatorBase {
+ public:
+ MockAsyncRegionLocator() : AsyncRegionLocatorBase() {}
+ explicit MockAsyncRegionLocator(std::shared_ptr<RegionLocation> region_location)
+ : AsyncRegionLocatorBase(region_location) {}
+ virtual ~MockAsyncRegionLocator() {}
+};
+
+class MockWrongRegionAsyncRegionLocator : public AsyncRegionLocatorBase {
+ private:
+ uint32_t tries_ = 0;
+ uint32_t num_fails_ = 0;
+
+ public:
+ explicit MockWrongRegionAsyncRegionLocator(uint32_t num_fails)
+ : AsyncRegionLocatorBase(), num_fails_(num_fails) {}
+ explicit MockWrongRegionAsyncRegionLocator(std::shared_ptr<RegionLocation> region_location)
+ : AsyncRegionLocatorBase(region_location) {}
+ virtual ~MockWrongRegionAsyncRegionLocator() {}
+
+ folly::Future<std::shared_ptr<hbase::RegionLocation>> LocateRegion(
+ const hbase::pb::TableName &tn, const std::string &row,
+ const RegionLocateType locate_type = RegionLocateType::kCurrent,
+ const int64_t locate_ns = 0) override {
+ // Fail for num_fails_ times, then delegate to the super class which will give the correct
+ // region location.
+ if (tries_++ > num_fails_) {
+ return AsyncRegionLocatorBase::LocateRegion(tn, row, locate_type, locate_ns);
+ }
+ folly::Promise<std::shared_ptr<RegionLocation>> promise;
+ /* set random region name, simulating invalid region */
+ auto result = std::make_shared<RegionLocation>(
+ "whatever-region-name", region_location_->region_info(), region_location_->server_name(),
+ region_location_->service());
+ promise.setValue(result);
+ return promise.getFuture();
+ }
+};
+
+class MockFailingAsyncRegionLocator : public AsyncRegionLocatorBase {
+ private:
+ uint32_t tries_ = 0;
+ uint32_t num_fails_ = 0;
+
+ public:
+ explicit MockFailingAsyncRegionLocator(uint32_t num_fails)
+ : AsyncRegionLocatorBase(), num_fails_(num_fails) {}
+ explicit MockFailingAsyncRegionLocator(std::shared_ptr<RegionLocation> region_location)
+ : AsyncRegionLocatorBase(region_location) {}
+ virtual ~MockFailingAsyncRegionLocator() {}
+ folly::Future<std::shared_ptr<hbase::RegionLocation>> LocateRegion(
+ const hbase::pb::TableName &tn, const std::string &row,
+ const RegionLocateType locate_type = RegionLocateType::kCurrent,
+ const int64_t locate_ns = 0) override {
+ // Fail for num_fails_ times, then delegate to the super class which will give the correct
+ // region location.
+ if (tries_++ > num_fails_) {
+ return AsyncRegionLocatorBase::LocateRegion(tn, row, locate_type, locate_ns);
+ }
+ folly::Promise<std::shared_ptr<RegionLocation>> promise;
+ promise.setException(std::runtime_error{"Failed to look up region location"});
+ return promise.getFuture();
+ }
+};
+
class MockAsyncConnection : public AsyncConnection,
public std::enable_shared_from_this<MockAsyncConnection> {
public:
MockAsyncConnection(std::shared_ptr<ConnectionConfiguration> conn_conf,
+ std::shared_ptr<folly::HHWheelTimer> retry_timer,
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
+ std::shared_ptr<wangle::IOThreadPoolExecutor> retry_executor,
std::shared_ptr<RpcClient> rpc_client,
std::shared_ptr<AsyncRegionLocator> region_locator)
- : conn_conf_(conn_conf), rpc_client_(rpc_client), region_locator_(region_locator) {}
+ : conn_conf_(conn_conf),
+ retry_timer_(retry_timer),
+ cpu_executor_(cpu_executor),
+ io_executor_(io_executor),
+ retry_executor_(retry_executor),
+ rpc_client_(rpc_client),
+ region_locator_(region_locator) {}
~MockAsyncConnection() {}
void Init() {
- caller_factory_ = std::make_shared<AsyncRpcRetryingCallerFactory>(shared_from_this());
+ caller_factory_ =
+ std::make_shared<AsyncRpcRetryingCallerFactory>(shared_from_this(), retry_timer_);
}
std::shared_ptr<Configuration> conf() override { return nullptr; }
@@ -108,6 +209,11 @@ class MockAsyncConnection : public AsyncConnection,
}
std::shared_ptr<RpcClient> rpc_client() override { return rpc_client_; }
std::shared_ptr<AsyncRegionLocator> region_locator() override { return region_locator_; }
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor() override { return cpu_executor_; }
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor() override { return io_executor_; }
+ std::shared_ptr<wangle::IOThreadPoolExecutor> retry_executor() override {
+ return retry_executor_;
+ }
void Close() override {}
std::shared_ptr<HBaseRpcController> CreateRpcController() override {
@@ -115,17 +221,20 @@ class MockAsyncConnection : public AsyncConnection,
}
private:
+ std::shared_ptr<folly::HHWheelTimer> retry_timer_;
std::shared_ptr<ConnectionConfiguration> conn_conf_;
std::shared_ptr<AsyncRpcRetryingCallerFactory> caller_factory_;
std::shared_ptr<RpcClient> rpc_client_;
std::shared_ptr<AsyncRegionLocator> region_locator_;
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_;
+ std::shared_ptr<wangle::IOThreadPoolExecutor> retry_executor_;
};
template <typename CONN>
class MockRawAsyncTableImpl {
public:
- explicit MockRawAsyncTableImpl(std::shared_ptr<CONN> conn)
- : conn_(conn), promise_(std::make_shared<folly::Promise<std::shared_ptr<hbase::Result>>>()) {}
+ explicit MockRawAsyncTableImpl(std::shared_ptr<CONN> conn) : conn_(conn) {}
virtual ~MockRawAsyncTableImpl() = default;
/* implement this in real RawAsyncTableImpl. */
@@ -133,11 +242,13 @@ class MockRawAsyncTableImpl {
/* in real RawAsyncTableImpl, this should be private. */
folly::Future<std::shared_ptr<hbase::Result>> GetCall(
std::shared_ptr<hbase::RpcClient> rpc_client, std::shared_ptr<HBaseRpcController> controller,
- std::shared_ptr<RegionLocation> loc, const hbase::Get& get) {
+ std::shared_ptr<RegionLocation> loc, const hbase::Get &get) {
hbase::RpcCall<hbase::Request, hbase::Response> rpc_call = [](
std::shared_ptr<hbase::RpcClient> rpc_client, std::shared_ptr<RegionLocation> loc,
std::shared_ptr<HBaseRpcController> controller,
std::unique_ptr<hbase::Request> preq) -> folly::Future<std::unique_ptr<hbase::Response>> {
+ VLOG(1) << "entering MockRawAsyncTableImpl#GetCall, calling AsyncCall, loc:"
+ << loc->DebugString();
return rpc_client->AsyncCall(loc->server_name().host_name(), loc->server_name().port(),
std::move(preq), User::defaultUser(), "ClientService");
};
@@ -151,17 +262,25 @@ class MockRawAsyncTableImpl {
template <typename REQ, typename PREQ, typename PRESP, typename RESP>
folly::Future<RESP> Call(std::shared_ptr<hbase::RpcClient> rpc_client,
std::shared_ptr<HBaseRpcController> controller,
- std::shared_ptr<RegionLocation> loc, const REQ& req,
+ std::shared_ptr<RegionLocation> loc, const REQ &req,
ReqConverter<std::unique_ptr<PREQ>, REQ, std::string> req_converter,
- const hbase::RpcCall<PREQ, PRESP>& rpc_call,
+ hbase::RpcCall<PREQ, PRESP> rpc_call,
RespConverter<RESP, PRESP> resp_converter) {
+ promise_ = std::make_shared<folly::Promise<std::shared_ptr<hbase::Result>>>();
+ auto f = promise_->getFuture();
+ VLOG(1) << "calling rpc_call";
rpc_call(rpc_client, loc, controller, std::move(req_converter(req, loc->region_name())))
.then([&, this, resp_converter](std::unique_ptr<PRESP> presp) {
+ VLOG(1) << "MockRawAsyncTableImpl#call succeded: ";
RESP result = resp_converter(*presp);
promise_->setValue(result);
})
- .onError([this](const std::exception& e) { promise_->setException(e); });
- return promise_->getFuture();
+ .onError([this](const exception_wrapper &e) {
+ VLOG(1) << "entering MockRawAsyncTableImpl#call, exception: " << e.what();
+ VLOG(1) << "entering MockRawAsyncTableImpl#call, error typeinfo: " << typeid(e).name();
+ promise_->setException(e);
+ });
+ return f;
}
private:
@@ -169,22 +288,19 @@ class MockRawAsyncTableImpl {
std::shared_ptr<folly::Promise<std::shared_ptr<hbase::Result>>> promise_;
};
-TEST(AsyncRpcRetryTest, TestGetBasic) {
- // Using TestUtil to populate test data
- auto test_util = std::make_unique<hbase::TestUtil>();
- test_util->StartMiniCluster(2);
-
- test_util->CreateTable("t", "d");
+void runTest(std::shared_ptr<AsyncRegionLocatorBase> region_locator, std::string tableName,
+ uint32_t operation_timeout_millis = 1200000) {
+ AsyncRpcRetryTest::test_util->CreateTable(tableName, "d");
// Create TableName and Row to be fetched from HBase
- auto tn = folly::to<hbase::pb::TableName>("t");
+ auto tn = folly::to<hbase::pb::TableName>(tableName);
auto row = "test2";
// Get to be performed on above HBase Table
hbase::Get get(row);
// Create a client
- Client client(*(test_util->conf()));
+ Client client(*(AsyncRpcRetryTest::test_util->conf()));
// Get connection to HBase Table
auto table = client.Table(tn);
@@ -196,24 +312,32 @@ TEST(AsyncRpcRetryTest, TestGetBasic) {
/* init region location and rpc channel */
auto region_location = table->GetRegionLocation(row);
- auto io_executor_ = std::make_shared<wangle::IOThreadPoolExecutor>(1);
+ // auto io_executor_ = std::make_shared<wangle::IOThreadPoolExecutor>(4);
+ auto cpu_executor_ = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
+ auto io_executor_ = client.async_connection()->io_executor();
+ auto retry_executor_ = std::make_shared<wangle::IOThreadPoolExecutor>(1);
auto codec = std::make_shared<hbase::KeyValueCodec>();
auto rpc_client = std::make_shared<RpcClient>(io_executor_, codec);
+ // auto retry_event_base_ = std::make_shared<folly::ScopedEventBaseThread>(true);
+ std::shared_ptr<folly::HHWheelTimer> retry_timer =
+ folly::HHWheelTimer::newTimer(retry_executor_->getEventBase());
/* init connection configuration */
auto connection_conf = std::make_shared<ConnectionConfiguration>(
- TimeUtil::SecondsToNanos(20), // connect_timeout
- TimeUtil::SecondsToNanos(1200), // operation_timeout
- TimeUtil::SecondsToNanos(60), // rpc_timeout
- TimeUtil::MillisToNanos(100), // pause
- 31, // max retries
- 9); // start log errors count
+ TimeUtil::SecondsToNanos(20), // connect_timeout
+ TimeUtil::MillisToNanos(operation_timeout_millis), // operation_timeout
+ TimeUtil::SecondsToNanos(60), // rpc_timeout
+ TimeUtil::MillisToNanos(100), // pause
+ 5, // max retries
+ 9); // start log errors count
- /* init region locator */
- auto region_locator = std::make_shared<MockAsyncRegionLocator>(region_location);
+ /* set region locator */
+ region_locator->set_region_location(region_location);
/* init hbase client connection */
- auto conn = std::make_shared<MockAsyncConnection>(connection_conf, rpc_client, region_locator);
+ auto conn = std::make_shared<MockAsyncConnection>(connection_conf, retry_timer, cpu_executor_,
+ io_executor_, retry_executor_, rpc_client,
+ region_locator);
conn->Init();
/* init retry caller factory */
@@ -237,7 +361,9 @@ TEST(AsyncRpcRetryTest, TestGetBasic) {
})
->Build();
- auto result = async_caller->Call().get();
+ auto promise = std::make_shared<folly::Promise<std::shared_ptr<hbase::Result>>>();
+
+ auto result = async_caller->Call().get(milliseconds(500000));
// Test the values, should be same as in put executed on hbase shell
ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty.";
@@ -245,6 +371,50 @@ TEST(AsyncRpcRetryTest, TestGetBasic) {
EXPECT_EQ("value2", *(result->Value("d", "2")));
EXPECT_EQ("value for extra", *(result->Value("d", "extra")));
+ retry_timer->destroy();
table->Close();
client.Close();
+ retry_executor_->stop();
+}
+
+// Test successful case
+TEST_F(AsyncRpcRetryTest, TestGetBasic) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockAsyncRegionLocator>());
+ runTest(region_locator, "table1");
+}
+
+// Tests the RPC failing 3 times, then succeeding
+TEST_F(AsyncRpcRetryTest, TestHandleException) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockWrongRegionAsyncRegionLocator>(3));
+ runTest(region_locator, "table2");
+}
+
+// Tests the RPC failing 5 times, throwing an exception
+TEST_F(AsyncRpcRetryTest, TestFailWithException) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockWrongRegionAsyncRegionLocator>(5));
+ EXPECT_ANY_THROW(runTest(region_locator, "table3"));
+}
+
+// Tests the region location lookup failing 3 times, then succeeding
+TEST_F(AsyncRpcRetryTest, TestHandleExceptionFromRegionLocationLookup) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockFailingAsyncRegionLocator>(3));
+ runTest(region_locator, "table4");
+}
+
+// Tests the region location lookup failing 5 times, throwing an exception
+TEST_F(AsyncRpcRetryTest, TestFailWithExceptionFromRegionLocationLookup) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockFailingAsyncRegionLocator>(5));
+ EXPECT_ANY_THROW(runTest(region_locator, "table5"));
+}
+
+// Tests hitting operation timeout, thus not retrying anymore
+TEST_F(AsyncRpcRetryTest, TestFailWithOperationTimeout) {
+ std::shared_ptr<AsyncRegionLocatorBase> region_locator(
+ std::make_shared<MockFailingAsyncRegionLocator>(3));
+ EXPECT_ANY_THROW(runTest(region_locator, "table6", 200));
}
diff --git a/core/client-test.cc b/core/client-test.cc
index ff4879a..274168f 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -156,6 +156,7 @@ TEST_F(ClientTest, GetForNonExistentTable) {
// Get to be performed on above HBase Table
hbase::Get get(row);
+ ClientTest::test_util->conf()->SetInt("hbase.client.retries.number", 5);
// Create a client
hbase::Client client(*ClientTest::test_util->conf());
diff --git a/core/client.h b/core/client.h
index 0e11278..2719470 100644
--- a/core/client.h
+++ b/core/client.h
@@ -61,6 +61,11 @@ class Client {
*/
void Close();
+ /**
+ * @brief Internal. DO NOT USE.
+ */
+ std::shared_ptr<AsyncConnectionImpl> async_connection() { return async_connection_; }
+
private:
/** Data */
std::shared_ptr<AsyncConnectionImpl> async_connection_;
diff --git a/core/location-cache.cc b/core/location-cache.cc
index 07c3d61..e0afcfb 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -284,7 +284,8 @@ void LocationCache::ClearCachedLocation(const hbase::pb::TableName &tn, const st
table_locs->erase(row);
}
-void LocationCache::UpdateCachedLocation(const RegionLocation &loc, const std::exception &error) {
+void LocationCache::UpdateCachedLocation(const RegionLocation &loc,
+ const folly::exception_wrapper &error) {
// TODO: just clears the location for now. We can inspect RegionMovedExceptions, etc later.
ClearCachedLocation(loc.region_info().table_name(), loc.region_info().start_key());
}
diff --git a/core/location-cache.h b/core/location-cache.h
index 5e79213..a3c15cb 100644
--- a/core/location-cache.h
+++ b/core/location-cache.h
@@ -18,6 +18,7 @@
*/
#pragma once
+#include <folly/ExceptionWrapper.h>
#include <folly/Executor.h>
#include <folly/SharedMutex.h>
#include <folly/futures/Future.h>
@@ -180,7 +181,7 @@ class LocationCache : public AsyncRegionLocator {
* Update cached region location, possibly using the information from exception.
*/
virtual void UpdateCachedLocation(const RegionLocation &loc,
- const std::exception &error) override;
+ const folly::exception_wrapper &error) override;
const std::string &zk_quorum() { return zk_quorum_; }
diff --git a/exceptions/BUCK b/exceptions/BUCK
index a23654c..eef4437 100644
--- a/exceptions/BUCK
+++ b/exceptions/BUCK
@@ -21,4 +21,4 @@ cxx_library(
srcs=[],
deps=["//third-party:folly",],
compiler_flags=['-Weffc++'],
- visibility=['//core/...'],)
\ No newline at end of file
+ visibility=['//core/...','//connection//...'],)
\ No newline at end of file
diff --git a/exceptions/exception.h b/exceptions/exception.h
index c0c4142..2943d57 100644
--- a/exceptions/exception.h
+++ b/exceptions/exception.h
@@ -22,52 +22,55 @@
#include <string>
#include <vector>
#include <folly/io/IOBuf.h>
+#include <folly/ExceptionWrapper.h>
namespace hbase {
class ThrowableWithExtraContext {
public:
- ThrowableWithExtraContext(std::shared_ptr<std::exception> cause,
+ ThrowableWithExtraContext(folly::exception_wrapper cause,
const long& when) :
cause_(cause), when_(when), extras_("") {
}
- ThrowableWithExtraContext(std::shared_ptr<std::exception> cause,
+ ThrowableWithExtraContext(folly::exception_wrapper cause,
const long& when, const std::string& extras) :
cause_(cause), when_(when), extras_(extras) {
}
- std::string ToString() {
+ virtual std::string ToString() {
// TODO:
// return new Date(this.when).toString() + ", " + extras + ", " + t.toString();
- return extras_ + ", " + cause_->what();
+ return extras_ + ", " + cause_.what().toStdString();
}
- std::shared_ptr<std::exception> cause() {
+ virtual folly::exception_wrapper cause() {
return cause_;
}
private:
- std::shared_ptr<std::exception> cause_;
+ folly::exception_wrapper cause_;
long when_;
std::string extras_;
};
class IOException: public std::logic_error {
public:
+ IOException() : logic_error("") {}
+
IOException(
const std::string& what) :
- logic_error(what), cause_(nullptr) {}
+ logic_error(what) {}
IOException(
const std::string& what,
- std::shared_ptr<std::exception> cause) :
+ folly::exception_wrapper cause) :
logic_error(what), cause_(cause) {}
virtual ~IOException() = default;
- std::shared_ptr<std::exception> cause() {
+ virtual folly::exception_wrapper cause() {
return cause_;
}
private:
- const std::shared_ptr<std::exception> cause_;
+ folly::exception_wrapper cause_;
};
class RetriesExhaustedException: public IOException {
@@ -77,7 +80,7 @@ public:
std::shared_ptr<std::vector<ThrowableWithExtraContext>> exceptions) :
IOException(
GetMessage(num_retries, exceptions),
- exceptions->empty() ? nullptr : (*exceptions)[exceptions->size() - 1].cause()){
+ exceptions->empty() ? folly::exception_wrapper{} : (*exceptions)[exceptions->size() - 1].cause()){
}
virtual ~RetriesExhaustedException() = default;
@@ -99,6 +102,71 @@ private:
class HBaseIOException : public IOException {
};
-class DoNotRetryIOException : public HBaseIOException {
+class RemoteException : public IOException {
+public:
+
+ RemoteException() : port_(0), do_not_retry_(false) {}
+
+ RemoteException(const std::string& what) :
+ IOException(what), port_(0), do_not_retry_(false) {}
+
+ RemoteException(
+ const std::string& what,
+ folly::exception_wrapper cause) :
+ IOException(what, cause), port_(0), do_not_retry_(false) {}
+
+ virtual ~RemoteException() = default;
+
+ std::string exception_class_name() const {
+ return exception_class_name_;
+ }
+
+ RemoteException* set_exception_class_name(const std::string& value) {
+ exception_class_name_ = value;
+ return this;
+ }
+
+ std::string stack_trace() const {
+ return stack_trace_;
+ }
+
+ RemoteException* set_stack_trace(const std::string& value) {
+ stack_trace_ = value;
+ return this;
+ }
+
+ std::string hostname() const {
+ return hostname_;
+ }
+
+ RemoteException* set_hostname(const std::string& value) {
+ hostname_ = value;
+ return this;
+ }
+
+ int port() const {
+ return port_;
+ }
+
+ RemoteException* set_port(int value) {
+ port_ = value;
+ return this;
+ }
+
+ bool do_not_retry() const {
+ return do_not_retry_;
+ }
+
+ RemoteException* set_do_not_retry(bool value) {
+ do_not_retry_ = value;
+ return this;
+ }
+
+private:
+ std::string exception_class_name_;
+ std::string stack_trace_;
+ std::string hostname_;
+ int port_;
+ bool do_not_retry_;
};
} // namespace hbase
[hbase] 17/137: HBASE-15718 Add on TableName implementation and
tests
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 0f8c1f8d73e8123e73932b3761e144d464fd48b9
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Tue Apr 26 14:51:06 2016 -0700
HBASE-15718 Add on TableName implementation and tests
Summary:
Table name will be needed to look up a row in meta. So this
patch adds on the implementation for creating that lookup
row. It also adds tests for TableName
Test Plan: Added on a unit test
Differential Revision: https://reviews.facebook.net/D57285
---
connection/client-handler.cc | 1 -
core/BUCK | 69 +++++++++++++++++++----------------
core/{table-name.cc => meta-utils.cc} | 12 ++++++
core/{table-name.h => meta-utils.h} | 8 ++--
core/simple-client.cc | 4 +-
core/table-name-test.cc | 50 +++++++++++++++++++++++++
core/table-name.cc | 15 ++++++++
core/table-name.h | 25 ++++++++++++-
if/Master.proto | 13 +++++++
if/ZooKeeper.proto | 10 ++++-
10 files changed, 165 insertions(+), 42 deletions(-)
diff --git a/connection/client-handler.cc b/connection/client-handler.cc
index 7b7794d..205993a7 100644
--- a/connection/client-handler.cc
+++ b/connection/client-handler.cc
@@ -60,7 +60,6 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
}
}
-
// TODO(eclark): Figure out how to handle the
// network errors that are going to come.
Future<Unit> ClientHandler::write(Context *ctx, Request r) {
diff --git a/core/BUCK b/core/BUCK
index 195fc5c..9db6fda 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -16,44 +16,49 @@
# limitations under the License.
# This is the main library.
-cxx_library(name="core",
- exported_headers=[
- "client.h",
- "connection.h",
- "get-request.h",
- "get-result.h",
- "hbase_macros.h",
- "location-cache.h",
- "table-name.h",
- ],
- srcs=[
- "client.cc",
- "get-request.cc",
- "get-result.cc",
- "location-cache.cc",
- "table-name.cc",
- ],
- deps=[
- "//connection:connection",
- "//if:if",
- "//serde:serde",
- "//third-party:folly",
- "//third-party:wangle",
- "//third-party:zookeeper_mt",
- ],
- visibility=[
- 'PUBLIC',
- ], )
-
+cxx_library(
+ name="core",
+ exported_headers=[
+ "client.h",
+ "connection.h",
+ "get-request.h",
+ "get-result.h",
+ "hbase_macros.h",
+ "location-cache.h",
+ "table-name.h",
+ # TODO: move this out of exported
+ # Once meta lookup works
+ "meta-utils.h",
+ ],
+ srcs=[
+ "client.cc",
+ "get-request.cc",
+ "get-result.cc",
+ "location-cache.cc",
+ "meta-utils.cc",
+ "table-name.cc",
+ ],
+ deps=[
+ "//connection:connection",
+ "//if:if",
+ "//serde:serde",
+ "//third-party:folly",
+ "//third-party:wangle",
+ "//third-party:zookeeper_mt",
+ ],
+ visibility=[
+ 'PUBLIC',
+ ], )
cxx_test(name="location-cache-test",
srcs=[
"test-env.cc",
"location-cache-test.cc",
],
- deps=[
- ":core",
- ],
+ deps=[":core", ],
run_test_separately=True, )
+cxx_test(name="table-name-test",
+ srcs=["table-name-test.cc", ],
+ deps=[":core", ], )
cxx_binary(name="simple-client",
srcs=["simple-client.cc", ],
deps=[":core", "//connection:connection"], )
diff --git a/core/table-name.cc b/core/meta-utils.cc
similarity index 74%
copy from core/table-name.cc
copy to core/meta-utils.cc
index ffaaed0..7c7d45f 100644
--- a/core/table-name.cc
+++ b/core/meta-utils.cc
@@ -16,4 +16,16 @@
* limitations under the License.
*
*/
+
+#include "core/meta-utils.h"
+
+#include <folly/Conv.h>
+
#include "core/table-name.h"
+
+using namespace hbase;
+
+std::string MetaUtil::region_lookup_rowkey(const TableName &tn,
+ const std::string &row) {
+ return folly::to<std::string>(tn, ",", row, ",", "999999999999999999");
+}
diff --git a/core/table-name.h b/core/meta-utils.h
similarity index 81%
copy from core/table-name.h
copy to core/meta-utils.h
index 37c3461..391d0ea 100644
--- a/core/table-name.h
+++ b/core/meta-utils.h
@@ -18,15 +18,13 @@
*/
#pragma once
-#include <memory>
#include <string>
namespace hbase {
+class TableName;
-// This is the core class of a HBase client.
-class TableName {
+class MetaUtil {
public:
- explicit TableName(std::string tableName);
- explicit TableName(std::string namespaceName, std::string tableName);
+ std::string region_lookup_rowkey(const TableName &tn, const std::string &row);
};
} // namespace hbase
diff --git a/core/simple-client.cc b/core/simple-client.cc
index 5e6f82e..8b2fae5 100644
--- a/core/simple-client.cc
+++ b/core/simple-client.cc
@@ -69,10 +69,12 @@ int main(int argc, char *argv[]) {
msg->mutable_region()->set_type(
RegionSpecifier_RegionSpecifierType::
RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
+
// What row.
msg->mutable_get()->set_row(FLAGS_row);
+
// Send it.
- r.set_msg(std::move(msg));
+ r.set_msg(msg);
auto resp = (*conn)(r).get(milliseconds(5000));
auto get_resp = std::static_pointer_cast<GetResponse>(resp.response());
diff --git a/core/table-name-test.cc b/core/table-name-test.cc
new file mode 100644
index 0000000..8aff375
--- /dev/null
+++ b/core/table-name-test.cc
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <folly/Conv.h>
+#include <gtest/gtest.h>
+
+#include <string>
+
+#include "core/table-name.h"
+
+using namespace hbase;
+
+TEST(TestTableName, TestToStringNoDefault) {
+ TableName tn{"TestTableName"};
+ std::string result = folly::to<std::string>(tn);
+ ASSERT_EQ(result.find("default"), std::string::npos);
+ ASSERT_EQ("TestTableName", result);
+}
+TEST(TestTableName, TestToStringIncludeNS) {
+ TableName tn{"hbase", "acl"};
+ std::string result = folly::to<std::string>(tn);
+ ASSERT_EQ(result.find("hbase"), 0);
+ ASSERT_EQ("hbase:acl", result);
+}
+TEST(TestTableName, TestIsDefault) {
+ TableName default_t1{"in_default"};
+ ASSERT_TRUE(default_t1.is_default_name_space());
+
+ TableName default_t2{"default", "in_also"};
+ ASSERT_TRUE(default_t2.is_default_name_space());
+
+ TableName non_default{"testing", "hmm"};
+ ASSERT_FALSE(non_default.is_default_name_space());
+}
diff --git a/core/table-name.cc b/core/table-name.cc
index ffaaed0..33087fd 100644
--- a/core/table-name.cc
+++ b/core/table-name.cc
@@ -17,3 +17,18 @@
*
*/
#include "core/table-name.h"
+
+#include <folly/Conv.h>
+
+using namespace hbase;
+
+TableName::TableName(std::string table_name)
+ : name_space_("default"), table_(table_name) {}
+TableName::TableName(std::string name_space, std::string table_name)
+ : name_space_(name_space), table_(table_name) {}
+bool TableName::is_default_name_space() const {
+ return name_space_.length() == 0 || name_space_ == "default";
+}
+bool TableName::operator==(const TableName &other) const {
+ return name_space_ == other.name_space_ && table_ == other.table_;
+}
diff --git a/core/table-name.h b/core/table-name.h
index 37c3461..816f7da 100644
--- a/core/table-name.h
+++ b/core/table-name.h
@@ -21,12 +21,33 @@
#include <memory>
#include <string>
+#include <folly/Conv.h>
+
namespace hbase {
// This is the core class of a HBase client.
class TableName {
public:
- explicit TableName(std::string tableName);
- explicit TableName(std::string namespaceName, std::string tableName);
+ explicit TableName(std::string table_name);
+ explicit TableName(std::string name_space, std::string table_name);
+
+ std::string name_space() const { return name_space_; };
+ std::string table() const { return table_; };
+ bool is_default_name_space() const;
+ bool operator==(const TableName &other) const;
+
+private:
+ std::string name_space_;
+ std::string table_;
};
+
+// Provide folly::to<std::string>(TableName);
+template <class String> void toAppend(const TableName &in, String *result) {
+ if (in.is_default_name_space()) {
+ folly::toAppend(in.table(), result);
+ } else {
+ folly::toAppend(in.name_space(), ':', in.table(), result);
+ }
+}
+
} // namespace hbase
diff --git a/if/Master.proto b/if/Master.proto
index 1f7a3b7..ad8111e 100644
--- a/if/Master.proto
+++ b/if/Master.proto
@@ -288,6 +288,7 @@ message SetSplitOrMergeEnabledRequest {
required bool enabled = 1;
optional bool synchronous = 2;
repeated MasterSwitchType switch_types = 3;
+ optional bool skip_lock = 4;
}
message SetSplitOrMergeEnabledResponse {
@@ -302,6 +303,12 @@ message IsSplitOrMergeEnabledResponse {
required bool enabled = 1;
}
+message ReleaseSplitOrMergeLockAndRollbackRequest {
+}
+
+message ReleaseSplitOrMergeLockAndRollbackResponse {
+}
+
message NormalizeRequest {
}
@@ -672,6 +679,12 @@ service MasterService {
returns(IsSplitOrMergeEnabledResponse);
/**
+ * Release lock and rollback state.
+ */
+ rpc ReleaseSplitOrMergeLockAndRollback(ReleaseSplitOrMergeLockAndRollbackRequest)
+ returns(ReleaseSplitOrMergeLockAndRollbackResponse);
+
+ /**
* Run region normalizer. Can NOT run for various reasons. Check logs.
*/
rpc Normalize(NormalizeRequest)
diff --git a/if/ZooKeeper.proto b/if/ZooKeeper.proto
index 186d183..41c0e0e 100644
--- a/if/ZooKeeper.proto
+++ b/if/ZooKeeper.proto
@@ -165,4 +165,12 @@ message TableLock {
*/
message SwitchState {
optional bool enabled = 1;
-}
\ No newline at end of file
+}
+
+/**
+ * State for split and merge, used in hbck
+ */
+message SplitAndMergeState {
+ optional bool split_enabled = 1;
+ optional bool merge_enabled = 2;
+}
[hbase] 91/137: HBASE-18126 Increment class
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit c0f829eaa74a2c70a4e67c2edcc23ee7f730b1c2
Author: tedyu <yu...@gmail.com>
AuthorDate: Mon Jun 5 16:33:48 2017 -0700
HBASE-18126 Increment class
---
core/BUCK | 11 +++
core/client-test.cc | 30 ++++++++
core/increment-test.cc | 129 +++++++++++++++++++++++++++++++++
core/increment.cc | 57 +++++++++++++++
utils/bytes-util.h => core/increment.h | 44 +++++++----
core/raw-async-table.cc | 14 ++++
core/raw-async-table.h | 2 +
core/request-converter.cc | 13 ++++
core/request-converter.h | 4 +
core/response-converter.cc | 7 ++
core/response-converter.h | 2 +
core/table.cc | 5 ++
core/table.h | 5 ++
utils/bytes-util-test.cc | 9 +++
utils/bytes-util.cc | 36 +++++++++
utils/bytes-util.h | 3 +
16 files changed, 355 insertions(+), 16 deletions(-)
diff --git a/core/BUCK b/core/BUCK
index 870c63f..203e00d 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -36,6 +36,7 @@ cxx_library(
# Once meta lookup works
"meta-utils.h",
"get.h",
+ "increment.h",
"mutation.h",
"put.h",
"delete.h",
@@ -66,6 +67,7 @@ cxx_library(
"keyvalue-codec.cc",
"location-cache.cc",
"meta-utils.cc",
+ "increment.cc",
"get.cc",
"mutation.cc",
"put.cc",
@@ -158,6 +160,15 @@ cxx_test(
deps=[":core",],
run_test_separately=True,)
cxx_test(
+ name="increment-test",
+ srcs=[
+ "increment-test.cc",
+ ],
+ deps=[
+ ":core",
+ ],
+ run_test_separately=True,)
+cxx_test(
name="put-test",
srcs=[
"put-test.cc",
diff --git a/core/client-test.cc b/core/client-test.cc
index c378f81..f3c5150 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -24,12 +24,14 @@
#include "core/configuration.h"
#include "core/delete.h"
#include "core/get.h"
+#include "core/increment.h"
#include "core/hbase-configuration-loader.h"
#include "core/put.h"
#include "core/result.h"
#include "core/table.h"
#include "serde/table-name.h"
#include "test-util/test-util.h"
+#include "utils/bytes-util.h"
using hbase::Cell;
using hbase::Configuration;
@@ -187,6 +189,34 @@ TEST_F(ClientTest, PutGetDelete) {
client.Close();
}
+TEST_F(ClientTest, Increment) {
+ // Using TestUtil to populate test data
+ ClientTest::test_util->CreateTable("t", "d");
+
+ // Create TableName and Row to be fetched from HBase
+ auto tn = folly::to<hbase::pb::TableName>("t");
+ auto row = "test1";
+
+ // Create a client
+ hbase::Client client(*ClientTest::test_util->conf());
+
+ // Get connection to HBase Table
+ auto table = client.Table(tn);
+ ASSERT_TRUE(table) << "Unable to get connection to Table.";
+ int64_t incr1 = 1235;
+ auto result = table->Increment(hbase::Increment{row}.AddColumn("d", "1", incr1));
+ EXPECT_EQ(row, result->Row());
+
+ long l = hbase::BytesUtil::ToInt64(*(result->Value("d", "1")));
+ EXPECT_EQ(incr1, l);
+
+ int64_t incr2 = -2;
+ result = table->Increment(hbase::Increment{row}.AddColumn("d", "1", incr2));
+
+ EXPECT_EQ(row, result->Row());
+ EXPECT_EQ(incr1+incr2, hbase::BytesUtil::ToInt64(*(result->Value("d", "1"))));
+}
+
TEST_F(ClientTest, PutGet) {
// Using TestUtil to populate test data
ClientTest::test_util->CreateTable("t", "d");
diff --git a/core/increment-test.cc b/core/increment-test.cc
new file mode 100644
index 0000000..3d7b1d4
--- /dev/null
+++ b/core/increment-test.cc
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include "core/mutation.h"
+#include "core/increment.h"
+#include "core/put.h"
+#include "utils/time-util.h"
+
+using hbase::Increment;
+using hbase::Increment;
+using hbase::Cell;
+using hbase::CellType;
+using hbase::Mutation;
+using hbase::TimeUtil;
+
+const constexpr int64_t Mutation::kLatestTimestamp;
+
+TEST(Increment, Row) {
+ Increment incr{"foo"};
+ EXPECT_EQ("foo", incr.row());
+}
+
+TEST(Increment, Durability) {
+ Increment incr{"row"};
+ EXPECT_EQ(hbase::pb::MutationProto_Durability_USE_DEFAULT, incr.Durability());
+
+ auto skipWal = hbase::pb::MutationProto_Durability_SKIP_WAL;
+ incr.SetDurability(skipWal);
+ EXPECT_EQ(skipWal, incr.Durability());
+}
+
+TEST(Increment, Timestamp) {
+ Increment incr{"row"};
+
+ // test default timestamp
+ EXPECT_EQ(Mutation::kLatestTimestamp, incr.TimeStamp());
+
+ // set custom timestamp
+ auto ts = TimeUtil::ToMillis(TimeUtil::GetNowNanos());
+ incr.SetTimeStamp(ts);
+ EXPECT_EQ(ts, incr.TimeStamp());
+
+ // Add a column with custom timestamp
+ incr.AddColumn("f", "q", 5l);
+ auto &cell = incr.FamilyMap().at("f")[0];
+ EXPECT_EQ(ts, cell->Timestamp());
+}
+
+TEST(Increment, HasFamilies) {
+ Increment incr{"row"};
+
+ EXPECT_EQ(false, incr.HasFamilies());
+
+ incr.AddColumn("f", "q", 5l);
+ EXPECT_EQ(true, incr.HasFamilies());
+}
+
+TEST(Increment, Add) {
+ CellType cell_type = CellType::PUT;
+ std::string row = "row";
+ std::string family = "family";
+ std::string column = "column";
+ std::string value = "value";
+ int64_t timestamp = std::numeric_limits<int64_t>::max();
+ auto cell = std::make_unique<Cell>(row, family, column, timestamp, value, cell_type);
+
+ // add first cell
+ Increment incr{"row"};
+ incr.Add(std::move(cell));
+ EXPECT_EQ(1, incr.FamilyMap().size());
+ EXPECT_EQ(1, incr.FamilyMap().at(family).size());
+
+ // add a non-matching row
+ auto cell2 = std::make_unique<Cell>(row, family, column, timestamp, value, cell_type);
+ Increment incr2{"foo"};
+ ASSERT_THROW(incr2.Add(std::move(cell2)), std::runtime_error); // rows don't match
+
+ // add a second cell with same family
+ auto cell3 = std::make_unique<Cell>(row, family, "column-2", timestamp, value, cell_type);
+ incr.Add(std::move(cell3));
+ EXPECT_EQ(1, incr.FamilyMap().size());
+ EXPECT_EQ(2, incr.FamilyMap().at(family).size());
+
+ // add a cell to a different family
+ auto cell4 = std::make_unique<Cell>(row, "family-2", "column-2", timestamp, value, cell_type);
+ incr.Add(std::move(cell4));
+ EXPECT_EQ(2, incr.FamilyMap().size());
+ EXPECT_EQ(1, incr.FamilyMap().at("family-2").size());
+}
+
+TEST(Increment, AddColumn) {
+ std::string row = "row";
+ std::string family = "family";
+ std::string column = "column";
+ std::string value = "value";
+
+ Increment incr{"row"};
+ incr.AddColumn(family, column, 5l);
+ EXPECT_EQ(1, incr.FamilyMap().size());
+ EXPECT_EQ(1, incr.FamilyMap().at(family).size());
+
+ // add a second cell with same family
+ incr.AddColumn(family, "column-2", 6l);
+ EXPECT_EQ(1, incr.FamilyMap().size());
+ EXPECT_EQ(2, incr.FamilyMap().at(family).size());
+
+ // add a cell to a different family
+ incr.AddColumn("family-2", column, 7l);
+ EXPECT_EQ(2, incr.FamilyMap().size());
+ EXPECT_EQ(1, incr.FamilyMap().at("family-2").size());
+}
diff --git a/core/increment.cc b/core/increment.cc
new file mode 100644
index 0000000..1a84266
--- /dev/null
+++ b/core/increment.cc
@@ -0,0 +1,57 @@
+
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/increment.h"
+#include <folly/Conv.h>
+#include <algorithm>
+#include <limits>
+#include <stdexcept>
+#include <utility>
+
+#include "utils/bytes-util.h"
+
+namespace hbase {
+
+/**
+ * @brief Increment the column from the specific family with the specified qualifier
+ * by the specified amount.
+ * @param family family name
+ * @param qualifier column qualifier
+ * @param amount amount to increment by
+ */
+Increment& Increment::AddColumn(const std::string& family, const std::string& qualifier,
+ int64_t amount) {
+ family_map_[family].push_back(std::move(
+ std::make_unique<Cell>(row_, family, qualifier, timestamp_, BytesUtil::ToString(amount),
+ hbase::CellType::PUT)));
+ return *this;
+}
+Increment& Increment::Add(std::unique_ptr<Cell> cell) {
+ if (cell->Row() != row_) {
+ throw std::runtime_error("The row in " + cell->DebugString() +
+ " doesn't match the original one " + row_);
+ }
+
+ family_map_[cell->Family()].push_back(std::move(cell));
+ return *this;
+}
+
+} // namespace hbase
diff --git a/utils/bytes-util.h b/core/increment.h
similarity index 51%
copy from utils/bytes-util.h
copy to core/increment.h
index 541b2d7..330f44a 100644
--- a/utils/bytes-util.h
+++ b/core/increment.h
@@ -19,27 +19,39 @@
#pragma once
+#include <cstdint>
+#include <map>
#include <memory>
#include <string>
+#include <vector>
+#include "core/cell.h"
+#include "core/mutation.h"
namespace hbase {
-class BytesUtil {
- private:
- static const constexpr char kHexChars[] = "0123456789ABCDEF";
-
+class Increment : public Mutation {
public:
- static std::string ToStringBinary(const std::string& b) { return ToStringBinary(b, 0, b.size()); }
/**
- * Write a printable representation of a byte array. Non-printable
- * characters are hex escaped in the format \\x%02X, eg:
- * \x00 \x05 etc
- *
- * @param b array to write out
- * @param off offset to start at
- * @param len length to write
- * @return string output
- */
- static std::string ToStringBinary(const std::string& b, size_t off, size_t len);
+ * Constructors
+ */
+ explicit Increment(const std::string& row) : Mutation(row) {}
+ Increment(const Increment& cincrement) : Mutation(cincrement) {}
+ Increment& operator=(const Increment& cincrement) {
+ Mutation::operator=(cincrement);
+ return *this;
+ }
+
+ ~Increment() = default;
+
+ /**
+ * @brief Increment the column from the specific family with the specified qualifier
+ * by the specified amount.
+ * @param family family name
+ * @param qualifier column qualifier
+ * @param amount amount to increment by
+ */
+ Increment& AddColumn(const std::string& family, const std::string& qualifier, int64_t amount);
+ Increment& Add(std::unique_ptr<Cell> cell);
};
-} /* namespace hbase */
+
+} // namespace hbase
diff --git a/core/raw-async-table.cc b/core/raw-async-table.cc
index 26967eb..f71fbba 100644
--- a/core/raw-async-table.cc
+++ b/core/raw-async-table.cc
@@ -77,6 +77,20 @@ folly::Future<std::shared_ptr<Result>> RawAsyncTable::Get(const hbase::Get& get)
// ensure that the lifecycle of the Caller object is longer than the retry lambdas.
return caller->Call().then([caller](const auto r) { return r; });
}
+folly::Future<std::shared_ptr<Result>> RawAsyncTable::Increment(const hbase::Increment& incr) {
+ auto caller =
+ CreateCallerBuilder<std::shared_ptr<Result>>(incr.row(), connection_conf_->write_rpc_timeout())
+ ->action([=, &incr](std::shared_ptr<hbase::HBaseRpcController> controller,
+ std::shared_ptr<hbase::RegionLocation> loc,
+ std::shared_ptr<hbase::RpcClient> rpc_client) -> folly::Future<std::shared_ptr<Result>> {
+ return Call<hbase::Increment, hbase::Request, hbase::Response, std::shared_ptr<Result>>(
+ rpc_client, controller, loc, incr, &hbase::RequestConverter::IncrementToMutateRequest,
+ &hbase::ResponseConverter::FromMutateResponse);
+ })
+ ->Build();
+
+ return caller->Call().then([caller](const auto r) { return r; });
+}
folly::Future<folly::Unit> RawAsyncTable::Put(const hbase::Put& put) {
auto caller =
diff --git a/core/raw-async-table.h b/core/raw-async-table.h
index d042d27..c8e9f2f 100644
--- a/core/raw-async-table.h
+++ b/core/raw-async-table.h
@@ -31,6 +31,7 @@
#include "core/connection-configuration.h"
#include "core/delete.h"
#include "core/get.h"
+#include "core/increment.h"
#include "core/put.h"
#include "core/result.h"
@@ -53,6 +54,7 @@ class RawAsyncTable {
folly::Future<std::shared_ptr<Result>> Get(const hbase::Get& get);
folly::Future<folly::Unit> Delete(const hbase::Delete& del);
+ folly::Future<std::shared_ptr<hbase::Result>> Increment(const hbase::Increment& increment);
folly::Future<folly::Unit> Put(const hbase::Put& put);
void Close() {}
diff --git a/core/request-converter.cc b/core/request-converter.cc
index 85b4d6d..a1e63fe 100644
--- a/core/request-converter.cc
+++ b/core/request-converter.cc
@@ -232,4 +232,17 @@ std::unique_ptr<Request> RequestConverter::DeleteToMutateRequest(const Delete &d
VLOG(3) << "Req is " << pb_req->req_msg()->ShortDebugString();
return pb_req;
}
+std::unique_ptr<Request> RequestConverter::IncrementToMutateRequest(const Increment &incr,
+ const std::string ®ion_name) {
+ auto pb_req = Request::mutate();
+ auto pb_msg = std::static_pointer_cast<hbase::pb::MutateRequest>(pb_req->req_msg());
+ RequestConverter::SetRegion(region_name, pb_msg->mutable_region());
+
+ pb_msg->set_allocated_mutation(
+ ToMutation(MutationType::MutationProto_MutationType_INCREMENT, incr, -1).release());
+
+ VLOG(3) << "Req is " << pb_req->req_msg()->ShortDebugString();
+ return pb_req;
+}
+
} /* namespace hbase */
diff --git a/core/request-converter.h b/core/request-converter.h
index 6db1962..6d57161 100644
--- a/core/request-converter.h
+++ b/core/request-converter.h
@@ -27,6 +27,7 @@
#include "core/cell.h"
#include "core/delete.h"
#include "core/get.h"
+#include "core/increment.h"
#include "core/mutation.h"
#include "core/put.h"
#include "core/region-request.h"
@@ -71,6 +72,9 @@ class RequestConverter {
static std::unique_ptr<Request> ToMutateRequest(const Put &put, const std::string ®ion_name);
+ static std::unique_ptr<Request> IncrementToMutateRequest(const Increment &incr,
+ const std::string ®ion_name);
+
static std::unique_ptr<pb::MutationProto> ToMutation(const MutationType type,
const Mutation &mutation,
const int64_t nonce);
diff --git a/core/response-converter.cc b/core/response-converter.cc
index b29a819..94b7875 100644
--- a/core/response-converter.cc
+++ b/core/response-converter.cc
@@ -29,6 +29,7 @@
#include "exceptions/exception.h"
using hbase::pb::GetResponse;
+using hbase::pb::MutateResponse;
using hbase::pb::ScanResponse;
using hbase::pb::RegionLoadStats;
@@ -46,6 +47,12 @@ std::shared_ptr<Result> ResponseConverter::FromGetResponse(const Response& resp)
return ToResult(get_resp->result(), resp.cell_scanner());
}
+std::shared_ptr<Result> ResponseConverter::FromMutateResponse(const Response& resp) {
+ auto mutate_resp = std::static_pointer_cast<MutateResponse>(resp.resp_msg());
+ hbase::pb::Result result = mutate_resp->result();
+ return ToResult(mutate_resp->result(), resp.cell_scanner());
+}
+
std::shared_ptr<Result> ResponseConverter::ToResult(
const hbase::pb::Result& result, const std::unique_ptr<CellScanner>& cell_scanner) {
std::vector<std::shared_ptr<Cell>> vcells;
diff --git a/core/response-converter.h b/core/response-converter.h
index 443527d..0fdde89 100644
--- a/core/response-converter.h
+++ b/core/response-converter.h
@@ -47,6 +47,8 @@ class ResponseConverter {
*/
static std::shared_ptr<hbase::Result> FromGetResponse(const Response& resp);
+ static std::shared_ptr<hbase::Result> FromMutateResponse(const Response& resp);
+
static std::vector<std::shared_ptr<Result>> FromScanResponse(const Response& resp);
static std::unique_ptr<hbase::MultiResponse> GetResults(std::shared_ptr<Request> req,
diff --git a/core/table.cc b/core/table.cc
index dd26adf..aa51989 100644
--- a/core/table.cc
+++ b/core/table.cc
@@ -62,6 +62,11 @@ void Table::Delete(const hbase::Delete &del) {
future.get(operation_timeout());
}
+std::shared_ptr<hbase::Result> Table::Increment(const hbase::Increment &increment) {
+ auto context = async_table_->Increment(increment);
+ return context.get(operation_timeout());
+}
+
milliseconds Table::operation_timeout() const {
return TimeUtil::ToMillis(async_connection_->connection_conf()->operation_timeout());
}
diff --git a/core/table.h b/core/table.h
index c4217d3..81ddc8e 100644
--- a/core/table.h
+++ b/core/table.h
@@ -67,6 +67,11 @@ class Table {
*/
void Delete(const hbase::Delete &del);
+ /**
+ * @brief - Increments some data in the table.
+ * @param - increment Increment object to perform HBase Increment operation.
+ */
+ std::shared_ptr<hbase::Result> Increment(const hbase::Increment &increment);
// TODO: Batch Puts
/**
diff --git a/utils/bytes-util-test.cc b/utils/bytes-util-test.cc
index d997ee2..676b441 100644
--- a/utils/bytes-util-test.cc
+++ b/utils/bytes-util-test.cc
@@ -46,4 +46,13 @@ TEST(TestBytesUtil, TestToStringBinary) {
EXPECT_EQ("foo_\\x00\\xFF_bar",
BytesUtil::ToStringBinary("foo_" + std::string{zero} + std::string{max} + "_bar"));
+
+ int64_t num = 761235;
+ EXPECT_EQ(num, BytesUtil::ToInt64(BytesUtil::ToString(num)));
+
+ num = -56125;
+ EXPECT_EQ(num, BytesUtil::ToInt64(BytesUtil::ToString(num)));
+
+ num = 0;
+ EXPECT_EQ(num, BytesUtil::ToInt64(BytesUtil::ToString(num)));
}
diff --git a/utils/bytes-util.cc b/utils/bytes-util.cc
index 5e4d728..e201d55 100644
--- a/utils/bytes-util.cc
+++ b/utils/bytes-util.cc
@@ -21,6 +21,8 @@
#include <memory>
#include <string>
+#include<bits/stdc++.h>
+#include <boost/predef.h>
#include <glog/logging.h>
@@ -28,6 +30,40 @@ namespace hbase {
constexpr char BytesUtil::kHexChars[];
+std::string BytesUtil::ToString(int64_t val) {
+ std::string res;
+#if BOOST_ENDIAN_BIG_BYTE || BOOST_ENDIAN_BIG_WORD
+ for (int i = 7; i > 0; i--) {
+ res += (int8_t) (val & 0xffu);
+ val = val >> 8;
+ }
+ res += (int8_t) val;
+#else
+ int64_t mask = 0xff00000000000000u;
+ for (int i = 56; i >= 1; i -= 8) {
+ auto num = ((val & mask) >> i);
+ res += num;
+ mask = mask >> 8;
+ }
+ res += (val & 0xff);
+#endif
+ return res;
+}
+
+int64_t BytesUtil::ToInt64(std::string str) {
+ if (str.length() < 8) {
+ throw std::runtime_error("There are not enough bytes. Expected: 8, actual: " + str.length());
+ }
+ const unsigned char *bytes = reinterpret_cast<unsigned char *>(const_cast<char*>(str.c_str()));
+ int64_t l = 0;
+ for(int i = 0; i < 8; i++) {
+ l <<= 8;
+ l ^= bytes[i];
+ }
+ return l;
+
+}
+
std::string BytesUtil::ToStringBinary(const std::string& b, size_t off, size_t len) {
std::string result;
// Just in case we are passed a 'len' that is > buffer length...
diff --git a/utils/bytes-util.h b/utils/bytes-util.h
index 541b2d7..3566d62 100644
--- a/utils/bytes-util.h
+++ b/utils/bytes-util.h
@@ -41,5 +41,8 @@ class BytesUtil {
* @return string output
*/
static std::string ToStringBinary(const std::string& b, size_t off, size_t len);
+
+ static std::string ToString(int64_t amt);
+ static long ToInt64(std::string str);
};
} /* namespace hbase */
[hbase] 128/137: HBASE-18579 Enable core dump by default for docker
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 0713a7479ccdb913feab214251dfaab70700aeac
Author: tedyu <yu...@gmail.com>
AuthorDate: Thu Sep 7 19:31:59 2017 -0700
HBASE-18579 Enable core dump by default for docker
---
bin/start-docker.sh | 2 +-
docker-files/Dockerfile | 4 +++-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/bin/start-docker.sh b/bin/start-docker.sh
index 53325c1..391238d 100755
--- a/bin/start-docker.sh
+++ b/bin/start-docker.sh
@@ -55,7 +55,7 @@ fi;
docker build -t hbase_native -f docker-files/Dockerfile .
# After the image is built run the thing
-docker run -h="securecluster" -p 16050:16050/tcp \
+docker run --privileged=true -h="securecluster" -p 16050:16050/tcp \
-v ${BASE_DIR}/..:/usr/src/hbase \
-v ~/.m2:/root/.m2 \
-it hbase_native /bin/bash
diff --git a/docker-files/Dockerfile b/docker-files/Dockerfile
index 1265b83..c8844ac 100644
--- a/docker-files/Dockerfile
+++ b/docker-files/Dockerfile
@@ -117,6 +117,8 @@ RUN cd /usr/src/ && \
ctest && \
make install
-ENTRYPOINT /usr/sbin/krb5kdc -P /var/run/krb5kdc.pid && /bin/bash
+RUN echo "enabled=1" >> /etc/default/apport
+
+ENTRYPOINT /usr/sbin/krb5kdc -P /var/run/krb5kdc.pid && echo "/tmp/core.%h.%e.%t" >> /proc/sys/kernel/core_pattern && sysctl -p && ulimit -c unlimited && /bin/bash
WORKDIR /usr/src/hbase/hbase-native-client
[hbase] 56/137: HBASE-17451 [C++] HBase Request and Response
Converter
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit df9fa3d0d9c6b6890657c0ea17842b52e8798fba
Author: Sudeep Sunthankar <su...@hashmapinc.com>
AuthorDate: Wed Jan 11 20:44:39 2017 +1100
HBASE-17451 [C++] HBase Request and Response Converter
Signed-off-by: Enis Soztutar <en...@apache.org>
---
core/BUCK | 13 +++++
core/request_converter-test.cc | 127 +++++++++++++++++++++++++++++++++++++++++
core/request_converter.cc | 118 ++++++++++++++++++++++++++++++++++++++
core/request_converter.h | 69 ++++++++++++++++++++++
core/response_converter.cc | 50 ++++++++++++++++
core/response_converter.h | 47 +++++++++++++++
6 files changed, 424 insertions(+)
diff --git a/core/BUCK b/core/BUCK
index a5ea5c0..b7db41a 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -33,6 +33,8 @@ cxx_library(
"hbase_configuration_loader.h",
"scan.h",
"result.h",
+ "request_converter.h",
+ "response_converter.h",
],
srcs=[
"cell.cc",
@@ -45,6 +47,8 @@ cxx_library(
"hbase_configuration_loader.cc",
"scan.cc",
"result.cc",
+ "request_converter.cc",
+ "response_converter.cc",
],
deps=[
"//connection:connection",
@@ -94,6 +98,15 @@ cxx_test(
srcs=["result-test.cc",],
deps=[":core",],
run_test_separately=True,)
+cxx_test(
+ name="request_converter-test",
+ srcs=["request_converter-test.cc",],
+ deps=[
+ ":core",
+ "//connection:connection",
+ "//if:if",
+ ],
+ run_test_separately=True,)
cxx_binary(
name="simple-client",
srcs=["simple-client.cc",],
diff --git a/core/request_converter-test.cc b/core/request_converter-test.cc
new file mode 100644
index 0000000..84d5e8b
--- /dev/null
+++ b/core/request_converter-test.cc
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/request_converter.h"
+
+#include <gtest/gtest.h>
+#include <limits>
+#include "connection/request.h"
+#include "core/get.h"
+#include "core/scan.h"
+
+using hbase::Get;
+using hbase::Scan;
+
+using hbase::pb::GetRequest;
+using hbase::pb::RegionSpecifier;
+using hbase::pb::RegionSpecifier_RegionSpecifierType;
+using hbase::pb::ScanRequest;
+
+TEST(RequestConverter, ToGet) {
+ std::string row_str = "row-test";
+ Get get(row_str);
+ get.AddFamily("family-1");
+ get.AddFamily("family-2");
+ get.AddFamily("family-3");
+ get.AddColumn("family-2", "qualifier-1");
+ get.AddColumn("family-2", "qualifier-2");
+ get.AddColumn("family-2", "qualifier-3");
+ get.SetCacheBlocks(false);
+ get.SetConsistency(hbase::pb::Consistency::TIMELINE);
+ get.SetMaxVersions(2);
+ get.SetTimeRange(10000, 20000);
+ std::string region_name("RegionName");
+
+ auto req = hbase::RequestConverter::ToGetRequest(get, region_name);
+ auto msg = std::static_pointer_cast<GetRequest>(req->req_msg());
+
+ // Tests whether the PB object is properly set or not.
+ ASSERT_TRUE(msg->has_region());
+ ASSERT_TRUE(msg->region().has_value());
+ EXPECT_EQ(msg->region().value(), region_name);
+
+ ASSERT_TRUE(msg->has_get());
+ EXPECT_EQ(msg->get().row(), row_str);
+ EXPECT_FALSE(msg->get().cache_blocks());
+ EXPECT_EQ(msg->get().consistency(), hbase::pb::Consistency::TIMELINE);
+ EXPECT_EQ(msg->get().max_versions(), 2);
+ EXPECT_EQ(msg->get().column_size(), 3);
+ for (int i = 0; i < msg->get().column_size(); ++i) {
+ EXPECT_EQ(msg->get().column(i).family(), "family-" + std::to_string(i + 1));
+ for (int j = 0; j < msg->get().column(i).qualifier_size(); ++j) {
+ EXPECT_EQ(msg->get().column(i).qualifier(j), "qualifier-" + std::to_string(j + 1));
+ }
+ }
+}
+
+TEST(RequestConverter, ToScan) {
+ std::string start_row("start-row");
+ std::string stop_row("stop-row");
+ hbase::Scan scan;
+ scan.AddFamily("family-1");
+ scan.AddFamily("family-2");
+ scan.AddFamily("family-3");
+ scan.AddColumn("family-2", "qualifier-1");
+ scan.AddColumn("family-2", "qualifier-2");
+ scan.AddColumn("family-2", "qualifier-3");
+ scan.SetReversed(true);
+ scan.SetStartRow(start_row);
+ scan.SetStopRow(stop_row);
+ scan.SetSmall(true);
+ scan.SetCaching(3);
+ scan.SetConsistency(hbase::pb::Consistency::TIMELINE);
+ scan.SetCacheBlocks(true);
+ scan.SetAllowPartialResults(true);
+ scan.SetLoadColumnFamiliesOnDemand(true);
+ scan.SetMaxVersions(5);
+ scan.SetTimeRange(10000, 20000);
+ std::string region_name("RegionName");
+
+ auto req = hbase::RequestConverter::ToScanRequest(scan, region_name);
+ auto msg = std::static_pointer_cast<ScanRequest>(req->req_msg());
+
+ // Tests whether the PB object is properly set or not.
+ ASSERT_TRUE(msg->has_region());
+ ASSERT_TRUE(msg->region().has_value());
+ EXPECT_EQ(msg->region().value(), region_name);
+
+ ASSERT_TRUE(msg->has_scan());
+ EXPECT_TRUE(msg->scan().reversed());
+ EXPECT_EQ(msg->scan().start_row(), start_row);
+ EXPECT_EQ(msg->scan().stop_row(), stop_row);
+ EXPECT_TRUE(msg->scan().small());
+ EXPECT_EQ(msg->scan().caching(), 3);
+ EXPECT_EQ(msg->scan().consistency(), hbase::pb::Consistency::TIMELINE);
+ EXPECT_TRUE(msg->scan().cache_blocks());
+ EXPECT_TRUE(msg->scan().allow_partial_results());
+ EXPECT_TRUE(msg->scan().load_column_families_on_demand());
+ EXPECT_EQ(msg->scan().max_versions(), 5);
+ EXPECT_EQ(msg->scan().max_result_size(), std::numeric_limits<uint64_t>::max());
+
+ EXPECT_EQ(msg->scan().column_size(), 3);
+ for (int i = 0; i < msg->scan().column_size(); ++i) {
+ EXPECT_EQ(msg->scan().column(i).family(), "family-" + std::to_string(i + 1));
+ for (int j = 0; j < msg->scan().column(i).qualifier_size(); ++j) {
+ EXPECT_EQ(msg->scan().column(i).qualifier(j), "qualifier-" + std::to_string(j + 1));
+ }
+ }
+ ASSERT_FALSE(msg->client_handles_partials());
+ ASSERT_FALSE(msg->client_handles_heartbeats());
+ ASSERT_FALSE(msg->track_scan_metrics());
+}
diff --git a/core/request_converter.cc b/core/request_converter.cc
new file mode 100644
index 0000000..eba07df
--- /dev/null
+++ b/core/request_converter.cc
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/request_converter.h"
+#include <utility>
+#include "if/Client.pb.h"
+
+using hbase::Request;
+using hbase::pb::GetRequest;
+using hbase::pb::RegionSpecifier;
+using hbase::pb::RegionSpecifier_RegionSpecifierType;
+using hbase::pb::ScanRequest;
+
+namespace hbase {
+
+RequestConverter::~RequestConverter() {}
+
+RequestConverter::RequestConverter() {}
+
+void RequestConverter::SetRegion(const std::string ®ion_name,
+ RegionSpecifier *region_specifier) {
+ region_specifier->set_type(
+ RegionSpecifier_RegionSpecifierType::RegionSpecifier_RegionSpecifierType_REGION_NAME);
+ region_specifier->set_value(region_name);
+}
+
+std::unique_ptr<Request> RequestConverter::ToGetRequest(const Get &get,
+ const std::string ®ion_name) {
+ auto pb_req = Request::get();
+
+ auto pb_msg = std::static_pointer_cast<GetRequest>(pb_req->req_msg());
+ RequestConverter::SetRegion(region_name, pb_msg->mutable_region());
+
+ auto pb_get = pb_msg->mutable_get();
+ pb_get->set_max_versions(get.MaxVersions());
+ pb_get->set_cache_blocks(get.CacheBlocks());
+ pb_get->set_consistency(get.Consistency());
+
+ if (!get.Timerange().IsAllTime()) {
+ hbase::pb::TimeRange *pb_time_range = pb_get->mutable_time_range();
+ pb_time_range->set_from(get.Timerange().MinTimeStamp());
+ pb_time_range->set_to(get.Timerange().MaxTimeStamp());
+ }
+ pb_get->set_row(get.Row());
+ if (get.HasFamilies()) {
+ for (const auto &family : get.Family()) {
+ auto column = pb_get->add_column();
+ column->set_family(family.first);
+ for (const auto &qualifier : family.second) {
+ column->add_qualifier(qualifier);
+ }
+ }
+ }
+
+ return pb_req;
+}
+
+std::unique_ptr<Request> RequestConverter::ToScanRequest(const Scan &scan,
+ const std::string ®ion_name) {
+ auto pb_req = Request::scan();
+
+ auto pb_msg = std::static_pointer_cast<ScanRequest>(pb_req->req_msg());
+
+ RequestConverter::SetRegion(region_name, pb_msg->mutable_region());
+
+ auto pb_scan = pb_msg->mutable_scan();
+ pb_scan->set_max_versions(scan.MaxVersions());
+ pb_scan->set_cache_blocks(scan.CacheBlocks());
+ pb_scan->set_reversed(scan.IsReversed());
+ pb_scan->set_small(scan.IsSmall());
+ pb_scan->set_caching(scan.Caching());
+ pb_scan->set_start_row(scan.StartRow());
+ pb_scan->set_stop_row(scan.StopRow());
+ pb_scan->set_consistency(scan.Consistency());
+ pb_scan->set_max_result_size(scan.MaxResultSize());
+ pb_scan->set_allow_partial_results(scan.AllowPartialResults());
+ pb_scan->set_load_column_families_on_demand(scan.LoadColumnFamiliesOnDemand());
+
+ if (!scan.Timerange().IsAllTime()) {
+ hbase::pb::TimeRange *pb_time_range = pb_scan->mutable_time_range();
+ pb_time_range->set_from(scan.Timerange().MinTimeStamp());
+ pb_time_range->set_to(scan.Timerange().MaxTimeStamp());
+ }
+
+ if (scan.HasFamilies()) {
+ for (const auto &family : scan.Family()) {
+ auto column = pb_scan->add_column();
+ column->set_family(family.first);
+ for (const auto &qualifier : family.second) {
+ column->add_qualifier(qualifier);
+ }
+ }
+ }
+
+ // TODO We will change this later.
+ pb_msg->set_client_handles_partials(false);
+ pb_msg->set_client_handles_heartbeats(false);
+ pb_msg->set_track_scan_metrics(false);
+
+ return pb_req;
+}
+} /* namespace hbase */
diff --git a/core/request_converter.h b/core/request_converter.h
new file mode 100644
index 0000000..57f08cc
--- /dev/null
+++ b/core/request_converter.h
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include "connection/request.h"
+#include "core/get.h"
+#include "core/scan.h"
+#include "if/HBase.pb.h"
+
+using hbase::pb::RegionSpecifier;
+namespace hbase {
+
+/**
+ * RequestConverter class
+ * This class converts a Client side Get, Scan, Mutate operation to corresponding PB message.
+ */
+class RequestConverter {
+ public:
+ ~RequestConverter();
+
+ /**
+ * @brief Returns a Request object comprising of PB GetRequest created using
+ * passed 'get'
+ * @param get - Get object used for creating GetRequest
+ * @param region_name - table region
+ */
+ static std::unique_ptr<Request> ToGetRequest(const Get &get, const std::string ®ion_name);
+
+ /**
+ * @brief Returns a Request object comprising of PB ScanRequest created using
+ * passed 'scan'
+ * @param scan - Scan object used for creating ScanRequest
+ * @param region_name - table region
+ */
+ static std::unique_ptr<Request> ToScanRequest(const Scan &scan, const std::string ®ion_name);
+
+ private:
+ // Constructor not required. We have all static methods to create PB requests.
+ RequestConverter();
+
+ /**
+ * @brief fills region_specifier with region values.
+ * @param region_name - table region
+ * @param region_specifier - RegionSpecifier to be filled and passed in PB
+ * Request.
+ */
+ static void SetRegion(const std::string ®ion_name, RegionSpecifier *region_specifier);
+};
+
+} /* namespace hbase */
diff --git a/core/response_converter.cc b/core/response_converter.cc
new file mode 100644
index 0000000..3fe2ba9
--- /dev/null
+++ b/core/response_converter.cc
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/response_converter.h"
+
+#include <vector>
+
+#include "core/cell.h"
+#include "if/Client.pb.h"
+
+using hbase::pb::GetResponse;
+
+namespace hbase {
+
+ResponseConverter::ResponseConverter() {}
+
+ResponseConverter::~ResponseConverter() {}
+
+std::unique_ptr<hbase::Result> ResponseConverter::FromGetResponse(const Response& resp) {
+ auto get_resp = std::static_pointer_cast<GetResponse>(resp.resp_msg());
+
+ std::vector<std::shared_ptr<Cell>> vcells;
+ for (auto cell : get_resp->result().cell()) {
+ std::shared_ptr<Cell> pcell =
+ std::make_shared<Cell>(cell.row(), cell.family(), cell.qualifier(), cell.timestamp(),
+ cell.value(), static_cast<hbase::CellType>(cell.cell_type()));
+ vcells.push_back(pcell);
+ }
+
+ return std::make_unique<hbase::Result>(vcells, get_resp->result().exists(),
+ get_resp->result().stale(), get_resp->result().partial());
+}
+
+} /* namespace hbase */
diff --git a/core/response_converter.h b/core/response_converter.h
new file mode 100644
index 0000000..7cbde81
--- /dev/null
+++ b/core/response_converter.h
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <memory>
+#include "core/result.h"
+#include "connection/response.h"
+
+namespace hbase {
+
+/**
+ * ResponseConverter class
+ * This class converts a PB Response to corresponding Result or other objects.
+ */
+class ResponseConverter {
+ public:
+ ~ResponseConverter();
+
+ /**
+ * @brief Returns a Result object created by PB Message in passed Response object.
+ * @param resp - Response object having the PB message.
+ */
+ static std::unique_ptr<hbase::Result> FromGetResponse(const Response &resp);
+
+ private:
+ // Constructor not required. We have all static methods to extract response from PB messages.
+ ResponseConverter();
+};
+
+} /* namespace hbase */
[hbase] 120/137: HBASE-18579 Enable core dump by default for docker
-revert
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 003df13b790579428602cf6e3f84c869b6801f76
Author: tedyu <yu...@gmail.com>
AuthorDate: Mon Aug 14 16:15:22 2017 -0700
HBASE-18579 Enable core dump by default for docker -revert
---
bin/start-docker.sh | 2 +-
docker-files/Dockerfile | 4 +---
2 files changed, 2 insertions(+), 4 deletions(-)
diff --git a/bin/start-docker.sh b/bin/start-docker.sh
index 391238d..53325c1 100755
--- a/bin/start-docker.sh
+++ b/bin/start-docker.sh
@@ -55,7 +55,7 @@ fi;
docker build -t hbase_native -f docker-files/Dockerfile .
# After the image is built run the thing
-docker run --privileged=true -h="securecluster" -p 16050:16050/tcp \
+docker run -h="securecluster" -p 16050:16050/tcp \
-v ${BASE_DIR}/..:/usr/src/hbase \
-v ~/.m2:/root/.m2 \
-it hbase_native /bin/bash
diff --git a/docker-files/Dockerfile b/docker-files/Dockerfile
index ac3cb47..efd9a9d 100644
--- a/docker-files/Dockerfile
+++ b/docker-files/Dockerfile
@@ -117,8 +117,6 @@ RUN cd /usr/src/ && \
ctest && \
make install
-RUN echo "enabled=1" >> /etc/default/apport
-
-ENTRYPOINT /usr/sbin/krb5kdc -P /var/run/krb5kdc.pid && echo "/tmp/core.%h.%e.%t" >> /proc/sys/kernel/core_pattern && sysctl -p && ulimit -c unlimited && /bin/bash
+ENTRYPOINT /usr/sbin/krb5kdc -P /var/run/krb5kdc.pid && /bin/bash
WORKDIR /usr/src/hbase/hbase-native-client
[hbase] 103/137: HBASE-18400 [C++] ConnectionId Equals/Hash should
consider service_name (Xiaobing Zhou)
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit b6a32146396f719aa1f2664a381a7f08c34b11e4
Author: tedyu <yu...@gmail.com>
AuthorDate: Tue Jul 18 14:11:43 2017 -0700
HBASE-18400 [C++] ConnectionId Equals/Hash should consider service_name (Xiaobing Zhou)
---
connection/connection-id.h | 7 ++++++-
connection/connection-pool-test.cc | 26 ++++++++++++++++++++++++++
2 files changed, 32 insertions(+), 1 deletion(-)
diff --git a/connection/connection-id.h b/connection/connection-id.h
index 7381103..059469f 100644
--- a/connection/connection-id.h
+++ b/connection/connection-id.h
@@ -39,6 +39,10 @@ class ConnectionId {
const std::string &service_name)
: user_(user), service_name_(service_name), host_(host), port_(port) {}
+ ConnectionId(const std::string &host, uint16_t port,
+ const std::string &service_name)
+ : user_(security::User::defaultUser()), service_name_(service_name), host_(host), port_(port) {}
+
virtual ~ConnectionId() = default;
std::shared_ptr<security::User> user() const { return user_; }
@@ -59,7 +63,7 @@ struct ConnectionIdEquals {
bool operator()(const std::shared_ptr<ConnectionId> &lhs,
const std::shared_ptr<ConnectionId> &rhs) const {
return userEquals(lhs->user(), rhs->user()) && lhs->host() == rhs->host() &&
- lhs->port() == rhs->port();
+ lhs->port() == rhs->port() && lhs->service_name() == rhs->service_name();
}
private:
@@ -78,6 +82,7 @@ struct ConnectionIdHash {
boost::hash_combine(h, ci->user() == nullptr ? 0 : ci->user()->user_name());
boost::hash_combine(h, ci->host());
boost::hash_combine(h, ci->port());
+ boost::hash_combine(h, ci->service_name());
return h;
}
};
diff --git a/connection/connection-pool-test.cc b/connection/connection-pool-test.cc
index 04ec7f1..63f774b 100644
--- a/connection/connection-pool-test.cc
+++ b/connection/connection-pool-test.cc
@@ -101,3 +101,29 @@ TEST(TestConnectionPool, TestOnlyCreateMultipleDispose) {
auto remote_id2 = std::make_shared<ConnectionId>(hostname_two, port);
auto result_two = cp.GetConnection(remote_id2);
}
+
+TEST(TestConnectionPool, TestCreateOneConnectionForOneService) {
+ std::string hostname{"hostname"};
+ uint32_t port{999};
+ std::string service1{"service1"};
+ std::string service2{"service2"};
+
+ auto mock_boot = std::make_shared<MockBootstrap>();
+ auto mock_service = std::make_shared<MockService>();
+ auto mock_cf = std::make_shared<MockConnectionFactory>();
+
+ EXPECT_CALL((*mock_cf), Connect(_, _, _)).Times(2).WillRepeatedly(Return(mock_service));
+ EXPECT_CALL((*mock_cf), MakeBootstrap()).Times(2).WillRepeatedly(Return(mock_boot));
+ ConnectionPool cp{mock_cf};
+
+ {
+ auto remote_id = std::make_shared<ConnectionId>(hostname, port, service1);
+ auto result_one = cp.GetConnection(remote_id);
+ auto remote_id2 = std::make_shared<ConnectionId>(hostname, port, service2);
+ auto result_two = cp.GetConnection(remote_id2);
+ }
+ auto remote_id = std::make_shared<ConnectionId>(hostname, port, service1);
+ auto result_one = cp.GetConnection(remote_id);
+ auto remote_id2 = std::make_shared<ConnectionId>(hostname, port, service2);
+ auto result_two = cp.GetConnection(remote_id2);
+}
[hbase] 43/137: HBASE-15893 Get object (Sudeep Sunthankar)
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 0972e6b231c02a8b34b1fd7d63a5248a0304421c
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Thu Nov 3 15:15:16 2016 -0700
HBASE-15893 Get object (Sudeep Sunthankar)
---
core/BUCK | 16 ++++
core/get-test.cc | 150 ++++++++++++++++++++++++++++++++++++
core/get.cc | 153 +++++++++++++++++++++++++++++++++++++
core/get.h | 137 ++++++++++++++++++++++++++++++++-
core/{get.h => time_range-test.cc} | 32 +++++++-
core/time_range.cc | 88 +++++++++++++++++++++
core/time_range.h | 55 +++++++++++++
7 files changed, 628 insertions(+), 3 deletions(-)
diff --git a/core/BUCK b/core/BUCK
index c615426..7e9044a 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -27,12 +27,16 @@ cxx_library(
# TODO: move this out of exported
# Once meta lookup works
"meta-utils.h",
+ "get.h",
+ "time_range.h",
],
srcs=[
"cell.cc",
"client.cc",
"location-cache.cc",
"meta-utils.cc",
+ "get.cc",
+ "time_range.cc",
],
deps=[
"//connection:connection",
@@ -59,6 +63,18 @@ cxx_test(name="cell-test",
],
deps=[":core", ],
run_test_separately=True, )
+cxx_test(name="get-test",
+ srcs=[
+ "get-test.cc",
+ ],
+ deps=[":core", ],
+ run_test_separately=True, )
+cxx_test(name="time_range-test",
+ srcs=[
+ "time_range-test.cc",
+ ],
+ deps=[":core", ],
+ run_test_separately=True, )
cxx_binary(name="simple-client",
srcs=["simple-client.cc", ],
deps=[":core", "//connection:connection"], )
diff --git a/core/get-test.cc b/core/get-test.cc
new file mode 100644
index 0000000..8ed5f2f
--- /dev/null
+++ b/core/get-test.cc
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/get.h"
+
+#include <gtest/gtest.h>
+#include <glog/logging.h>
+
+using namespace hbase;
+const int NUMBER_OF_GETS = 5;
+
+TEST (Get, SingleGet) {
+
+ std::string row_str = "row-test";
+ ASSERT_NO_THROW(Get tmp = Get(row_str));
+ Get get = Get(row_str);
+
+ get.SetCacheBlocks(true);
+ get.SetConsistency(hbase::pb::Consistency::STRONG);
+ get.SetMaxResultsPerColumnFamily(1);
+
+ ASSERT_THROW(get.SetMaxVersions(0), std::runtime_error);
+ ASSERT_NO_THROW(get.SetMaxVersions(-10));
+ ASSERT_THROW(get.SetMaxVersions(std::numeric_limits<unsigned int>::max() + 1),
+ std::runtime_error);
+
+ ASSERT_THROW(get.SetTimeRange(-100, 2000), std::runtime_error);
+ ASSERT_THROW(get.SetTimeRange(100, -2000), std::runtime_error);
+ ASSERT_THROW(get.SetTimeRange(1000, 200), std::runtime_error);
+
+ ASSERT_NO_THROW(get.SetMaxVersions());
+ ASSERT_NO_THROW(get.SetMaxVersions(2));
+ ASSERT_NO_THROW(get.SetTimeRange(0, std::numeric_limits<long>::max()));
+
+ EXPECT_EQ(true, get.CacheBlocks());
+ EXPECT_EQ(hbase::pb::Consistency::STRONG, get.Consistency());
+ EXPECT_EQ(1, get.MaxResultsPerColumnFamily());
+ EXPECT_EQ(2, get.MaxVersions());
+
+ TimeRange tr = get.Timerange();
+ EXPECT_EQ(0, tr.MinTimeStamp());
+ EXPECT_EQ(std::numeric_limits<long>::max(), tr.MaxTimeStamp());
+
+ EXPECT_EQ("row-test", get.Row());
+
+ EXPECT_EQ(false, get.HasFamilies());
+
+ get.AddFamily("family-1");
+ EXPECT_EQ(true, get.HasFamilies());
+
+ get.AddColumn("family-1", "column-1");
+ get.AddColumn("family-1", "column-2");
+ get.AddColumn("family-1", "");
+ get.AddColumn("family-1", "column-3");
+ get.AddFamily("family-1");
+ get.AddFamily("family-2");
+ get.AddFamily("family-3");
+
+}
+
+TEST (Get, MultiGet) {
+
+ std::vector<Get *> gets;
+ for (int i = 0; i < NUMBER_OF_GETS; i++) {
+ std::string row_str = "row-test";
+ row_str += std::to_string(i);
+ ASSERT_NO_THROW(Get tmp = Get(row_str));
+ Get *get = new Get(row_str);
+
+ get->SetCacheBlocks(true);
+ get->SetConsistency(hbase::pb::Consistency::STRONG);
+ get->SetMaxResultsPerColumnFamily(1);
+
+ ASSERT_THROW(get->SetMaxVersions(0), std::runtime_error);
+ ASSERT_NO_THROW(get->SetMaxVersions(-10));
+ ASSERT_THROW(
+ get->SetMaxVersions(std::numeric_limits<unsigned int>::max() + 1),
+ std::runtime_error);
+
+ ASSERT_THROW(get->SetTimeRange(-100, 2000), std::runtime_error);
+ ASSERT_THROW(get->SetTimeRange(100, -2000), std::runtime_error);
+ ASSERT_THROW(get->SetTimeRange(1000, 200), std::runtime_error);
+
+ get->SetMaxVersions();
+ get->SetMaxVersions(2);
+ get->SetTimeRange(0, std::numeric_limits<long>::max());
+
+ EXPECT_EQ(true, get->CacheBlocks());
+ EXPECT_EQ(hbase::pb::Consistency::STRONG, get->Consistency());
+ EXPECT_EQ(1, get->MaxResultsPerColumnFamily());
+ EXPECT_EQ(2, get->MaxVersions());
+
+ TimeRange tr = get->Timerange();
+ EXPECT_EQ(0, tr.MinTimeStamp());
+ EXPECT_EQ(std::numeric_limits<long>::max(), tr.MaxTimeStamp());
+
+ EXPECT_EQ(false, get->HasFamilies());
+
+ get->AddFamily("family-1");
+ EXPECT_EQ(true, get->HasFamilies());
+
+ get->AddColumn("family-1", "column-1");
+ get->AddColumn("family-1", "column-2");
+ get->AddColumn("family-1", "");
+ get->AddColumn("family-1", "column-3");
+ get->AddFamily("family-1");
+ get->AddFamily("family-2");
+ get->AddFamily("family-3");
+
+ gets.push_back(get);
+ }
+ EXPECT_EQ(NUMBER_OF_GETS, gets.size());
+
+ int i = 0;
+ for (const auto &get : gets) {
+ std::string row_str = "row-test";
+ row_str += std::to_string(i);
+ EXPECT_EQ(row_str, get->Row());
+ i++;
+ }
+
+ for (const auto &get : gets) {
+ delete get;
+ }
+ gets.clear();
+
+}
+
+TEST (Get, Exception) {
+
+ std::string row(std::numeric_limits<short>::max() + 1, 'X');
+ ASSERT_THROW(Get tmp = Get(row), std::runtime_error);
+ ASSERT_THROW(Get tmp = Get(""), std::runtime_error);
+}
diff --git a/core/get.cc b/core/get.cc
index 9e11332..b0d91ec 100644
--- a/core/get.cc
+++ b/core/get.cc
@@ -18,3 +18,156 @@
*/
#include "core/get.h"
+#include <algorithm>
+#include <limits>
+#include <stdexcept>
+
+namespace hbase {
+
+Get::~Get() {
+
+}
+
+Get::Get(const std::string &row)
+ : row_(row),
+ max_versions_(1),
+ cache_blocks_(true),
+ store_limit_(-1),
+ store_offset_(0),
+ check_existence_only_(false),
+ consistency_(hbase::pb::Consistency::STRONG),
+ tr_(TimeRange()) {
+ Get::CheckRow(&row_);
+ family_map_.clear();
+}
+
+Get::Get(const Get &cget) {
+ this->row_ = cget.row_;
+ this->max_versions_ = cget.max_versions_;
+ this->cache_blocks_ = cget.cache_blocks_;
+ this->store_limit_ = cget.store_limit_;
+ this->store_offset_ = cget.store_offset_;
+ this->check_existence_only_ = cget.check_existence_only_;
+ this->consistency_ = cget.consistency_;
+ this->tr_ = cget.tr_;
+}
+
+Get& Get::operator=(const Get &cget) {
+ this->row_ = cget.row_;
+ this->max_versions_ = cget.max_versions_;
+ this->cache_blocks_ = cget.cache_blocks_;
+ this->store_limit_ = cget.store_limit_;
+ this->store_offset_ = cget.store_offset_;
+ this->check_existence_only_ = cget.check_existence_only_;
+ this->consistency_ = cget.consistency_;
+ this->tr_ = cget.tr_;
+ return *this;
+}
+
+Get& Get::AddFamily(const std::string &family) {
+ const auto &it = family_map_.find(family);
+
+ /**
+ * Check if any qualifiers are already present or not.
+ * Remove all existing qualifiers if the given family is already present in the map
+ */
+ if (family_map_.end() != it) {
+ it->second.clear();
+ } else {
+ family_map_[family];
+ }
+ return *this;
+}
+
+Get& Get::AddColumn(const std::string &family, const std::string &qualifier) {
+ const auto &it = std::find(this->family_map_[family].begin(),
+ this->family_map_[family].end(), qualifier);
+
+ /**
+ * Check if any qualifiers are already present or not.
+ * Add only if qualifiers for a given family are not present
+ */
+ if (it == this->family_map_[family].end()) {
+ this->family_map_[family].push_back(qualifier);
+ }
+ return *this;
+}
+
+const std::string& Get::Row() const {
+ return row_;
+}
+
+hbase::pb::Consistency Get::Consistency() const {
+ return this->consistency_;
+}
+
+Get &Get::SetConsistency(hbase::pb::Consistency consistency) {
+ this->consistency_ = consistency;
+ return *this;
+}
+
+bool Get::HasFamilies() {
+ return !this->family_map_.empty();
+}
+
+const FAMILY_MAP &Get::FamilyMap() const {
+ return this->family_map_;
+}
+
+int Get::MaxVersions() const {
+ return this->max_versions_;
+}
+
+Get& Get::SetMaxVersions(uint32_t max_versions) {
+ if (0 == max_versions)
+ throw std::runtime_error("max_versions must be positive");
+
+ this->max_versions_ = max_versions;
+ return *this;
+}
+
+bool Get::CacheBlocks() const {
+ return this->cache_blocks_;
+}
+
+Get & Get::SetCacheBlocks(bool cache_blocks) {
+ this->cache_blocks_ = cache_blocks;
+ return *this;
+}
+
+int Get::MaxResultsPerColumnFamily() const {
+ return this->store_limit_;
+}
+
+Get& Get::SetMaxResultsPerColumnFamily(int store_limit) {
+ this->store_limit_ = store_limit;
+ return *this;
+}
+
+Get& Get::SetTimeRange(long min_timestamp, long max_timestamp) {
+ this->tr_ = TimeRange(min_timestamp, max_timestamp);
+ return *this;
+}
+
+Get& Get::SetTimeStamp(long timestamp) {
+ this->tr_ = TimeRange(timestamp, timestamp + 1);
+ return *this;
+}
+
+const TimeRange& Get::Timerange() const {
+ return this->tr_;
+}
+
+void Get::CheckRow(const std::string *row) {
+ int MAX_ROW_LENGTH = std::numeric_limits<short>::max();
+ int row_length = row->size();
+ if (0 == row_length) {
+ throw std::runtime_error("Row length can't be 0");
+ }
+ if (row_length > MAX_ROW_LENGTH) {
+ throw std::runtime_error(
+ "Length of " + *row + " is greater than max row size: "
+ + std::to_string(MAX_ROW_LENGTH));
+ }
+}
+}
diff --git a/core/get.h b/core/get.h
index b4b5912..28141f2 100644
--- a/core/get.h
+++ b/core/get.h
@@ -19,4 +19,139 @@
#pragma once
-class Get {};
+#include <map>
+#include <string>
+#include <vector>
+#include <cstdint>
+#include "core/time_range.h"
+#include "if/Client.pb.h"
+
+namespace hbase {
+
+/**
+ * @brief Map consisting of column families and qualifiers to be used for Get operation
+ */
+using FAMILY_MAP = std::map<std::string, std::vector<std::string>>;
+
+class Get {
+
+ public:
+
+ /**
+ * Constructors
+ */
+ Get(const std::string &row);
+ Get(const Get &cget);
+ Get& operator=(const Get &cget);
+
+ ~Get();
+
+ /**
+ * @brief Returns the maximum number of values to fetch per CF
+ */
+ int MaxVersions() const;
+
+ /**
+ * @brief Get up to the specified number of versions of each column. default is 1.
+ * @param max_versions max_versons to set
+ */
+ Get& SetMaxVersions(uint32_t max_versions = 1);
+
+ /**
+ * @brief Returns whether blocks should be cached for this Get operation.
+ */
+ bool CacheBlocks() const;
+
+ /**
+ * @brief Set whether blocks should be cached for this Get operation.
+ * @param cache_blocks to set
+ */
+ Get& SetCacheBlocks(bool cache_blocks);
+
+ /**
+ * @brief Method for retrieving the get's maximum number of values to return per Column Family
+ */
+ int MaxResultsPerColumnFamily() const;
+
+ /**
+ * @brief Set the maximum number of values to return per row per Column Family
+ * @param the store_limit to be set
+ */
+ Get& SetMaxResultsPerColumnFamily(int store_limit);
+
+ /**
+ * @brief Returns the Get family map (FAMILY_MAP) for this Get operation.
+ */
+ const FAMILY_MAP &FamilyMap() const;
+
+ /**
+ * @brief Returns the timerange for this Get
+ */
+ const TimeRange& Timerange() const;
+
+ /**
+ * @brief Get versions of columns only within the specified timestamp range, [minStamp, maxStamp).
+ * @param minStamp the minimum timestamp, inclusive
+ * @param maxStamp the maximum timestamp, exclusive
+ */
+ Get& SetTimeRange(long min_timestamp, long max_timestamp);
+
+ /**
+ * @brief Get versions of columns with the specified timestamp.
+ * @param The timestamp to be set
+ */
+ Get& SetTimeStamp(long timestamp);
+
+ /**
+ * @brief Get all columns from the specified family.
+ * @param family to be retrieved
+ */
+ Get& AddFamily(const std::string &family);
+
+ /**
+ * @brief Get the column from the specific family with the specified qualifier.
+ * @param family to be retrieved
+ * @param qualifier to be retrieved
+ */
+ Get& AddColumn(const std::string &family, const std::string &qualifier);
+
+ /**
+ * @brief Returns the row for this Get operation
+ */
+ const std::string& Row() const;
+
+ /**
+ * @brief Returns true if family map (FAMILY_MAP) is non empty false otherwise
+ */
+ bool HasFamilies();
+
+ /**
+ * @brief Returns the consistency level for this Get operation
+ */
+ hbase::pb::Consistency Consistency() const;
+
+ /**
+ * @brief Sets the consistency level for this Get operation
+ * @param Consistency to be set
+ */
+ Get& SetConsistency(hbase::pb::Consistency consistency);
+
+ private:
+ std::string row_;
+ uint32_t max_versions_;
+ bool cache_blocks_;
+ int store_limit_;
+ int store_offset_;
+ bool check_existence_only_;
+ FAMILY_MAP family_map_;
+ hbase::pb::Consistency consistency_;
+ TimeRange tr_;
+
+ /**
+ * @brief Checks if the row for this Get operation is proper or not
+ * @param row Row to check
+ * @throws std::runtime_error if row is empty or greater than MAX_ROW_LENGTH(i.e. std::numeric_limits<short>::max())
+ */
+ void CheckRow(const std::string *row);
+};
+}
diff --git a/core/get.h b/core/time_range-test.cc
similarity index 50%
copy from core/get.h
copy to core/time_range-test.cc
index b4b5912..c08d52a 100644
--- a/core/get.h
+++ b/core/time_range-test.cc
@@ -17,6 +17,34 @@
*
*/
-#pragma once
+#include "core/time_range.h"
-class Get {};
+#include <gtest/gtest.h>
+#include <glog/logging.h>
+
+using namespace hbase;
+
+TEST (TimeRange, DefaultObject) {
+
+ TimeRange *timerange_def = nullptr;
+ ASSERT_NO_THROW(timerange_def = new TimeRange());
+
+ EXPECT_EQ(0, timerange_def->MinTimeStamp());
+ EXPECT_EQ(std::numeric_limits<long>::max(), timerange_def->MaxTimeStamp());
+ EXPECT_NE(1000, timerange_def->MinTimeStamp());
+ EXPECT_NE(2000, timerange_def->MaxTimeStamp());
+ delete timerange_def;
+ timerange_def = nullptr;
+}
+
+TEST (TimeRange, Exception) {
+
+ // Negative Min TS
+ ASSERT_THROW(TimeRange(-1000, 2000), std::runtime_error);
+
+ // Negative Max TS
+ ASSERT_THROW(TimeRange(1000, -2000), std::runtime_error);
+
+ // Min TS > Max TS
+ ASSERT_THROW(TimeRange(10000, 2000), std::runtime_error);
+}
diff --git a/core/time_range.cc b/core/time_range.cc
new file mode 100644
index 0000000..9e6ed99
--- /dev/null
+++ b/core/time_range.cc
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/time_range.h"
+#include <limits>
+#include <stdexcept>
+#include <string>
+
+namespace hbase {
+
+TimeRange::TimeRange()
+ : min_timestamp_(0L),
+ max_timestamp_(std::numeric_limits<long>::max()),
+ all_time_(true) {
+
+}
+
+TimeRange::TimeRange(const TimeRange &tr) {
+ this->all_time_ = tr.all_time_;
+ this->max_timestamp_ = tr.max_timestamp_;
+ this->min_timestamp_ = tr.min_timestamp_;
+}
+
+TimeRange &TimeRange::operator =(const TimeRange &tr) {
+ this->all_time_ = tr.all_time_;
+ this->max_timestamp_ = tr.max_timestamp_;
+ this->min_timestamp_ = tr.min_timestamp_;
+ return *this;
+}
+
+TimeRange::~TimeRange() {
+
+}
+
+TimeRange::TimeRange(long min_timestamp) {
+ this->min_timestamp_ = min_timestamp;
+ this->max_timestamp_ = std::numeric_limits<long>::max();
+ this->all_time_ = false;
+}
+
+TimeRange::TimeRange(long min_timestamp, long max_timestamp) {
+ if (min_timestamp < 0 || max_timestamp < 0) {
+ throw std::runtime_error(
+ "Timestamp cannot be negative. min_timestamp: "
+ + std::to_string(min_timestamp) + ", max_timestamp:"
+ + std::to_string(max_timestamp));
+ }
+ if (max_timestamp < min_timestamp) {
+ throw std::runtime_error(
+ "max_timestamp [" + std::to_string(max_timestamp)
+ + "] should be greater than min_timestamp ["
+ + std::to_string(min_timestamp) + "]");
+ }
+
+ this->min_timestamp_ = min_timestamp;
+ this->max_timestamp_ = max_timestamp;
+ this->all_time_ = false;
+}
+
+long TimeRange::MinTimeStamp() const {
+ return this->min_timestamp_;
+
+}
+
+long TimeRange::MaxTimeStamp() const {
+ return this->max_timestamp_;
+}
+
+bool TimeRange::IsAllTime() const {
+ return this->all_time_;
+}
+}
diff --git a/core/time_range.h b/core/time_range.h
new file mode 100644
index 0000000..5c07b4b
--- /dev/null
+++ b/core/time_range.h
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+namespace hbase {
+class TimeRange {
+
+ public:
+
+ /**
+ * @brief Default constructor. Represents interval [0, Long.MAX_VALUE) (allTime)
+ */
+ TimeRange();
+ TimeRange(const TimeRange &tr);
+ TimeRange &operator=(const TimeRange &tr);
+ /**
+ * @brief Represents interval [minStamp, Long.MAX_VALUE)
+ * @param minStamp the minimum timestamp value, inclusive
+ */
+ TimeRange(long min_timestamp);
+ /**
+ * @brief Represents interval [minStamp, maxStamp)
+ * @param minStamp the minimum timestamp, inclusive
+ * @param maxStamp the maximum timestamp, exclusive
+ * @throws std::runtime_error if min_timestamp < 0 or max_timestamp < 0 or max_timestamp < min_timestamp
+ */
+ TimeRange(long min_timestamp, long max_timestamp);
+ long MinTimeStamp() const;
+ long MaxTimeStamp() const;
+ bool IsAllTime() const;
+ ~TimeRange();
+
+ private:
+ long min_timestamp_;
+ long max_timestamp_;
+ bool all_time_;
+};
+}
[hbase] 68/137: HBASE-17629 [C++] Timeouts and retry configuration
for connections
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit effe9667705838537b53942159f128c7afa013e5
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Fri Feb 24 16:30:52 2017 -0800
HBASE-17629 [C++] Timeouts and retry configuration for connections
---
bin/cpplint.sh | 3 +-
connection/connection-factory.cc | 19 +++-
connection/connection-factory.h | 10 +-
connection/connection-pool.cc | 5 +-
connection/connection-pool.h | 19 ++--
connection/rpc-client.cc | 4 +-
connection/rpc-client.h | 5 +-
core/BUCK | 1 +
core/client.cc | 12 ++-
core/client.h | 24 +++--
core/connection-configuration.h | 195 +++++++++++++++++++++++++++++++++++++++
11 files changed, 268 insertions(+), 29 deletions(-)
diff --git a/bin/cpplint.sh b/bin/cpplint.sh
index 78a00bd..81795fd 100755
--- a/bin/cpplint.sh
+++ b/bin/cpplint.sh
@@ -26,4 +26,5 @@ wget -nc $CPPLINT_LOC -O $OUTPUT
# Execute the script
# Exclude the following rules: build/header_guard (We use #pragma once instead)
# readability/todo (TODOs are generic)
-find core connection serde utils test-util security -name "*.h" -or -name "*.cc" | xargs -P8 python $OUTPUT --filter=-build/header_guard,-readability/todo --linelength=100
+# build/c++11 (We are building with c++14)
+find core connection serde utils test-util security -name "*.h" -or -name "*.cc" | xargs -P8 python $OUTPUT --filter=-build/header_guard,-readability/todo,-build/c++11 --linelength=100
diff --git a/connection/connection-factory.cc b/connection/connection-factory.cc
index 6aba351..2f7e75c 100644
--- a/connection/connection-factory.cc
+++ b/connection/connection-factory.cc
@@ -19,22 +19,32 @@
#include "connection/connection-factory.h"
+#include <chrono>
+
#include "connection/client-dispatcher.h"
#include "connection/pipeline.h"
#include "connection/service.h"
using namespace folly;
using namespace hbase;
+using std::chrono::milliseconds;
+using std::chrono::nanoseconds;
ConnectionFactory::ConnectionFactory(std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool,
- std::shared_ptr<Codec> codec)
- : io_pool_(io_pool), pipeline_factory_(std::make_shared<RpcPipelineFactory>(codec)) {}
+ std::shared_ptr<Codec> codec,
+ nanoseconds connect_timeout)
+ : connect_timeout_(connect_timeout),
+ io_pool_(io_pool),
+ pipeline_factory_(std::make_shared<RpcPipelineFactory>(codec)) {}
std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> ConnectionFactory::MakeBootstrap() {
auto client = std::make_shared<wangle::ClientBootstrap<SerializePipeline>>();
client->group(io_pool_);
client->pipelineFactory(pipeline_factory_);
+ // TODO: Opened https://github.com/facebook/wangle/issues/85 in wangle so that we can set socket
+ // options like TCP_NODELAY, SO_KEEPALIVE, CONNECT_TIMEOUT_MILLIS, etc.
+
return client;
}
std::shared_ptr<HBaseService> ConnectionFactory::Connect(
@@ -43,7 +53,10 @@ std::shared_ptr<HBaseService> ConnectionFactory::Connect(
// Yes this will block however it makes dealing with connection pool soooooo
// much nicer.
// TODO see about using shared promise for this.
- auto pipeline = client->connect(SocketAddress(hostname, port, true)).get();
+ auto pipeline = client
+ ->connect(SocketAddress(hostname, port, true),
+ std::chrono::duration_cast<milliseconds>(connect_timeout_))
+ .get();
auto dispatcher = std::make_shared<ClientDispatcher>();
dispatcher->setPipeline(pipeline);
return dispatcher;
diff --git a/connection/connection-factory.h b/connection/connection-factory.h
index 0d1e0d0..fbcb6ef 100644
--- a/connection/connection-factory.h
+++ b/connection/connection-factory.h
@@ -20,6 +20,7 @@
#include <wangle/service/Service.h>
+#include <chrono>
#include <memory>
#include <string>
@@ -28,6 +29,8 @@
#include "connection/response.h"
#include "connection/service.h"
+using std::chrono::nanoseconds;
+
namespace hbase {
/**
@@ -41,8 +44,10 @@ class ConnectionFactory {
* There should only be one ConnectionFactory per client.
*/
ConnectionFactory(std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool,
- std::shared_ptr<Codec> codec);
- /** Default Desctructor */
+ std::shared_ptr<Codec> codec,
+ nanoseconds connect_timeout = nanoseconds(0));
+
+ /** Default Destructor */
virtual ~ConnectionFactory() = default;
/**
@@ -60,6 +65,7 @@ class ConnectionFactory {
const std::string &hostname, int port);
private:
+ nanoseconds connect_timeout_;
std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool_;
std::shared_ptr<RpcPipelineFactory> pipeline_factory_;
};
diff --git a/connection/connection-pool.cc b/connection/connection-pool.cc
index 6635a6d..b18ee89 100644
--- a/connection/connection-pool.cc
+++ b/connection/connection-pool.cc
@@ -34,8 +34,9 @@ using folly::SharedMutexWritePriority;
using folly::SocketAddress;
ConnectionPool::ConnectionPool(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
- std::shared_ptr<Codec> codec)
- : cf_(std::make_shared<ConnectionFactory>(io_executor, codec)),
+ std::shared_ptr<Codec> codec,
+ nanoseconds connect_timeout)
+ : cf_(std::make_shared<ConnectionFactory>(io_executor, codec, connect_timeout)),
clients_(),
connections_(),
map_mutex_() {}
diff --git a/connection/connection-pool.h b/connection/connection-pool.h
index 23e5e9a..2a8f195 100644
--- a/connection/connection-pool.h
+++ b/connection/connection-pool.h
@@ -20,6 +20,7 @@
#include <folly/SharedMutex.h>
#include <boost/functional/hash.hpp>
+#include <chrono>
#include <memory>
#include <mutex>
#include <unordered_map>
@@ -35,6 +36,8 @@ using hbase::ConnectionIdEquals;
using hbase::ConnectionIdHash;
using hbase::RpcConnection;
+using std::chrono::nanoseconds;
+
namespace hbase {
/**
@@ -47,14 +50,7 @@ class ConnectionPool {
public:
/** Create connection pool wit default connection factory */
ConnectionPool(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
- std::shared_ptr<Codec> codec);
-
- /**
- * Desctructor.
- * All connections will be close.
- * All connections will be released
- */
- ~ConnectionPool();
+ std::shared_ptr<Codec> codec, nanoseconds connect_timeout = nanoseconds(0));
/**
* Constructor that allows specifiying the connetion factory.
@@ -63,6 +59,13 @@ class ConnectionPool {
explicit ConnectionPool(std::shared_ptr<ConnectionFactory> cf);
/**
+ * Destructor.
+ * All connections will be close.
+ * All connections will be released
+ */
+ ~ConnectionPool();
+
+ /**
* Get a connection to the server name. Start time is ignored.
* This can be a blocking operation for a short time.
*/
diff --git a/connection/rpc-client.cc b/connection/rpc-client.cc
index cfbda3a..c61a73e 100644
--- a/connection/rpc-client.cc
+++ b/connection/rpc-client.cc
@@ -40,9 +40,9 @@ class RpcChannelImplementation : public AbstractRpcChannel {
} // namespace hbase
RpcClient::RpcClient(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
- std::shared_ptr<Codec> codec)
+ std::shared_ptr<Codec> codec, nanoseconds connect_timeout)
: io_executor_(io_executor) {
- cp_ = std::make_shared<ConnectionPool>(io_executor_, codec);
+ cp_ = std::make_shared<ConnectionPool>(io_executor_, codec, connect_timeout);
}
void RpcClient::Close() { io_executor_->stop(); }
diff --git a/connection/rpc-client.h b/connection/rpc-client.h
index f4645a0..5c11ab5 100644
--- a/connection/rpc-client.h
+++ b/connection/rpc-client.h
@@ -26,6 +26,7 @@
#include <google/protobuf/service.h>
+#include <chrono>
#include <utility>
using hbase::security::User;
@@ -43,6 +44,8 @@ using google::protobuf::Message;
using google::protobuf::RpcController;
using google::protobuf::Closure;
+using std::chrono::nanoseconds;
+
class RpcChannelImplementation;
namespace hbase {
@@ -52,7 +55,7 @@ class RpcClient : public std::enable_shared_from_this<RpcClient> {
public:
RpcClient(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
- std::shared_ptr<Codec> codec);
+ std::shared_ptr<Codec> codec, nanoseconds connect_timeout);
virtual ~RpcClient() { Close(); }
diff --git a/core/BUCK b/core/BUCK
index ce0c733..e541d8f 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -27,6 +27,7 @@ cxx_library(
"keyvalue-codec.h",
"region-location.h",
"location-cache.h",
+ "connection-configuration.h",
# TODO: move this out of exported
# Once meta lookup works
"meta-utils.h",
diff --git a/core/client.cc b/core/client.cc
index 685524f..240da72 100644
--- a/core/client.cc
+++ b/core/client.cc
@@ -20,6 +20,7 @@
#include "core/client.h"
#include <glog/logging.h>
+#include <chrono>
#include <exception>
#include <utility>
@@ -42,9 +43,12 @@ void Client::init(const hbase::Configuration &conf) {
conf_ = std::make_shared<hbase::Configuration>(conf);
auto zk_quorum = conf_->Get(kHBaseZookeeperQuorum_, kDefHBaseZookeeperQuorum_);
- cpu_executor_ =
- std::make_shared<wangle::CPUThreadPoolExecutor>(4); // TODO: read num threads from conf
- io_executor_ = std::make_shared<wangle::IOThreadPoolExecutor>(sysconf(_SC_NPROCESSORS_ONLN));
+ conn_conf_ = std::make_shared<hbase::ConnectionConfiguration>(*conf_);
+ // start thread pools
+ auto io_threads = conf_->GetInt(kClientIoThreadPoolSize, sysconf(_SC_NPROCESSORS_ONLN));
+ auto cpu_threads = conf_->GetInt(kClientCpuThreadPoolSize, 2 * sysconf(_SC_NPROCESSORS_ONLN));
+ cpu_executor_ = std::make_shared<wangle::CPUThreadPoolExecutor>(cpu_threads);
+ io_executor_ = std::make_shared<wangle::IOThreadPoolExecutor>(io_threads);
std::shared_ptr<Codec> codec = nullptr;
if (conf.Get(kRpcCodec, hbase::KeyValueCodec::kJavaClassName) ==
@@ -53,7 +57,7 @@ void Client::init(const hbase::Configuration &conf) {
} else {
LOG(WARNING) << "Not using RPC Cell Codec";
}
- rpc_client_ = std::make_shared<hbase::RpcClient>(io_executor_, codec);
+ rpc_client_ = std::make_shared<hbase::RpcClient>(io_executor_, codec, conn_conf_->connect_timeout());
location_cache_ =
std::make_shared<hbase::LocationCache>(conf_, cpu_executor_, rpc_client_->connection_pool());
}
diff --git a/core/client.h b/core/client.h
index 0e436ba..a96d6f3 100644
--- a/core/client.h
+++ b/core/client.h
@@ -29,6 +29,7 @@
#include "connection/rpc-client.h"
#include "core/configuration.h"
+#include "core/connection-configuration.h"
#include "core/hbase_configuration_loader.h"
#include "core/keyvalue-codec.h"
#include "core/location-cache.h"
@@ -54,13 +55,13 @@ class Client {
* @param quorum_spec Where to connect to get Zookeeper bootstrap information.
*/
Client();
- explicit Client(const hbase::Configuration &conf);
+ explicit Client(const hbase::Configuration& conf);
~Client();
/**
* @brief Retrieve a Table implementation for accessing a table.
* @param - table_name
*/
- std::unique_ptr<hbase::Table> Table(const TableName &table_name);
+ std::unique_ptr<hbase::Table> Table(const TableName& table_name);
/**
* @brief Close the Client connection.
@@ -68,16 +69,27 @@ class Client {
void Close();
private:
- void init(const hbase::Configuration &conf);
- const std::string kHBaseZookeeperQuorum_ = "hbase.zookeeper.quorum";
- const std::string kDefHBaseZookeeperQuorum_ = "localhost:2181";
- const std::string kRpcCodec = "hbase.client.rpc.codec";
+ /** Constants */
+ static constexpr const char* kHBaseZookeeperQuorum_ = "hbase.zookeeper.quorum";
+ static constexpr const char* kDefHBaseZookeeperQuorum_ = "localhost:2181";
+ /** Parameter name for HBase client IO thread pool size. Defaults to num cpus */
+ static constexpr const char* kClientIoThreadPoolSize = "hbase.client.io.thread.pool.size";
+ /** Parameter name for HBase client CPU thread pool size. Defaults to (2 * num cpus) */
+ static constexpr const char* kClientCpuThreadPoolSize = "hbase.client.cpu.thread.pool.size";
+ /** The RPC codec to encode cells. For now it is KeyValueCodec */
+ static constexpr const char* kRpcCodec = "hbase.client.rpc.codec";
+
+ /** Data */
std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_;
std::shared_ptr<hbase::LocationCache> location_cache_;
std::shared_ptr<hbase::RpcClient> rpc_client_;
std::shared_ptr<hbase::Configuration> conf_;
+ std::shared_ptr<hbase::ConnectionConfiguration> conn_conf_;
bool is_closed_ = false;
+
+ /** Methods */
+ void init(const hbase::Configuration &conf);
};
} // namespace hbase
diff --git a/core/connection-configuration.h b/core/connection-configuration.h
new file mode 100644
index 0000000..e1e9f87
--- /dev/null
+++ b/core/connection-configuration.h
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <chrono>
+#include <climits>
+#include <string>
+
+#include "core/configuration.h"
+
+using std::chrono::nanoseconds;
+using std::chrono::milliseconds;
+
+namespace hbase {
+
+/**
+ * Timeout configs.
+ */
+class ConnectionConfiguration {
+ public:
+ explicit ConnectionConfiguration(const Configuration& conf) {
+ connect_timeout_ =
+ ToNanos(conf.GetInt(kClientSocketConnectTimeout, kDefaultClientSocketConnectTimeout));
+ meta_operation_timeout_ =
+ ToNanos(conf.GetLong(kClientMetaOperationTimeout, kDefaultClientOperationTimeout));
+ operation_timeout_ =
+ ToNanos(conf.GetLong(kClientOperationTimeout, kDefaultClientOperationTimeout));
+ rpc_timeout_ = ToNanos(conf.GetLong(kRpcTimeout, kDefaultRpcTimeout));
+ read_rpc_timeout_ = ToNanos(conf.GetLong(kRpcReadTimeout, ToMillis(rpc_timeout_)));
+ write_rpc_timeout_ = ToNanos(conf.GetLong(kRpcWriteTimeout, ToMillis(rpc_timeout_)));
+ pause_ = ToNanos(conf.GetLong(kClientPause, kDefaultClientPause));
+ max_retries_ = conf.GetInt(kClientRetriesNumber, kDefaultClientRetriesNumber);
+ start_log_errors_count_ =
+ conf.GetInt(kStartLogErrorsAfterCount, kDefaultStartLogErrorsAfterCount);
+ scan_timeout_ =
+ ToNanos(conf.GetLong(kClientScannerTimeoutPeriod, kDefaultClientScannerTimeoutPeriod));
+ scanner_caching_ = conf.GetInt(kClientScannerCaching, kDefaultClientScannerCaching);
+ scanner_max_result_size_ =
+ conf.GetLong(kClientScannerMaxResultsSize, kDefaultClientScannerMaxResultsSize);
+ }
+
+ nanoseconds connect_timeout() const { return connect_timeout_; }
+
+ nanoseconds meta_operation_timeout() const { return meta_operation_timeout_; }
+
+ // timeout for a whole operation such as get, put or delete. Notice that scan will not be effected
+ // by this value, see scanTimeoutNs.
+ nanoseconds operation_timeout() const { return operation_timeout_; }
+
+ // timeout for each rpc request. Can be overridden by a more specific config, such as
+ // readRpcTimeout or writeRpcTimeout.
+ nanoseconds rpc_timeout() const { return rpc_timeout_; }
+
+ // timeout for each read rpc request
+ nanoseconds read_rpc_timeout() const { return read_rpc_timeout_; }
+
+ // timeout for each write rpc request
+ nanoseconds write_rpc_timeout() const { return write_rpc_timeout_; }
+
+ nanoseconds pause_nanos() const { return pause_; }
+
+ uint32_t max_retries() const { return max_retries_; }
+
+ /** How many retries are allowed before we start to log */
+ uint32_t start_log_errors_count() const { return start_log_errors_count_; }
+
+ // The scan timeout is used as operation timeout for every
+ // operations in a scan, such as openScanner or next.
+ nanoseconds scan_timeout() const { return scan_timeout_; }
+
+ uint32_t scanner_caching() const { return scanner_caching_; }
+
+ uint64_t scanner_max_result_size() const { return scanner_max_result_size_; }
+
+ private:
+ /** Parameter name for HBase client CPU thread pool size. Defaults to (2 * num cpus) */
+ static constexpr const char* kClientSocketConnectTimeout =
+ "hbase.ipc.client.socket.timeout.connect";
+ /** Parameter name for HBase client CPU thread pool size. Defaults to (2 * num cpus) */
+ static constexpr const uint32_t kDefaultClientSocketConnectTimeout = 10000; // 10 secs
+
+ /** Parameter name for HBase client operation timeout. */
+ static constexpr const char* kClientOperationTimeout = "hbase.client.operation.timeout";
+
+ /** Parameter name for HBase client meta operation timeout. */
+ static constexpr const char* kClientMetaOperationTimeout = "hbase.client.meta.operation.timeout";
+
+ /** Default HBase client operation timeout, which is tantamount to a blocking call */
+ static constexpr const uint32_t kDefaultClientOperationTimeout = 1200000;
+
+ /** timeout for each RPC */
+ static constexpr const char* kRpcTimeout = "hbase.rpc.timeout";
+
+ /** timeout for each read RPC */
+ static constexpr const char* kRpcReadTimeout = "hbase.rpc.read.timeout";
+
+ /** timeout for each write RPC */
+ static constexpr const char* kRpcWriteTimeout = "hbase.rpc.write.timeout";
+
+ static constexpr const uint32_t kDefaultRpcTimeout = 60000;
+
+ /**
+ * Parameter name for client pause value, used mostly as value to wait
+ * before running a retry of a failed get, region lookup, etc.
+ */
+ static constexpr const char* kClientPause = "hbase.client.pause";
+
+ static constexpr const uint64_t kDefaultClientPause = 100;
+
+ /**
+ * Parameter name for maximum retries, used as maximum for all retryable
+ * operations such as fetching of the root region from root region server,
+ * getting a cell's value, starting a row update, etc.
+ */
+ static constexpr const char* kClientRetriesNumber = "hbase.client.retries.number";
+
+ static constexpr const uint32_t kDefaultClientRetriesNumber = 31;
+
+ /**
+ * Configure the number of failures after which the client will start logging. A few failures
+ * is fine: region moved, then is not opened, then is overloaded. We try to have an acceptable
+ * heuristic for the number of errors we don't log. 9 was chosen because we wait for 1s at
+ * this stage.
+ */
+ static constexpr const char* kStartLogErrorsAfterCount = "hbase.client.start.log.errors.counter";
+ static constexpr const uint32_t kDefaultStartLogErrorsAfterCount = 9;
+
+ /** The client scanner timeout period in milliseconds. */
+ static constexpr const char* kClientScannerTimeoutPeriod = "hbase.client.scanner.timeout.period";
+
+ static constexpr const uint32_t kDefaultClientScannerTimeoutPeriod = 60000;
+
+ /**
+ * Parameter name to set the default scanner caching for all clients.
+ */
+ static constexpr const char* kClientScannerCaching = "hbase.client.scanner.caching";
+
+ static constexpr const uint32_t kDefaultClientScannerCaching = INT_MAX;
+
+ /**
+ * Parameter name for maximum number of bytes returned when calling a scanner's next method.
+ * Controlled by the client.
+ */
+ static constexpr const char* kClientScannerMaxResultsSize =
+ "hbase.client.scanner.max.result.size";
+
+ /**
+ * Maximum number of bytes returned when calling a scanner's next method.
+ * Note that when a single row is larger than this limit the row is still
+ * returned completely.
+ *
+ * The default value is 2MB.
+ */
+ static constexpr const uint64_t kDefaultClientScannerMaxResultsSize = 2 * 1024 * 1024;
+
+ nanoseconds connect_timeout_;
+ nanoseconds meta_operation_timeout_;
+ nanoseconds operation_timeout_;
+ nanoseconds rpc_timeout_;
+ nanoseconds read_rpc_timeout_;
+ nanoseconds write_rpc_timeout_;
+ nanoseconds pause_;
+ uint32_t max_retries_;
+ uint32_t start_log_errors_count_;
+ nanoseconds scan_timeout_;
+ uint32_t scanner_caching_;
+ uint64_t scanner_max_result_size_;
+
+ static nanoseconds ToNanos(const uint64_t& millis) {
+ return std::chrono::duration_cast<nanoseconds>(milliseconds(millis));
+ }
+
+ static uint64_t ToMillis(const nanoseconds& nanos) {
+ return std::chrono::duration_cast<milliseconds>(nanos).count();
+ }
+};
+
+} // namespace hbase
[hbase] 61/137: HBASE-17481 [C++] cpplint should use line wrapping
of 100
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 0c67147b09d765745ffe6c3a37a57f058270b072
Author: Sudeep Sunthankar <su...@hashmapinc.com>
AuthorDate: Tue Jan 17 23:57:16 2017 +1100
HBASE-17481 [C++] cpplint should use line wrapping of 100
Signed-off-by: Enis Soztutar <en...@apache.org>
---
bin/cpplint.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bin/cpplint.sh b/bin/cpplint.sh
index 169c382..78a00bd 100755
--- a/bin/cpplint.sh
+++ b/bin/cpplint.sh
@@ -26,4 +26,4 @@ wget -nc $CPPLINT_LOC -O $OUTPUT
# Execute the script
# Exclude the following rules: build/header_guard (We use #pragma once instead)
# readability/todo (TODOs are generic)
-find core connection serde utils test-util -name "*.h" -or -name "*.cc"| xargs -P8 python $OUTPUT --filter=-build/header_guard,-readability/todo
+find core connection serde utils test-util security -name "*.h" -or -name "*.cc" | xargs -P8 python $OUTPUT --filter=-build/header_guard,-readability/todo --linelength=100
[hbase] 58/137: HBASE-17315 [C++] HBase Client and Table
Implementation (Sudeep Sunthankar)
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit a8bc1f6c3362dfd0bbf85c1ef1d3f11590f103de
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Fri Jan 13 16:03:30 2017 -0800
HBASE-17315 [C++] HBase Client and Table Implementation (Sudeep Sunthankar)
---
connection/BUCK | 2 +
core/BUCK | 12 +++
core/client-test.cc | 240 ++++++++++++++++++++++++++++++++++++++++++++++++++++
core/client.cc | 46 +++++++---
core/client.h | 37 ++++++--
core/table.cc | 74 ++++++++++++++++
core/table.h | 69 +++++++++++++++
7 files changed, 462 insertions(+), 18 deletions(-)
diff --git a/connection/BUCK b/connection/BUCK
index c22cc89..19536d5 100644
--- a/connection/BUCK
+++ b/connection/BUCK
@@ -30,6 +30,7 @@ cxx_library(
"rpc-connection.h",
"response.h",
"service.h",
+ "rpc-client.h",
],
srcs=[
"client-dispatcher.cc",
@@ -38,6 +39,7 @@ cxx_library(
"connection-pool.cc",
"pipeline.cc",
"request.cc",
+ "rpc-client.cc",
],
deps=[
"//if:if",
diff --git a/core/BUCK b/core/BUCK
index b7db41a..0d1bc93 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -35,6 +35,7 @@ cxx_library(
"result.h",
"request_converter.h",
"response_converter.h",
+ "table.h",
],
srcs=[
"cell.cc",
@@ -49,6 +50,7 @@ cxx_library(
"result.cc",
"request_converter.cc",
"response_converter.cc",
+ "table.cc",
],
deps=[
"//connection:connection",
@@ -107,6 +109,16 @@ cxx_test(
"//if:if",
],
run_test_separately=True,)
+cxx_test(
+ name="client-test",
+ srcs=["client-test.cc",],
+ deps=[
+ ":core",
+ "//if:if",
+ "//serde:serde",
+ "//test-util:test-util",
+ ],
+ run_test_separately=True,)
cxx_binary(
name="simple-client",
srcs=["simple-client.cc",],
diff --git a/core/client-test.cc b/core/client-test.cc
new file mode 100644
index 0000000..0fe0225
--- /dev/null
+++ b/core/client-test.cc
@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <gtest/gtest.h>
+#include "core/client.h"
+#include "core/configuration.h"
+#include "core/get.h"
+#include "core/hbase_configuration_loader.h"
+#include "core/result.h"
+#include "core/table.h"
+#include "serde/table-name.h"
+#include "test-util/test-util.h"
+
+class ClientTest {
+ public:
+ const static std::string kDefHBaseConfPath;
+
+ const static std::string kHBaseDefaultXml;
+ const static std::string kHBaseSiteXml;
+
+ const static std::string kHBaseXmlData;
+
+ static void WriteDataToFile(const std::string &file, const std::string &xml_data) {
+ std::ofstream hbase_conf;
+ hbase_conf.open(file.c_str());
+ hbase_conf << xml_data;
+ hbase_conf.close();
+ }
+
+ static void CreateHBaseConf(const std::string &dir, const std::string &file,
+ const std::string xml_data) {
+ // Directory will be created if not present
+ if (!boost::filesystem::exists(dir)) {
+ boost::filesystem::create_directories(dir);
+ }
+ // Remove temp file always
+ boost::filesystem::remove((dir + file).c_str());
+ WriteDataToFile((dir + file), xml_data);
+ }
+
+ static void CreateHBaseConfWithEnv() {
+ // Creating Empty Config Files so that we dont get a Configuration exception @Client
+ CreateHBaseConf(kDefHBaseConfPath, kHBaseDefaultXml, kHBaseXmlData);
+ CreateHBaseConf(kDefHBaseConfPath, kHBaseSiteXml, kHBaseXmlData);
+ setenv("HBASE_CONF", kDefHBaseConfPath.c_str(), 1);
+ }
+};
+
+const std::string ClientTest::kDefHBaseConfPath("./build/test-data/client-test/conf/");
+
+const std::string ClientTest::kHBaseDefaultXml("hbase-default.xml");
+const std::string ClientTest::kHBaseSiteXml("hbase-site.xml");
+
+const std::string ClientTest::kHBaseXmlData(
+ "<?xml version=\"1.0\"?>\n<?xml-stylesheet type=\"text/xsl\" "
+ "href=\"configuration.xsl\"?>\n<!--\n/**\n *\n * Licensed to the Apache "
+ "Software Foundation (ASF) under one\n * or more contributor license "
+ "agreements. See the NOTICE file\n * distributed with this work for "
+ "additional information\n * regarding copyright ownership. The ASF "
+ "licenses this file\n * to you under the Apache License, Version 2.0 "
+ "(the\n * \"License\"); you may not use this file except in compliance\n * "
+ "with the License. You may obtain a copy of the License at\n *\n * "
+ "http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by "
+ "applicable law or agreed to in writing, software\n * distributed under "
+ "the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES "
+ "OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License "
+ "for the specific language governing permissions and\n * limitations under "
+ "the License.\n "
+ "*/\n-->\n<configuration>\n\n</configuration>");
+
+TEST(Client, EmptyConfigurationPassedToClient) {
+ ASSERT_ANY_THROW(hbase::Client client);
+}
+
+TEST(Client, ConfigurationPassedToClient) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+ ClientTest::CreateHBaseConfWithEnv();
+
+ // Create Configuration
+ hbase::HBaseConfigurationLoader loader;
+ auto conf = loader.LoadDefaultResources();
+ // Create a client
+ hbase::Client client(conf.value());
+ client.Close();
+}
+
+TEST(Client, DefaultConfiguration) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+ ClientTest::CreateHBaseConfWithEnv();
+
+ // Create Configuration
+ hbase::Client client;
+ client.Close();
+}
+
+TEST(Client, Get) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+ ClientTest::CreateHBaseConfWithEnv();
+
+ // Using TestUtil to populate test data
+ hbase::TestUtil *test_util = new hbase::TestUtil();
+ test_util->RunShellCmd("create 't', 'd'");
+ test_util->RunShellCmd("put 't', 'test2', 'd:2', 'value2'");
+ test_util->RunShellCmd("put 't', 'test2', 'd:extra', 'value for extra'");
+
+ // Create TableName and Row to be fetched from HBase
+ auto tn = folly::to<hbase::pb::TableName>("t");
+ auto row = "test2";
+
+ // Get to be performed on above HBase Table
+ hbase::Get get(row);
+
+ // Create Configuration
+ hbase::HBaseConfigurationLoader loader;
+ auto conf = loader.LoadDefaultResources();
+
+ // Create a client
+ hbase::Client client(conf.value());
+
+ // Get connection to HBase Table
+ auto table = client.Table(tn);
+ ASSERT_TRUE(table) << "Unable to get connection to Table.";
+
+ // Perform the Get
+ auto result = table->Get(get);
+
+ // Stopping the connection as we are getting segfault due to some folly issue
+ // The connection stays open and we don't want that.
+ // So we are stopping the connection.
+ // We can remove this once we have fixed the folly part
+ delete test_util;
+
+ // Test the values, should be same as in put executed on hbase shell
+ ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty.";
+ EXPECT_EQ("test2", result->Row());
+ EXPECT_EQ("value2", *(result->Value("d", "2")));
+ EXPECT_EQ("value for extra", *(result->Value("d", "extra")));
+
+ table->Close();
+ client.Close();
+}
+
+TEST(Client, GetForNonExistentTable) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+ ClientTest::CreateHBaseConfWithEnv();
+
+ // Using TestUtil to populate test data
+ hbase::TestUtil *test_util = new hbase::TestUtil();
+
+ // Create TableName and Row to be fetched from HBase
+ auto tn = folly::to<hbase::pb::TableName>("t_not_exists");
+ auto row = "test2";
+
+ // Get to be performed on above HBase Table
+ hbase::Get get(row);
+
+ // Create Configuration
+ hbase::HBaseConfigurationLoader loader;
+ auto conf = loader.LoadDefaultResources();
+
+ // Create a client
+ hbase::Client client(conf.value());
+
+ // Get connection to HBase Table
+ auto table = client.Table(tn);
+ ASSERT_TRUE(table) << "Unable to get connection to Table.";
+
+ // Perform the Get
+ ASSERT_ANY_THROW(table->Get(get)) << "Table does not exist. We should get an exception";
+
+ // Stopping the connection as we are getting segfault due to some folly issue
+ // The connection stays open and we don't want that.
+ // So we are stopping the connection.
+ // We can remove this once we have fixed the folly part
+ delete test_util;
+
+ table->Close();
+ client.Close();
+}
+
+TEST(Client, GetForNonExistentRow) {
+ // Remove already configured env if present.
+ unsetenv("HBASE_CONF");
+ ClientTest::CreateHBaseConfWithEnv();
+
+ // Using TestUtil to populate test data
+ hbase::TestUtil *test_util = new hbase::TestUtil();
+ test_util->RunShellCmd("create 't_exists', 'd'");
+
+ // Create TableName and Row to be fetched from HBase
+ auto tn = folly::to<hbase::pb::TableName>("t_exists");
+ auto row = "row_not_exists";
+
+ // Get to be performed on above HBase Table
+ hbase::Get get(row);
+
+ // Create Configuration
+ hbase::HBaseConfigurationLoader loader;
+ auto conf = loader.LoadDefaultResources();
+
+ // Create a client
+ hbase::Client client(conf.value());
+
+ // Get connection to HBase Table
+ auto table = client.Table(tn);
+ ASSERT_TRUE(table) << "Unable to get connection to Table.";
+
+ // Perform the Get
+ auto result = table->Get(get);
+ ASSERT_TRUE(result->IsEmpty()) << "Result should be empty.";
+
+ // Stopping the connection as we are getting segfault due to some folly issue
+ // The connection stays open and we don't want that.
+ // So we are stopping the connection.
+ // We can remove this once we have fixed the folly part
+ delete test_util;
+
+ table->Close();
+ client.Close();
+}
diff --git a/core/client.cc b/core/client.cc
index 0389b24..6eb3d8f 100644
--- a/core/client.cc
+++ b/core/client.cc
@@ -20,27 +20,49 @@
#include "core/client.h"
#include <glog/logging.h>
-
-#include <unistd.h>
-#include <string>
-
-using namespace folly;
-using namespace std;
-using namespace hbase::pb;
+#include <exception>
+#include <utility>
namespace hbase {
-Client::Client(std::string zk_quorum)
- : cpu_executor_(std::make_shared<wangle::CPUThreadPoolExecutor>(4)),
- io_executor_(std::make_shared<wangle::IOThreadPoolExecutor>(
- sysconf(_SC_NPROCESSORS_ONLN))),
- location_cache_(zk_quorum, cpu_executor_, io_executor_) {}
+Client::Client() {
+ HBaseConfigurationLoader loader;
+ auto conf = loader.LoadDefaultResources();
+ if (!conf) {
+ LOG(ERROR) << "Unable to create default Configuration object. Either hbase-default.xml or "
+ "hbase-site.xml is absent in the search path or problems in XML parsing";
+ throw std::runtime_error("Configuration object not present.");
+ }
+ conf_ = std::make_shared<hbase::Configuration>(conf.value());
+ auto zk_quorum = conf_->Get(kHBaseZookeeperQuorum_, kDefHBaseZookeeperQuorum_);
+ location_cache_ = std::make_shared<hbase::LocationCache>(zk_quorum, cpu_executor_, io_executor_);
+}
+
+Client::Client(const hbase::Configuration &conf) {
+ conf_ = std::make_shared<hbase::Configuration>(conf);
+ auto zk_quorum = conf_->Get(kHBaseZookeeperQuorum_, kDefHBaseZookeeperQuorum_);
+ location_cache_ = std::make_shared<hbase::LocationCache>(zk_quorum, cpu_executor_, io_executor_);
+}
// We can't have the threads continue running after everything is done
// that leads to an error.
Client::~Client() {
cpu_executor_->stop();
io_executor_->stop();
+ if (rpc_client_.get()) rpc_client_->Close();
+}
+
+std::unique_ptr<hbase::Table> Client::Table(const TableName &table_name) {
+ return std::make_unique<hbase::Table>(table_name, location_cache_, rpc_client_, conf_);
+}
+
+void Client::Close() {
+ if (is_closed_) return;
+
+ cpu_executor_->stop();
+ io_executor_->stop();
+ if (rpc_client_.get()) rpc_client_->Close();
+ is_closed_ = true;
}
} // namespace hbase
diff --git a/core/client.h b/core/client.h
index 0ba1276..2bb506b 100644
--- a/core/client.h
+++ b/core/client.h
@@ -27,11 +27,18 @@
#include <memory>
#include <string>
+#include "core/configuration.h"
+#include "core/hbase_configuration_loader.h"
#include "core/location-cache.h"
+#include "connection/rpc-client.h"
+#include "core/table.h"
+#include "serde/table-name.h"
#include "if/Cell.pb.h"
-namespace hbase {
+using hbase::pb::TableName;
+namespace hbase {
+class Table;
/**
* Client.
*
@@ -42,16 +49,34 @@ namespace hbase {
class Client {
public:
/**
- * Create a new client.
+ * @brief Create a new client.
* @param quorum_spec Where to connect to get Zookeeper bootstrap information.
*/
- explicit Client(std::string quorum_spec);
+ Client();
+ explicit Client(const hbase::Configuration &conf);
~Client();
+ /**
+ * @brief Retrieve a Table implementation for accessing a table.
+ * @param - table_name
+ */
+ std::unique_ptr<hbase::Table> Table(const TableName &table_name);
+
+ /**
+ * @brief Close the Client connection.
+ */
+ void Close();
private:
- std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
- std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_;
- LocationCache location_cache_;
+ const std::string kHBaseZookeeperQuorum_ = "hbase.zookeeper.quorum";
+ const std::string kDefHBaseZookeeperQuorum_ = "localhost:2181";
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_ =
+ std::make_shared<wangle::CPUThreadPoolExecutor>(4);
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_ =
+ std::make_shared<wangle::IOThreadPoolExecutor>(sysconf(_SC_NPROCESSORS_ONLN));
+ std::shared_ptr<hbase::LocationCache> location_cache_;
+ std::shared_ptr<hbase::RpcClient> rpc_client_ = std::make_shared<hbase::RpcClient>();
+ std::shared_ptr<hbase::Configuration> conf_;
+ bool is_closed_ = false;
};
} // namespace hbase
diff --git a/core/table.cc b/core/table.cc
new file mode 100644
index 0000000..58125f9
--- /dev/null
+++ b/core/table.cc
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/table.h"
+
+#include <folly/futures/Future.h>
+#include <chrono>
+#include <limits>
+#include <utility>
+#include <vector>
+
+#include "core/request_converter.h"
+#include "core/response_converter.h"
+#include "if/Client.pb.h"
+#include "security/user.h"
+#include "serde/server-name.h"
+
+using folly::Future;
+using hbase::pb::TableName;
+using hbase::security::User;
+using std::chrono::milliseconds;
+
+namespace hbase {
+
+Table::Table(const TableName &table_name,
+ const std::shared_ptr<hbase::LocationCache> &location_cache,
+ const std::shared_ptr<hbase::RpcClient> &rpc_client,
+ const std::shared_ptr<hbase::Configuration> &conf)
+ : table_name_(std::make_shared<TableName>(table_name)),
+ location_cache_(location_cache),
+ rpc_client_(rpc_client),
+ conf_(conf) {
+ client_retries_ = (conf_) ? conf_->GetInt("hbase.client.retries", client_retries_) : 5;
+}
+
+Table::~Table() {}
+
+std::unique_ptr<hbase::Result> Table::Get(const hbase::Get &get) {
+ auto loc = location_cache_->LocateFromMeta(*table_name_, get.Row()).get(milliseconds(1000));
+ auto req = hbase::RequestConverter::ToGetRequest(get, loc->region_name());
+ auto user = User::defaultUser(); // TODO: make User::current() similar to UserUtil
+
+ Future<Response> f =
+ rpc_client_->AsyncCall(loc->server_name().host_name(), loc->server_name().port(),
+ std::move(req), user, "ClientService");
+ auto resp = f.get();
+
+ return hbase::ResponseConverter::FromGetResponse(resp);
+}
+
+void Table::Close() {
+ if (is_closed_) return;
+
+ if (rpc_client_.get()) rpc_client_->Close();
+ is_closed_ = true;
+}
+
+} /* namespace hbase */
diff --git a/core/table.h b/core/table.h
new file mode 100644
index 0000000..0e98cd2
--- /dev/null
+++ b/core/table.h
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "connection/rpc-client.h"
+#include "core/client.h"
+#include "core/configuration.h"
+#include "core/get.h"
+#include "core/location-cache.h"
+#include "core/result.h"
+#include "serde/table-name.h"
+
+using hbase::pb::TableName;
+
+namespace hbase {
+class Client;
+
+class Table {
+ public:
+ /**
+ * Constructors
+ */
+ Table(const TableName &table_name, const std::shared_ptr<hbase::LocationCache> &location_cache,
+ const std::shared_ptr<hbase::RpcClient> &rpc_client,
+ const std::shared_ptr<hbase::Configuration> &conf);
+ ~Table();
+
+ /**
+ * @brief - Returns a Result object for the constructed Get.
+ * @param - get Get object to perform HBase Get operation.
+ */
+ std::unique_ptr<hbase::Result> Get(const hbase::Get &get);
+
+ /**
+ * @brief - Close the client connection.
+ */
+ void Close();
+
+ private:
+ std::shared_ptr<TableName> table_name_;
+ std::shared_ptr<hbase::LocationCache> location_cache_;
+ std::shared_ptr<hbase::RpcClient> rpc_client_;
+ std::shared_ptr<hbase::Configuration> conf_;
+ bool is_closed_ = false;
+ // default 5 retries. over-ridden in constructor.
+ int client_retries_ = 5;
+};
+} /* namespace hbase */
[hbase] 97/137: HBASE-18174 Implement Table#checkAndPut()
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit bad90d06bb65b1e1cff6db838c1fb7b2eb0b4020
Author: tedyu <yu...@gmail.com>
AuthorDate: Fri Jun 9 17:19:57 2017 -0700
HBASE-18174 Implement Table#checkAndPut()
---
core/async-rpc-retrying-caller.cc | 1 +
core/client-test.cc | 29 +++++++++++++++++++++++++++++
core/raw-async-table.cc | 26 ++++++++++++++++++++++++++
core/raw-async-table.h | 5 +++++
core/request-converter.cc | 21 +++++++++++++++++++++
core/request-converter.h | 5 +++++
core/response-converter.cc | 5 +++++
core/response-converter.h | 2 ++
core/table.cc | 7 +++++++
core/table.h | 17 +++++++++++++++++
10 files changed, 118 insertions(+)
diff --git a/core/async-rpc-retrying-caller.cc b/core/async-rpc-retrying-caller.cc
index b9d01bb..0302ad3 100644
--- a/core/async-rpc-retrying-caller.cc
+++ b/core/async-rpc-retrying-caller.cc
@@ -216,4 +216,5 @@ class OpenScannerResponse;
template class AsyncSingleRequestRpcRetryingCaller<std::shared_ptr<hbase::Result>>;
template class AsyncSingleRequestRpcRetryingCaller<folly::Unit>;
template class AsyncSingleRequestRpcRetryingCaller<std::shared_ptr<OpenScannerResponse>>;
+template class AsyncSingleRequestRpcRetryingCaller<bool>;
} /* namespace hbase */
diff --git a/core/client-test.cc b/core/client-test.cc
index 4972e05..ed413e8 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -247,6 +247,35 @@ TEST_F(ClientTest, Increment) {
EXPECT_EQ(incr1 + incr2, hbase::BytesUtil::ToInt64(*(result->Value("d", "1"))));
}
+TEST_F(ClientTest, CheckAndPut) {
+ // Using TestUtil to populate test data
+ ClientTest::test_util->CreateTable("check", "d");
+
+ // Create TableName and Row to be fetched from HBase
+ auto tn = folly::to<hbase::pb::TableName>("check");
+ auto row = "test1";
+
+ // Create a client
+ hbase::Client client(*ClientTest::test_util->conf());
+
+ // Get connection to HBase Table
+ auto table = client.Table(tn);
+ ASSERT_TRUE(table) << "Unable to get connection to Table.";
+
+ // Perform Puts
+ table->Put(Put{row}.AddColumn("d", "1", "value1"));
+ auto result = table->CheckAndPut(row, "d", "1", "value1", Put{row}.AddColumn("d", "1", "value2"));
+ ASSERT_TRUE(result) << "CheckAndPut didn't replace value";
+
+ result = table->CheckAndPut(row, "d", "1", "value1", Put{row}.AddColumn("d", "1", "value3"));
+
+ // Perform the Get
+ hbase::Get get(row);
+ auto result1 = table->Get(get);
+ EXPECT_EQ("value2", *(result1->Value("d", "1")));
+ ASSERT_FALSE(result) << "CheckAndPut shouldn't replace value";
+}
+
TEST_F(ClientTest, PutGet) {
// Using TestUtil to populate test data
ClientTest::test_util->CreateTable("t", "d");
diff --git a/core/raw-async-table.cc b/core/raw-async-table.cc
index 413dc6c..46d9dfd 100644
--- a/core/raw-async-table.cc
+++ b/core/raw-async-table.cc
@@ -111,6 +111,32 @@ folly::Future<folly::Unit> RawAsyncTable::Put(const hbase::Put& put) {
return caller->Call().then([caller](const auto r) { return r; });
}
+folly::Future<bool> RawAsyncTable::CheckAndPut(const std::string& row, const std::string& family,
+ const std::string& qualifier,
+ const std::string& value, const hbase::Put& put,
+ const pb::CompareType& compare_op) {
+ auto caller =
+ CreateCallerBuilder<bool>(row, connection_conf_->write_rpc_timeout())
+ ->action([=, &put](std::shared_ptr<hbase::HBaseRpcController> controller,
+ std::shared_ptr<hbase::RegionLocation> loc,
+ std::shared_ptr<hbase::RpcClient> rpc_client) -> folly::Future<bool> {
+ return Call<hbase::Put, hbase::Request, hbase::Response, bool>(
+ rpc_client, controller, loc, put,
+ // request conversion
+ [=, &put](const hbase::Put& put,
+ const std::string& region_name) -> std::unique_ptr<Request> {
+ auto checkReq = RequestConverter::CheckAndPutToMutateRequest(
+ row, family, qualifier, value, compare_op, put, region_name);
+ return checkReq;
+ },
+ // response conversion
+ &ResponseConverter::BoolFromMutateResponse);
+ })
+ ->Build();
+
+ return caller->Call().then([caller](const auto r) { return r; });
+}
+
folly::Future<folly::Unit> RawAsyncTable::Delete(const hbase::Delete& del) {
auto caller =
CreateCallerBuilder<folly::Unit>(del.row(), connection_conf_->write_rpc_timeout())
diff --git a/core/raw-async-table.h b/core/raw-async-table.h
index 6088d1b..068f230 100644
--- a/core/raw-async-table.h
+++ b/core/raw-async-table.h
@@ -67,6 +67,11 @@ class RawAsyncTable {
folly::Future<folly::Unit> Put(const hbase::Put& put);
+ folly::Future<bool> CheckAndPut(const std::string& row, const std::string& family,
+ const std::string& qualifier, const std::string& value,
+ const hbase::Put& put,
+ const pb::CompareType& compare_op = pb::CompareType::EQUAL);
+
void Scan(const hbase::Scan& scan, std::shared_ptr<RawScanResultConsumer> consumer);
void Close() {}
diff --git a/core/request-converter.cc b/core/request-converter.cc
index 042d5b1..0cd9c7c 100644
--- a/core/request-converter.cc
+++ b/core/request-converter.cc
@@ -276,6 +276,27 @@ std::unique_ptr<Request> RequestConverter::ToMutateRequest(const Put &put,
return pb_req;
}
+std::unique_ptr<Request> RequestConverter::CheckAndPutToMutateRequest(
+ const std::string &row, const std::string &family, const std::string &qualifier,
+ const std::string &value, const pb::CompareType compare_op, const hbase::Put &put,
+ const std::string ®ion_name) {
+ auto pb_req = Request::mutate();
+ auto pb_msg = std::static_pointer_cast<hbase::pb::MutateRequest>(pb_req->req_msg());
+
+ pb_msg->set_allocated_mutation(
+ ToMutation(MutationType::MutationProto_MutationType_PUT, put, -1).release());
+ ::hbase::pb::Condition *cond = pb_msg->mutable_condition();
+ cond->set_row(row);
+ cond->set_family(family);
+ cond->set_qualifier(qualifier);
+ cond->set_allocated_comparator(
+ Comparator::ToProto(*(ComparatorFactory::BinaryComparator(value).get())).release());
+ cond->set_compare_type(compare_op);
+
+ RequestConverter::SetRegion(region_name, pb_msg->mutable_region());
+ return pb_req;
+}
+
std::unique_ptr<Request> RequestConverter::DeleteToMutateRequest(const Delete &del,
const std::string ®ion_name) {
auto pb_req = Request::mutate();
diff --git a/core/request-converter.h b/core/request-converter.h
index 51440d8..0755f42 100644
--- a/core/request-converter.h
+++ b/core/request-converter.h
@@ -84,6 +84,11 @@ class RequestConverter {
static std::unique_ptr<Request> ToMutateRequest(const Put &put, const std::string ®ion_name);
+ static std::unique_ptr<Request> CheckAndPutToMutateRequest(
+ const std::string &row, const std::string &family, const std::string &qualifier,
+ const std::string &value, const pb::CompareType compare_op, const hbase::Put &put,
+ const std::string ®ion_name);
+
static std::unique_ptr<Request> IncrementToMutateRequest(const Increment &incr,
const std::string ®ion_name);
diff --git a/core/response-converter.cc b/core/response-converter.cc
index d2d719b..9bc4892 100644
--- a/core/response-converter.cc
+++ b/core/response-converter.cc
@@ -53,6 +53,11 @@ std::shared_ptr<Result> ResponseConverter::FromMutateResponse(const Response& re
return ToResult(mutate_resp->result(), resp.cell_scanner());
}
+bool ResponseConverter::BoolFromMutateResponse(const Response& resp) {
+ auto mutate_resp = std::static_pointer_cast<MutateResponse>(resp.resp_msg());
+ return mutate_resp->processed();
+}
+
std::shared_ptr<Result> ResponseConverter::ToResult(
const hbase::pb::Result& result, const std::shared_ptr<CellScanner> cell_scanner) {
std::vector<std::shared_ptr<Cell>> vcells;
diff --git a/core/response-converter.h b/core/response-converter.h
index b518d1c..2f8f279 100644
--- a/core/response-converter.h
+++ b/core/response-converter.h
@@ -49,6 +49,8 @@ class ResponseConverter {
static std::shared_ptr<hbase::Result> FromMutateResponse(const Response& resp);
+ static bool BoolFromMutateResponse(const Response& resp);
+
static std::vector<std::shared_ptr<Result>> FromScanResponse(const Response& resp);
static std::vector<std::shared_ptr<Result>> FromScanResponse(
diff --git a/core/table.cc b/core/table.cc
index bf22169..f20078d 100644
--- a/core/table.cc
+++ b/core/table.cc
@@ -73,6 +73,13 @@ void Table::Put(const hbase::Put &put) {
future.get(operation_timeout());
}
+bool Table::CheckAndPut(const std::string &row, const std::string &family,
+ const std::string &qualifier, const std::string &value,
+ const hbase::Put &put, const pb::CompareType &compare_op) {
+ auto context = async_table_->CheckAndPut(row, family, qualifier, value, put, compare_op);
+ return context.get(operation_timeout());
+}
+
void Table::Delete(const hbase::Delete &del) {
auto future = async_table_->Delete(del);
future.get(operation_timeout());
diff --git a/core/table.h b/core/table.h
index 781c6f1..3c486af 100644
--- a/core/table.h
+++ b/core/table.h
@@ -63,6 +63,23 @@ class Table {
void Put(const hbase::Put &put);
/**
+ * Atomically checks if a row/family/qualifier value matches the expected
+ * value. If it does, it adds the put. If the passed value is null, the check
+ * is for the lack of column (ie: non-existance)
+ *
+ * @param row to check
+ * @param family column family to check
+ * @param qualifier column qualifier to check
+ * @param value the expected value
+ * @param put data to put if check succeeds
+ * @param compare_op comparison operator to use
+ * @throws IOException e
+ * @return true if the new put was executed, false otherwise
+ */
+ bool CheckAndPut(const std::string &row, const std::string &family, const std::string &qualifier,
+ const std::string &value, const hbase::Put &put,
+ const pb::CompareType &compare_op = pb::CompareType::EQUAL);
+ /**
* @brief - Deletes some data in the table.
* @param - del Delete object to perform HBase Delete operation.
*/
[hbase] 55/137: HBASE-17395 [C++] Use custom line wrapping in
formatting - ADDENDUM to remove an extra '=' sign.
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit e1ce4b93652b922149e87172e23cffe764ea3038
Author: Sudeep Sunthankar <su...@hashmapinc.com>
AuthorDate: Thu Jan 5 18:17:52 2017 +1100
HBASE-17395 [C++] Use custom line wrapping in formatting - ADDENDUM to remove an extra '=' sign.
Signed-off-by: Enis Soztutar <en...@apache.org>
---
bin/format-code.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bin/format-code.sh b/bin/format-code.sh
index 6796acd..8a19930 100755
--- a/bin/format-code.sh
+++ b/bin/format-code.sh
@@ -19,5 +19,5 @@ set -euo pipefail
IFS=$'\n\t'
-find core connection serde utils test-util security -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i --style=='{BasedOnStyle: Google, ColumnLimit: 100}'
+find core connection serde utils test-util security -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i --style='{BasedOnStyle: Google, ColumnLimit: 100}'
find core connection serde utils third-party security -name "BUCK" | xargs -P8 yapf -i --style=google
[hbase] 62/137: HBASE-17503 [C++] Configuration should be settable
and used w/o XML files
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 93210389ee90c5b53937bd120aee588374911497
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Tue Jan 24 15:38:43 2017 -0800
HBASE-17503 [C++] Configuration should be settable and used w/o XML files
---
core/BUCK | 5 ++++
core/configuration-test.cc | 63 ++++++++++++++++++++++++++++++++++++++++++++++
core/configuration.cc | 22 ++++++++++++++++
core/configuration.h | 43 +++++++++++++++++++++++++++++++
4 files changed, 133 insertions(+)
diff --git a/core/BUCK b/core/BUCK
index 0d1bc93..f1880a4 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -86,6 +86,11 @@ cxx_test(
deps=[":core",],
run_test_separately=True,)
cxx_test(
+ name="configuration-test",
+ srcs=["configuration-test.cc",],
+ deps=[":core",],
+ run_test_separately=True,)
+cxx_test(
name="hbase_configuration-test",
srcs=["hbase_configuration-test.cc",],
deps=[":core",],
diff --git a/core/configuration-test.cc b/core/configuration-test.cc
new file mode 100644
index 0000000..192ed46
--- /dev/null
+++ b/core/configuration-test.cc
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/configuration.h"
+#include <gtest/gtest.h>
+
+using hbase::Configuration;
+
+TEST(Configuration, SetGet) {
+ Configuration conf;
+
+ EXPECT_EQ(conf.Get("foo", "default"), "default");
+ conf.Set("foo", "bar");
+ EXPECT_EQ(conf.Get("foo", "default"), "bar");
+}
+
+TEST(Configuration, SetGetInt) {
+ Configuration conf;
+
+ EXPECT_EQ(conf.GetInt("foo", 0), 0);
+ conf.SetInt("foo", 42);
+ EXPECT_EQ(conf.GetInt("foo", 0), 42);
+}
+
+TEST(Configuration, SetGetLong) {
+ Configuration conf;
+
+ EXPECT_EQ(conf.GetLong("foo", 0), 0);
+ conf.SetLong("foo", 42);
+ EXPECT_EQ(conf.GetLong("foo", 0), 42);
+}
+
+TEST(Configuration, SetGetDouble) {
+ Configuration conf;
+
+ EXPECT_EQ(conf.GetDouble("foo", 0), 0);
+ conf.SetDouble("foo", 42.0);
+ EXPECT_EQ(conf.GetDouble("foo", 0), 42.0);
+}
+
+TEST(Configuration, SetGetBool) {
+ Configuration conf;
+
+ EXPECT_EQ(conf.GetBool("foo", false), false);
+ conf.SetInt("foo", true);
+ EXPECT_EQ(conf.GetInt("foo", false), true);
+}
diff --git a/core/configuration.cc b/core/configuration.cc
index bd582d8..afca122 100644
--- a/core/configuration.cc
+++ b/core/configuration.cc
@@ -28,6 +28,8 @@
namespace hbase {
+Configuration::Configuration() : hb_property_() {}
+
Configuration::Configuration(ConfigMap &config_map) : hb_property_(std::move(config_map)) {}
Configuration::~Configuration() {}
@@ -215,4 +217,24 @@ bool Configuration::GetBool(const std::string &key, bool default_value) const {
return GetBool(key).value_or(default_value);
}
+void Configuration::Set(const std::string &key, const std::string &value) {
+ hb_property_[key] = value;
+}
+
+void Configuration::SetInt(const std::string &key, int32_t value) {
+ Set(key, boost::lexical_cast<std::string>(value));
+}
+
+void Configuration::SetLong(const std::string &key, int64_t value) {
+ Set(key, boost::lexical_cast<std::string>(value));
+}
+
+void Configuration::SetDouble(const std::string &key, double value) {
+ Set(key, boost::lexical_cast<std::string>(value));
+}
+
+void Configuration::SetBool(const std::string &key, bool value) {
+ Set(key, boost::lexical_cast<std::string>(value));
+}
+
} /* namespace hbase */
diff --git a/core/configuration.h b/core/configuration.h
index 69bef75..d70941c 100644
--- a/core/configuration.h
+++ b/core/configuration.h
@@ -33,6 +33,14 @@ using optional = std::experimental::optional<T>;
class Configuration {
public:
+ /**
+ * @brief Create Configuration object using an empty map. This
+ * object WILL NOT be initialized from reading XML configuration (hbase-site.xml, etc).
+ * Use HBaseConfigurationLoader to initialize the configuration from
+ * hbase-default.xml and hbase-site.xml files.
+ */
+ Configuration();
+
~Configuration();
/**
@@ -79,6 +87,41 @@ class Configuration {
*/
bool GetBool(const std::string &key, bool default_value) const;
+ /**
+ * @brief This method sets the given key to the value.
+ * @param key property name
+ * @param value property value
+ */
+ void Set(const std::string &key, const std::string &value);
+
+ /**
+ * @brief This method sets the given key to the value.
+ * @param key property name
+ * @param value property value
+ */
+ void SetInt(const std::string &key, int32_t value);
+
+ /**
+ * @brief This method sets the given key to the value.
+ * @param key property name
+ * @param value property value
+ */
+ void SetLong(const std::string &key, int64_t value);
+
+ /**
+ * @brief This method sets the given key to the value.
+ * @param key property name
+ * @param value property value
+ */
+ void SetDouble(const std::string &key, double value);
+
+ /**
+ * @brief This method sets the given key to the value.
+ * @param key property name
+ * @param value property value
+ */
+ void SetBool(const std::string &key, bool value);
+
private:
friend class HBaseConfigurationLoader;
[hbase] 72/137: HBASE-17728 Create separate build target for
Configuration classes
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 4c5a0896def97b402e5c067c8238473076dc25c8
Author: tedyu <yu...@gmail.com>
AuthorDate: Wed Mar 8 12:11:57 2017 -0800
HBASE-17728 Create separate build target for Configuration classes
---
core/BUCK | 20 ++++++++++++++++----
security/BUCK | 2 +-
security/user.h | 6 ++++++
3 files changed, 23 insertions(+), 5 deletions(-)
diff --git a/core/BUCK b/core/BUCK
index e40236f..fd4c0dc 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -33,8 +33,6 @@ cxx_library(
"meta-utils.h",
"get.h",
"time-range.h",
- "configuration.h",
- "hbase-configuration-loader.h",
"scan.h",
"result.h",
"request-converter.h",
@@ -52,8 +50,6 @@ cxx_library(
"meta-utils.cc",
"get.cc",
"time-range.cc",
- "configuration.cc",
- "hbase-configuration-loader.cc",
"scan.cc",
"result.cc",
"request-converter.cc",
@@ -64,6 +60,7 @@ cxx_library(
"//exceptions:exceptions",
"//utils:utils",
"//connection:connection",
+ "//core:conf",
"//if:if",
"//serde:serde",
"//third-party:folly",
@@ -74,6 +71,21 @@ cxx_library(
visibility=[
'PUBLIC',
],)
+cxx_library(
+ name="conf",
+ exported_headers=[
+ "configuration.h",
+ "hbase-configuration-loader.h",
+ ],
+ srcs=[
+ "configuration.cc",
+ "hbase-configuration-loader.cc",
+ ],
+ deps=["//third-party:folly"],
+ compiler_flags=['-Weffc++', '-ggdb'],
+ visibility=[
+ 'PUBLIC',
+ ],)
cxx_test(
name="location-cache-test",
srcs=[
diff --git a/security/BUCK b/security/BUCK
index d602ff3..1199797 100644
--- a/security/BUCK
+++ b/security/BUCK
@@ -23,6 +23,6 @@ cxx_library(
"user.h",
],
srcs=[],
- deps=[],
+ deps=["//core:conf"],
compiler_flags=['-Weffc++'],
visibility=['//core/...', '//connection/...'],)
diff --git a/security/user.h b/security/user.h
index 8a15891..372ab44 100644
--- a/security/user.h
+++ b/security/user.h
@@ -18,10 +18,13 @@
*/
#pragma once
+#include <mutex>
#include <string>
+#include "core/configuration.h"
namespace hbase {
namespace security {
+static constexpr const char* kKerberos = "kerberos";
class User {
public:
explicit User(const std::string& user_name) : user_name_(user_name) {}
@@ -31,6 +34,9 @@ class User {
static std::shared_ptr<User> defaultUser() { return std::make_shared<User>("__drwho"); }
+ static bool IsSecurityEnabled(const Configuration& conf) {
+ return conf.Get("hbase.security.authentication", "").compare(kKerberos) == 0;
+ }
private:
std::string user_name_;
};
[hbase] 80/137: HBASE-16365 [C++] End to end Table::Put()
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit ab47d94c061e02c7ec7f1a9ebf0023817f9c1d33
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Wed Mar 29 16:53:45 2017 -0700
HBASE-16365 [C++] End to end Table::Put()
---
core/BUCK | 5 ++
core/async-rpc-retrying-test.cc | 5 +-
core/cell-test.cc | 175 +++++++++++++++++++---------------------
core/cell.cc | 22 +++++
core/cell.h | 2 +
core/client-test.cc | 119 ++++++++++++++++++---------
core/filter-test.cc | 18 +++--
core/get-test.cc | 14 ++--
core/mutation.h | 9 +--
core/put-test.cc | 135 +++++++++++++++++++++++++++++++
core/put.cc | 5 +-
core/raw-async-table.cc | 15 ++++
core/raw-async-table.h | 5 ++
core/request-converter.cc | 68 ++++++++++++++++
core/request-converter.h | 16 +++-
core/result-test.cc | 8 +-
core/result.cc | 22 +++++
core/result.h | 2 +
core/simple-client.cc | 124 ++++++++++++----------------
core/table.cc | 5 ++
core/table.h | 8 ++
test-util/mini-cluster.cc | 19 -----
test-util/mini-cluster.h | 3 -
test-util/test-util.cc | 6 +-
test-util/test-util.h | 2 -
25 files changed, 548 insertions(+), 264 deletions(-)
diff --git a/core/BUCK b/core/BUCK
index 03fc0fc..30c3390 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -130,6 +130,11 @@ cxx_test(
deps=[":core",],
run_test_separately=True,)
cxx_test(
+ name="put-test",
+ srcs=["put-test.cc",],
+ deps=[":core",],
+ run_test_separately=True,)
+cxx_test(
name="retry-test",
srcs=["async-rpc-retrying-test.cc",],
deps=[
diff --git a/core/async-rpc-retrying-test.cc b/core/async-rpc-retrying-test.cc
index 5086286..3ed6866 100644
--- a/core/async-rpc-retrying-test.cc
+++ b/core/async-rpc-retrying-test.cc
@@ -163,8 +163,6 @@ TEST(AsyncRpcRetryTest, TestGetBasic) {
test_util->StartMiniCluster(2);
test_util->CreateTable("t", "d");
- test_util->TablePut("t", "test2", "d", "2", "value2");
- test_util->TablePut("t", "test2", "d", "extra", "value for extra");
// Create TableName and Row to be fetched from HBase
auto tn = folly::to<hbase::pb::TableName>("t");
@@ -180,6 +178,9 @@ TEST(AsyncRpcRetryTest, TestGetBasic) {
auto table = client.Table(tn);
ASSERT_TRUE(table) << "Unable to get connection to Table.";
+ table->Put(Put{"test2"}.AddColumn("d", "2", "value2"));
+ table->Put(Put{"test2"}.AddColumn("d", "extra", "value for extra"));
+
/* init region location and rpc channel */
auto region_location = table->GetRegionLocation(row);
diff --git a/core/cell-test.cc b/core/cell-test.cc
index 2ec983b..efb835d 100644
--- a/core/cell-test.cc
+++ b/core/cell-test.cc
@@ -23,75 +23,65 @@
#include <gtest/gtest.h>
#include <memory>
-using namespace hbase;
-TEST(CellTest, CellFailureTest) {
- CellType cell_type = CellType::PUT;
- std::string row = "row";
- std::string family = "family";
- std::string column = "column";
- std::string value = "value";
+using hbase::Cell;
+using hbase::CellType;
+
+TEST(CellTest, Constructor) {
+ std::string row = "row-value";
+ std::string family = "family-value";
+ std::string column = "column-value";
+ std::string value = "value-value";
int64_t timestamp = std::numeric_limits<int64_t>::max();
- std::unique_ptr<Cell> cell(new Cell(row, family, column, timestamp, value, cell_type));
- if (cell.get()) {
- EXPECT_NE("row-value", cell.get()->Row());
- EXPECT_NE("family-value", cell.get()->Family());
- EXPECT_NE("column-value", cell.get()->Qualifier());
- EXPECT_NE("value-value", cell.get()->Value());
- EXPECT_NE(8975431260, cell.get()->Timestamp());
- EXPECT_NE(CellType::MAXIMUM, cell.get()->Type());
- }
+ CellType cell_type = CellType::PUT;
+
+ Cell cell{row, family, column, timestamp, value, cell_type};
+
+ EXPECT_EQ(row, cell.Row());
+ EXPECT_EQ(family, cell.Family());
+ EXPECT_EQ(column, cell.Qualifier());
+ EXPECT_EQ(value, cell.Value());
+ EXPECT_EQ(timestamp, cell.Timestamp());
+ EXPECT_EQ(cell_type, cell.Type());
}
-TEST(CellTest, CellSuceessTest) {
+TEST(CellTest, CopyConstructor) {
std::string row = "row-value";
std::string family = "family-value";
std::string column = "column-value";
std::string value = "value-value";
int64_t timestamp = std::numeric_limits<int64_t>::max();
CellType cell_type = CellType::PUT;
- const std::unique_ptr<Cell> cell(new Cell(row, family, column, timestamp, value, cell_type));
- if (cell.get()) {
- EXPECT_EQ(row, cell.get()->Row());
- EXPECT_EQ(family, cell.get()->Family());
- EXPECT_EQ(column, cell.get()->Qualifier());
- EXPECT_EQ(value, cell.get()->Value());
- EXPECT_EQ(timestamp, cell.get()->Timestamp());
- EXPECT_EQ(cell_type, cell.get()->Type());
- }
+
+ auto cell = std::make_unique<Cell>(row, family, column, timestamp, value, cell_type);
+ Cell cell2{*cell};
+ cell = nullptr;
+
+ EXPECT_EQ(row, cell2.Row());
+ EXPECT_EQ(family, cell2.Family());
+ EXPECT_EQ(column, cell2.Qualifier());
+ EXPECT_EQ(value, cell2.Value());
+ EXPECT_EQ(timestamp, cell2.Timestamp());
+ EXPECT_EQ(cell_type, cell2.Type());
}
-TEST(CellTest, MultipleCellsTest) {
- std::vector<const Cell *> cells;
- for (int i = 0; i < 5; i++) {
- std::string row = "row-value";
- std::string family = "family-value";
- std::string column = "column-value";
- std::string value = "value-value";
- int64_t timestamp = std::numeric_limits<int64_t>::max();
- row += std::to_string(i);
- value += std::to_string(i);
- CellType cell_type = CellType::PUT;
- const Cell *cell = new Cell(row, family, column, timestamp, value, cell_type);
- cells.push_back(cell);
- }
- int i = 0;
- for (const auto cell : cells) {
- std::string row = "row-value";
- std::string value = "value-value";
- row += std::to_string(i);
- value += std::to_string(i);
- EXPECT_EQ(row, cell->Row());
- EXPECT_EQ("family-value", cell->Family());
- EXPECT_EQ("column-value", cell->Qualifier());
- EXPECT_EQ(value, cell->Value());
- EXPECT_EQ(std::numeric_limits<int64_t>::max(), cell->Timestamp());
- EXPECT_EQ(CellType::PUT, cell->Type());
- i += 1;
- }
- for (const auto cell : cells) {
- delete cell;
- }
- cells.clear();
+TEST(CellTest, CopyAssignment) {
+ std::string row = "row-value";
+ std::string family = "family-value";
+ std::string column = "column-value";
+ std::string value = "value-value";
+ int64_t timestamp = std::numeric_limits<int64_t>::max();
+ CellType cell_type = CellType::PUT;
+
+ auto cell = std::make_unique<Cell>(row, family, column, timestamp, value, cell_type);
+ Cell cell2 = *cell;
+ cell = nullptr;
+
+ EXPECT_EQ(row, cell2.Row());
+ EXPECT_EQ(family, cell2.Family());
+ EXPECT_EQ(column, cell2.Qualifier());
+ EXPECT_EQ(value, cell2.Value());
+ EXPECT_EQ(timestamp, cell2.Timestamp());
+ EXPECT_EQ(cell_type, cell2.Type());
}
TEST(CellTest, CellRowTest) {
@@ -101,15 +91,14 @@ TEST(CellTest, CellRowTest) {
std::string value = "";
int64_t timestamp = std::numeric_limits<int64_t>::max();
CellType cell_type = CellType::PUT;
- std::unique_ptr<Cell> cell(new Cell(row, family, column, timestamp, value, cell_type));
- if (cell.get()) {
- EXPECT_EQ(row, cell.get()->Row());
- EXPECT_EQ(family, cell.get()->Family());
- EXPECT_EQ(column, cell.get()->Qualifier());
- EXPECT_EQ(value, cell.get()->Value());
- EXPECT_EQ(timestamp, cell.get()->Timestamp());
- EXPECT_EQ(cell_type, cell.get()->Type());
- }
+ Cell cell{row, family, column, timestamp, value, cell_type};
+
+ EXPECT_EQ(row, cell.Row());
+ EXPECT_EQ(family, cell.Family());
+ EXPECT_EQ(column, cell.Qualifier());
+ EXPECT_EQ(value, cell.Value());
+ EXPECT_EQ(timestamp, cell.Timestamp());
+ EXPECT_EQ(cell_type, cell.Type());
}
TEST(CellTest, CellRowFamilyTest) {
@@ -119,15 +108,14 @@ TEST(CellTest, CellRowFamilyTest) {
std::string value = "";
int64_t timestamp = std::numeric_limits<int64_t>::max();
CellType cell_type = CellType::PUT;
- const std::unique_ptr<Cell> cell(new Cell(row, family, column, timestamp, value, cell_type));
- if (cell.get()) {
- EXPECT_EQ(row, cell.get()->Row());
- EXPECT_EQ(family, cell.get()->Family());
- EXPECT_EQ(column, cell.get()->Qualifier());
- EXPECT_EQ(value, cell.get()->Value());
- EXPECT_EQ(timestamp, cell.get()->Timestamp());
- EXPECT_EQ(cell_type, cell.get()->Type());
- }
+ Cell cell{row, family, column, timestamp, value, cell_type};
+
+ EXPECT_EQ(row, cell.Row());
+ EXPECT_EQ(family, cell.Family());
+ EXPECT_EQ(column, cell.Qualifier());
+ EXPECT_EQ(value, cell.Value());
+ EXPECT_EQ(timestamp, cell.Timestamp());
+ EXPECT_EQ(cell_type, cell.Type());
}
TEST(CellTest, CellRowFamilyValueTest) {
@@ -137,15 +125,15 @@ TEST(CellTest, CellRowFamilyValueTest) {
std::string value = "only-value";
int64_t timestamp = std::numeric_limits<int64_t>::max();
CellType cell_type = CellType::PUT;
- const std::unique_ptr<Cell> cell(new Cell(row, family, column, timestamp, value, cell_type));
- if (cell.get()) {
- EXPECT_EQ(row, cell.get()->Row());
- EXPECT_EQ(family, cell.get()->Family());
- EXPECT_EQ(column, cell.get()->Qualifier());
- EXPECT_EQ(value, cell.get()->Value());
- EXPECT_EQ(timestamp, cell.get()->Timestamp());
- EXPECT_EQ(cell_type, cell.get()->Type());
- }
+
+ Cell cell{row, family, column, timestamp, value, cell_type};
+
+ EXPECT_EQ(row, cell.Row());
+ EXPECT_EQ(family, cell.Family());
+ EXPECT_EQ(column, cell.Qualifier());
+ EXPECT_EQ(value, cell.Value());
+ EXPECT_EQ(timestamp, cell.Timestamp());
+ EXPECT_EQ(cell_type, cell.Type());
}
TEST(CellTest, CellRowFamilyColumnValueTest) {
@@ -155,15 +143,14 @@ TEST(CellTest, CellRowFamilyColumnValueTest) {
std::string value = "only-value";
int64_t timestamp = std::numeric_limits<int64_t>::max();
CellType cell_type = CellType::PUT;
- std::unique_ptr<Cell> cell(new Cell(row, family, column, timestamp, value, cell_type));
- if (cell.get()) {
- EXPECT_EQ(row, cell.get()->Row());
- EXPECT_EQ(family, cell.get()->Family());
- EXPECT_EQ(column, cell.get()->Qualifier());
- EXPECT_EQ(value, cell.get()->Value());
- EXPECT_EQ(timestamp, cell.get()->Timestamp());
- EXPECT_EQ(cell_type, cell.get()->Type());
- }
+ Cell cell{row, family, column, timestamp, value, cell_type};
+
+ EXPECT_EQ(row, cell.Row());
+ EXPECT_EQ(family, cell.Family());
+ EXPECT_EQ(column, cell.Qualifier());
+ EXPECT_EQ(value, cell.Value());
+ EXPECT_EQ(timestamp, cell.Timestamp());
+ EXPECT_EQ(cell_type, cell.Type());
}
TEST(CellTest, CellDebugString) {
diff --git a/core/cell.cc b/core/cell.cc
index 14a8014..24788ab 100644
--- a/core/cell.cc
+++ b/core/cell.cc
@@ -19,6 +19,7 @@
#include "core/cell.h"
#include <climits>
+#include <limits>
#include <stdexcept>
#include "folly/Conv.h"
@@ -40,6 +41,27 @@ Cell::Cell(const std::string &row, const std::string &family, const std::string
if (0 >= timestamp) throw std::runtime_error("Timestamp should be greater than 0");
}
+Cell::Cell(const Cell &cell)
+ : row_(cell.row_),
+ family_(cell.family_),
+ qualifier_(cell.qualifier_),
+ timestamp_(cell.timestamp_),
+ cell_type_(cell.cell_type_),
+ value_(cell.value_),
+ sequence_id_(cell.sequence_id_) {}
+
+Cell &Cell::operator=(const Cell &cell) {
+ row_ = cell.row_;
+ family_ = cell.family_;
+ qualifier_ = cell.qualifier_;
+ timestamp_ = cell.timestamp_;
+ cell_type_ = cell.cell_type_;
+ value_ = cell.value_;
+ sequence_id_ = cell.sequence_id_;
+
+ return *this;
+}
+
Cell::~Cell() {}
const std::string &Cell::Row() const { return row_; }
diff --git a/core/cell.h b/core/cell.h
index c062e45..acedd96 100644
--- a/core/cell.h
+++ b/core/cell.h
@@ -38,6 +38,8 @@ class Cell {
public:
Cell(const std::string &row, const std::string &family, const std::string &qualifier,
const int64_t timestamp, const std::string &value, const hbase::CellType &cell_type);
+ Cell(const Cell &cell);
+ Cell &operator=(const Cell &cell);
virtual ~Cell();
const std::string &Row() const;
const std::string &Family() const;
diff --git a/core/client-test.cc b/core/client-test.cc
index 184a6a7..ff4879a 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -17,24 +17,47 @@
*
*/
-#include "core/client.h"
#include <gtest/gtest.h>
+
+#include "core/cell.h"
+#include "core/client.h"
#include "core/configuration.h"
#include "core/get.h"
#include "core/hbase-configuration-loader.h"
+#include "core/put.h"
#include "core/result.h"
#include "core/table.h"
#include "serde/table-name.h"
#include "test-util/test-util.h"
+using hbase::Cell;
+using hbase::Configuration;
+using hbase::Get;
+using hbase::Put;
+using hbase::Table;
+using hbase::TestUtil;
+
class ClientTest : public ::testing::Test {
public:
- const static std::string kDefHBaseConfPath;
-
- const static std::string kHBaseDefaultXml;
- const static std::string kHBaseSiteXml;
-
- const static std::string kHBaseXmlData;
+ static const constexpr char *kDefHBaseConfPath = "./build/test-data/client-test/conf/";
+ static const constexpr char *kHBaseDefaultXml = "hbase-default.xml";
+ static const constexpr char *kHBaseSiteXml = "hbase-site.xml";
+ static const constexpr char *kHBaseXmlData =
+ "<?xml version=\"1.0\"?>\n<?xml-stylesheet type=\"text/xsl\" "
+ "href=\"configuration.xsl\"?>\n<!--\n/**\n *\n * Licensed to the Apache "
+ "Software Foundation (ASF) under one\n * or more contributor license "
+ "agreements. See the NOTICE file\n * distributed with this work for "
+ "additional information\n * regarding copyright ownership. The ASF "
+ "licenses this file\n * to you under the Apache License, Version 2.0 "
+ "(the\n * \"License\"); you may not use this file except in compliance\n * "
+ "with the License. You may obtain a copy of the License at\n *\n * "
+ "http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by "
+ "applicable law or agreed to in writing, software\n * distributed under "
+ "the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES "
+ "OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License "
+ "for the specific language governing permissions and\n * limitations under "
+ "the License.\n "
+ "*/\n-->\n<configuration>\n\n</configuration>";
static void WriteDataToFile(const std::string &file, const std::string &xml_data) {
std::ofstream hbase_conf;
@@ -55,7 +78,7 @@ class ClientTest : public ::testing::Test {
// Creating Empty Config Files so that we dont get a Configuration exception @Client
CreateHBaseConf(kDefHBaseConfPath, kHBaseDefaultXml, kHBaseXmlData);
// the hbase-site.xml would be persisted by MiniCluster
- setenv("HBASE_CONF", kDefHBaseConfPath.c_str(), 1);
+ setenv("HBASE_CONF", kDefHBaseConfPath, 1);
}
static std::unique_ptr<hbase::TestUtil> test_util;
@@ -67,28 +90,6 @@ class ClientTest : public ::testing::Test {
};
std::unique_ptr<hbase::TestUtil> ClientTest::test_util = nullptr;
-const std::string ClientTest::kDefHBaseConfPath("./build/test-data/client-test/conf/");
-
-const std::string ClientTest::kHBaseDefaultXml("hbase-default.xml");
-const std::string ClientTest::kHBaseSiteXml("hbase-site.xml");
-
-const std::string ClientTest::kHBaseXmlData(
- "<?xml version=\"1.0\"?>\n<?xml-stylesheet type=\"text/xsl\" "
- "href=\"configuration.xsl\"?>\n<!--\n/**\n *\n * Licensed to the Apache "
- "Software Foundation (ASF) under one\n * or more contributor license "
- "agreements. See the NOTICE file\n * distributed with this work for "
- "additional information\n * regarding copyright ownership. The ASF "
- "licenses this file\n * to you under the Apache License, Version 2.0 "
- "(the\n * \"License\"); you may not use this file except in compliance\n * "
- "with the License. You may obtain a copy of the License at\n *\n * "
- "http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by "
- "applicable law or agreed to in writing, software\n * distributed under "
- "the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES "
- "OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License "
- "for the specific language governing permissions and\n * limitations under "
- "the License.\n "
- "*/\n-->\n<configuration>\n\n</configuration>");
-
TEST_F(ClientTest, EmptyConfigurationPassedToClient) { ASSERT_ANY_THROW(hbase::Client client); }
TEST_F(ClientTest, ConfigurationPassedToClient) {
@@ -114,18 +115,13 @@ TEST_F(ClientTest, DefaultConfiguration) {
client.Close();
}
-TEST_F(ClientTest, Get) {
+TEST_F(ClientTest, PutGet) {
// Using TestUtil to populate test data
ClientTest::test_util->CreateTable("t", "d");
- ClientTest::test_util->TablePut("t", "test2", "d", "2", "value2");
- ClientTest::test_util->TablePut("t", "test2", "d", "extra", "value for extra");
// Create TableName and Row to be fetched from HBase
auto tn = folly::to<hbase::pb::TableName>("t");
- auto row = "test2";
-
- // Get to be performed on above HBase Table
- hbase::Get get(row);
+ auto row = "test1";
// Create a client
hbase::Client client(*ClientTest::test_util->conf());
@@ -134,13 +130,18 @@ TEST_F(ClientTest, Get) {
auto table = client.Table(tn);
ASSERT_TRUE(table) << "Unable to get connection to Table.";
+ // Perform Puts
+ table->Put(Put{"test1"}.AddColumn("d", "1", "value1"));
+ table->Put(Put{"test1"}.AddColumn("d", "extra", "value for extra"));
+
// Perform the Get
+ hbase::Get get(row);
auto result = table->Get(get);
// Test the values, should be same as in put executed on hbase shell
ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty.";
- EXPECT_EQ("test2", result->Row());
- EXPECT_EQ("value2", *(result->Value("d", "2")));
+ EXPECT_EQ("test1", result->Row());
+ EXPECT_EQ("value1", *(result->Value("d", "1")));
EXPECT_EQ("value for extra", *(result->Value("d", "extra")));
table->Close();
@@ -150,7 +151,7 @@ TEST_F(ClientTest, Get) {
TEST_F(ClientTest, GetForNonExistentTable) {
// Create TableName and Row to be fetched from HBase
auto tn = folly::to<hbase::pb::TableName>("t_not_exists");
- auto row = "test2";
+ auto row = "test1";
// Get to be performed on above HBase Table
hbase::Get get(row);
@@ -194,3 +195,41 @@ TEST_F(ClientTest, GetForNonExistentRow) {
table->Close();
client.Close();
}
+
+TEST_F(ClientTest, PutsWithTimestamp) {
+ // Using TestUtil to populate test data
+ ClientTest::test_util->CreateTable("t_puts_with_timestamp", "d");
+
+ // Create TableName and Row to be fetched from HBase
+ auto tn = folly::to<hbase::pb::TableName>("t_puts_with_timestamp");
+ auto row = "test1";
+
+ // Create a client
+ hbase::Client client(*ClientTest::test_util->conf());
+
+ // Get connection to HBase Table
+ auto table = client.Table(tn);
+ ASSERT_TRUE(table) << "Unable to get connection to Table.";
+
+ int64_t ts = 42;
+ // Perform Puts
+ table->Put(Put{"test1"}.AddColumn("d", "1", ts, "value1"));
+ auto cell =
+ std::make_unique<Cell>("test1", "d", "extra", ts, "value for extra", hbase::CellType::PUT);
+ table->Put(Put{"test1"}.Add(std::move(cell)));
+
+ // Perform the Get
+ hbase::Get get(row);
+ auto result = table->Get(get);
+
+ // Test the values, should be same as in put executed on hbase shell
+ ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty.";
+ EXPECT_EQ("test1", result->Row());
+ EXPECT_EQ("value1", *(result->Value("d", "1")));
+ EXPECT_EQ("value for extra", *(result->Value("d", "extra")));
+ EXPECT_EQ(ts, result->ColumnLatestCell("d", "1")->Timestamp());
+ EXPECT_EQ(ts, result->ColumnLatestCell("d", "extra")->Timestamp());
+
+ table->Close();
+ client.Close();
+}
diff --git a/core/filter-test.cc b/core/filter-test.cc
index 40081cc..f401698 100644
--- a/core/filter-test.cc
+++ b/core/filter-test.cc
@@ -21,6 +21,7 @@
#include "core/client.h"
#include "core/configuration.h"
#include "core/get.h"
+#include "core/put.h"
#include "core/result.h"
#include "core/table.h"
#include "if/Comparator.pb.h"
@@ -30,6 +31,7 @@
using hbase::Configuration;
using hbase::Get;
+using hbase::Put;
using hbase::FilterFactory;
using hbase::Table;
using hbase::TestUtil;
@@ -57,9 +59,6 @@ std::unique_ptr<TestUtil> FilterTest::test_util_ = nullptr;
TEST_F(FilterTest, GetWithColumnPrefixFilter) {
// write row1 with 3 columns (column_1, column_2, and foo_column)
FilterTest::test_util_->CreateTable("t", "d");
- FilterTest::test_util_->TablePut("t", "row1", "d", "column_1", "value1");
- FilterTest::test_util_->TablePut("t", "row1", "d", "column_2", "value2");
- FilterTest::test_util_->TablePut("t", "row1", "d", "foo_column", "value3");
// Create TableName and Row to be fetched from HBase
auto tn = folly::to<hbase::pb::TableName>("t");
@@ -75,11 +74,13 @@ TEST_F(FilterTest, GetWithColumnPrefixFilter) {
// Create a client
hbase::Client client(*(FilterTest::test_util_->conf()));
-
- // Get connection to HBase Table
auto table = client.Table(tn);
ASSERT_TRUE(table) << "Unable to get connection to Table.";
+ table->Put(Put{"row1"}.AddColumn("d", "column_1", "value1"));
+ table->Put(Put{"row1"}.AddColumn("d", "column_2", "value2"));
+ table->Put(Put{"row1"}.AddColumn("d", "foo_column", "value3"));
+
// Perform the Get
auto result_all = table->Get(get_all);
auto result_one = table->Get(get_one);
@@ -106,9 +107,6 @@ TEST_F(FilterTest, GetWithColumnPrefixFilter) {
TEST_F(FilterTest, GetWithQualifierFilter) {
// write row1 with 3 columns (a,b,c)
FilterTest::test_util_->CreateTable("t1", "d");
- FilterTest::test_util_->TablePut("t1", "row1", "d", "a", "value1");
- FilterTest::test_util_->TablePut("t1", "row1", "d", "b", "value2");
- FilterTest::test_util_->TablePut("t1", "row1", "d", "c", "value3");
// Create TableName and Row to be fetched from HBase
auto tn = folly::to<hbase::pb::TableName>("t1");
@@ -126,6 +124,10 @@ TEST_F(FilterTest, GetWithQualifierFilter) {
auto table = client.Table(tn);
ASSERT_TRUE(table) << "Unable to get connection to Table.";
+ table->Put(Put{"row1"}.AddColumn("d", "a", "value1"));
+ table->Put(Put{"row1"}.AddColumn("d", "b", "value2"));
+ table->Put(Put{"row1"}.AddColumn("d", "c", "value3"));
+
// Perform the Get
auto result = table->Get(get);
diff --git a/core/get-test.cc b/core/get-test.cc
index fafff39..6127e23 100644
--- a/core/get-test.cc
+++ b/core/get-test.cc
@@ -17,12 +17,15 @@
*
*/
+#include "core/cell.h"
#include "core/get.h"
#include <glog/logging.h>
#include <gtest/gtest.h>
+using hbase::Cell;
using hbase::Get;
+
const int NUMBER_OF_GETS = 5;
void CheckFamilies(Get &get) {
@@ -199,21 +202,16 @@ TEST(Get, SingleGet) {
}
TEST(Get, MultiGet) {
- std::vector<Get *> gets;
+ std::vector<std::unique_ptr<Get>> gets;
for (int i = 0; i < NUMBER_OF_GETS; i++) {
std::string row_str = "row-test";
row_str += std::to_string(i);
- Get *get = new Get(row_str);
+ auto get = std::make_unique<Get>(row_str);
GetMethods(*get, row_str);
- gets.push_back(get);
+ gets.push_back(std::move(get));
}
EXPECT_EQ(NUMBER_OF_GETS, gets.size());
-
- for (const auto &get : gets) {
- delete get;
- }
- gets.clear();
}
TEST(Get, Exception) {
diff --git a/core/mutation.h b/core/mutation.h
index 83392e1..5e0381b 100644
--- a/core/mutation.h
+++ b/core/mutation.h
@@ -65,11 +65,6 @@ class Mutation: public Row {
}
/**
- * @brief Returns the row for this operation
- */
- const std::string& row() const;
-
- /**
* @brief Returns true if family map is non empty false otherwise
*/
bool HasFamilies() const;
@@ -85,8 +80,10 @@ class Mutation: public Row {
*/
Mutation& SetDurability(pb::MutationProto_Durability durability);
- protected:
+ public:
static const constexpr int64_t kLatestTimestamp = std::numeric_limits<int64_t>::max();
+
+ protected:
std::map<std::string, std::vector<std::unique_ptr<Cell>>> family_map_;
pb::MutationProto_Durability durability_ =
hbase::pb::MutationProto_Durability::MutationProto_Durability_USE_DEFAULT;
diff --git a/core/put-test.cc b/core/put-test.cc
new file mode 100644
index 0000000..d4ed00a
--- /dev/null
+++ b/core/put-test.cc
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include "core/mutation.h"
+#include "core/put.h"
+#include "utils/time-util.h"
+
+using hbase::Put;
+using hbase::Cell;
+using hbase::CellType;
+using hbase::Mutation;
+using hbase::TimeUtil;
+
+const constexpr int64_t Mutation::kLatestTimestamp;
+
+TEST(Put, Row) {
+ Put put{"foo"};
+ EXPECT_EQ("foo", put.row());
+}
+
+TEST(Put, Durability) {
+ Put put{"row"};
+ EXPECT_EQ(hbase::pb::MutationProto_Durability_USE_DEFAULT, put.Durability());
+
+ auto skipWal = hbase::pb::MutationProto_Durability_SKIP_WAL;
+ put.SetDurability(skipWal);
+ EXPECT_EQ(skipWal, put.Durability());
+}
+
+TEST(Put, Timestamp) {
+ Put put{"row"};
+
+ // test default timestamp
+ EXPECT_EQ(Mutation::kLatestTimestamp, put.TimeStamp());
+
+ // set custom timestamp
+ auto ts = TimeUtil::ToMillis(TimeUtil::GetNowNanos());
+ put.SetTimeStamp(ts);
+ EXPECT_EQ(ts, put.TimeStamp());
+
+ // Add a column with custom timestamp
+ put.AddColumn("f", "q", "v");
+ auto &cell = put.FamilyMap().at("f")[0];
+ EXPECT_EQ(ts, cell->Timestamp());
+}
+
+TEST(Put, HasFamilies) {
+ Put put{"row"};
+
+ EXPECT_EQ(false, put.HasFamilies());
+
+ put.AddColumn("f", "q", "v");
+ EXPECT_EQ(true, put.HasFamilies());
+}
+
+TEST(Put, Add) {
+ CellType cell_type = CellType::PUT;
+ std::string row = "row";
+ std::string family = "family";
+ std::string column = "column";
+ std::string value = "value";
+ int64_t timestamp = std::numeric_limits<int64_t>::max();
+ auto cell = std::make_unique<Cell>(row, family, column, timestamp, value, cell_type);
+
+ // add first cell
+ Put put{"row"};
+ put.Add(std::move(cell));
+ EXPECT_EQ(1, put.FamilyMap().size());
+ EXPECT_EQ(1, put.FamilyMap().at(family).size());
+
+ // add a non-matching row
+ auto cell2 = std::make_unique<Cell>(row, family, column, timestamp, value, cell_type);
+ Put put2{"foo"};
+ ASSERT_THROW(put2.Add(std::move(cell2)), std::runtime_error); // rows don't match
+
+ // add a second cell with same family
+ auto cell3 = std::make_unique<Cell>(row, family, "column-2", timestamp, value, cell_type);
+ put.Add(std::move(cell3));
+ EXPECT_EQ(1, put.FamilyMap().size());
+ EXPECT_EQ(2, put.FamilyMap().at(family).size());
+
+ // add a cell to a different family
+ auto cell4 = std::make_unique<Cell>(row, "family-2", "column-2", timestamp, value, cell_type);
+ put.Add(std::move(cell4));
+ EXPECT_EQ(2, put.FamilyMap().size());
+ EXPECT_EQ(1, put.FamilyMap().at("family-2").size());
+}
+
+TEST(Put, AddColumn) {
+ std::string row = "row";
+ std::string family = "family";
+ std::string column = "column";
+ std::string value = "value";
+
+ Put put{"row"};
+ put.AddColumn(family, column, value);
+ EXPECT_EQ(1, put.FamilyMap().size());
+ EXPECT_EQ(1, put.FamilyMap().at(family).size());
+
+ // add a second cell with same family
+ put.AddColumn(family, "column-2", value);
+ EXPECT_EQ(1, put.FamilyMap().size());
+ EXPECT_EQ(2, put.FamilyMap().at(family).size());
+
+ // add a cell to a different family
+ put.AddColumn("family-2", column, value);
+ EXPECT_EQ(2, put.FamilyMap().size());
+ EXPECT_EQ(1, put.FamilyMap().at("family-2").size());
+
+ // use the AddColumn overload
+ auto ts = TimeUtil::ToMillis(TimeUtil::GetNowNanos());
+ put.AddColumn(family, column, ts, value);
+ EXPECT_EQ(2, put.FamilyMap().size());
+ EXPECT_EQ(3, put.FamilyMap().at(family).size());
+ auto &cell = put.FamilyMap().at(family)[2];
+ EXPECT_EQ(ts, cell->Timestamp());
+}
diff --git a/core/put.cc b/core/put.cc
index b81051e..bb20c5c 100644
--- a/core/put.cc
+++ b/core/put.cc
@@ -24,6 +24,7 @@
#include <algorithm>
#include <limits>
#include <stdexcept>
+#include <utility>
namespace hbase {
@@ -35,7 +36,7 @@ namespace hbase {
*/
Put& Put::AddColumn(const std::string& family, const std::string& qualifier,
const std::string& value) {
- return AddColumn(family, qualifier, kLatestTimestamp, value);
+ return AddColumn(family, qualifier, timestamp_, value);
}
/**
@@ -57,7 +58,7 @@ Put& Put::AddColumn(const std::string& family, const std::string& qualifier, int
Put& Put::Add(std::unique_ptr<Cell> cell) {
if (cell->Row() != row_) {
- throw std::runtime_error("The row in" + cell->DebugString() +
+ throw std::runtime_error("The row in " + cell->DebugString() +
" doesn't match the original one " + row_);
}
diff --git a/core/raw-async-table.cc b/core/raw-async-table.cc
index 9a680ed..2bc9f36 100644
--- a/core/raw-async-table.cc
+++ b/core/raw-async-table.cc
@@ -76,4 +76,19 @@ Future<std::shared_ptr<Result>> RawAsyncTable::Get(const hbase::Get& get) {
return caller->Call().then([caller](const auto r) { return r; });
}
+Future<Unit> RawAsyncTable::Put(const hbase::Put& put) {
+ auto caller =
+ CreateCallerBuilder<Unit>(put.row(), connection_conf_->write_rpc_timeout())
+ ->action([=, &put](std::shared_ptr<hbase::HBaseRpcController> controller,
+ std::shared_ptr<hbase::RegionLocation> loc,
+ std::shared_ptr<hbase::RpcClient> rpc_client) -> folly::Future<Unit> {
+ return Call<hbase::Put, hbase::Request, hbase::Response, Unit>(
+ rpc_client, controller, loc, put, &hbase::RequestConverter::ToMutateRequest,
+ [](const Response& r) -> Unit { return folly::unit; });
+ })
+ ->Build();
+
+ return caller->Call().then([caller](const auto r) { return r; });
+}
+
} /* namespace hbase */
diff --git a/core/raw-async-table.h b/core/raw-async-table.h
index bbdc6bd..978a2b8 100644
--- a/core/raw-async-table.h
+++ b/core/raw-async-table.h
@@ -19,6 +19,7 @@
#pragma once
#include <folly/futures/Future.h>
+#include <folly/futures/Unit.h>
#include <chrono>
#include <memory>
@@ -29,9 +30,11 @@
#include "core/async-rpc-retrying-caller.h"
#include "core/connection-configuration.h"
#include "core/get.h"
+#include "core/put.h"
#include "core/result.h"
using folly::Future;
+using folly::Unit;
using hbase::pb::TableName;
using std::chrono::nanoseconds;
using std::chrono::milliseconds;
@@ -52,6 +55,8 @@ class RawAsyncTable {
virtual ~RawAsyncTable() = default;
Future<std::shared_ptr<Result>> Get(const hbase::Get& get);
+
+ Future<Unit> Put(const hbase::Put& put);
void Close() {}
private:
diff --git a/core/request-converter.cc b/core/request-converter.cc
index c40a56e..4c12ee7 100644
--- a/core/request-converter.cc
+++ b/core/request-converter.cc
@@ -18,6 +18,9 @@
*/
#include "core/request-converter.h"
+
+#include <folly/Conv.h>
+
#include <utility>
#include "if/Client.pb.h"
@@ -152,4 +155,69 @@ std::unique_ptr<hbase::pb::Get> RequestConverter::ToGet(const Get &get) {
}
return pb_get;
}
+
+std::unique_ptr<MutationProto> RequestConverter::ToMutation(const MutationType type,
+ const Mutation &mutation,
+ const int64_t nonce) {
+ auto pb_mut = std::make_unique<MutationProto>();
+ pb_mut->set_row(mutation.row());
+ pb_mut->set_mutate_type(type);
+ pb_mut->set_durability(mutation.Durability());
+ pb_mut->set_timestamp(mutation.TimeStamp());
+ // TODO: set attributes from the mutation (key value pairs).
+
+ if (nonce > 0) {
+ pb_mut->set_nonce(nonce);
+ }
+
+ for (const auto &family : mutation.FamilyMap()) {
+ for (const auto &cell : family.second) {
+ auto column = pb_mut->add_column_value();
+ column->set_family(cell->Family());
+ auto qual = column->add_qualifier_value();
+ qual->set_qualifier(cell->Qualifier());
+ qual->set_timestamp(cell->Timestamp());
+ auto cell_type = cell->Type();
+ if (type == pb::MutationProto_MutationType_DELETE ||
+ (type == pb::MutationProto_MutationType_PUT && IsDelete(cell_type))) {
+ qual->set_delete_type(ToDeleteType(cell_type));
+ }
+
+ qual->set_value(cell->Value());
+ }
+ }
+ return std::move(pb_mut);
+}
+
+DeleteType RequestConverter::ToDeleteType(const CellType type) {
+ switch (type) {
+ case DELETE:
+ return pb::MutationProto_DeleteType_DELETE_ONE_VERSION;
+ case DELETE_COLUMN:
+ return pb::MutationProto_DeleteType_DELETE_MULTIPLE_VERSIONS;
+ case DELETE_FAMILY:
+ return pb::MutationProto_DeleteType_DELETE_FAMILY;
+ case DELETE_FAMILY_VERSION:
+ return pb::MutationProto_DeleteType_DELETE_FAMILY_VERSION;
+ default:
+ throw std::runtime_error("Unknown delete type: " + folly::to<std::string>(type));
+ }
+}
+
+bool RequestConverter::IsDelete(const CellType type) {
+ return CellType::DELETE <= type && type <= CellType::DELETE_FAMILY;
+}
+
+std::unique_ptr<Request> RequestConverter::ToMutateRequest(const Put &put,
+ const std::string ®ion_name) {
+ auto pb_req = Request::mutate();
+ auto pb_msg = std::static_pointer_cast<hbase::pb::MutateRequest>(pb_req->req_msg());
+ RequestConverter::SetRegion(region_name, pb_msg->mutable_region());
+
+ pb_msg->set_allocated_mutation(
+ ToMutation(MutationType::MutationProto_MutationType_PUT, put, -1).release());
+
+ VLOG(3) << "Req is " << pb_req->req_msg()->ShortDebugString();
+ return pb_req;
+}
} /* namespace hbase */
diff --git a/core/request-converter.h b/core/request-converter.h
index 003afaa..ff6b290 100644
--- a/core/request-converter.h
+++ b/core/request-converter.h
@@ -24,16 +24,23 @@
#include <vector>
#include "connection/request.h"
#include "core/action.h"
+#include "core/cell.h"
#include "core/get.h"
#include "core/region-request.h"
#include "core/scan.h"
#include "core/server-request.h"
+#include "core/mutation.h"
+#include "core/put.h"
+#include "if/Client.pb.h"
#include "if/HBase.pb.h"
-using hbase::pb::RegionSpecifier;
+using hbase::pb::MutationProto;
using hbase::pb::RegionAction;
+using hbase::pb::RegionSpecifier;
using hbase::pb::ServerName;
using hbase::ServerRequest;
+using MutationType = hbase::pb::MutationProto_MutationType;
+using DeleteType = hbase::pb::MutationProto_DeleteType;
namespace hbase {
@@ -64,6 +71,11 @@ class RequestConverter {
static std::unique_ptr<Request> ToMultiRequest(const ActionsByRegion ®ion_requests);
+ static std::unique_ptr<Request> ToMutateRequest(const Put &put, const std::string ®ion_name);
+
+ static std::unique_ptr<MutationProto> ToMutation(const MutationType type,
+ const Mutation &mutation, const int64_t nonce);
+
private:
// Constructor not required. We have all static methods to create PB requests.
RequestConverter();
@@ -76,6 +88,8 @@ class RequestConverter {
*/
static void SetRegion(const std::string ®ion_name, RegionSpecifier *region_specifier);
static std::unique_ptr<hbase::pb::Get> ToGet(const Get &get);
+ static DeleteType ToDeleteType(const CellType type);
+ static bool IsDelete(const CellType type);
};
} /* namespace hbase */
diff --git a/core/result-test.cc b/core/result-test.cc
index 09d4e15..520f4b9 100644
--- a/core/result-test.cc
+++ b/core/result-test.cc
@@ -17,8 +17,6 @@
*
*/
-#include "core/result.h"
-
#include <gtest/gtest.h>
#include <limits>
#include <memory>
@@ -26,7 +24,11 @@
#include <vector>
#include "core/cell.h"
-using namespace hbase;
+#include "core/result.h"
+
+using hbase::Cell;
+using hbase::CellType;
+using hbase::Result;
void PopulateCells(std::vector<std::shared_ptr<Cell> > &cells) {
// Populate some Results
diff --git a/core/result.cc b/core/result.cc
index eb28b32..9d9ddb3 100644
--- a/core/result.cc
+++ b/core/result.cc
@@ -109,4 +109,26 @@ const std::map<std::string, std::string> Result::FamilyMap(const std::string &fa
}
return family_map;
}
+
+std::string Result::DebugString() const {
+ std::string ret{"keyvalues="};
+ if (IsEmpty()) {
+ ret += "NONE";
+ return ret;
+ }
+ ret += "{";
+ bool is_first = true;
+ for (const auto &cell : cells_) {
+ if (is_first) {
+ is_first = false;
+ } else {
+ ret += ", ";
+ }
+ ret += cell->DebugString();
+ }
+ ret += "}";
+
+ return ret;
+}
+
} /* namespace hbase */
diff --git a/core/result.h b/core/result.h
index 8064197..627d161 100644
--- a/core/result.h
+++ b/core/result.h
@@ -113,6 +113,8 @@ class Result {
*/
const std::map<std::string, std::string> FamilyMap(const std::string &family) const;
+ std::string DebugString() const;
+
private:
bool exists_ = false;
bool stale_ = false;
diff --git a/core/simple-client.cc b/core/simple-client.cc
index 4b1144c..b417353 100644
--- a/core/simple-client.cc
+++ b/core/simple-client.cc
@@ -19,111 +19,91 @@
#include <folly/Logging.h>
#include <folly/Random.h>
-#include <folly/futures/Future.h>
#include <gflags/gflags.h>
-#include <wangle/concurrent/CPUThreadPoolExecutor.h>
-#include <wangle/concurrent/IOThreadPoolExecutor.h>
#include <atomic>
#include <chrono>
#include <iostream>
#include <thread>
-#include "connection/connection-pool.h"
#include "core/client.h"
-#include "core/keyvalue-codec.h"
-#include "if/Client.pb.h"
-#include "if/ZooKeeper.pb.h"
+#include "core/get.h"
+#include "core/put.h"
+#include "core/table.h"
#include "serde/server-name.h"
#include "serde/table-name.h"
+#include "utils/time-util.h"
-using namespace folly;
-using namespace std;
-using namespace std::chrono;
+using hbase::Client;
using hbase::Configuration;
-using hbase::Response;
-using hbase::Request;
-using hbase::HBaseService;
-using hbase::KeyValueCodec;
-using hbase::LocationCache;
-using hbase::ConnectionPool;
-using hbase::ConnectionFactory;
+using hbase::Get;
+using hbase::Put;
+using hbase::Table;
using hbase::pb::TableName;
using hbase::pb::ServerName;
-using hbase::pb::RegionSpecifier_RegionSpecifierType;
-using hbase::pb::MutateRequest;
-using hbase::pb::MutationProto_MutationType;
+using hbase::TimeUtil;
-// TODO(eclark): remove the need for this.
-DEFINE_string(table, "t", "What region to send a get");
-DEFINE_string(row, "test", "What row to get");
+DEFINE_string(table, "test_table", "What table to do the reads or writes");
+DEFINE_string(row, "row_", "row prefix");
DEFINE_string(zookeeper, "localhost:2181", "What zk quorum to talk to");
-DEFINE_uint64(columns, 10000, "How many columns to write");
+DEFINE_uint64(num_rows, 10000, "How many rows to write and read");
+DEFINE_bool(display_results, false, "Whether to display the Results from Gets");
DEFINE_int32(threads, 6, "How many cpu threads");
-std::unique_ptr<Request> MakeRequest(uint64_t col, std::string region_name) {
- auto req = Request::mutate();
- auto msg = std::static_pointer_cast<MutateRequest>(req->req_msg());
- auto region = msg->mutable_region();
- auto suf = folly::to<std::string>(col);
-
- region->set_value(region_name);
- region->set_type(
- RegionSpecifier_RegionSpecifierType::RegionSpecifier_RegionSpecifierType_REGION_NAME);
- auto mutation = msg->mutable_mutation();
- mutation->set_row(FLAGS_row + suf);
- mutation->set_mutate_type(MutationProto_MutationType::MutationProto_MutationType_PUT);
- auto column = mutation->add_column_value();
- column->set_family("d");
- auto qual = column->add_qualifier_value();
- qual->set_qualifier(suf);
- qual->set_value(".");
-
- return std::move(req);
+std::unique_ptr<Put> MakePut(const std::string &row) {
+ auto put = std::make_unique<Put>(row);
+ put->AddColumn("f", "q", row);
+ return std::move(put);
+}
+
+std::string Row(const std::string &prefix, uint64_t i) {
+ auto suf = folly::to<std::string>(i);
+ return prefix + suf;
}
int main(int argc, char *argv[]) {
google::SetUsageMessage("Simple client to get a single row from HBase on the comamnd line");
google::ParseCommandLineFlags(&argc, &argv, true);
google::InitGoogleLogging(argv[0]);
-
- // Set up thread pools.
- auto cpu_pool = std::make_shared<wangle::CPUThreadPoolExecutor>(FLAGS_threads);
- auto io_pool = std::make_shared<wangle::IOThreadPoolExecutor>(5);
- auto codec = std::make_shared<KeyValueCodec>();
- auto cp = std::make_shared<ConnectionPool>(io_pool, codec);
+ google::InstallFailureSignalHandler();
+ FLAGS_logtostderr = 1;
+ FLAGS_stderrthreshold = 1;
// Configuration
auto conf = std::make_shared<Configuration>();
conf->Set("hbase.zookeeper.quorum", FLAGS_zookeeper);
-
- // Create the cache.
- LocationCache cache{conf, cpu_pool, cp};
+ conf->SetInt("hbase.client.cpu.thread.pool.size", FLAGS_threads);
auto row = FLAGS_row;
- auto tn = folly::to<TableName>(FLAGS_table);
-
- auto loc = cache.LocateRegion(tn, row).get(milliseconds(5000));
- auto connection = loc->service();
-
- auto num_puts = FLAGS_columns;
-
- auto results = std::vector<Future<std::unique_ptr<Response>>>{};
- auto col = uint64_t{0};
- for (; col < num_puts; col++) {
- results.push_back(
- folly::makeFuture(col)
- .via(cpu_pool.get())
- .then([loc](uint64_t col) { return MakeRequest(col, loc->region_name()); })
- .then([connection](std::unique_ptr<Request> req) {
- return (*connection)(std::move(req));
- }));
+ auto tn = std::make_shared<TableName>(folly::to<TableName>(FLAGS_table));
+ auto num_puts = FLAGS_num_rows;
+
+ auto client = std::make_unique<Client>(*conf);
+ auto table = client->Table(*tn);
+
+ // Do the Put requests
+ auto start_ns = TimeUtil::GetNowNanos();
+ for (uint64_t i = 0; i < num_puts; i++) {
+ table->Put(*MakePut(Row(FLAGS_row, i)));
+ }
+
+ LOG(INFO) << "Successfully sent " << num_puts << " Put requests in "
+ << TimeUtil::ElapsedMillis(start_ns) << " ms.";
+
+ // Do the Get requests
+ start_ns = TimeUtil::GetNowNanos();
+ for (uint64_t i = 0; i < num_puts; i++) {
+ auto result = table->Get(Get{Row(FLAGS_row, i)});
+ if (FLAGS_display_results) {
+ LOG(INFO) << result->DebugString();
+ }
}
- auto allf = folly::collect(results).get();
- LOG(ERROR) << "Successfully sent " << allf.size() << " requests.";
+ LOG(INFO) << "Successfully sent " << num_puts << " Get requests in "
+ << TimeUtil::ElapsedMillis(start_ns) << " ms.";
- io_pool->stop();
+ table->Close();
+ client->Close();
return 0;
}
diff --git a/core/table.cc b/core/table.cc
index 3c54d78..8ace4af 100644
--- a/core/table.cc
+++ b/core/table.cc
@@ -54,6 +54,11 @@ std::shared_ptr<hbase::Result> Table::Get(const hbase::Get &get) {
return context.get(operation_timeout());
}
+void Table::Put(const hbase::Put &put) {
+ auto future = async_table_->Put(put);
+ future.get(operation_timeout());
+}
+
milliseconds Table::operation_timeout() const {
return TimeUtil::ToMillis(async_connection_->connection_conf()->operation_timeout());
}
diff --git a/core/table.h b/core/table.h
index 803befe..cbb95b7 100644
--- a/core/table.h
+++ b/core/table.h
@@ -58,6 +58,14 @@ class Table {
// std::vector<std::unique_ptr<hbase::Result>> Get(const std::vector<hbase::Get> &gets);
/**
+ * @brief - Puts some data in the table.
+ * @param - put Put object to perform HBase Put operation.
+ */
+ void Put(const hbase::Put &put);
+
+ // TODO: Batch Puts
+
+ /**
* @brief - Close the client connection.
*/
void Close();
diff --git a/test-util/mini-cluster.cc b/test-util/mini-cluster.cc
index fb58fc1..34da54c 100644
--- a/test-util/mini-cluster.cc
+++ b/test-util/mini-cluster.cc
@@ -239,25 +239,6 @@ jobject MiniCluster::admin() {
return admin;
}
-jobject MiniCluster::TablePut(const std::string &table, const std::string &row,
- const std::string &family, const std::string &column,
- const std::string &value) {
- env();
- jobject conn = env_->CallObjectMethod(htu(), get_conn_mid_);
- jobject put = env_->NewObject(put_class_, put_ctor_, StrToByteChar(row));
- if (put == NULL) {
- LOG(INFO) << "Couldn't create Put";
- exit(-1);
- }
- env_->CallObjectMethod(put, add_col_mid_, StrToByteChar(family), StrToByteChar(column),
- StrToByteChar(value));
- jobject table_name_obj = env_->CallStaticObjectMethod(table_name_class_, tbl_name_value_of_mid_,
- env_->NewStringUTF(table.c_str()));
- jobject table_obj = env_->CallObjectMethod(conn, get_table_mid_, table_name_obj);
- env_->CallObjectMethod(table_obj, put_mid_, put);
- return table_obj;
-}
-
// moves region to server
void MiniCluster::MoveRegion(const std::string ®ion, const std::string &server) {
jobject admin_ = admin();
diff --git a/test-util/mini-cluster.h b/test-util/mini-cluster.h
index a9502b5..4119cb5 100644
--- a/test-util/mini-cluster.h
+++ b/test-util/mini-cluster.h
@@ -39,9 +39,6 @@ class MiniCluster {
jobject GetConf();
// returns the value for config key retrieved from cluster
const std::string GetConfValue(const std::string &key);
- // Does Put into table for family fam, qualifier col with value
- jobject TablePut(const std::string &table, const std::string &row, const std::string &family,
- const std::string &column, const std::string &value);
private:
JNIEnv *env_;
diff --git a/test-util/test-util.cc b/test-util/test-util.cc
index 54d2c83..c4e6ed2 100644
--- a/test-util/test-util.cc
+++ b/test-util/test-util.cc
@@ -58,19 +58,17 @@ void TestUtil::StartMiniCluster(int32_t num_region_servers) {
conf()->Set(ZKUtil::kHBaseZookeeperClientPort_,
mini_->GetConfValue(ZKUtil::kHBaseZookeeperClientPort_));
}
+
void TestUtil::StopMiniCluster() { mini_->StopCluster(); }
void TestUtil::CreateTable(const std::string &table, const std::string &family) {
mini_->CreateTable(table, family);
}
+
void TestUtil::CreateTable(const std::string &table, const std::string &family,
const std::vector<std::string> &keys) {
mini_->CreateTable(table, family, keys);
}
-void TestUtil::TablePut(const std::string &table, const std::string &row, const std::string &family,
- const std::string &column, const std::string &value) {
- mini_->TablePut(table, row, family, column, value);
-}
void TestUtil::StartStandAloneInstance() {
auto p = temp_dir_.path().string();
diff --git a/test-util/test-util.h b/test-util/test-util.h
index b5714ee..5729674 100644
--- a/test-util/test-util.h
+++ b/test-util/test-util.h
@@ -61,8 +61,6 @@ class TestUtil {
void CreateTable(const std::string &table, const std::string &family);
void CreateTable(const std::string &table, const std::string &family,
const std::vector<std::string> &keys);
- void TablePut(const std::string &table, const std::string &row, const std::string &family,
- const std::string &column, const std::string &value);
void StartStandAloneInstance();
void StopStandAloneInstance();
[hbase] 100/137: HBASE-18210 Implement Table#checkAndDelete()
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit a924ead2846d5ed4722f95a5dd1ec9231ed4cae9
Author: tedyu <yu...@gmail.com>
AuthorDate: Tue Jun 20 18:41:14 2017 -0700
HBASE-18210 Implement Table#checkAndDelete()
---
core/client-test.cc | 31 +++++++++++++++++++++++++++++++
core/raw-async-table.cc | 27 +++++++++++++++++++++++++++
core/raw-async-table.h | 5 +++++
core/request-converter.cc | 21 +++++++++++++++++++++
core/request-converter.h | 5 +++++
core/table.cc | 7 +++++++
core/table.h | 23 +++++++++++++++++++++++
7 files changed, 119 insertions(+)
diff --git a/core/client-test.cc b/core/client-test.cc
index 6462e6a..1a9fe11 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -278,6 +278,37 @@ TEST_F(ClientTest, CheckAndPut) {
ASSERT_FALSE(result) << "CheckAndPut shouldn't replace value";
}
+TEST_F(ClientTest, CheckAndDelete) {
+ // Using TestUtil to populate test data
+ ClientTest::test_util->CreateTable("checkDel", "d");
+
+ // Create TableName and Row to be fetched from HBase
+ auto tn = folly::to<hbase::pb::TableName>("checkDel");
+ auto row = "test1";
+
+ // Create a client
+ hbase::Client client(*ClientTest::test_util->conf());
+
+ // Get connection to HBase Table
+ auto table = client.Table(tn);
+ ASSERT_TRUE(table) << "Unable to get connection to Table.";
+
+ auto val1 = "value1";
+
+ // Perform Puts
+ table->Put(Put{row}.AddColumn("d", "1", val1));
+ table->Put(Put{row}.AddColumn("d", "2", "value2"));
+ auto result =
+ table->CheckAndDelete(row, "d", "1", val1, hbase::Delete{row}.AddColumn("d", "2"));
+ ASSERT_TRUE(result) << "CheckAndDelete didn't replace value";
+
+ // Perform the Get
+ hbase::Get get(row);
+ auto result1 = table->Get(get);
+ EXPECT_EQ(val1, *(result1->Value("d", "1")));
+ ASSERT_FALSE(result1->Value("d", "2")) << "Column 2 should be gone";
+}
+
TEST_F(ClientTest, PutGet) {
// Using TestUtil to populate test data
ClientTest::test_util->CreateTable("t", "d");
diff --git a/core/raw-async-table.cc b/core/raw-async-table.cc
index 46d9dfd..53ab526 100644
--- a/core/raw-async-table.cc
+++ b/core/raw-async-table.cc
@@ -137,6 +137,33 @@ folly::Future<bool> RawAsyncTable::CheckAndPut(const std::string& row, const std
return caller->Call().then([caller](const auto r) { return r; });
}
+folly::Future<bool> RawAsyncTable::CheckAndDelete(const std::string& row, const std::string& family,
+ const std::string& qualifier,
+ const std::string& value,
+ const hbase::Delete& del,
+ const pb::CompareType& compare_op) {
+ auto caller =
+ CreateCallerBuilder<bool>(row, connection_conf_->write_rpc_timeout())
+ ->action([=, &del](std::shared_ptr<hbase::HBaseRpcController> controller,
+ std::shared_ptr<hbase::RegionLocation> loc,
+ std::shared_ptr<hbase::RpcClient> rpc_client) -> folly::Future<bool> {
+ return Call<hbase::Delete, hbase::Request, hbase::Response, bool>(
+ rpc_client, controller, loc, del,
+ // request conversion
+ [=, &del](const hbase::Delete& del,
+ const std::string& region_name) -> std::unique_ptr<Request> {
+ auto checkReq = RequestConverter::CheckAndDeleteToMutateRequest(
+ row, family, qualifier, value, compare_op, del, region_name);
+ return checkReq;
+ },
+ // response conversion
+ &ResponseConverter::BoolFromMutateResponse);
+ })
+ ->Build();
+
+ return caller->Call().then([caller](const auto r) { return r; });
+}
+
folly::Future<folly::Unit> RawAsyncTable::Delete(const hbase::Delete& del) {
auto caller =
CreateCallerBuilder<folly::Unit>(del.row(), connection_conf_->write_rpc_timeout())
diff --git a/core/raw-async-table.h b/core/raw-async-table.h
index 068f230..ca12be6 100644
--- a/core/raw-async-table.h
+++ b/core/raw-async-table.h
@@ -72,6 +72,11 @@ class RawAsyncTable {
const hbase::Put& put,
const pb::CompareType& compare_op = pb::CompareType::EQUAL);
+ folly::Future<bool> CheckAndDelete(const std::string& row, const std::string& family,
+ const std::string& qualifier, const std::string& value,
+ const hbase::Delete& del,
+ const pb::CompareType& compare_op = pb::CompareType::EQUAL);
+
void Scan(const hbase::Scan& scan, std::shared_ptr<RawScanResultConsumer> consumer);
void Close() {}
diff --git a/core/request-converter.cc b/core/request-converter.cc
index 0cd9c7c..47c09d1 100644
--- a/core/request-converter.cc
+++ b/core/request-converter.cc
@@ -297,6 +297,27 @@ std::unique_ptr<Request> RequestConverter::CheckAndPutToMutateRequest(
return pb_req;
}
+std::unique_ptr<Request> RequestConverter::CheckAndDeleteToMutateRequest(
+ const std::string &row, const std::string &family, const std::string &qualifier,
+ const std::string &value, const pb::CompareType compare_op, const hbase::Delete &del,
+ const std::string ®ion_name) {
+ auto pb_req = Request::mutate();
+ auto pb_msg = std::static_pointer_cast<hbase::pb::MutateRequest>(pb_req->req_msg());
+
+ pb_msg->set_allocated_mutation(
+ ToMutation(MutationType::MutationProto_MutationType_DELETE, del, -1).release());
+ ::hbase::pb::Condition *cond = pb_msg->mutable_condition();
+ cond->set_row(row);
+ cond->set_family(family);
+ cond->set_qualifier(qualifier);
+ cond->set_allocated_comparator(
+ Comparator::ToProto(*(ComparatorFactory::BinaryComparator(value).get())).release());
+ cond->set_compare_type(compare_op);
+
+ RequestConverter::SetRegion(region_name, pb_msg->mutable_region());
+ return pb_req;
+}
+
std::unique_ptr<Request> RequestConverter::DeleteToMutateRequest(const Delete &del,
const std::string ®ion_name) {
auto pb_req = Request::mutate();
diff --git a/core/request-converter.h b/core/request-converter.h
index 0755f42..bcea278 100644
--- a/core/request-converter.h
+++ b/core/request-converter.h
@@ -89,6 +89,11 @@ class RequestConverter {
const std::string &value, const pb::CompareType compare_op, const hbase::Put &put,
const std::string ®ion_name);
+ static std::unique_ptr<Request> CheckAndDeleteToMutateRequest(
+ const std::string &row, const std::string &family, const std::string &qualifier,
+ const std::string &value, const pb::CompareType compare_op, const hbase::Delete &del,
+ const std::string ®ion_name);
+
static std::unique_ptr<Request> IncrementToMutateRequest(const Increment &incr,
const std::string ®ion_name);
diff --git a/core/table.cc b/core/table.cc
index f20078d..b89c1a2 100644
--- a/core/table.cc
+++ b/core/table.cc
@@ -80,6 +80,13 @@ bool Table::CheckAndPut(const std::string &row, const std::string &family,
return context.get(operation_timeout());
}
+bool Table::CheckAndDelete(const std::string &row, const std::string &family,
+ const std::string &qualifier, const std::string &value,
+ const hbase::Delete &del, const pb::CompareType &compare_op) {
+ auto context = async_table_->CheckAndDelete(row, family, qualifier, value, del, compare_op);
+ return context.get(operation_timeout());
+}
+
void Table::Delete(const hbase::Delete &del) {
auto future = async_table_->Delete(del);
future.get(operation_timeout());
diff --git a/core/table.h b/core/table.h
index 3c486af..cc37182 100644
--- a/core/table.h
+++ b/core/table.h
@@ -86,6 +86,29 @@ class Table {
void Delete(const hbase::Delete &del);
/**
+ * Atomically checks if a row/family/qualifier value matches the expected
+ * value. If it does, it adds the delete. If the passed value is null, the
+ * check is for the lack of column (ie: non-existence)
+ *
+ * The expected value argument of this call is on the left and the current
+ * value of the cell is on the right side of the comparison operator.
+ *
+ * Ie. eg. GREATER operator means expected value > existing <=> add the delete.
+ *
+ * @param row to check
+ * @param family column family to check
+ * @param qualifier column qualifier to check
+ * @param compare_op comparison operator to use
+ * @param value the expected value
+ * @param del data to delete if check succeeds
+ * @return true if the new delete was executed, false otherwise
+ */
+ bool CheckAndDelete(const std::string &row, const std::string &family,
+ const std::string &qualifier, const std::string &value,
+ const hbase::Delete &del,
+ const pb::CompareType &compare_op = pb::CompareType::EQUAL);
+
+ /**
* @brief - Increments some data in the table.
* @param - increment Increment object to perform HBase Increment operation.
*/
[hbase] 108/137: HBASE-18086 Create native client which creates
load on selected cluster
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 4cfb6e9182fd84aa8276a93a4821292efb9de8ac
Author: tedyu <yu...@gmail.com>
AuthorDate: Sun Jul 23 23:52:58 2017 -0700
HBASE-18086 Create native client which creates load on selected cluster
---
core/BUCK | 6 +
core/load-client.cc | 337 ++++++++++++++++++++++++++++++++++++++++++++++++++++
2 files changed, 343 insertions(+)
diff --git a/core/BUCK b/core/BUCK
index f9db0bd..76c836b 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -340,3 +340,9 @@ cxx_binary(
"simple-client.cc",
],
deps=[":core", "//connection:connection"],)
+cxx_binary(
+ name="load-client",
+ srcs=[
+ "load-client.cc",
+ ],
+ deps=[":core", "//connection:connection"],)
diff --git a/core/load-client.cc b/core/load-client.cc
new file mode 100644
index 0000000..67f0d57
--- /dev/null
+++ b/core/load-client.cc
@@ -0,0 +1,337 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <folly/Logging.h>
+#include <folly/Random.h>
+#include <gflags/gflags.h>
+
+#include <atomic>
+#include <chrono>
+#include <iostream>
+#include <thread>
+
+#include "core/client.h"
+#include "core/get.h"
+#include "core/put.h"
+#include "core/table.h"
+#include "serde/table-name.h"
+#include "utils/time-util.h"
+
+using hbase::Client;
+using hbase::Configuration;
+using hbase::Get;
+using hbase::Put;
+using hbase::Table;
+using hbase::pb::TableName;
+using hbase::TimeUtil;
+using folly::Random;
+
+DEFINE_string(table, "load_test_table", "What table to do the reads and writes with");
+DEFINE_string(families, "f", "comma separated list of column family names");
+DEFINE_string(conf, "", "Conf directory to read the config from (optional)");
+DEFINE_string(zookeeper, "localhost:2181", "What zk quorum to talk to");
+DEFINE_string(znode, "/hbase", "parent znode");
+DEFINE_uint64(num_rows, 1'000'000, "How many rows to write and read");
+DEFINE_uint64(num_cols, 1000, "How many columns there are in a row");
+DEFINE_int32(threads, 10, "How many client threads");
+DEFINE_int32(batch_num_rows, 100, "number of rows in one multi-get / multi-put");
+DEFINE_uint64(report_num_rows, 5000, "How frequent we should report the progress");
+DEFINE_bool(gets, true, "perform gets");
+DEFINE_bool(scans, true, "perform scans");
+DEFINE_bool(puts, true, "perform put's");
+DEFINE_bool(appends, true, "perform append's");
+
+static constexpr const char *kNumColumn = "num";
+static constexpr const char *incrPrefix = "i";
+static constexpr const char *appendPrefix = "a";
+
+std::string PrefixZero(int total_width, int num) {
+ std::string str = std::to_string(num);
+ auto prefix_len = total_width - str.length();
+ if (prefix_len > 0) return std::string(total_width - str.length(), '0') + str;
+ return str;
+}
+
+bool Verify(std::shared_ptr<hbase::Result> result, std::string family, int m) {
+ auto col = std::to_string(m);
+ auto int_val = hbase::BytesUtil::ToInt64(*(result->Value(family, incrPrefix + col)));
+ if (int_val != m) {
+ LOG(ERROR) << "value is not " << col << " for " << result->Row();
+ return false;
+ }
+ auto l = *(result->Value(family, col));
+ if (l != col) {
+ LOG(ERROR) << "value " << *(result->Value(family, "1")) << " is not " << col;
+ return false;
+ }
+ l = *(result->Value(family, appendPrefix + col));
+ if (l != col) {
+ LOG(ERROR) << "value " << *(result->Value(family, "1")) << " is not " << col;
+ return false;
+ }
+ return true;
+}
+
+bool DoScan(int iteration, uint64_t max_row, uint64_t rows, std::unique_ptr<Table> table,
+ const std::vector<std::string> &families) {
+ hbase::Scan scan{};
+ auto start = iteration * rows;
+ auto end = start + rows;
+ auto width = std::to_string(max_row).length();
+ scan.SetStartRow(PrefixZero(width, start));
+ scan.SetStopRow(PrefixZero(width, end));
+ auto scanner = table->Scan(scan);
+
+ auto cnt = start;
+ auto r = scanner->Next();
+ while (r != nullptr) {
+ auto row = PrefixZero(width, cnt);
+ if (r->Row().compare(row) != 0) {
+ LOG(ERROR) << "row " << r->Row() << " is not the expected: " << row;
+ return false;
+ }
+ for (auto family : families) {
+ auto cols = std::stoi(*(r->Value(family, kNumColumn)));
+ VLOG(3) << "scan gets " << std::to_string(cols) << " columns";
+ for (int m = 1; m <= cols; m++) {
+ if (!Verify(r, family, m)) return false;
+ }
+ }
+ cnt++;
+ r = scanner->Next();
+ }
+ LOG(INFO) << "Iteration " << iteration << " scanned " << std::to_string(cnt - start) << " rows";
+ return true;
+}
+
+bool DoGet(int iteration, uint64_t max_row, uint64_t rows, std::unique_ptr<Table> table,
+ const std::vector<std::string> &families, int batch_num_rows) {
+ auto width = std::to_string(max_row).length();
+ for (uint64_t k = iteration; k <= max_row;) {
+ std::vector<hbase::Get> gets;
+ for (uint64_t i = 0; i < batch_num_rows && k <= max_row; ++i, k += FLAGS_threads) {
+ std::string row = PrefixZero(width, k);
+ hbase::Get get(row);
+ gets.push_back(get);
+ }
+ VLOG(3) << "getting for " << batch_num_rows << " rows";
+ auto results = table->Get(gets);
+ for (uint64_t i = 0; i < batch_num_rows && i < results.size(); ++i) {
+ auto result = results[i];
+ if (result == nullptr) {
+ LOG(ERROR) << "didn't get result";
+ return false;
+ }
+ // Test the values
+ for (auto family : families) {
+ auto cols = std::stoi(*(result->Value(family, kNumColumn)));
+ VLOG(3) << "gets " << std::to_string(cols) << " columns";
+ for (int m = 1; m <= cols; m++) {
+ if (!Verify(result, family, m)) return false;
+ }
+ }
+ }
+ k += batch_num_rows;
+ }
+ LOG(INFO) << "Sent " << rows << " gets in iteration " << iteration;
+ return true;
+}
+
+void DoPut(int iteration, uint64_t max_row, uint64_t rows, int cols, std::unique_ptr<Table> table,
+ const std::vector<std::string> &families) {
+ auto width = std::to_string(max_row).length();
+ for (uint64_t j = 0; j < rows; j++) {
+ std::string row = PrefixZero(width, iteration * rows + j);
+ auto put = Put{row};
+ for (auto family : families) {
+ auto n_cols = Random::rand32(1, cols);
+ put.AddColumn(family, kNumColumn, std::to_string(n_cols));
+ for (unsigned int k = 1; k <= n_cols; k++) {
+ put.AddColumn(family, std::to_string(k), std::to_string(k));
+ }
+ }
+ table->Put(put);
+ if ((j + 1) % FLAGS_report_num_rows == 0)
+ LOG(INFO) << "Written " << std::to_string(j + 1) << " rows";
+ }
+ LOG(INFO) << "written " << std::to_string(rows) << " rows in " << std::to_string(iteration)
+ << " iteration";
+}
+
+bool DoAppendIncrement(int iteration, uint64_t max_row, uint64_t rows, int cols,
+ std::unique_ptr<Table> table, const std::vector<std::string> &families) {
+ auto width = std::to_string(max_row).length();
+ for (uint64_t j = 0; j < rows; j++) {
+ std::string row = PrefixZero(width, iteration * rows + j);
+ hbase::Get get(row);
+ auto result = table->Get(get);
+ for (auto family : families) {
+ auto n_cols = std::stoi(*(result->Value(family, kNumColumn)));
+ for (unsigned int k = 1; k <= n_cols; k++) {
+ table->Increment(
+ hbase::Increment{row}.AddColumn(family, incrPrefix + std::to_string(k), k));
+ if (!table->Append(hbase::Append{row}.Add(family, appendPrefix + std::to_string(k),
+ std::to_string(k)))) {
+ LOG(ERROR) << "append for " << row << " family: " << family << " failed";
+ return false;
+ }
+ }
+ }
+ if ((j + 1) % FLAGS_report_num_rows == 0)
+ LOG(INFO) << "Written " << std::to_string(j + 1) << " increments";
+ }
+ return true;
+}
+
+int main(int argc, char *argv[]) {
+ google::SetUsageMessage("Load client to manipulate multiple rows from HBase on the comamnd line");
+ google::ParseCommandLineFlags(&argc, &argv, true);
+ google::InitGoogleLogging(argv[0]);
+ google::InstallFailureSignalHandler();
+ FLAGS_logtostderr = 1;
+ FLAGS_stderrthreshold = 1;
+
+ if (FLAGS_batch_num_rows < 1) {
+ LOG(ERROR) << "size of multi get should be positive";
+ return -1;
+ }
+ if (!FLAGS_gets && !FLAGS_scans && !FLAGS_puts) {
+ LOG(ERROR) << "Must perform at least Get or Put operations";
+ return -1;
+ }
+ std::shared_ptr<Configuration> conf = nullptr;
+ if (FLAGS_conf == "") {
+ // Configuration
+ conf = std::make_shared<Configuration>();
+ conf->Set("hbase.zookeeper.quorum", FLAGS_zookeeper);
+ conf->Set("zookeeper.znode.parent", FLAGS_znode);
+ } else {
+ setenv("HBASE_CONF", FLAGS_conf.c_str(), 1);
+ hbase::HBaseConfigurationLoader loader;
+ conf = std::make_shared<Configuration>(loader.LoadDefaultResources().value());
+ }
+ auto tn = std::make_shared<TableName>(folly::to<TableName>(FLAGS_table));
+ auto num_puts = FLAGS_num_rows;
+
+ auto client = std::make_unique<Client>(*conf);
+
+ // Do the Put requests
+
+ std::vector<std::string> families;
+ std::size_t pos = 0, found;
+ while ((found = FLAGS_families.find_first_of(',', pos)) != std::string::npos) {
+ families.push_back(FLAGS_families.substr(pos, found - pos));
+ pos = found + 1;
+ }
+ families.push_back(FLAGS_families.substr(pos));
+
+ int rows = FLAGS_num_rows / FLAGS_threads;
+ if (FLAGS_num_rows % FLAGS_threads != 0) rows++;
+ int cols = FLAGS_num_cols;
+ bool succeeded = true;
+ if (FLAGS_puts) {
+ auto start_ns = TimeUtil::GetNowNanos();
+ std::vector<std::thread> writer_threads;
+ for (int i = 0; i < FLAGS_threads; i++) {
+ writer_threads.push_back(std::thread([=, &client, &tn, &families, &succeeded] {
+ // Get connection to HBase Table
+ auto table = client->Table(*tn);
+
+ DoPut(i, FLAGS_num_rows - 1, rows, cols, std::move(table), families);
+ table->Close();
+ }));
+ }
+ for (std::vector<std::thread>::iterator it = writer_threads.begin(); it != writer_threads.end();
+ it++) {
+ std::thread thread = std::move(*it);
+ thread.join();
+ }
+ LOG(INFO) << "Successfully sent " << num_puts << " Put requests in "
+ << TimeUtil::ElapsedMillis(start_ns) << " ms.";
+ }
+ if (FLAGS_appends) {
+ auto start_ns = TimeUtil::GetNowNanos();
+ std::vector<std::thread> writer_threads;
+ for (int i = 0; i < FLAGS_threads; i++) {
+ writer_threads.push_back(std::thread([=, &client, &tn, &families, &succeeded] {
+ // Get connection to HBase Table
+ auto table = client->Table(*tn);
+
+ succeeded &=
+ DoAppendIncrement(i, FLAGS_num_rows - 1, rows, cols, std::move(table), families);
+ table->Close();
+ }));
+ }
+ for (std::vector<std::thread>::iterator it = writer_threads.begin(); it != writer_threads.end();
+ it++) {
+ std::thread thread = std::move(*it);
+ thread.join();
+ }
+ LOG(INFO) << "Successfully sent " << num_puts << " append requests in "
+ << TimeUtil::ElapsedMillis(start_ns) << " ms.";
+ }
+
+ if (FLAGS_scans) {
+ auto start_ns = TimeUtil::GetNowNanos();
+ std::vector<std::thread> reader_threads;
+ for (int i = 0; i < FLAGS_threads; i++) {
+ reader_threads.push_back(std::thread([=, &client, &tn, &families, &succeeded] {
+ // Get connection to HBase Table
+ auto table1 = client->Table(*tn);
+
+ succeeded &= DoScan(i, FLAGS_num_rows - 1, rows, std::move(table1), families);
+ table1->Close();
+ }));
+ }
+ for (std::vector<std::thread>::iterator it = reader_threads.begin(); it != reader_threads.end();
+ it++) {
+ std::thread thread = std::move(*it);
+ thread.join();
+ }
+
+ LOG(INFO) << (succeeded ? "Successful. " : "Failed. ") << "Spent "
+ << TimeUtil::ElapsedMillis(start_ns) << " ms.";
+ }
+
+ if (FLAGS_gets) {
+ auto start_ns = TimeUtil::GetNowNanos();
+ std::vector<std::thread> reader_threads;
+ for (int i = 0; i < FLAGS_threads; i++) {
+ reader_threads.push_back(std::thread([=, &client, &tn, &families, &succeeded] {
+ // Get connection to HBase Table
+ auto table1 = client->Table(*tn);
+
+ succeeded &=
+ DoGet(i, FLAGS_num_rows - 1, rows, std::move(table1), families, FLAGS_batch_num_rows);
+ table1->Close();
+ }));
+ }
+ for (std::vector<std::thread>::iterator it = reader_threads.begin(); it != reader_threads.end();
+ it++) {
+ std::thread thread = std::move(*it);
+ thread.join();
+ }
+
+ LOG(INFO) << (succeeded ? "Successful. " : "Failed. ") << "Spent "
+ << TimeUtil::ElapsedMillis(start_ns) << " ms.";
+ }
+ client->Close();
+
+ return succeeded ? 0 : -1;
+}
[hbase] 95/137: HBASE-18173 Append class
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 348842d22a6b35c384202e867c2c8fb773b4491b
Author: tedyu <yu...@gmail.com>
AuthorDate: Tue Jun 6 16:42:45 2017 -0700
HBASE-18173 Append class
---
core/BUCK | 11 +++++
core/append-test.cc | 106 ++++++++++++++++++++++++++++++++++++++++++++++
core/append.cc | 54 +++++++++++++++++++++++
core/append.h | 56 ++++++++++++++++++++++++
core/client-test.cc | 30 +++++++++++++
core/raw-async-table.cc | 14 ++++++
core/raw-async-table.h | 2 +
core/request-converter.cc | 12 ++++++
core/request-converter.h | 3 ++
core/table.cc | 5 +++
core/table.h | 6 +++
11 files changed, 299 insertions(+)
diff --git a/core/BUCK b/core/BUCK
index 81fd4a7..47e97f5 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -43,6 +43,7 @@ cxx_library(
"put.h",
"delete.h",
"scan.h",
+ "append.h",
"result.h",
"result-scanner.h",
"request-converter.h",
@@ -82,6 +83,7 @@ cxx_library(
"put.cc",
"delete.cc",
"scan.cc",
+ "append.cc",
"scan-result-cache.cc",
"raw-async-table.cc",
"result.cc",
@@ -192,6 +194,15 @@ cxx_test(
],
run_test_separately=True,)
cxx_test(
+ name="append-test",
+ srcs=[
+ "append-test.cc",
+ ],
+ deps=[
+ ":core",
+ ],
+ run_test_separately=True,)
+cxx_test(
name="retry-test",
srcs=[
"async-rpc-retrying-test.cc",
diff --git a/core/append-test.cc b/core/append-test.cc
new file mode 100644
index 0000000..619826c
--- /dev/null
+++ b/core/append-test.cc
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include <glog/logging.h>
+#include <gtest/gtest.h>
+
+#include "core/mutation.h"
+#include "core/append.h"
+#include "utils/time-util.h"
+
+using hbase::Append;
+using hbase::Cell;
+using hbase::CellType;
+using hbase::Mutation;
+using hbase::TimeUtil;
+
+const constexpr int64_t Mutation::kLatestTimestamp;
+
+TEST(Append, Row) {
+ Append append{"foo"};
+ EXPECT_EQ("foo", append.row());
+}
+
+TEST(Append, Durability) {
+ Append append{"row"};
+ EXPECT_EQ(hbase::pb::MutationProto_Durability_USE_DEFAULT, append.Durability());
+
+ auto skipWal = hbase::pb::MutationProto_Durability_SKIP_WAL;
+ append.SetDurability(skipWal);
+ EXPECT_EQ(skipWal, append.Durability());
+}
+
+TEST(Append, Timestamp) {
+ Append append{"row"};
+
+ // test default timestamp
+ EXPECT_EQ(Mutation::kLatestTimestamp, append.TimeStamp());
+
+ // set custom timestamp
+ auto ts = TimeUtil::ToMillis(TimeUtil::GetNowNanos());
+ append.SetTimeStamp(ts);
+ EXPECT_EQ(ts, append.TimeStamp());
+
+ // Add a column with custom timestamp
+ append.Add("f", "q", "v");
+ auto &cell = append.FamilyMap().at("f")[0];
+ EXPECT_EQ(ts, cell->Timestamp());
+}
+
+TEST(Append, HasFamilies) {
+ Append append{"row"};
+
+ EXPECT_EQ(false, append.HasFamilies());
+
+ append.Add("f", "q", "v");
+ EXPECT_EQ(true, append.HasFamilies());
+}
+
+TEST(Append, Add) {
+ CellType cell_type = CellType::PUT;
+ std::string row = "row";
+ std::string family = "family";
+ std::string column = "column";
+ std::string value = "value";
+ int64_t timestamp = std::numeric_limits<int64_t>::max();
+ auto cell = std::make_unique<Cell>(row, family, column, timestamp, value, cell_type);
+
+ // add first cell
+ Append append{"row"};
+ append.Add(std::move(cell));
+ EXPECT_EQ(1, append.FamilyMap().size());
+ EXPECT_EQ(1, append.FamilyMap().at(family).size());
+
+ // add a non-matching row
+ auto cell2 = std::make_unique<Cell>(row, family, column, timestamp, value, cell_type);
+ Append append2{"foo"};
+ ASSERT_THROW(append2.Add(std::move(cell2)), std::runtime_error); // rows don't match
+
+ // add a second cell with same family
+ auto cell3 = std::make_unique<Cell>(row, family, "column-2", timestamp, value, cell_type);
+ append.Add(std::move(cell3));
+ EXPECT_EQ(1, append.FamilyMap().size());
+ EXPECT_EQ(2, append.FamilyMap().at(family).size());
+
+ // add a cell to a different family
+ auto cell4 = std::make_unique<Cell>(row, "family-2", "column-2", timestamp, value, cell_type);
+ append.Add(std::move(cell4));
+ EXPECT_EQ(2, append.FamilyMap().size());
+ EXPECT_EQ(1, append.FamilyMap().at("family-2").size());
+}
+
diff --git a/core/append.cc b/core/append.cc
new file mode 100644
index 0000000..18ee45a
--- /dev/null
+++ b/core/append.cc
@@ -0,0 +1,54 @@
+
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/append.h"
+#include <folly/Conv.h>
+#include <algorithm>
+#include <limits>
+#include <stdexcept>
+#include <utility>
+
+namespace hbase {
+
+/**
+ * @brief Append to the column from the specific family with the specified qualifier
+ * @param family family name
+ * @param qualifier column qualifier
+ * @param value value to append
+ */
+Append& Append::Add(const std::string& family, const std::string& qualifier,
+ const std::string& value) {
+ family_map_[family].push_back(std::move(
+ std::make_unique<Cell>(row_, family, qualifier, timestamp_, value,
+ hbase::CellType::PUT)));
+ return *this;
+}
+Append& Append::Add(std::unique_ptr<Cell> cell) {
+ if (cell->Row() != row_) {
+ throw std::runtime_error("The row in " + cell->DebugString() +
+ " doesn't match the original one " + row_);
+ }
+
+ family_map_[cell->Family()].push_back(std::move(cell));
+ return *this;
+}
+
+} // namespace hbase
diff --git a/core/append.h b/core/append.h
new file mode 100644
index 0000000..cf9ac24
--- /dev/null
+++ b/core/append.h
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <cstdint>
+#include <map>
+#include <memory>
+#include <string>
+#include <vector>
+#include "core/cell.h"
+#include "core/mutation.h"
+
+namespace hbase {
+
+class Append : public Mutation {
+ public:
+ /**
+ * Constructors
+ */
+ explicit Append(const std::string& row) : Mutation(row) {}
+ Append(const Append& cappend) : Mutation(cappend) {}
+ Append& operator=(const Append& cappend) {
+ Mutation::operator=(cappend);
+ return *this;
+ }
+
+ ~Append() = default;
+
+ /**
+ * @brief Add the specified column and value to this Append operation.
+ * @param family family name
+ * @param qualifier column qualifier
+ * @param value value to append
+ */
+ Append& Add(const std::string& family, const std::string& qualifier, const std::string& value);
+ Append& Add(std::unique_ptr<Cell> cell);
+};
+
+} // namespace hbase
diff --git a/core/client-test.cc b/core/client-test.cc
index d166f1c..1ee0a83 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -19,6 +19,7 @@
#include <gtest/gtest.h>
+#include "core/append.h"
#include "core/cell.h"
#include "core/client.h"
#include "core/configuration.h"
@@ -118,6 +119,35 @@ TEST_F(ClientTest, DefaultConfiguration) {
client.Close();
}
+TEST_F(ClientTest, Append) {
+ // Using TestUtil to populate test data
+ ClientTest::test_util->CreateTable("t", "d");
+
+ // Create TableName and Row to be fetched from HBase
+ auto tn = folly::to<hbase::pb::TableName>("t");
+ auto row = "test1";
+
+ // Create a client
+ hbase::Client client(*ClientTest::test_util->conf());
+
+ // Get connection to HBase Table
+ auto table = client.Table(tn);
+ ASSERT_TRUE(table) << "Unable to get connection to Table.";
+ std::string val1 = "a";
+ auto result = table->Append(hbase::Append{row}.Add("d", "1", val1));
+
+ ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty.";
+ EXPECT_EQ(row, result->Row());
+ EXPECT_EQ(val1, *(result->Value("d", "1")));
+
+ std::string val2 = "b";
+ result = table->Append(hbase::Append{row}.Add("d", "1", val2));
+
+ ASSERT_TRUE(!result->IsEmpty()) << "Result shouldn't be empty.";
+ EXPECT_EQ(row, result->Row());
+ EXPECT_EQ("ab", *(result->Value("d", "1")));
+}
+
TEST_F(ClientTest, PutGetDelete) {
// Using TestUtil to populate test data
std::string tableName = "t1";
diff --git a/core/raw-async-table.cc b/core/raw-async-table.cc
index 998e2f1..2a98d54 100644
--- a/core/raw-async-table.cc
+++ b/core/raw-async-table.cc
@@ -124,6 +124,20 @@ folly::Future<folly::Unit> RawAsyncTable::Delete(const hbase::Delete& del) {
return caller->Call().then([caller](const auto r) { return r; });
}
+folly::Future<std::shared_ptr<Result>> RawAsyncTable::Append(const hbase::Append& append) {
+ auto caller =
+ CreateCallerBuilder<std::shared_ptr<Result>>(append.row(), connection_conf_->write_rpc_timeout())
+ ->action([=, &append](std::shared_ptr<hbase::HBaseRpcController> controller,
+ std::shared_ptr<hbase::RegionLocation> loc,
+ std::shared_ptr<hbase::RpcClient> rpc_client) -> folly::Future<std::shared_ptr<Result>> {
+ return Call<hbase::Append, hbase::Request, hbase::Response, std::shared_ptr<Result>>(
+ rpc_client, controller, loc, append, &hbase::RequestConverter::AppendToMutateRequest,
+ &hbase::ResponseConverter::FromMutateResponse);
+ })
+ ->Build();
+
+ return caller->Call().then([caller](const auto r) { return r; });
+}
folly::Future<std::vector<folly::Try<std::shared_ptr<Result>>>> RawAsyncTable::Get(
const std::vector<hbase::Get>& gets) {
return this->Batch(gets);
diff --git a/core/raw-async-table.h b/core/raw-async-table.h
index 8c40dae..6088d1b 100644
--- a/core/raw-async-table.h
+++ b/core/raw-async-table.h
@@ -61,6 +61,8 @@ class RawAsyncTable {
folly::Future<folly::Unit> Delete(const hbase::Delete& del);
+ folly::Future<std::shared_ptr<hbase::Result>> Append(const hbase::Append& append);
+
folly::Future<std::shared_ptr<hbase::Result>> Increment(const hbase::Increment& increment);
folly::Future<folly::Unit> Put(const hbase::Put& put);
diff --git a/core/request-converter.cc b/core/request-converter.cc
index 6eb2f04..54fdfc5 100644
--- a/core/request-converter.cc
+++ b/core/request-converter.cc
@@ -301,4 +301,16 @@ std::unique_ptr<Request> RequestConverter::IncrementToMutateRequest(const Increm
return pb_req;
}
+std::unique_ptr<Request> RequestConverter::AppendToMutateRequest(const Append &append,
+ const std::string ®ion_name) {
+ auto pb_req = Request::mutate();
+ auto pb_msg = std::static_pointer_cast<hbase::pb::MutateRequest>(pb_req->req_msg());
+ RequestConverter::SetRegion(region_name, pb_msg->mutable_region());
+
+ pb_msg->set_allocated_mutation(
+ ToMutation(MutationType::MutationProto_MutationType_APPEND, append, -1).release());
+
+ VLOG(3) << "Req is " << pb_req->req_msg()->ShortDebugString();
+ return pb_req;
+}
} /* namespace hbase */
diff --git a/core/request-converter.h b/core/request-converter.h
index c807f45..a9d65d6 100644
--- a/core/request-converter.h
+++ b/core/request-converter.h
@@ -25,6 +25,7 @@
#include "connection/request.h"
#include "core/action.h"
#include "core/cell.h"
+#include "core/append.h"
#include "core/delete.h"
#include "core/get.h"
#include "core/increment.h"
@@ -90,6 +91,8 @@ class RequestConverter {
const Mutation &mutation,
const int64_t nonce);
+ static std::unique_ptr<Request> AppendToMutateRequest(const Append &append,
+ const std::string ®ion_name);
private:
// Constructor not required. We have all static methods to create PB requests.
RequestConverter();
diff --git a/core/table.cc b/core/table.cc
index 9cdd8b0..bf22169 100644
--- a/core/table.cc
+++ b/core/table.cc
@@ -83,6 +83,11 @@ std::shared_ptr<hbase::Result> Table::Increment(const hbase::Increment &incremen
return context.get(operation_timeout());
}
+std::shared_ptr<hbase::Result> Table::Append(const hbase::Append &append) {
+ auto context = async_table_->Append(append);
+ return context.get(operation_timeout());
+}
+
milliseconds Table::operation_timeout() const {
return TimeUtil::ToMillis(async_connection_->connection_conf()->operation_timeout());
}
diff --git a/core/table.h b/core/table.h
index 1f6d9b7..781c6f1 100644
--- a/core/table.h
+++ b/core/table.h
@@ -73,6 +73,12 @@ class Table {
* @param - increment Increment object to perform HBase Increment operation.
*/
std::shared_ptr<hbase::Result> Increment(const hbase::Increment &increment);
+
+ /**
+ * @brief - Appends some data in the table.
+ * @param - append Append object to perform HBase Append operation.
+ */
+ std::shared_ptr<hbase::Result> Append(const hbase::Append &append);
// TODO: Batch Puts
std::shared_ptr<ResultScanner> Scan(const hbase::Scan &scan);
[hbase] 109/137: HBASE-18371 [C++] Update folly and wangle
dependencies
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 323a774687788b34252f7090285c12f77d07fd53
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Wed Jul 26 12:18:05 2017 -0700
HBASE-18371 [C++] Update folly and wangle dependencies
---
core/async-batch-rpc-retrying-caller.h | 2 +-
core/async-rpc-retrying-caller.cc | 43 ++++++++++++++++++++--------------
core/raw-async-table.h | 2 +-
core/table.cc | 4 ++--
docker-files/Dockerfile | 33 ++++++++++++++++++++++++++
5 files changed, 62 insertions(+), 22 deletions(-)
diff --git a/core/async-batch-rpc-retrying-caller.h b/core/async-batch-rpc-retrying-caller.h
index 4c04b91..194c439 100644
--- a/core/async-batch-rpc-retrying-caller.h
+++ b/core/async-batch-rpc-retrying-caller.h
@@ -21,9 +21,9 @@
#include <folly/ExceptionWrapper.h>
#include <folly/Format.h>
+#include <folly/Try.h>
#include <folly/futures/Future.h>
#include <folly/futures/Promise.h>
-#include <folly/futures/Try.h>
#include <folly/io/IOBuf.h>
#include <folly/io/async/HHWheelTimer.h>
#include <wangle/concurrent/CPUThreadPoolExecutor.h>
diff --git a/core/async-rpc-retrying-caller.cc b/core/async-rpc-retrying-caller.cc
index aee7d0b..cb058b1 100644
--- a/core/async-rpc-retrying-caller.cc
+++ b/core/async-rpc-retrying-caller.cc
@@ -23,7 +23,7 @@
#include <folly/ExceptionWrapper.h>
#include <folly/Format.h>
#include <folly/Logging.h>
-#include <folly/futures/Unit.h>
+#include <folly/Unit.h>
#include "connection/rpc-client.h"
#include "core/async-connection.h"
@@ -94,9 +94,10 @@ void AsyncSingleRequestRpcRetryingCaller<RESP>::LocateThenCall() {
.then([this](std::shared_ptr<RegionLocation> loc) { Call(*loc); })
.onError([this](const exception_wrapper& e) {
OnError(e,
- [this]() -> std::string {
+ [this, e]() -> std::string {
return "Locate '" + row_ + "' in " + table_name_->namespace_() + "::" +
- table_name_->qualifier() + " failed, tries = " + std::to_string(tries_) +
+ table_name_->qualifier() + " failed with e.what()=" +
+ e.what().toStdString() + ", tries = " + std::to_string(tries_) +
", maxAttempts = " + std::to_string(max_attempts_) + ", timeout = " +
TimeUtil::ToMillisStr(operation_timeout_nanos_) + " ms, time elapsed = " +
TimeUtil::ElapsedMillisStr(this->start_ns_) + " ms";
@@ -116,6 +117,12 @@ void AsyncSingleRequestRpcRetryingCaller<RESP>::OnError(
return;
}
+ if (tries_ > start_log_errors_count_) {
+ LOG(WARNING) << err_msg();
+ } else {
+ VLOG(1) << err_msg();
+ }
+
int64_t delay_ns;
if (operation_timeout_nanos_.count() > 0) {
int64_t max_delay_ns = RemainingTimeNs() - ConnectionUtils::kSleepDeltaNs;
@@ -174,21 +181,21 @@ void AsyncSingleRequestRpcRetryingCaller<RESP>::Call(const RegionLocation& loc)
callable_(controller_, loc_ptr, rpc_client)
.then([loc_ptr, this](const RESP& resp) { this->promise_->setValue(std::move(resp)); })
.onError([&, loc_ptr, this](const exception_wrapper& e) {
- OnError(e,
- [&, this]() -> std::string {
- return "Call to " + folly::sformat("{0}:{1}", loc_ptr->server_name().host_name(),
- loc_ptr->server_name().port()) +
- " for '" + row_ + "' in " + loc_ptr->DebugString() + " of " +
- table_name_->namespace_() + "::" + table_name_->qualifier() +
- " failed, tries = " + std::to_string(tries_) + ", maxAttempts = " +
- std::to_string(max_attempts_) + ", timeout = " +
- TimeUtil::ToMillisStr(this->operation_timeout_nanos_) +
- " ms, time elapsed = " + TimeUtil::ElapsedMillisStr(this->start_ns_) +
- " ms";
- },
- [&, this](const exception_wrapper& error) {
- conn_->region_locator()->UpdateCachedLocation(*loc_ptr, error);
- });
+ OnError(
+ e,
+ [&, this, e]() -> std::string {
+ return "Call to " + folly::sformat("{0}:{1}", loc_ptr->server_name().host_name(),
+ loc_ptr->server_name().port()) +
+ " for '" + row_ + "' in " + loc_ptr->DebugString() + " of " +
+ table_name_->namespace_() + "::" + table_name_->qualifier() +
+ " failed with e.what()=" + e.what().toStdString() + ", tries = " +
+ std::to_string(tries_) + ", maxAttempts = " + std::to_string(max_attempts_) +
+ ", timeout = " + TimeUtil::ToMillisStr(this->operation_timeout_nanos_) +
+ " ms, time elapsed = " + TimeUtil::ElapsedMillisStr(this->start_ns_) + " ms";
+ },
+ [&, this](const exception_wrapper& error) {
+ conn_->region_locator()->UpdateCachedLocation(*loc_ptr, error);
+ });
});
}
diff --git a/core/raw-async-table.h b/core/raw-async-table.h
index ca12be6..e651f8a 100644
--- a/core/raw-async-table.h
+++ b/core/raw-async-table.h
@@ -18,8 +18,8 @@
*/
#pragma once
+#include <folly/Unit.h>
#include <folly/futures/Future.h>
-#include <folly/futures/Unit.h>
#include <chrono>
#include <memory>
#include <string>
diff --git a/core/table.cc b/core/table.cc
index b89c1a2..3b7a87b 100644
--- a/core/table.cc
+++ b/core/table.cc
@@ -120,9 +120,9 @@ std::vector<std::shared_ptr<hbase::Result>> Table::Get(const std::vector<hbase::
if (tresult.hasValue()) {
results.push_back(tresult.value());
} else if (tresult.hasException()) {
- LOG(ERROR) << "Caught exception:- " << tresult.exception().getCopied()->what() << " for "
+ LOG(ERROR) << "Caught exception:- " << tresult.exception().what() << " for "
<< gets[num++].row();
- throw tresult.exception().getCopied();
+ throw tresult.exception();
}
}
return results;
diff --git a/docker-files/Dockerfile b/docker-files/Dockerfile
index d5a1eab..efd9a9d 100644
--- a/docker-files/Dockerfile
+++ b/docker-files/Dockerfile
@@ -84,6 +84,39 @@ RUN git clone https://github.com/google/protobuf.git /usr/src/protobuf && \
make clean && \
ldconfig
+# Update folly
+RUN cd /usr/src/ && \
+ ver=2017.06.19.00 && \
+ wget https://github.com/facebook/folly/archive/v$ver.tar.gz && \
+ tar zxf v$ver.tar.gz && \
+ rm -rf v$ver.tar.gz && \
+ cd folly-$ver/folly/test && \
+ rm -rf gtest && \
+ wget https://github.com/google/googletest/archive/release-1.8.0.tar.gz && \
+ tar zxf release-1.8.0.tar.gz && \
+ rm -f release-1.8.0.tar.gz && \
+ mv googletest-release-1.8.0 gtest && \
+ cd ../ && \
+ ldconfig && \
+ autoreconf -ivf && \
+ ./configure && \
+ make && \
+ make check && \
+ make install
+
+# Update wangle
+RUN cd /usr/src/ && \
+ ver=2017.06.26.00 && \
+ wget https://github.com/facebook/wangle/archive/v$ver.tar.gz && \
+ tar zxf v$ver.tar.gz && \
+ rm -rf v$ver.tar.gz && \
+ cd wangle-$ver/wangle && \
+ ldconfig && \
+ cmake . -DBUILD_TESTS=OFF && \
+ make && \
+ ctest && \
+ make install
+
ENTRYPOINT /usr/sbin/krb5kdc -P /var/run/krb5kdc.pid && /bin/bash
WORKDIR /usr/src/hbase/hbase-native-client
[hbase] 06/137: HBASE-14853 Add on protobuf to c++ chain
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 5e604ca2d7268b7513cc5d4ad1a18cdd628b4fe1
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Mon Dec 28 15:33:52 2015 -0800
HBASE-14853 Add on protobuf to c++ chain
---
bin/start-docker.sh | 1 +
core/BUCK | 1 +
core/client.cc | 4 +
core/client.h | 2 +
if/AccessControl.proto | 123 +++++
if/Admin.proto | 309 +++++++++++
if/Aggregate.proto | 63 +++
if/Authentication.proto | 82 +++
if/BUCK | 36 ++
if/Cell.proto | 67 +++
if/Client.proto | 472 +++++++++++++++++
core/client.cc => if/ClusterId.proto | 32 +-
if/ClusterStatus.proto | 224 ++++++++
if/Comparator.proto | 74 +++
core/client.cc => if/Encryption.proto | 31 +-
if/ErrorHandling.proto | 58 +++
core/client.cc => if/FS.proto | 39 +-
if/Filter.proto | 170 ++++++
if/HBase.proto | 258 +++++++++
if/HFile.proto | 49 ++
core/client.h => if/LoadBalancer.proto | 15 +-
core/client.cc => if/MapReduce.proto | 31 +-
if/Master.proto | 778 ++++++++++++++++++++++++++++
if/MasterProcedure.proto | 245 +++++++++
core/client.cc => if/MultiRowMutation.proto | 39 +-
if/Procedure.proto | 119 +++++
if/Quota.proto | 76 +++
if/RPC.proto | 136 +++++
core/client.h => if/RegionNormalizer.proto | 14 +-
if/RegionServerStatus.proto | 158 ++++++
if/RowProcessor.proto | 45 ++
if/SecureBulkLoad.proto | 72 +++
if/Snapshot.proto | 66 +++
core/client.cc => if/Tracing.proto | 33 +-
if/VisibilityLabels.proto | 83 +++
if/WAL.proto | 172 ++++++
if/ZooKeeper.proto | 155 ++++++
rpc/CMakeLists.txt | 17 -
third-party/BUCK | 4 +-
39 files changed, 4222 insertions(+), 131 deletions(-)
diff --git a/bin/start-docker.sh b/bin/start-docker.sh
index bf38912..4426705 100755
--- a/bin/start-docker.sh
+++ b/bin/start-docker.sh
@@ -20,6 +20,7 @@ set -e
set -x
eval "$(docker-machine env docker-vm)"
+eval "$(docker-machine env dinghy)"
docker build -t hbase_native .
diff --git a/core/BUCK b/core/BUCK
index 2e4e755..ef027a1 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -41,6 +41,7 @@ cxx_binary(
"scanner.cc",
],
deps = [
+ "//if:if",
"//third-party:folly",
"//third-party:wangle",
],
diff --git a/core/client.cc b/core/client.cc
index 98cf38a..a04daee 100644
--- a/core/client.cc
+++ b/core/client.cc
@@ -24,9 +24,13 @@
#include <glog/logging.h>
#include <gflags/gflags.h>
+#include "if/ZooKeeper.pb.h"
+
using namespace folly;
+using namespace hbase::pb;
int main(int argc, char *argv[]) {
+ MetaRegionServer mrs;
google::ParseCommandLineFlags(&argc, &argv, true);
google::InitGoogleLogging(argv[0]);
diff --git a/core/client.h b/core/client.h
index 921cecc..35a3bd8 100644
--- a/core/client.h
+++ b/core/client.h
@@ -21,4 +21,6 @@
#include <folly/io/IOBuf.h>
+#include "if/Cell.pb.h"
+
class Client {};
diff --git a/if/AccessControl.proto b/if/AccessControl.proto
new file mode 100644
index 0000000..e67540b
--- /dev/null
+++ b/if/AccessControl.proto
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "AccessControlProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+
+message Permission {
+ enum Action {
+ READ = 0;
+ WRITE = 1;
+ EXEC = 2;
+ CREATE = 3;
+ ADMIN = 4;
+ }
+ enum Type {
+ Global = 1;
+ Namespace = 2;
+ Table = 3;
+ }
+ required Type type = 1;
+ optional GlobalPermission global_permission = 2;
+ optional NamespacePermission namespace_permission = 3;
+ optional TablePermission table_permission = 4;
+}
+
+message TablePermission {
+ optional TableName table_name = 1;
+ optional bytes family = 2;
+ optional bytes qualifier = 3;
+ repeated Permission.Action action = 4;
+}
+
+message NamespacePermission {
+ optional bytes namespace_name = 1;
+ repeated Permission.Action action = 2;
+}
+
+message GlobalPermission {
+ repeated Permission.Action action = 1;
+}
+
+message UserPermission {
+ required bytes user = 1;
+ required Permission permission = 3;
+}
+
+/**
+ * Content of the /hbase/acl/<table or namespace> znode.
+ */
+message UsersAndPermissions {
+ message UserPermissions {
+ required bytes user = 1;
+ repeated Permission permissions = 2;
+ }
+
+ repeated UserPermissions user_permissions = 1;
+}
+
+message GrantRequest {
+ required UserPermission user_permission = 1;
+}
+
+message GrantResponse {
+}
+
+message RevokeRequest {
+ required UserPermission user_permission = 1;
+}
+
+message RevokeResponse {
+}
+
+message GetUserPermissionsRequest {
+ optional Permission.Type type = 1;
+ optional TableName table_name = 2;
+ optional bytes namespace_name = 3;
+}
+
+message GetUserPermissionsResponse {
+ repeated UserPermission user_permission = 1;
+}
+
+message CheckPermissionsRequest {
+ repeated Permission permission = 1;
+}
+
+message CheckPermissionsResponse {
+}
+
+service AccessControlService {
+ rpc Grant(GrantRequest)
+ returns (GrantResponse);
+
+ rpc Revoke(RevokeRequest)
+ returns (RevokeResponse);
+
+ rpc GetUserPermissions(GetUserPermissionsRequest)
+ returns (GetUserPermissionsResponse);
+
+ rpc CheckPermissions(CheckPermissionsRequest)
+ returns (CheckPermissionsResponse);
+}
diff --git a/if/Admin.proto b/if/Admin.proto
new file mode 100644
index 0000000..e905340
--- /dev/null
+++ b/if/Admin.proto
@@ -0,0 +1,309 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers that are used for Admin service.
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "AdminProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "WAL.proto";
+
+message GetRegionInfoRequest {
+ required RegionSpecifier region = 1;
+ optional bool compaction_state = 2;
+}
+
+message GetRegionInfoResponse {
+ required RegionInfo region_info = 1;
+ optional CompactionState compaction_state = 2;
+ optional bool isRecovering = 3;
+
+ enum CompactionState {
+ NONE = 0;
+ MINOR = 1;
+ MAJOR = 2;
+ MAJOR_AND_MINOR = 3;
+ }
+}
+
+/**
+ * Get a list of store files for a set of column families in a particular region.
+ * If no column family is specified, get the store files for all column families.
+ */
+message GetStoreFileRequest {
+ required RegionSpecifier region = 1;
+ repeated bytes family = 2;
+}
+
+message GetStoreFileResponse {
+ repeated string store_file = 1;
+}
+
+message GetOnlineRegionRequest {
+}
+
+message GetOnlineRegionResponse {
+ repeated RegionInfo region_info = 1;
+}
+
+message OpenRegionRequest {
+ repeated RegionOpenInfo open_info = 1;
+ // the intended server for this RPC.
+ optional uint64 serverStartCode = 2;
+ // wall clock time from master
+ optional uint64 master_system_time = 5;
+
+ message RegionOpenInfo {
+ required RegionInfo region = 1;
+ optional uint32 version_of_offline_node = 2;
+ repeated ServerName favored_nodes = 3;
+ // open region for distributedLogReplay
+ optional bool openForDistributedLogReplay = 4;
+ }
+}
+
+message OpenRegionResponse {
+ repeated RegionOpeningState opening_state = 1;
+
+ enum RegionOpeningState {
+ OPENED = 0;
+ ALREADY_OPENED = 1;
+ FAILED_OPENING = 2;
+ }
+}
+
+message WarmupRegionRequest {
+
+ required RegionInfo regionInfo = 1;
+}
+
+message WarmupRegionResponse {
+}
+
+/**
+ * Closes the specified region and will use or not use ZK during the close
+ * according to the specified flag.
+ */
+message CloseRegionRequest {
+ required RegionSpecifier region = 1;
+ optional uint32 version_of_closing_node = 2;
+ optional bool transition_in_ZK = 3 [default = true];
+ optional ServerName destination_server = 4;
+ // the intended server for this RPC.
+ optional uint64 serverStartCode = 5;
+}
+
+message CloseRegionResponse {
+ required bool closed = 1;
+}
+
+/**
+ * Flushes the MemStore of the specified region.
+ * <p>
+ * This method is synchronous.
+ */
+message FlushRegionRequest {
+ required RegionSpecifier region = 1;
+ optional uint64 if_older_than_ts = 2;
+ optional bool write_flush_wal_marker = 3; // whether to write a marker to WAL even if not flushed
+}
+
+message FlushRegionResponse {
+ required uint64 last_flush_time = 1;
+ optional bool flushed = 2;
+ optional bool wrote_flush_wal_marker = 3;
+}
+
+/**
+ * Splits the specified region.
+ * <p>
+ * This method currently flushes the region and then forces a compaction which
+ * will then trigger a split. The flush is done synchronously but the
+ * compaction is asynchronous.
+ */
+message SplitRegionRequest {
+ required RegionSpecifier region = 1;
+ optional bytes split_point = 2;
+}
+
+message SplitRegionResponse {
+}
+
+/**
+ * Compacts the specified region. Performs a major compaction if specified.
+ * <p>
+ * This method is asynchronous.
+ */
+message CompactRegionRequest {
+ required RegionSpecifier region = 1;
+ optional bool major = 2;
+ optional bytes family = 3;
+}
+
+message CompactRegionResponse {
+}
+
+message UpdateFavoredNodesRequest {
+ repeated RegionUpdateInfo update_info = 1;
+
+ message RegionUpdateInfo {
+ required RegionInfo region = 1;
+ repeated ServerName favored_nodes = 2;
+ }
+}
+
+message UpdateFavoredNodesResponse {
+ optional uint32 response = 1;
+}
+
+/**
+ * Merges the specified regions.
+ * <p>
+ * This method currently closes the regions and then merges them
+ */
+message MergeRegionsRequest {
+ required RegionSpecifier region_a = 1;
+ required RegionSpecifier region_b = 2;
+ optional bool forcible = 3 [default = false];
+ // wall clock time from master
+ optional uint64 master_system_time = 4;
+}
+
+message MergeRegionsResponse {
+}
+
+// Protocol buffer version of WAL for replication
+message WALEntry {
+ required WALKey key = 1;
+ // Following may be null if the KVs/Cells are carried along the side in a cellblock (See
+ // RPC for more on cellblocks). If Cells/KVs are in a cellblock, this next field is null
+ // and associated_cell_count has count of Cells associated w/ this WALEntry
+ repeated bytes key_value_bytes = 2;
+ // If Cell data is carried alongside in a cellblock, this is count of Cells in the cellblock.
+ optional int32 associated_cell_count = 3;
+}
+
+/**
+ * Replicates the given entries. The guarantee is that the given entries
+ * will be durable on the slave cluster if this method returns without
+ * any exception. hbase.replication has to be set to true for this to work.
+ */
+message ReplicateWALEntryRequest {
+ repeated WALEntry entry = 1;
+ optional string replicationClusterId = 2;
+ optional string sourceBaseNamespaceDirPath = 3;
+ optional string sourceHFileArchiveDirPath = 4;
+}
+
+message ReplicateWALEntryResponse {
+}
+
+message RollWALWriterRequest {
+}
+
+/*
+ * Roll request responses no longer include regions to flush
+ * this list will always be empty when talking to a 1.0 server
+ */
+message RollWALWriterResponse {
+ // A list of encoded name of regions to flush
+ repeated bytes region_to_flush = 1;
+}
+
+message StopServerRequest {
+ required string reason = 1;
+}
+
+message StopServerResponse {
+}
+
+message GetServerInfoRequest {
+}
+
+message ServerInfo {
+ required ServerName server_name = 1;
+ optional uint32 webui_port = 2;
+}
+
+message GetServerInfoResponse {
+ required ServerInfo server_info = 1;
+}
+
+message UpdateConfigurationRequest {
+}
+
+message UpdateConfigurationResponse {
+}
+
+service AdminService {
+ rpc GetRegionInfo(GetRegionInfoRequest)
+ returns(GetRegionInfoResponse);
+
+ rpc GetStoreFile(GetStoreFileRequest)
+ returns(GetStoreFileResponse);
+
+ rpc GetOnlineRegion(GetOnlineRegionRequest)
+ returns(GetOnlineRegionResponse);
+
+ rpc OpenRegion(OpenRegionRequest)
+ returns(OpenRegionResponse);
+
+ rpc WarmupRegion(WarmupRegionRequest)
+ returns(WarmupRegionResponse);
+
+ rpc CloseRegion(CloseRegionRequest)
+ returns(CloseRegionResponse);
+
+ rpc FlushRegion(FlushRegionRequest)
+ returns(FlushRegionResponse);
+
+ rpc SplitRegion(SplitRegionRequest)
+ returns(SplitRegionResponse);
+
+ rpc CompactRegion(CompactRegionRequest)
+ returns(CompactRegionResponse);
+
+ rpc MergeRegions(MergeRegionsRequest)
+ returns(MergeRegionsResponse);
+
+ rpc ReplicateWALEntry(ReplicateWALEntryRequest)
+ returns(ReplicateWALEntryResponse);
+
+ rpc Replay(ReplicateWALEntryRequest)
+ returns(ReplicateWALEntryResponse);
+
+ rpc RollWALWriter(RollWALWriterRequest)
+ returns(RollWALWriterResponse);
+
+ rpc GetServerInfo(GetServerInfoRequest)
+ returns(GetServerInfoResponse);
+
+ rpc StopServer(StopServerRequest)
+ returns(StopServerResponse);
+
+ rpc UpdateFavoredNodes(UpdateFavoredNodesRequest)
+ returns(UpdateFavoredNodesResponse);
+
+ rpc UpdateConfiguration(UpdateConfigurationRequest)
+ returns(UpdateConfigurationResponse);
+}
diff --git a/if/Aggregate.proto b/if/Aggregate.proto
new file mode 100644
index 0000000..4d32e70
--- /dev/null
+++ b/if/Aggregate.proto
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "AggregateProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "Client.proto";
+
+message AggregateRequest {
+ /** The request passed to the AggregateService consists of three parts
+ * (1) the (canonical) classname of the ColumnInterpreter implementation
+ * (2) the Scan query
+ * (3) any bytes required to construct the ColumnInterpreter object
+ * properly
+ */
+ required string interpreter_class_name = 1;
+ required Scan scan = 2;
+ optional bytes interpreter_specific_bytes = 3;
+}
+
+message AggregateResponse {
+ /**
+ * The AggregateService methods all have a response that either is a Pair
+ * or a simple object. When it is a Pair both first_part and second_part
+ * have defined values (and the second_part is not present in the response
+ * when the response is not a pair). Refer to the AggregateImplementation
+ * class for an overview of the AggregateResponse object constructions.
+ */
+ repeated bytes first_part = 1;
+ optional bytes second_part = 2;
+}
+
+/** Refer to the AggregateImplementation class for an overview of the
+ * AggregateService method implementations and their functionality.
+ */
+service AggregateService {
+ rpc GetMax (AggregateRequest) returns (AggregateResponse);
+ rpc GetMin (AggregateRequest) returns (AggregateResponse);
+ rpc GetSum (AggregateRequest) returns (AggregateResponse);
+ rpc GetRowNum (AggregateRequest) returns (AggregateResponse);
+ rpc GetAvg (AggregateRequest) returns (AggregateResponse);
+ rpc GetStd (AggregateRequest) returns (AggregateResponse);
+ rpc GetMedian (AggregateRequest) returns (AggregateResponse);
+}
diff --git a/if/Authentication.proto b/if/Authentication.proto
new file mode 100644
index 0000000..2f64799
--- /dev/null
+++ b/if/Authentication.proto
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "AuthenticationProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+message AuthenticationKey {
+ required int32 id = 1;
+ required int64 expiration_date = 2;
+ required bytes key = 3;
+}
+
+
+message TokenIdentifier {
+ enum Kind {
+ HBASE_AUTH_TOKEN = 0;
+ }
+ required Kind kind = 1;
+ required bytes username = 2;
+ required int32 key_id = 3;
+ optional int64 issue_date = 4;
+ optional int64 expiration_date = 5;
+ optional int64 sequence_number = 6;
+}
+
+
+// Serialization of the org.apache.hadoop.security.token.Token class
+// Note that this is a Hadoop class, so fields may change!
+message Token {
+ // the TokenIdentifier in serialized form
+ // Note: we can't use the protobuf directly because the Hadoop Token class
+ // only stores the serialized bytes
+ optional bytes identifier = 1;
+ optional bytes password = 2;
+ optional bytes service = 3;
+}
+
+
+// RPC request & response messages
+message GetAuthenticationTokenRequest {
+}
+
+message GetAuthenticationTokenResponse {
+ optional Token token = 1;
+}
+
+message WhoAmIRequest {
+}
+
+message WhoAmIResponse {
+ optional string username = 1;
+ optional string auth_method = 2;
+}
+
+
+// RPC service
+service AuthenticationService {
+ rpc GetAuthenticationToken(GetAuthenticationTokenRequest)
+ returns (GetAuthenticationTokenResponse);
+
+ rpc WhoAmI(WhoAmIRequest)
+ returns (WhoAmIResponse);
+}
diff --git a/if/BUCK b/if/BUCK
new file mode 100644
index 0000000..3490a05
--- /dev/null
+++ b/if/BUCK
@@ -0,0 +1,36 @@
+PROTO_SRCS = glob(['*.proto'])
+HEADER_FILENAMES = [ x.replace('.proto','.pb.h') for x in PROTO_SRCS]
+CC_FILENAMES = [ x.replace('.proto', '.pb.cc') for x in PROTO_SRCS]
+
+genrule(
+ name = 'generate-proto-sources',
+ srcs = PROTO_SRCS,
+ cmd = 'mkdir -p $OUT && pwd && protoc --proto_path=. --cpp_out=$OUT *.proto',
+ out = 'output',
+)
+
+for header_filename in HEADER_FILENAMES:
+ genrule(
+ name = header_filename,
+ cmd = 'mkdir -p `dirname $OUT` '
+ ' && cp $(location :generate-proto-sources)/{} $OUT'.format(header_filename),
+ out = header_filename,
+ )
+for cc_filename in CC_FILENAMES:
+ genrule(
+ name = cc_filename,
+ cmd = 'mkdir -p `dirname $OUT` '
+ ' && cp $(location :generate-proto-sources)/{} $OUT '
+ ' && cp $(location :generate-proto-sources)/*.h `dirname $OUT`'.format(cc_filename),
+ out = cc_filename,
+ )
+
+cxx_library(
+ name = 'if',
+ exported_headers = [':' + x for x in HEADER_FILENAMES],
+ srcs = [':' + x for x in CC_FILENAMES],
+ deps = [ '//third-party:protobuf']
+ + [':' + x for x in CC_FILENAMES]
+ + [ ':' + x for x in HEADER_FILENAMES ],
+ visibility = [ 'PUBLIC', ],
+)
diff --git a/if/Cell.proto b/if/Cell.proto
new file mode 100644
index 0000000..2c61035
--- /dev/null
+++ b/if/Cell.proto
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Cell and KeyValue protos
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "CellProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+/**
+ * The type of the key in a Cell
+ */
+enum CellType {
+ MINIMUM = 0;
+ PUT = 4;
+
+ DELETE = 8;
+ DELETE_COLUMN = 12;
+ DELETE_FAMILY = 14;
+
+ // MAXIMUM is used when searching; you look from maximum on down.
+ MAXIMUM = 255;
+}
+
+/**
+ * Protocol buffer version of Cell.
+ */
+message Cell {
+ optional bytes row = 1;
+ optional bytes family = 2;
+ optional bytes qualifier = 3;
+ optional uint64 timestamp = 4;
+ optional CellType cell_type = 5;
+ optional bytes value = 6;
+ optional bytes tags = 7;
+}
+
+/**
+ * Protocol buffer version of KeyValue.
+ * It doesn't have those transient parameters
+ */
+message KeyValue {
+ required bytes row = 1;
+ required bytes family = 2;
+ required bytes qualifier = 3;
+ optional uint64 timestamp = 4;
+ optional CellType key_type = 5;
+ optional bytes value = 6;
+ optional bytes tags = 7;
+}
diff --git a/if/Client.proto b/if/Client.proto
new file mode 100644
index 0000000..ca9abdc
--- /dev/null
+++ b/if/Client.proto
@@ -0,0 +1,472 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers that are used for Client service.
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "ClientProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "Filter.proto";
+import "Cell.proto";
+import "Comparator.proto";
+import "MapReduce.proto";
+
+/**
+ * The protocol buffer version of Authorizations.
+ */
+message Authorizations {
+ repeated string label = 1;
+}
+
+/**
+ * The protocol buffer version of CellVisibility.
+ */
+message CellVisibility {
+ required string expression = 1;
+}
+
+/**
+ * Container for a list of column qualifier names of a family.
+ */
+message Column {
+ required bytes family = 1;
+ repeated bytes qualifier = 2;
+}
+
+/**
+ * Consistency defines the expected consistency level for an operation.
+ */
+enum Consistency {
+ STRONG = 0;
+ TIMELINE = 1;
+}
+
+/**
+ * The protocol buffer version of Get.
+ * Unless existence_only is specified, return all the requested data
+ * for the row that matches exactly.
+ */
+message Get {
+ required bytes row = 1;
+ repeated Column column = 2;
+ repeated NameBytesPair attribute = 3;
+ optional Filter filter = 4;
+ optional TimeRange time_range = 5;
+ optional uint32 max_versions = 6 [default = 1];
+ optional bool cache_blocks = 7 [default = true];
+ optional uint32 store_limit = 8;
+ optional uint32 store_offset = 9;
+
+ // The result isn't asked for, just check for
+ // the existence.
+ optional bool existence_only = 10 [default = false];
+
+ optional Consistency consistency = 12 [default = STRONG];
+ repeated ColumnFamilyTimeRange cf_time_range = 13;
+}
+
+message Result {
+ // Result includes the Cells or else it just has a count of Cells
+ // that are carried otherwise.
+ repeated Cell cell = 1;
+ // The below count is set when the associated cells are
+ // not part of this protobuf message; they are passed alongside
+ // and then this Message is just a placeholder with metadata.
+ // The count is needed to know how many to peel off the block of Cells as
+ // ours. NOTE: This is different from the pb managed cell_count of the
+ // 'cell' field above which is non-null when the cells are pb'd.
+ optional int32 associated_cell_count = 2;
+
+ // used for Get to check existence only. Not set if existence_only was not set to true
+ // in the query.
+ optional bool exists = 3;
+
+ // Whether or not the results are coming from possibly stale data
+ optional bool stale = 4 [default = false];
+
+ // Whether or not the entire result could be returned. Results will be split when
+ // the RPC chunk size limit is reached. Partial results contain only a subset of the
+ // cells for a row and must be combined with a result containing the remaining cells
+ // to form a complete result
+ optional bool partial = 5 [default = false];
+}
+
+/**
+ * The get request. Perform a single Get operation.
+ */
+message GetRequest {
+ required RegionSpecifier region = 1;
+ required Get get = 2;
+}
+
+message GetResponse {
+ optional Result result = 1;
+}
+
+/**
+ * Condition to check if the value of a given cell (row,
+ * family, qualifier) matches a value via a given comparator.
+ *
+ * Condition is used in check and mutate operations.
+ */
+message Condition {
+ required bytes row = 1;
+ required bytes family = 2;
+ required bytes qualifier = 3;
+ required CompareType compare_type = 4;
+ required Comparator comparator = 5;
+}
+
+
+/**
+ * A specific mutation inside a mutate request.
+ * It can be an append, increment, put or delete based
+ * on the mutation type. It can be fully filled in or
+ * only metadata present because data is being carried
+ * elsewhere outside of pb.
+ */
+message MutationProto {
+ optional bytes row = 1;
+ optional MutationType mutate_type = 2;
+ repeated ColumnValue column_value = 3;
+ optional uint64 timestamp = 4;
+ repeated NameBytesPair attribute = 5;
+ optional Durability durability = 6 [default = USE_DEFAULT];
+
+ // For some mutations, a result may be returned, in which case,
+ // time range can be specified for potential performance gain
+ optional TimeRange time_range = 7;
+ // The below count is set when the associated cells are NOT
+ // part of this protobuf message; they are passed alongside
+ // and then this Message is a placeholder with metadata. The
+ // count is needed to know how many to peel off the block of Cells as
+ // ours. NOTE: This is different from the pb managed cell_count of the
+ // 'cell' field above which is non-null when the cells are pb'd.
+ optional int32 associated_cell_count = 8;
+
+ optional uint64 nonce = 9;
+
+ enum Durability {
+ USE_DEFAULT = 0;
+ SKIP_WAL = 1;
+ ASYNC_WAL = 2;
+ SYNC_WAL = 3;
+ FSYNC_WAL = 4;
+ }
+
+ enum MutationType {
+ APPEND = 0;
+ INCREMENT = 1;
+ PUT = 2;
+ DELETE = 3;
+ }
+
+ enum DeleteType {
+ DELETE_ONE_VERSION = 0;
+ DELETE_MULTIPLE_VERSIONS = 1;
+ DELETE_FAMILY = 2;
+ DELETE_FAMILY_VERSION = 3;
+ }
+
+ message ColumnValue {
+ required bytes family = 1;
+ repeated QualifierValue qualifier_value = 2;
+
+ message QualifierValue {
+ optional bytes qualifier = 1;
+ optional bytes value = 2;
+ optional uint64 timestamp = 3;
+ optional DeleteType delete_type = 4;
+ optional bytes tags = 5;
+ }
+ }
+}
+
+/**
+ * The mutate request. Perform a single Mutate operation.
+ *
+ * Optionally, you can specify a condition. The mutate
+ * will take place only if the condition is met. Otherwise,
+ * the mutate will be ignored. In the response result,
+ * parameter processed is used to indicate if the mutate
+ * actually happened.
+ */
+message MutateRequest {
+ required RegionSpecifier region = 1;
+ required MutationProto mutation = 2;
+ optional Condition condition = 3;
+ optional uint64 nonce_group = 4;
+}
+
+message MutateResponse {
+ optional Result result = 1;
+
+ // used for mutate to indicate processed only
+ optional bool processed = 2;
+}
+
+/**
+ * Instead of get from a table, you can scan it with optional filters.
+ * You can specify the row key range, time range, the columns/families
+ * to scan and so on.
+ *
+ * This scan is used the first time in a scan request. The response of
+ * the initial scan will return a scanner id, which should be used to
+ * fetch result batches later on before it is closed.
+ */
+message Scan {
+ repeated Column column = 1;
+ repeated NameBytesPair attribute = 2;
+ optional bytes start_row = 3;
+ optional bytes stop_row = 4;
+ optional Filter filter = 5;
+ optional TimeRange time_range = 6;
+ optional uint32 max_versions = 7 [default = 1];
+ optional bool cache_blocks = 8 [default = true];
+ optional uint32 batch_size = 9;
+ optional uint64 max_result_size = 10;
+ optional uint32 store_limit = 11;
+ optional uint32 store_offset = 12;
+ optional bool load_column_families_on_demand = 13; /* DO NOT add defaults to load_column_families_on_demand. */
+ optional bool small = 14;
+ optional bool reversed = 15 [default = false];
+ optional Consistency consistency = 16 [default = STRONG];
+ optional uint32 caching = 17;
+ optional bool allow_partial_results = 18;
+ repeated ColumnFamilyTimeRange cf_time_range = 19;
+}
+
+/**
+ * A scan request. Initially, it should specify a scan. Later on, you
+ * can use the scanner id returned to fetch result batches with a different
+ * scan request.
+ *
+ * The scanner will remain open if there are more results, and it's not
+ * asked to be closed explicitly.
+ *
+ * You can fetch the results and ask the scanner to be closed to save
+ * a trip if you are not interested in remaining results.
+ */
+message ScanRequest {
+ optional RegionSpecifier region = 1;
+ optional Scan scan = 2;
+ optional uint64 scanner_id = 3;
+ optional uint32 number_of_rows = 4;
+ optional bool close_scanner = 5;
+ optional uint64 next_call_seq = 6;
+ optional bool client_handles_partials = 7;
+ optional bool client_handles_heartbeats = 8;
+ optional bool track_scan_metrics = 9;
+ optional bool renew = 10 [default = false];
+}
+
+/**
+ * The scan response. If there are no more results, more_results will
+ * be false. If it is not specified, it means there are more.
+ */
+message ScanResponse {
+ // This field is filled in if we are doing cellblocks. A cellblock is made up
+ // of all Cells serialized out as one cellblock BUT responses from a server
+ // have their Cells grouped by Result. So we can reconstitute the
+ // Results on the client-side, this field is a list of counts of Cells
+ // in each Result that makes up the response. For example, if this field
+ // has 3, 3, 3 in it, then we know that on the client, we are to make
+ // three Results each of three Cells each.
+ repeated uint32 cells_per_result = 1;
+
+ optional uint64 scanner_id = 2;
+ optional bool more_results = 3;
+ optional uint32 ttl = 4;
+ // If cells are not carried in an accompanying cellblock, then they are pb'd here.
+ // This field is mutually exclusive with cells_per_result (since the Cells will
+ // be inside the pb'd Result)
+ repeated Result results = 5;
+ optional bool stale = 6;
+
+ // This field is filled in if we are doing cellblocks. In the event that a row
+ // could not fit all of its cells into a single RPC chunk, the results will be
+ // returned as partials, and reconstructed into a complete result on the client
+ // side. This field is a list of flags indicating whether or not the result
+ // that the cells belong to is a partial result. For example, if this field
+ // has false, false, true in it, then we know that on the client side, we need to
+ // make another RPC request since the last result was only a partial.
+ repeated bool partial_flag_per_result = 7;
+
+ // A server may choose to limit the number of results returned to the client for
+ // reasons such as the size in bytes or quantity of results accumulated. This field
+ // will true when more results exist in the current region.
+ optional bool more_results_in_region = 8;
+
+ // This field is filled in if the server is sending back a heartbeat message.
+ // Heartbeat messages are sent back to the client to prevent the scanner from
+ // timing out. Seeing a heartbeat message communicates to the Client that the
+ // server would have continued to scan had the time limit not been reached.
+ optional bool heartbeat_message = 9;
+
+ // This field is filled in if the client has requested that scan metrics be tracked.
+ // The metrics tracked here are sent back to the client to be tracked together with
+ // the existing client side metrics.
+ optional ScanMetrics scan_metrics = 10;
+}
+
+/**
+ * Atomically bulk load multiple HFiles (say from different column families)
+ * into an open region.
+ */
+message BulkLoadHFileRequest {
+ required RegionSpecifier region = 1;
+ repeated FamilyPath family_path = 2;
+ optional bool assign_seq_num = 3;
+
+ message FamilyPath {
+ required bytes family = 1;
+ required string path = 2;
+ }
+}
+
+message BulkLoadHFileResponse {
+ required bool loaded = 1;
+}
+
+message CoprocessorServiceCall {
+ required bytes row = 1;
+ required string service_name = 2;
+ required string method_name = 3;
+ required bytes request = 4;
+}
+
+message CoprocessorServiceResult {
+ optional NameBytesPair value = 1;
+}
+
+message CoprocessorServiceRequest {
+ required RegionSpecifier region = 1;
+ required CoprocessorServiceCall call = 2;
+}
+
+message CoprocessorServiceResponse {
+ required RegionSpecifier region = 1;
+ required NameBytesPair value = 2;
+}
+
+// Either a Get or a Mutation
+message Action {
+ // If part of a multi action, useful aligning
+ // result with what was originally submitted.
+ optional uint32 index = 1;
+ optional MutationProto mutation = 2;
+ optional Get get = 3;
+ optional CoprocessorServiceCall service_call = 4;
+}
+
+/**
+ * Actions to run against a Region.
+ */
+message RegionAction {
+ required RegionSpecifier region = 1;
+ // When set, run mutations as atomic unit.
+ optional bool atomic = 2;
+ repeated Action action = 3;
+}
+
+/*
+* Statistics about the current load on the region
+*/
+message RegionLoadStats {
+ // Percent load on the memstore. Guaranteed to be positive, between 0 and 100.
+ optional int32 memstoreLoad = 1 [default = 0];
+ // Percent JVM heap occupancy. Guaranteed to be positive, between 0 and 100.
+ // We can move this to "ServerLoadStats" should we develop them.
+ optional int32 heapOccupancy = 2 [default = 0];
+ // Compaction pressure. Guaranteed to be positive, between 0 and 100.
+ optional int32 compactionPressure = 3 [default = 0];
+}
+
+/**
+ * Either a Result or an Exception NameBytesPair (keyed by
+ * exception name whose value is the exception stringified)
+ * or maybe empty if no result and no exception.
+ */
+message ResultOrException {
+ // If part of a multi call, save original index of the list of all
+ // passed so can align this response w/ original request.
+ optional uint32 index = 1;
+ optional Result result = 2;
+ optional NameBytesPair exception = 3;
+ // result if this was a coprocessor service call
+ optional CoprocessorServiceResult service_result = 4;
+ // current load on the region
+ optional RegionLoadStats loadStats = 5;
+}
+
+/**
+ * The result of a RegionAction.
+ */
+message RegionActionResult {
+ repeated ResultOrException resultOrException = 1;
+ // If the operation failed globally for this region, this exception is set
+ optional NameBytesPair exception = 2;
+}
+
+/**
+ * Execute a list of actions on a given region in order.
+ * Nothing prevents a request to contains a set of RegionAction on the same region.
+ * For this reason, the matching between the MultiRequest and the MultiResponse is not
+ * done by the region specifier but by keeping the order of the RegionActionResult vs.
+ * the order of the RegionAction.
+ */
+message MultiRequest {
+ repeated RegionAction regionAction = 1;
+ optional uint64 nonceGroup = 2;
+ optional Condition condition = 3;
+}
+
+message MultiResponse {
+ repeated RegionActionResult regionActionResult = 1;
+ // used for mutate to indicate processed only
+ optional bool processed = 2;
+}
+
+
+service ClientService {
+ rpc Get(GetRequest)
+ returns(GetResponse);
+
+ rpc Mutate(MutateRequest)
+ returns(MutateResponse);
+
+ rpc Scan(ScanRequest)
+ returns(ScanResponse);
+
+ rpc BulkLoadHFile(BulkLoadHFileRequest)
+ returns(BulkLoadHFileResponse);
+
+ rpc ExecService(CoprocessorServiceRequest)
+ returns(CoprocessorServiceResponse);
+
+ rpc ExecRegionServerService(CoprocessorServiceRequest)
+ returns(CoprocessorServiceResponse);
+
+ rpc Multi(MultiRequest)
+ returns(MultiResponse);
+}
diff --git a/core/client.cc b/if/ClusterId.proto
similarity index 61%
copy from core/client.cc
copy to if/ClusterId.proto
index 98cf38a..aed8cfc 100644
--- a/core/client.cc
+++ b/if/ClusterId.proto
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -14,25 +14,21 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
- *
*/
-#include "core/client.h"
-
-#include <folly/Logging.h>
-#include <folly/Random.h>
-#include <glog/logging.h>
-#include <gflags/gflags.h>
+// This file contains protocol buffers that are shared throughout HBase
+package hbase.pb;
-using namespace folly;
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "ClusterIdProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
-int main(int argc, char *argv[]) {
- google::ParseCommandLineFlags(&argc, &argv, true);
- google::InitGoogleLogging(argv[0]);
-
- FB_LOG_EVERY_MS(INFO, 10000) << "Hello";
- for (long i = 0; i < 10000000; i++) {
- FB_LOG_EVERY_MS(INFO, 1) << Random::rand32();
- }
- return 0;
+/**
+ * Content of the '/hbase/hbaseid', cluster id, znode.
+ * Also cluster of the ${HBASE_ROOTDIR}/hbase.id file.
+ */
+message ClusterId {
+ // This is the cluster id, a uuid as a String
+ required string cluster_id = 1;
}
diff --git a/if/ClusterStatus.proto b/if/ClusterStatus.proto
new file mode 100644
index 0000000..228be7e
--- /dev/null
+++ b/if/ClusterStatus.proto
@@ -0,0 +1,224 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers that are used for ClustStatus
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "ClusterStatusProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "ClusterId.proto";
+import "FS.proto";
+
+message RegionState {
+ required RegionInfo region_info = 1;
+ required State state = 2;
+ optional uint64 stamp = 3;
+ enum State {
+ OFFLINE = 0; // region is in an offline state
+ PENDING_OPEN = 1; // sent rpc to server to open but has not begun
+ OPENING = 2; // server has begun to open but not yet done
+ OPEN = 3; // server opened region and updated meta
+ PENDING_CLOSE = 4; // sent rpc to server to close but has not begun
+ CLOSING = 5; // server has begun to close but not yet done
+ CLOSED = 6; // server closed region and updated meta
+ SPLITTING = 7; // server started split of a region
+ SPLIT = 8; // server completed split of a region
+ FAILED_OPEN = 9; // failed to open, and won't retry any more
+ FAILED_CLOSE = 10; // failed to close, and won't retry any more
+ MERGING = 11; // server started merge a region
+ MERGED = 12; // server completed merge of a region
+ SPLITTING_NEW = 13; // new region to be created when RS splits a parent
+ // region but hasn't be created yet, or master doesn't
+ // know it's already created
+ MERGING_NEW = 14; // new region to be created when RS merges two
+ // daughter regions but hasn't be created yet, or
+ // master doesn't know it's already created
+ }
+}
+
+message RegionInTransition {
+ required RegionSpecifier spec = 1;
+ required RegionState region_state = 2;
+}
+
+/**
+ * sequence Id of a store
+ */
+message StoreSequenceId {
+ required bytes family_name = 1;
+ required uint64 sequence_id = 2;
+}
+
+/**
+ * contains a sequence id of a region which should be the minimum of its store sequence ids and
+ * list of sequence ids of the region's stores
+ */
+message RegionStoreSequenceIds {
+ required uint64 last_flushed_sequence_id = 1;
+ repeated StoreSequenceId store_sequence_id = 2;
+}
+
+message RegionLoad {
+ /** the region specifier */
+ required RegionSpecifier region_specifier = 1;
+
+ /** the number of stores for the region */
+ optional uint32 stores = 2;
+
+ /** the number of storefiles for the region */
+ optional uint32 storefiles = 3;
+
+ /** the total size of the store files for the region, uncompressed, in MB */
+ optional uint32 store_uncompressed_size_MB = 4;
+
+ /** the current total size of the store files for the region, in MB */
+ optional uint32 storefile_size_MB = 5;
+
+ /** the current size of the memstore for the region, in MB */
+ optional uint32 memstore_size_MB = 6;
+
+ /**
+ * The current total size of root-level store file indexes for the region,
+ * in MB. The same as {@link #rootIndexSizeKB} but in MB.
+ */
+ optional uint32 storefile_index_size_MB = 7;
+
+ /** the current total read requests made to region */
+ optional uint64 read_requests_count = 8;
+
+ /** the current total write requests made to region */
+ optional uint64 write_requests_count = 9;
+
+ /** the total compacting key values in currently running compaction */
+ optional uint64 total_compacting_KVs = 10;
+
+ /** the completed count of key values in currently running compaction */
+ optional uint64 current_compacted_KVs = 11;
+
+ /** The current total size of root-level indexes for the region, in KB. */
+ optional uint32 root_index_size_KB = 12;
+
+ /** The total size of all index blocks, not just the root level, in KB. */
+ optional uint32 total_static_index_size_KB = 13;
+
+ /**
+ * The total size of all Bloom filter blocks, not just loaded into the
+ * block cache, in KB.
+ */
+ optional uint32 total_static_bloom_size_KB = 14;
+
+ /** the most recent sequence Id from cache flush */
+ optional uint64 complete_sequence_id = 15;
+
+ /** The current data locality for region in the regionserver */
+ optional float data_locality = 16;
+
+ optional uint64 last_major_compaction_ts = 17 [default = 0];
+
+ /** the most recent sequence Id of store from cache flush */
+ repeated StoreSequenceId store_complete_sequence_id = 18;
+}
+
+/* Server-level protobufs */
+
+message ReplicationLoadSink {
+ required uint64 ageOfLastAppliedOp = 1;
+ required uint64 timeStampsOfLastAppliedOp = 2;
+}
+
+message ReplicationLoadSource {
+ required string peerID = 1;
+ required uint64 ageOfLastShippedOp = 2;
+ required uint32 sizeOfLogQueue = 3;
+ required uint64 timeStampOfLastShippedOp = 4;
+ required uint64 replicationLag = 5;
+}
+
+message ServerLoad {
+ /** Number of requests since last report. */
+ optional uint64 number_of_requests = 1;
+
+ /** Total Number of requests from the start of the region server. */
+ optional uint64 total_number_of_requests = 2;
+
+ /** the amount of used heap, in MB. */
+ optional uint32 used_heap_MB = 3;
+
+ /** the maximum allowable size of the heap, in MB. */
+ optional uint32 max_heap_MB = 4;
+
+ /** Information on the load of individual regions. */
+ repeated RegionLoad region_loads = 5;
+
+ /**
+ * Regionserver-level coprocessors, e.g., WALObserver implementations.
+ * Region-level coprocessors, on the other hand, are stored inside RegionLoad
+ * objects.
+ */
+ repeated Coprocessor coprocessors = 6;
+
+ /**
+ * Time when incremental (non-total) counts began being calculated (e.g. number_of_requests)
+ * time is measured as the difference, measured in milliseconds, between the current time
+ * and midnight, January 1, 1970 UTC.
+ */
+ optional uint64 report_start_time = 7;
+
+ /**
+ * Time when report was generated.
+ * time is measured as the difference, measured in milliseconds, between the current time
+ * and midnight, January 1, 1970 UTC.
+ */
+ optional uint64 report_end_time = 8;
+
+ /**
+ * The port number that this region server is hosing an info server on.
+ */
+ optional uint32 info_server_port = 9;
+
+ /**
+ * The replicationLoadSource for the replication Source status of this region server.
+ */
+ repeated ReplicationLoadSource replLoadSource = 10;
+
+ /**
+ * The replicationLoadSink for the replication Sink status of this region server.
+ */
+ optional ReplicationLoadSink replLoadSink = 11;
+}
+
+message LiveServerInfo {
+ required ServerName server = 1;
+ required ServerLoad server_load = 2;
+}
+
+message ClusterStatus {
+ optional HBaseVersionFileContent hbase_version = 1;
+ repeated LiveServerInfo live_servers = 2;
+ repeated ServerName dead_servers = 3;
+ repeated RegionInTransition regions_in_transition = 4;
+ optional ClusterId cluster_id = 5;
+ repeated Coprocessor master_coprocessors = 6;
+ optional ServerName master = 7;
+ repeated ServerName backup_masters = 8;
+ optional bool balancer_on = 9;
+}
diff --git a/if/Comparator.proto b/if/Comparator.proto
new file mode 100644
index 0000000..496b68d
--- /dev/null
+++ b/if/Comparator.proto
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers that are used for filters
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "ComparatorProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+// This file contains protocol buffers that are used for comparators (e.g. in filters)
+
+message Comparator {
+ required string name = 1;
+ optional bytes serialized_comparator = 2;
+}
+
+message ByteArrayComparable {
+ optional bytes value = 1;
+}
+
+message BinaryComparator {
+ required ByteArrayComparable comparable = 1;
+}
+
+message LongComparator {
+ required ByteArrayComparable comparable = 1;
+}
+
+message BinaryPrefixComparator {
+ required ByteArrayComparable comparable = 1;
+}
+
+message BitComparator {
+ required ByteArrayComparable comparable = 1;
+ required BitwiseOp bitwise_op = 2;
+
+ enum BitwiseOp {
+ AND = 1;
+ OR = 2;
+ XOR = 3;
+ }
+}
+
+message NullComparator {
+}
+
+message RegexStringComparator {
+ required string pattern = 1;
+ required int32 pattern_flags = 2;
+ required string charset = 3;
+ optional string engine = 4;
+}
+
+message SubstringComparator {
+ required string substr = 1;
+}
diff --git a/core/client.cc b/if/Encryption.proto
similarity index 64%
copy from core/client.cc
copy to if/Encryption.proto
index 98cf38a..97ab5b2 100644
--- a/core/client.cc
+++ b/if/Encryption.proto
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -14,25 +14,20 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
- *
*/
-#include "core/client.h"
-
-#include <folly/Logging.h>
-#include <folly/Random.h>
-#include <glog/logging.h>
-#include <gflags/gflags.h>
-
-using namespace folly;
+// This file contains protocol buffers used for encryption
+package hbase.pb;
-int main(int argc, char *argv[]) {
- google::ParseCommandLineFlags(&argc, &argv, true);
- google::InitGoogleLogging(argv[0]);
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "EncryptionProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
- FB_LOG_EVERY_MS(INFO, 10000) << "Hello";
- for (long i = 0; i < 10000000; i++) {
- FB_LOG_EVERY_MS(INFO, 1) << Random::rand32();
- }
- return 0;
+message WrappedKey {
+ required string algorithm = 1;
+ required uint32 length = 2;
+ required bytes data = 3;
+ optional bytes iv = 4;
+ optional bytes hash = 5;
}
diff --git a/if/ErrorHandling.proto b/if/ErrorHandling.proto
new file mode 100644
index 0000000..be9a743
--- /dev/null
+++ b/if/ErrorHandling.proto
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers that are used for error handling
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "ErrorHandlingProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+/**
+ * Protobuf version of a java.lang.StackTraceElement
+ * so we can serialize exceptions.
+ */
+message StackTraceElementMessage {
+ optional string declaring_class = 1;
+ optional string method_name = 2;
+ optional string file_name = 3;
+ optional int32 line_number = 4;
+}
+
+/**
+ * Cause of a remote failure for a generic exception. Contains
+ * all the information for a generic exception as well as
+ * optional info about the error for generic info passing
+ * (which should be another protobuffed class).
+ */
+message GenericExceptionMessage {
+ optional string class_name = 1;
+ optional string message = 2;
+ optional bytes error_info = 3;
+ repeated StackTraceElementMessage trace = 4;
+}
+
+/**
+ * Exception sent across the wire when a remote task needs
+ * to notify other tasks that it failed and why
+ */
+message ForeignExceptionMessage {
+ optional string source = 1;
+ optional GenericExceptionMessage generic_exception = 2;
+}
diff --git a/core/client.cc b/if/FS.proto
similarity index 57%
copy from core/client.cc
copy to if/FS.proto
index 98cf38a..9e93120 100644
--- a/core/client.cc
+++ b/if/FS.proto
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -14,25 +14,32 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
- *
*/
-#include "core/client.h"
-
-#include <folly/Logging.h>
-#include <folly/Random.h>
-#include <glog/logging.h>
-#include <gflags/gflags.h>
+// This file contains protocol buffers that are written into the filesystem
+package hbase.pb;
-using namespace folly;
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "FSProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
-int main(int argc, char *argv[]) {
- google::ParseCommandLineFlags(&argc, &argv, true);
- google::InitGoogleLogging(argv[0]);
+/**
+ * The ${HBASE_ROOTDIR}/hbase.version file content
+ */
+message HBaseVersionFileContent {
+ required string version = 1;
+}
- FB_LOG_EVERY_MS(INFO, 10000) << "Hello";
- for (long i = 0; i < 10000000; i++) {
- FB_LOG_EVERY_MS(INFO, 1) << Random::rand32();
+/**
+ * Reference file content used when we split an hfile under a region.
+ */
+message Reference {
+ required bytes splitkey = 1;
+ enum Range {
+ TOP = 0;
+ BOTTOM = 1;
}
- return 0;
+ required Range range = 2;
}
+
diff --git a/if/Filter.proto b/if/Filter.proto
new file mode 100644
index 0000000..67d5717
--- /dev/null
+++ b/if/Filter.proto
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers that are used for filters
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "FilterProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "Comparator.proto";
+
+message Filter {
+ required string name = 1;
+ optional bytes serialized_filter = 2;
+}
+
+message ColumnCountGetFilter {
+ required int32 limit = 1;
+}
+
+message ColumnPaginationFilter {
+ required int32 limit = 1;
+ optional int32 offset = 2;
+ optional bytes column_offset = 3;
+}
+
+message ColumnPrefixFilter {
+ required bytes prefix = 1;
+}
+
+message ColumnRangeFilter {
+ optional bytes min_column = 1;
+ optional bool min_column_inclusive = 2;
+ optional bytes max_column = 3;
+ optional bool max_column_inclusive = 4;
+}
+
+message CompareFilter {
+ required CompareType compare_op = 1;
+ optional Comparator comparator = 2;
+}
+
+message DependentColumnFilter {
+ required CompareFilter compare_filter = 1;
+ optional bytes column_family = 2;
+ optional bytes column_qualifier = 3;
+ optional bool drop_dependent_column = 4;
+}
+
+message FamilyFilter {
+ required CompareFilter compare_filter = 1;
+}
+
+message FilterList {
+ required Operator operator = 1;
+ repeated Filter filters = 2;
+
+ enum Operator {
+ MUST_PASS_ALL = 1;
+ MUST_PASS_ONE = 2;
+ }
+}
+
+message FilterWrapper {
+ required Filter filter = 1;
+}
+
+message FirstKeyOnlyFilter {
+}
+
+message FirstKeyValueMatchingQualifiersFilter {
+ repeated bytes qualifiers = 1;
+}
+
+message FuzzyRowFilter {
+ repeated BytesBytesPair fuzzy_keys_data = 1;
+}
+
+message InclusiveStopFilter {
+ optional bytes stop_row_key = 1;
+}
+
+message KeyOnlyFilter {
+ required bool len_as_val = 1;
+}
+
+message MultipleColumnPrefixFilter {
+ repeated bytes sorted_prefixes = 1;
+}
+
+message PageFilter {
+ required int64 page_size = 1;
+}
+
+message PrefixFilter {
+ optional bytes prefix = 1;
+}
+
+message QualifierFilter {
+ required CompareFilter compare_filter = 1;
+}
+
+message RandomRowFilter {
+ required float chance = 1;
+}
+
+message RowFilter {
+ required CompareFilter compare_filter = 1;
+}
+
+message SingleColumnValueExcludeFilter {
+ required SingleColumnValueFilter single_column_value_filter = 1;
+}
+
+message SingleColumnValueFilter {
+ optional bytes column_family = 1;
+ optional bytes column_qualifier = 2;
+ required CompareType compare_op = 3;
+ required Comparator comparator = 4;
+ optional bool filter_if_missing = 5;
+ optional bool latest_version_only = 6;
+}
+
+message SkipFilter {
+ required Filter filter = 1;
+}
+
+message TimestampsFilter {
+ repeated int64 timestamps = 1 [packed=true];
+}
+
+message ValueFilter {
+ required CompareFilter compare_filter = 1;
+}
+
+message WhileMatchFilter {
+ required Filter filter = 1;
+}
+message FilterAllFilter {
+}
+
+message RowRange {
+ optional bytes start_row = 1;
+ optional bool start_row_inclusive = 2;
+ optional bytes stop_row = 3;
+ optional bool stop_row_inclusive =4;
+}
+
+message MultiRowRangeFilter {
+ repeated RowRange row_range_list = 1;
+}
\ No newline at end of file
diff --git a/if/HBase.proto b/if/HBase.proto
new file mode 100644
index 0000000..e5c967a
--- /dev/null
+++ b/if/HBase.proto
@@ -0,0 +1,258 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers that are shared throughout HBase
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "HBaseProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "Cell.proto";
+
+/**
+ * Table Name
+ */
+message TableName {
+ required bytes namespace = 1;
+ required bytes qualifier = 2;
+}
+
+/**
+ * Table Schema
+ * Inspired by the rest TableSchema
+ */
+message TableSchema {
+ optional TableName table_name = 1;
+ repeated BytesBytesPair attributes = 2;
+ repeated ColumnFamilySchema column_families = 3;
+ repeated NameStringPair configuration = 4;
+}
+
+/** Denotes state of the table */
+message TableState {
+ // Table's current state
+ enum State {
+ ENABLED = 0;
+ DISABLED = 1;
+ DISABLING = 2;
+ ENABLING = 3;
+ }
+ // This is the table's state.
+ required State state = 1;
+}
+
+/** On HDFS representation of table state. */
+message TableDescriptor {
+ required TableSchema schema = 1;
+}
+
+/**
+ * Column Family Schema
+ * Inspired by the rest ColumSchemaMessage
+ */
+message ColumnFamilySchema {
+ required bytes name = 1;
+ repeated BytesBytesPair attributes = 2;
+ repeated NameStringPair configuration = 3;
+}
+
+/**
+ * Protocol buffer version of HRegionInfo.
+ */
+message RegionInfo {
+ required uint64 region_id = 1;
+ required TableName table_name = 2;
+ optional bytes start_key = 3;
+ optional bytes end_key = 4;
+ optional bool offline = 5;
+ optional bool split = 6;
+ optional int32 replica_id = 7 [default = 0];
+}
+
+/**
+ * Protocol buffer for favored nodes
+ */
+message FavoredNodes {
+ repeated ServerName favored_node = 1;
+}
+
+/**
+ * Container protocol buffer to specify a region.
+ * You can specify region by region name, or the hash
+ * of the region name, which is known as encoded
+ * region name.
+ */
+message RegionSpecifier {
+ required RegionSpecifierType type = 1;
+ required bytes value = 2;
+
+ enum RegionSpecifierType {
+ // <tablename>,<startkey>,<regionId>.<encodedName>
+ REGION_NAME = 1;
+
+ // hash of <tablename>,<startkey>,<regionId>
+ ENCODED_REGION_NAME = 2;
+ }
+}
+
+/**
+ * A range of time. Both from and to are Java time
+ * stamp in milliseconds. If you don't specify a time
+ * range, it means all time. By default, if not
+ * specified, from = 0, and to = Long.MAX_VALUE
+ */
+message TimeRange {
+ optional uint64 from = 1;
+ optional uint64 to = 2;
+}
+
+/* ColumnFamily Specific TimeRange */
+message ColumnFamilyTimeRange {
+ required bytes column_family = 1;
+ required TimeRange time_range = 2;
+}
+
+/* Comparison operators */
+enum CompareType {
+ LESS = 0;
+ LESS_OR_EQUAL = 1;
+ EQUAL = 2;
+ NOT_EQUAL = 3;
+ GREATER_OR_EQUAL = 4;
+ GREATER = 5;
+ NO_OP = 6;
+}
+
+/**
+ * Protocol buffer version of ServerName
+ */
+message ServerName {
+ required string host_name = 1;
+ optional uint32 port = 2;
+ optional uint64 start_code = 3;
+}
+
+// Comment data structures
+
+message Coprocessor {
+ required string name = 1;
+}
+
+message NameStringPair {
+ required string name = 1;
+ required string value = 2;
+}
+
+message NameBytesPair {
+ required string name = 1;
+ optional bytes value = 2;
+}
+
+message BytesBytesPair {
+ required bytes first = 1;
+ required bytes second = 2;
+}
+
+message NameInt64Pair {
+ optional string name = 1;
+ optional int64 value = 2;
+}
+
+/**
+ * Description of the snapshot to take
+ */
+message SnapshotDescription {
+ required string name = 1;
+ optional string table = 2; // not needed for delete, but checked for in taking snapshot
+ optional int64 creation_time = 3 [default = 0];
+ enum Type {
+ DISABLED = 0;
+ FLUSH = 1;
+ SKIPFLUSH = 2;
+ }
+ optional Type type = 4 [default = FLUSH];
+ optional int32 version = 5;
+ optional string owner = 6;
+}
+
+/**
+ * Description of the distributed procedure to take
+ */
+message ProcedureDescription {
+ required string signature = 1; // the unique signature of the procedure
+ optional string instance = 2; // the procedure instance name
+ optional int64 creation_time = 3 [default = 0];
+ repeated NameStringPair configuration = 4;
+}
+
+message EmptyMsg {
+}
+
+enum TimeUnit {
+ NANOSECONDS = 1;
+ MICROSECONDS = 2;
+ MILLISECONDS = 3;
+ SECONDS = 4;
+ MINUTES = 5;
+ HOURS = 6;
+ DAYS = 7;
+}
+
+message LongMsg {
+ required int64 long_msg = 1;
+}
+
+message DoubleMsg {
+ required double double_msg = 1;
+}
+
+message BigDecimalMsg {
+ required bytes bigdecimal_msg = 1;
+}
+
+message UUID {
+ required uint64 least_sig_bits = 1;
+ required uint64 most_sig_bits = 2;
+}
+
+message NamespaceDescriptor {
+ required bytes name = 1;
+ repeated NameStringPair configuration = 2;
+}
+
+// Rpc client version info proto. Included in ConnectionHeader on connection setup
+message VersionInfo {
+ required string version = 1;
+ required string url = 2;
+ required string revision = 3;
+ required string user = 4;
+ required string date = 5;
+ required string src_checksum = 6;
+ optional uint32 version_major = 7;
+ optional uint32 version_minor = 8;
+}
+
+/**
+ * Description of the region server info
+ */
+message RegionServerInfo {
+ optional int32 infoPort = 1;
+ optional VersionInfo version_info = 2;
+}
diff --git a/if/HFile.proto b/if/HFile.proto
new file mode 100644
index 0000000..5c5e4f3
--- /dev/null
+++ b/if/HFile.proto
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "HFileProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+
+// Map of name/values
+message FileInfoProto {
+ repeated BytesBytesPair map_entry = 1;
+}
+
+// HFile file trailer
+message FileTrailerProto {
+ optional uint64 file_info_offset = 1;
+ optional uint64 load_on_open_data_offset = 2;
+ optional uint64 uncompressed_data_index_size = 3;
+ optional uint64 total_uncompressed_bytes = 4;
+ optional uint32 data_index_count = 5;
+ optional uint32 meta_index_count = 6;
+ optional uint64 entry_count = 7;
+ optional uint32 num_data_index_levels = 8;
+ optional uint64 first_data_block_offset = 9;
+ optional uint64 last_data_block_offset = 10;
+ optional string comparator_class_name = 11;
+ optional uint32 compression_codec = 12;
+ optional bytes encryption_key = 13;
+}
diff --git a/core/client.h b/if/LoadBalancer.proto
similarity index 68%
copy from core/client.h
copy to if/LoadBalancer.proto
index 921cecc..f9c5d0d 100644
--- a/core/client.h
+++ b/if/LoadBalancer.proto
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -14,11 +14,16 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
- *
*/
-#pragma once
+// This file contains protocol buffers to represent the state of the load balancer.
+package hbase.pb;
-#include <folly/io/IOBuf.h>
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "LoadBalancerProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
-class Client {};
+message LoadBalancerState {
+ optional bool balancer_on = 1;
+}
diff --git a/core/client.cc b/if/MapReduce.proto
similarity index 62%
copy from core/client.cc
copy to if/MapReduce.proto
index 98cf38a..f96ffdf 100644
--- a/core/client.cc
+++ b/if/MapReduce.proto
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -14,25 +14,24 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
- *
*/
-#include "core/client.h"
+ //This file includes protocol buffers used in MapReduce only.
+package hbase.pb;
-#include <folly/Logging.h>
-#include <folly/Random.h>
-#include <glog/logging.h>
-#include <gflags/gflags.h>
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "MapReduceProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
-using namespace folly;
+import "HBase.proto";
-int main(int argc, char *argv[]) {
- google::ParseCommandLineFlags(&argc, &argv, true);
- google::InitGoogleLogging(argv[0]);
+message ScanMetrics {
+ repeated NameInt64Pair metrics = 1;
+}
- FB_LOG_EVERY_MS(INFO, 10000) << "Hello";
- for (long i = 0; i < 10000000; i++) {
- FB_LOG_EVERY_MS(INFO, 1) << Random::rand32();
- }
- return 0;
+message TableSnapshotRegionSplit {
+ repeated string locations = 2;
+ optional TableSchema table = 3;
+ optional RegionInfo region = 4;
}
diff --git a/if/Master.proto b/if/Master.proto
new file mode 100644
index 0000000..4d3a2e1
--- /dev/null
+++ b/if/Master.proto
@@ -0,0 +1,778 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// All to do with the Master. Includes schema management since these
+// changes are run by the Master process.
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "MasterProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "Client.proto";
+import "ClusterStatus.proto";
+import "ErrorHandling.proto";
+import "Procedure.proto";
+import "Quota.proto";
+
+/* Column-level protobufs */
+
+message AddColumnRequest {
+ required TableName table_name = 1;
+ required ColumnFamilySchema column_families = 2;
+ optional uint64 nonce_group = 3 [default = 0];
+ optional uint64 nonce = 4 [default = 0];
+}
+
+message AddColumnResponse {
+ optional uint64 proc_id = 1;
+}
+
+message DeleteColumnRequest {
+ required TableName table_name = 1;
+ required bytes column_name = 2;
+ optional uint64 nonce_group = 3 [default = 0];
+ optional uint64 nonce = 4 [default = 0];
+}
+
+message DeleteColumnResponse {
+ optional uint64 proc_id = 1;
+}
+
+message ModifyColumnRequest {
+ required TableName table_name = 1;
+ required ColumnFamilySchema column_families = 2;
+ optional uint64 nonce_group = 3 [default = 0];
+ optional uint64 nonce = 4 [default = 0];
+}
+
+message ModifyColumnResponse {
+ optional uint64 proc_id = 1;
+}
+
+/* Region-level Protos */
+
+message MoveRegionRequest {
+ required RegionSpecifier region = 1;
+ optional ServerName dest_server_name = 2;
+}
+
+message MoveRegionResponse {
+}
+
+/**
+ * Dispatch merging the specified regions.
+ */
+message DispatchMergingRegionsRequest {
+ required RegionSpecifier region_a = 1;
+ required RegionSpecifier region_b = 2;
+ optional bool forcible = 3 [default = false];
+}
+
+message DispatchMergingRegionsResponse {
+}
+
+message AssignRegionRequest {
+ required RegionSpecifier region = 1;
+}
+
+message AssignRegionResponse {
+}
+
+message UnassignRegionRequest {
+ required RegionSpecifier region = 1;
+ optional bool force = 2 [default = false];
+}
+
+message UnassignRegionResponse {
+}
+
+message OfflineRegionRequest {
+ required RegionSpecifier region = 1;
+}
+
+message OfflineRegionResponse {
+}
+
+/* Table-level protobufs */
+
+message CreateTableRequest {
+ required TableSchema table_schema = 1;
+ repeated bytes split_keys = 2;
+ optional uint64 nonce_group = 3 [default = 0];
+ optional uint64 nonce = 4 [default = 0];
+}
+
+message CreateTableResponse {
+ optional uint64 proc_id = 1;
+}
+
+message DeleteTableRequest {
+ required TableName table_name = 1;
+ optional uint64 nonce_group = 2 [default = 0];
+ optional uint64 nonce = 3 [default = 0];
+}
+
+message DeleteTableResponse {
+ optional uint64 proc_id = 1;
+}
+
+message TruncateTableRequest {
+ required TableName tableName = 1;
+ optional bool preserveSplits = 2 [default = false];
+ optional uint64 nonce_group = 3 [default = 0];
+ optional uint64 nonce = 4 [default = 0];
+}
+
+message TruncateTableResponse {
+ optional uint64 proc_id = 1;
+}
+
+message EnableTableRequest {
+ required TableName table_name = 1;
+ optional uint64 nonce_group = 2 [default = 0];
+ optional uint64 nonce = 3 [default = 0];
+}
+
+message EnableTableResponse {
+ optional uint64 proc_id = 1;
+}
+
+message DisableTableRequest {
+ required TableName table_name = 1;
+ optional uint64 nonce_group = 2 [default = 0];
+ optional uint64 nonce = 3 [default = 0];
+}
+
+message DisableTableResponse {
+ optional uint64 proc_id = 1;
+}
+
+message ModifyTableRequest {
+ required TableName table_name = 1;
+ required TableSchema table_schema = 2;
+ optional uint64 nonce_group = 3 [default = 0];
+ optional uint64 nonce = 4 [default = 0];
+}
+
+message ModifyTableResponse {
+ optional uint64 proc_id = 1;
+}
+
+/* Namespace-level protobufs */
+
+message CreateNamespaceRequest {
+ required NamespaceDescriptor namespaceDescriptor = 1;
+ optional uint64 nonce_group = 2 [default = 0];
+ optional uint64 nonce = 3 [default = 0];
+}
+
+message CreateNamespaceResponse {
+}
+
+message DeleteNamespaceRequest {
+ required string namespaceName = 1;
+ optional uint64 nonce_group = 2 [default = 0];
+ optional uint64 nonce = 3 [default = 0];
+}
+
+message DeleteNamespaceResponse {
+}
+
+message ModifyNamespaceRequest {
+ required NamespaceDescriptor namespaceDescriptor = 1;
+ optional uint64 nonce_group = 2 [default = 0];
+ optional uint64 nonce = 3 [default = 0];
+}
+
+message ModifyNamespaceResponse {
+}
+
+message GetNamespaceDescriptorRequest {
+ required string namespaceName = 1;
+}
+
+message GetNamespaceDescriptorResponse {
+ required NamespaceDescriptor namespaceDescriptor = 1;
+}
+
+message ListNamespaceDescriptorsRequest {
+}
+
+message ListNamespaceDescriptorsResponse {
+ repeated NamespaceDescriptor namespaceDescriptor = 1;
+}
+
+message ListTableDescriptorsByNamespaceRequest {
+ required string namespaceName = 1;
+}
+
+message ListTableDescriptorsByNamespaceResponse {
+ repeated TableSchema tableSchema = 1;
+}
+
+message ListTableNamesByNamespaceRequest {
+ required string namespaceName = 1;
+}
+
+message ListTableNamesByNamespaceResponse {
+ repeated TableName tableName = 1;
+}
+
+/* Cluster-level protobufs */
+
+
+message ShutdownRequest {
+}
+
+message ShutdownResponse {
+}
+
+message StopMasterRequest {
+}
+
+message StopMasterResponse {
+}
+
+message BalanceRequest {
+ optional bool force = 1;
+}
+
+message BalanceResponse {
+ required bool balancer_ran = 1;
+}
+
+message SetBalancerRunningRequest {
+ required bool on = 1;
+ optional bool synchronous = 2;
+}
+
+message SetBalancerRunningResponse {
+ optional bool prev_balance_value = 1;
+}
+
+message IsBalancerEnabledRequest {
+}
+
+message IsBalancerEnabledResponse {
+ required bool enabled = 1;
+}
+
+message NormalizeRequest {
+}
+
+message NormalizeResponse {
+ required bool normalizer_ran = 1;
+}
+
+message SetNormalizerRunningRequest {
+ required bool on = 1;
+}
+
+message SetNormalizerRunningResponse {
+ optional bool prev_normalizer_value = 1;
+}
+
+message IsNormalizerEnabledRequest {
+}
+
+message IsNormalizerEnabledResponse {
+ required bool enabled = 1;
+}
+
+message RunCatalogScanRequest {
+}
+
+message RunCatalogScanResponse {
+ optional int32 scan_result = 1;
+}
+
+message EnableCatalogJanitorRequest {
+ required bool enable = 1;
+}
+
+message EnableCatalogJanitorResponse {
+ optional bool prev_value = 1;
+}
+
+message IsCatalogJanitorEnabledRequest {
+}
+
+message IsCatalogJanitorEnabledResponse {
+ required bool value = 1;
+}
+
+message SnapshotRequest {
+ required SnapshotDescription snapshot = 1;
+}
+
+message SnapshotResponse {
+ required int64 expected_timeout = 1;
+}
+
+message GetCompletedSnapshotsRequest {
+}
+
+message GetCompletedSnapshotsResponse {
+ repeated SnapshotDescription snapshots = 1;
+}
+
+message DeleteSnapshotRequest {
+ required SnapshotDescription snapshot = 1;
+}
+
+message DeleteSnapshotResponse {
+}
+
+message RestoreSnapshotRequest {
+ required SnapshotDescription snapshot = 1;
+}
+
+message RestoreSnapshotResponse {
+}
+
+/* if you don't send the snapshot, then you will get it back
+ * in the response (if the snapshot is done) so you can check the snapshot
+ */
+message IsSnapshotDoneRequest {
+ optional SnapshotDescription snapshot = 1;
+}
+
+message IsSnapshotDoneResponse {
+ optional bool done = 1 [default = false];
+ optional SnapshotDescription snapshot = 2;
+}
+
+message IsRestoreSnapshotDoneRequest {
+ optional SnapshotDescription snapshot = 1;
+}
+
+message IsRestoreSnapshotDoneResponse {
+ optional bool done = 1 [default = false];
+}
+
+message GetSchemaAlterStatusRequest {
+ required TableName table_name = 1;
+}
+
+message GetSchemaAlterStatusResponse {
+ optional uint32 yet_to_update_regions = 1;
+ optional uint32 total_regions = 2;
+}
+
+message GetTableDescriptorsRequest {
+ repeated TableName table_names = 1;
+ optional string regex = 2;
+ optional bool include_sys_tables = 3 [default=false];
+ optional string namespace = 4;
+}
+
+message GetTableDescriptorsResponse {
+ repeated TableSchema table_schema = 1;
+}
+
+message GetTableNamesRequest {
+ optional string regex = 1;
+ optional bool include_sys_tables = 2 [default=false];
+ optional string namespace = 3;
+}
+
+message GetTableNamesResponse {
+ repeated TableName table_names = 1;
+}
+
+message GetTableStateRequest {
+ required TableName table_name = 1;
+}
+
+message GetTableStateResponse {
+ required TableState table_state = 1;
+}
+
+message GetClusterStatusRequest {
+}
+
+message GetClusterStatusResponse {
+ required ClusterStatus cluster_status = 1;
+}
+
+message IsMasterRunningRequest {
+}
+
+message IsMasterRunningResponse {
+ required bool is_master_running = 1;
+}
+
+message ExecProcedureRequest {
+ required ProcedureDescription procedure = 1;
+}
+
+message ExecProcedureResponse {
+ optional int64 expected_timeout = 1;
+ optional bytes return_data = 2;
+}
+
+message IsProcedureDoneRequest {
+ optional ProcedureDescription procedure = 1;
+}
+
+message IsProcedureDoneResponse {
+ optional bool done = 1 [default = false];
+ optional ProcedureDescription snapshot = 2;
+}
+
+message GetProcedureResultRequest {
+ required uint64 proc_id = 1;
+}
+
+message GetProcedureResultResponse {
+ enum State {
+ NOT_FOUND = 0;
+ RUNNING = 1;
+ FINISHED = 2;
+ }
+
+ required State state = 1;
+ optional uint64 start_time = 2;
+ optional uint64 last_update = 3;
+ optional bytes result = 4;
+ optional ForeignExceptionMessage exception = 5;
+}
+
+message AbortProcedureRequest {
+ required uint64 proc_id = 1;
+ optional bool mayInterruptIfRunning = 2 [default = true];
+}
+
+message AbortProcedureResponse {
+ required bool is_procedure_aborted = 1;
+}
+
+message ListProceduresRequest {
+}
+
+message ListProceduresResponse {
+ repeated Procedure procedure = 1;
+}
+
+message SetQuotaRequest {
+ optional string user_name = 1;
+ optional string user_group = 2;
+ optional string namespace = 3;
+ optional TableName table_name = 4;
+
+ optional bool remove_all = 5;
+ optional bool bypass_globals = 6;
+ optional ThrottleRequest throttle = 7;
+}
+
+message SetQuotaResponse {
+}
+
+message MajorCompactionTimestampRequest {
+ required TableName table_name = 1;
+}
+
+message MajorCompactionTimestampForRegionRequest {
+ required RegionSpecifier region = 1;
+}
+
+message MajorCompactionTimestampResponse {
+ required int64 compaction_timestamp = 1;
+}
+
+message SecurityCapabilitiesRequest {
+}
+
+message SecurityCapabilitiesResponse {
+ enum Capability {
+ SIMPLE_AUTHENTICATION = 0;
+ SECURE_AUTHENTICATION = 1;
+ AUTHORIZATION = 2;
+ CELL_AUTHORIZATION = 3;
+ CELL_VISIBILITY = 4;
+ }
+
+ repeated Capability capabilities = 1;
+}
+
+service MasterService {
+ /** Used by the client to get the number of regions that have received the updated schema */
+ rpc GetSchemaAlterStatus(GetSchemaAlterStatusRequest)
+ returns(GetSchemaAlterStatusResponse);
+
+ /** Get list of TableDescriptors for requested tables. */
+ rpc GetTableDescriptors(GetTableDescriptorsRequest)
+ returns(GetTableDescriptorsResponse);
+
+ /** Get the list of table names. */
+ rpc GetTableNames(GetTableNamesRequest)
+ returns(GetTableNamesResponse);
+
+ /** Return cluster status. */
+ rpc GetClusterStatus(GetClusterStatusRequest)
+ returns(GetClusterStatusResponse);
+
+ /** return true if master is available */
+ rpc IsMasterRunning(IsMasterRunningRequest) returns(IsMasterRunningResponse);
+
+ /** Adds a column to the specified table. */
+ rpc AddColumn(AddColumnRequest)
+ returns(AddColumnResponse);
+
+ /** Deletes a column from the specified table. Table must be disabled. */
+ rpc DeleteColumn(DeleteColumnRequest)
+ returns(DeleteColumnResponse);
+
+ /** Modifies an existing column on the specified table. */
+ rpc ModifyColumn(ModifyColumnRequest)
+ returns(ModifyColumnResponse);
+
+ /** Move the region region to the destination server. */
+ rpc MoveRegion(MoveRegionRequest)
+ returns(MoveRegionResponse);
+
+ /** Master dispatch merging the regions */
+ rpc DispatchMergingRegions(DispatchMergingRegionsRequest)
+ returns(DispatchMergingRegionsResponse);
+
+ /** Assign a region to a server chosen at random. */
+ rpc AssignRegion(AssignRegionRequest)
+ returns(AssignRegionResponse);
+
+ /**
+ * Unassign a region from current hosting regionserver. Region will then be
+ * assigned to a regionserver chosen at random. Region could be reassigned
+ * back to the same server. Use MoveRegion if you want
+ * to control the region movement.
+ */
+ rpc UnassignRegion(UnassignRegionRequest)
+ returns(UnassignRegionResponse);
+
+ /**
+ * Offline a region from the assignment manager's in-memory state. The
+ * region should be in a closed state and there will be no attempt to
+ * automatically reassign the region as in unassign. This is a special
+ * method, and should only be used by experts or hbck.
+ */
+ rpc OfflineRegion(OfflineRegionRequest)
+ returns(OfflineRegionResponse);
+
+ /** Deletes a table */
+ rpc DeleteTable(DeleteTableRequest)
+ returns(DeleteTableResponse);
+
+ /** Truncate a table */
+ rpc truncateTable(TruncateTableRequest)
+ returns(TruncateTableResponse);
+
+ /** Puts the table on-line (only needed if table has been previously taken offline) */
+ rpc EnableTable(EnableTableRequest)
+ returns(EnableTableResponse);
+
+ /** Take table offline */
+ rpc DisableTable(DisableTableRequest)
+ returns(DisableTableResponse);
+
+ /** Modify a table's metadata */
+ rpc ModifyTable(ModifyTableRequest)
+ returns(ModifyTableResponse);
+
+ /** Creates a new table asynchronously */
+ rpc CreateTable(CreateTableRequest)
+ returns(CreateTableResponse);
+
+ /** Shutdown an HBase cluster. */
+ rpc Shutdown(ShutdownRequest)
+ returns(ShutdownResponse);
+
+ /** Stop HBase Master only. Does not shutdown the cluster. */
+ rpc StopMaster(StopMasterRequest)
+ returns(StopMasterResponse);
+
+ /**
+ * Run the balancer. Will run the balancer and if regions to move, it will
+ * go ahead and do the reassignments. Can NOT run for various reasons.
+ * Check logs.
+ */
+ rpc Balance(BalanceRequest)
+ returns(BalanceResponse);
+
+ /**
+ * Turn the load balancer on or off.
+ * If synchronous is true, it waits until current balance() call, if outstanding, to return.
+ */
+ rpc SetBalancerRunning(SetBalancerRunningRequest)
+ returns(SetBalancerRunningResponse);
+
+ /**
+ * Query whether the Region Balancer is running.
+ */
+ rpc IsBalancerEnabled(IsBalancerEnabledRequest)
+ returns(IsBalancerEnabledResponse);
+
+ /**
+ * Run region normalizer. Can NOT run for various reasons. Check logs.
+ */
+ rpc Normalize(NormalizeRequest)
+ returns(NormalizeResponse);
+
+ /**
+ * Turn region normalizer on or off.
+ */
+ rpc SetNormalizerRunning(SetNormalizerRunningRequest)
+ returns(SetNormalizerRunningResponse);
+
+ /**
+ * Query whether region normalizer is enabled.
+ */
+ rpc IsNormalizerEnabled(IsNormalizerEnabledRequest)
+ returns(IsNormalizerEnabledResponse);
+
+ /** Get a run of the catalog janitor */
+ rpc RunCatalogScan(RunCatalogScanRequest)
+ returns(RunCatalogScanResponse);
+
+ /**
+ * Enable the catalog janitor on or off.
+ */
+ rpc EnableCatalogJanitor(EnableCatalogJanitorRequest)
+ returns(EnableCatalogJanitorResponse);
+
+ /**
+ * Query whether the catalog janitor is enabled.
+ */
+ rpc IsCatalogJanitorEnabled(IsCatalogJanitorEnabledRequest)
+ returns(IsCatalogJanitorEnabledResponse);
+
+ /**
+ * Call a master coprocessor endpoint
+ */
+ rpc ExecMasterService(CoprocessorServiceRequest)
+ returns(CoprocessorServiceResponse);
+
+ /**
+ * Create a snapshot for the given table.
+ */
+ rpc Snapshot(SnapshotRequest) returns(SnapshotResponse);
+
+ /**
+ * Get completed snapshots.
+ * Returns a list of snapshot descriptors for completed snapshots
+ */
+ rpc GetCompletedSnapshots(GetCompletedSnapshotsRequest) returns(GetCompletedSnapshotsResponse);
+
+ /**
+ * Delete an existing snapshot. This method can also be used to clean up an aborted snapshot.
+ */
+ rpc DeleteSnapshot(DeleteSnapshotRequest) returns(DeleteSnapshotResponse);
+
+ /**
+ * Determine if the snapshot is done yet.
+ */
+ rpc IsSnapshotDone(IsSnapshotDoneRequest) returns(IsSnapshotDoneResponse);
+
+ /**
+ * Restore a snapshot
+ */
+ rpc RestoreSnapshot(RestoreSnapshotRequest) returns(RestoreSnapshotResponse);
+
+ /**
+ * Determine if the snapshot restore is done yet.
+ */
+ rpc IsRestoreSnapshotDone(IsRestoreSnapshotDoneRequest) returns(IsRestoreSnapshotDoneResponse);
+
+ /**
+ * Execute a distributed procedure.
+ */
+ rpc ExecProcedure(ExecProcedureRequest) returns(ExecProcedureResponse);
+
+ /**
+ * Execute a distributed procedure with return data.
+ */
+ rpc ExecProcedureWithRet(ExecProcedureRequest) returns(ExecProcedureResponse);
+
+ /**
+ * Determine if the procedure is done yet.
+ */
+ rpc IsProcedureDone(IsProcedureDoneRequest) returns(IsProcedureDoneResponse);
+
+ /** return true if master is available */
+ /** rpc IsMasterRunning(IsMasterRunningRequest) returns(IsMasterRunningResponse); */
+
+ /** Modify a namespace's metadata */
+ rpc ModifyNamespace(ModifyNamespaceRequest)
+ returns(ModifyNamespaceResponse);
+
+ /** Creates a new namespace synchronously */
+ rpc CreateNamespace(CreateNamespaceRequest)
+ returns(CreateNamespaceResponse);
+
+ /** Deletes namespace synchronously */
+ rpc DeleteNamespace(DeleteNamespaceRequest)
+ returns(DeleteNamespaceResponse);
+
+ /** Get a namespace descriptor by name */
+ rpc GetNamespaceDescriptor(GetNamespaceDescriptorRequest)
+ returns(GetNamespaceDescriptorResponse);
+
+ /** returns a list of namespaces */
+ rpc ListNamespaceDescriptors(ListNamespaceDescriptorsRequest)
+ returns(ListNamespaceDescriptorsResponse);
+
+ /** returns a list of tables for a given namespace*/
+ rpc ListTableDescriptorsByNamespace(ListTableDescriptorsByNamespaceRequest)
+ returns(ListTableDescriptorsByNamespaceResponse);
+
+ /** returns a list of tables for a given namespace*/
+ rpc ListTableNamesByNamespace(ListTableNamesByNamespaceRequest)
+ returns(ListTableNamesByNamespaceResponse);
+
+ /** returns table state */
+ rpc GetTableState(GetTableStateRequest)
+ returns(GetTableStateResponse);
+
+ /** Apply the new quota settings */
+ rpc SetQuota(SetQuotaRequest) returns(SetQuotaResponse);
+
+ /** Returns the timestamp of the last major compaction */
+ rpc getLastMajorCompactionTimestamp(MajorCompactionTimestampRequest)
+ returns(MajorCompactionTimestampResponse);
+
+ /** Returns the timestamp of the last major compaction */
+ rpc getLastMajorCompactionTimestampForRegion(MajorCompactionTimestampForRegionRequest)
+ returns(MajorCompactionTimestampResponse);
+
+ rpc getProcedureResult(GetProcedureResultRequest)
+ returns(GetProcedureResultResponse);
+
+ /** Returns the security capabilities in effect on the cluster */
+ rpc getSecurityCapabilities(SecurityCapabilitiesRequest)
+ returns(SecurityCapabilitiesResponse);
+
+ /** Abort a procedure */
+ rpc AbortProcedure(AbortProcedureRequest)
+ returns(AbortProcedureResponse);
+
+ /** returns a list of procedures */
+ rpc ListProcedures(ListProceduresRequest)
+ returns(ListProceduresResponse);
+}
diff --git a/if/MasterProcedure.proto b/if/MasterProcedure.proto
new file mode 100644
index 0000000..2d2aff4
--- /dev/null
+++ b/if/MasterProcedure.proto
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "MasterProcedureProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "RPC.proto";
+
+// ============================================================================
+// WARNING - Compatibility rules
+// ============================================================================
+// This .proto contains the data serialized by the master procedures.
+// Each procedure has some state stored to know, which step were executed
+// and what were the parameters or data created by the previous steps.
+// new code should be able to handle the old format or at least fail cleanly
+// triggering a rollback/cleanup.
+//
+// Procedures that are inheriting from a StateMachineProcedure have an enum:
+// - Do not change the number of the 'State' enums.
+// doing so, will cause executing the wrong 'step' on the pending
+// procedures when they will be replayed.
+// - Do not remove items from the enum, new code must be able to handle
+// all the previous 'steps'. There may be pending procedure ready to be
+// recovered replayed. alternative you can make sure that not-known state
+// will result in a failure that will rollback the already executed steps.
+// ============================================================================
+
+enum CreateTableState {
+ CREATE_TABLE_PRE_OPERATION = 1;
+ CREATE_TABLE_WRITE_FS_LAYOUT = 2;
+ CREATE_TABLE_ADD_TO_META = 3;
+ CREATE_TABLE_ASSIGN_REGIONS = 4;
+ CREATE_TABLE_UPDATE_DESC_CACHE = 5;
+ CREATE_TABLE_POST_OPERATION = 6;
+}
+
+message CreateTableStateData {
+ required UserInformation user_info = 1;
+ required TableSchema table_schema = 2;
+ repeated RegionInfo region_info = 3;
+}
+
+enum ModifyTableState {
+ MODIFY_TABLE_PREPARE = 1;
+ MODIFY_TABLE_PRE_OPERATION = 2;
+ MODIFY_TABLE_UPDATE_TABLE_DESCRIPTOR = 3;
+ MODIFY_TABLE_REMOVE_REPLICA_COLUMN = 4;
+ MODIFY_TABLE_DELETE_FS_LAYOUT = 5;
+ MODIFY_TABLE_POST_OPERATION = 6;
+ MODIFY_TABLE_REOPEN_ALL_REGIONS = 7;
+}
+
+message ModifyTableStateData {
+ required UserInformation user_info = 1;
+ optional TableSchema unmodified_table_schema = 2;
+ required TableSchema modified_table_schema = 3;
+ required bool delete_column_family_in_modify = 4;
+}
+
+enum TruncateTableState {
+ TRUNCATE_TABLE_PRE_OPERATION = 1;
+ TRUNCATE_TABLE_REMOVE_FROM_META = 2;
+ TRUNCATE_TABLE_CLEAR_FS_LAYOUT = 3;
+ TRUNCATE_TABLE_CREATE_FS_LAYOUT = 4;
+ TRUNCATE_TABLE_ADD_TO_META = 5;
+ TRUNCATE_TABLE_ASSIGN_REGIONS = 6;
+ TRUNCATE_TABLE_POST_OPERATION = 7;
+}
+
+message TruncateTableStateData {
+ required UserInformation user_info = 1;
+ required bool preserve_splits = 2;
+ optional TableName table_name = 3;
+ optional TableSchema table_schema = 4;
+ repeated RegionInfo region_info = 5;
+}
+
+enum DeleteTableState {
+ DELETE_TABLE_PRE_OPERATION = 1;
+ DELETE_TABLE_REMOVE_FROM_META = 2;
+ DELETE_TABLE_CLEAR_FS_LAYOUT = 3;
+ DELETE_TABLE_UPDATE_DESC_CACHE = 4;
+ DELETE_TABLE_UNASSIGN_REGIONS = 5;
+ DELETE_TABLE_POST_OPERATION = 6;
+}
+
+message DeleteTableStateData {
+ required UserInformation user_info = 1;
+ required TableName table_name = 2;
+ repeated RegionInfo region_info = 3;
+}
+
+enum CreateNamespaceState {
+ CREATE_NAMESPACE_PREPARE = 1;
+ CREATE_NAMESPACE_CREATE_DIRECTORY = 2;
+ CREATE_NAMESPACE_INSERT_INTO_NS_TABLE = 3;
+ CREATE_NAMESPACE_UPDATE_ZK = 4;
+ CREATE_NAMESPACE_SET_NAMESPACE_QUOTA = 5;
+}
+
+message CreateNamespaceStateData {
+ required NamespaceDescriptor namespace_descriptor = 1;
+}
+
+enum ModifyNamespaceState {
+ MODIFY_NAMESPACE_PREPARE = 1;
+ MODIFY_NAMESPACE_UPDATE_NS_TABLE = 2;
+ MODIFY_NAMESPACE_UPDATE_ZK = 3;
+}
+
+message ModifyNamespaceStateData {
+ required NamespaceDescriptor namespace_descriptor = 1;
+ optional NamespaceDescriptor unmodified_namespace_descriptor = 2;
+}
+
+enum DeleteNamespaceState {
+ DELETE_NAMESPACE_PREPARE = 1;
+ DELETE_NAMESPACE_DELETE_FROM_NS_TABLE = 2;
+ DELETE_NAMESPACE_REMOVE_FROM_ZK = 3;
+ DELETE_NAMESPACE_DELETE_DIRECTORIES = 4;
+ DELETE_NAMESPACE_REMOVE_NAMESPACE_QUOTA = 5;
+}
+
+message DeleteNamespaceStateData {
+ required string namespace_name = 1;
+ optional NamespaceDescriptor namespace_descriptor = 2;
+}
+
+enum AddColumnFamilyState {
+ ADD_COLUMN_FAMILY_PREPARE = 1;
+ ADD_COLUMN_FAMILY_PRE_OPERATION = 2;
+ ADD_COLUMN_FAMILY_UPDATE_TABLE_DESCRIPTOR = 3;
+ ADD_COLUMN_FAMILY_POST_OPERATION = 4;
+ ADD_COLUMN_FAMILY_REOPEN_ALL_REGIONS = 5;
+}
+
+message AddColumnFamilyStateData {
+ required UserInformation user_info = 1;
+ required TableName table_name = 2;
+ required ColumnFamilySchema columnfamily_schema = 3;
+ optional TableSchema unmodified_table_schema = 4;
+}
+
+enum ModifyColumnFamilyState {
+ MODIFY_COLUMN_FAMILY_PREPARE = 1;
+ MODIFY_COLUMN_FAMILY_PRE_OPERATION = 2;
+ MODIFY_COLUMN_FAMILY_UPDATE_TABLE_DESCRIPTOR = 3;
+ MODIFY_COLUMN_FAMILY_POST_OPERATION = 4;
+ MODIFY_COLUMN_FAMILY_REOPEN_ALL_REGIONS = 5;
+}
+
+message ModifyColumnFamilyStateData {
+ required UserInformation user_info = 1;
+ required TableName table_name = 2;
+ required ColumnFamilySchema columnfamily_schema = 3;
+ optional TableSchema unmodified_table_schema = 4;
+}
+
+enum DeleteColumnFamilyState {
+ DELETE_COLUMN_FAMILY_PREPARE = 1;
+ DELETE_COLUMN_FAMILY_PRE_OPERATION = 2;
+ DELETE_COLUMN_FAMILY_UPDATE_TABLE_DESCRIPTOR = 3;
+ DELETE_COLUMN_FAMILY_DELETE_FS_LAYOUT = 4;
+ DELETE_COLUMN_FAMILY_POST_OPERATION = 5;
+ DELETE_COLUMN_FAMILY_REOPEN_ALL_REGIONS = 6;
+}
+
+message DeleteColumnFamilyStateData {
+ required UserInformation user_info = 1;
+ required TableName table_name = 2;
+ required bytes columnfamily_name = 3;
+ optional TableSchema unmodified_table_schema = 4;
+}
+
+enum EnableTableState {
+ ENABLE_TABLE_PREPARE = 1;
+ ENABLE_TABLE_PRE_OPERATION = 2;
+ ENABLE_TABLE_SET_ENABLING_TABLE_STATE = 3;
+ ENABLE_TABLE_MARK_REGIONS_ONLINE = 4;
+ ENABLE_TABLE_SET_ENABLED_TABLE_STATE = 5;
+ ENABLE_TABLE_POST_OPERATION = 6;
+}
+
+message EnableTableStateData {
+ required UserInformation user_info = 1;
+ required TableName table_name = 2;
+ required bool skip_table_state_check = 3;
+}
+
+enum DisableTableState {
+ DISABLE_TABLE_PREPARE = 1;
+ DISABLE_TABLE_PRE_OPERATION = 2;
+ DISABLE_TABLE_SET_DISABLING_TABLE_STATE = 3;
+ DISABLE_TABLE_MARK_REGIONS_OFFLINE = 4;
+ DISABLE_TABLE_SET_DISABLED_TABLE_STATE = 5;
+ DISABLE_TABLE_POST_OPERATION = 6;
+}
+
+message DisableTableStateData {
+ required UserInformation user_info = 1;
+ required TableName table_name = 2;
+ required bool skip_table_state_check = 3;
+}
+
+message ServerCrashStateData {
+ required ServerName server_name = 1;
+ optional bool distributed_log_replay = 2;
+ repeated RegionInfo regions_on_crashed_server = 3;
+ repeated RegionInfo regions_assigned = 4;
+ optional bool carrying_meta = 5;
+ optional bool should_split_wal = 6 [default = true];
+}
+
+enum ServerCrashState {
+ SERVER_CRASH_START = 1;
+ SERVER_CRASH_PROCESS_META = 2;
+ SERVER_CRASH_GET_REGIONS = 3;
+ SERVER_CRASH_NO_SPLIT_LOGS = 4;
+ SERVER_CRASH_SPLIT_LOGS = 5;
+ SERVER_CRASH_PREPARE_LOG_REPLAY = 6;
+ // Removed SERVER_CRASH_CALC_REGIONS_TO_ASSIGN = 7;
+ SERVER_CRASH_ASSIGN = 8;
+ SERVER_CRASH_WAIT_ON_ASSIGN = 9;
+ SERVER_CRASH_FINISH = 100;
+}
diff --git a/core/client.cc b/if/MultiRowMutation.proto
similarity index 55%
copy from core/client.cc
copy to if/MultiRowMutation.proto
index 98cf38a..747afac 100644
--- a/core/client.cc
+++ b/if/MultiRowMutation.proto
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -14,25 +14,32 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
- *
*/
+package hbase.pb;
-#include "core/client.h"
+import "Client.proto";
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "MultiRowMutationProtos";
+option java_generate_equals_and_hash = true;
+option java_generic_services = true;
+option optimize_for = SPEED;
-#include <folly/Logging.h>
-#include <folly/Random.h>
-#include <glog/logging.h>
-#include <gflags/gflags.h>
+message MultiRowMutationProcessorRequest{
+}
-using namespace folly;
+message MultiRowMutationProcessorResponse{
+}
-int main(int argc, char *argv[]) {
- google::ParseCommandLineFlags(&argc, &argv, true);
- google::InitGoogleLogging(argv[0]);
+message MutateRowsRequest {
+ repeated MutationProto mutation_request = 1;
+ optional uint64 nonce_group = 2;
+ optional uint64 nonce = 3;
+}
- FB_LOG_EVERY_MS(INFO, 10000) << "Hello";
- for (long i = 0; i < 10000000; i++) {
- FB_LOG_EVERY_MS(INFO, 1) << Random::rand32();
- }
- return 0;
+message MutateRowsResponse {
}
+
+service MultiRowMutationService {
+ rpc MutateRows(MutateRowsRequest)
+ returns(MutateRowsResponse);
+}
\ No newline at end of file
diff --git a/if/Procedure.proto b/if/Procedure.proto
new file mode 100644
index 0000000..55e44a4
--- /dev/null
+++ b/if/Procedure.proto
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "ProcedureProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "ErrorHandling.proto";
+
+enum ProcedureState {
+ INITIALIZING = 1; // Procedure in construction, not yet added to the executor
+ RUNNABLE = 2; // Procedure added to the executor, and ready to be executed
+ WAITING = 3; // The procedure is waiting on children to be completed
+ WAITING_TIMEOUT = 4; // The procedure is waiting a timout or an external event
+ ROLLEDBACK = 5; // The procedure failed and was rolledback
+ FINISHED = 6; // The procedure execution is completed. may need a rollback if failed.
+}
+
+/**
+ * Procedure metadata, serialized by the ProcedureStore to be able to recover the old state.
+ */
+message Procedure {
+ // internal "static" state
+ required string class_name = 1; // full classname to be able to instantiate the procedure
+ optional uint64 parent_id = 2; // parent if not a root-procedure otherwise not set
+ required uint64 proc_id = 3;
+ required uint64 start_time = 4;
+ optional string owner = 5;
+
+ // internal "runtime" state
+ required ProcedureState state = 6;
+ repeated uint32 stack_id = 7; // stack indices in case the procedure was running
+ required uint64 last_update = 8;
+ optional uint32 timeout = 9;
+
+ // user state/results
+ optional ForeignExceptionMessage exception = 10;
+ optional bytes result = 11; // opaque (user) result structure
+ optional bytes state_data = 12; // opaque (user) procedure internal-state
+
+ // Nonce to prevent same procedure submit by multiple times
+ optional uint64 nonce_group = 13 [default = 0];
+ optional uint64 nonce = 14 [default = 0];
+}
+
+/**
+ * SequentialProcedure data
+ */
+message SequentialProcedureData {
+ required bool executed = 1;
+}
+
+/**
+ * StateMachineProcedure data
+ */
+message StateMachineProcedureData {
+ repeated uint32 state = 1;
+}
+
+/**
+ * Procedure WAL header
+ */
+message ProcedureWALHeader {
+ required uint32 version = 1;
+ required uint32 type = 2;
+ required uint64 log_id = 3;
+ required uint64 min_proc_id = 4;
+}
+
+/**
+ * Procedure WAL trailer
+ */
+message ProcedureWALTrailer {
+ required uint32 version = 1;
+ required uint64 tracker_pos = 2;
+}
+
+message ProcedureStoreTracker {
+ message TrackerNode {
+ required uint64 start_id = 1;
+ repeated uint64 updated = 2;
+ repeated uint64 deleted = 3;
+ }
+
+ repeated TrackerNode node = 1;
+}
+
+message ProcedureWALEntry {
+ enum Type {
+ PROCEDURE_WAL_EOF = 1;
+ PROCEDURE_WAL_INIT = 2;
+ PROCEDURE_WAL_INSERT = 3;
+ PROCEDURE_WAL_UPDATE = 4;
+ PROCEDURE_WAL_DELETE = 5;
+ PROCEDURE_WAL_COMPACT = 6;
+ }
+
+ required Type type = 1;
+ repeated Procedure procedure = 2;
+ optional uint64 proc_id = 3;
+}
diff --git a/if/Quota.proto b/if/Quota.proto
new file mode 100644
index 0000000..a8303b1
--- /dev/null
+++ b/if/Quota.proto
@@ -0,0 +1,76 @@
+ /**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "QuotaProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+
+enum QuotaScope {
+ CLUSTER = 1;
+ MACHINE = 2;
+}
+
+message TimedQuota {
+ required TimeUnit time_unit = 1;
+ optional uint64 soft_limit = 2;
+ optional float share = 3;
+ optional QuotaScope scope = 4 [default = MACHINE];
+}
+
+enum ThrottleType {
+ REQUEST_NUMBER = 1;
+ REQUEST_SIZE = 2;
+ WRITE_NUMBER = 3;
+ WRITE_SIZE = 4;
+ READ_NUMBER = 5;
+ READ_SIZE = 6;
+}
+
+message Throttle {
+ optional TimedQuota req_num = 1;
+ optional TimedQuota req_size = 2;
+
+ optional TimedQuota write_num = 3;
+ optional TimedQuota write_size = 4;
+
+ optional TimedQuota read_num = 5;
+ optional TimedQuota read_size = 6;
+}
+
+message ThrottleRequest {
+ optional ThrottleType type = 1;
+ optional TimedQuota timed_quota = 2;
+}
+
+enum QuotaType {
+ THROTTLE = 1;
+}
+
+message Quotas {
+ optional bool bypass_globals = 1 [default = false];
+ optional Throttle throttle = 2;
+}
+
+message QuotaUsage {
+}
diff --git a/if/RPC.proto b/if/RPC.proto
new file mode 100644
index 0000000..59bb03d
--- /dev/null
+++ b/if/RPC.proto
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+import "Tracing.proto";
+import "HBase.proto";
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "RPCProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+// See https://issues.apache.org/jira/browse/HBASE-7898 for high-level
+// description of RPC specification.
+//
+// On connection setup, the client sends six bytes of preamble -- a four
+// byte magic, a byte of version, and a byte of authentication type.
+//
+// We then send a "ConnectionHeader" protobuf of user information and the
+// 'protocol' or 'service' that is to be run over this connection as well as
+// info such as codecs and compression to use when we send cell blocks(see below).
+// This connection header protobuf is prefaced by an int that holds the length
+// of this connection header (this is NOT a varint). The pb connection header
+// is sent with Message#writeTo. The server throws an exception if it doesn't
+// like what it was sent noting what it is objecting too. Otherwise, the server
+// says nothing and is open for business.
+//
+// Hereafter the client makes requests and the server returns responses.
+//
+// Requests look like this:
+//
+// <An int with the total length of the request>
+// <RequestHeader Message written out using Message#writeDelimitedTo>
+// <Optionally a Request Parameter Message written out using Message#writeDelimitedTo>
+// <Optionally a Cell block>
+//
+// ...where the Request Parameter Message is whatever the method name stipulated
+// in the RequestHeader expects; e.g. if the method is a scan, then the pb
+// Request Message is a GetRequest, or a ScanRequest. A block of Cells
+// optionally follows. The presence of a Request param Message and/or a
+// block of Cells will be noted in the RequestHeader.
+//
+// Response is the mirror of the request:
+//
+// <An int with the total length of the response>
+// <ResponseHeader Message written out using Message#writeDelimitedTo>
+// <Optionally a Response Result Message written out using Message#writeDelimitedTo>
+// <Optionally a Cell block>
+//
+// ...where the Response Message is the response type that goes with the
+// method specified when making the request and the follow on Cell blocks may
+// or may not be there -- read the response header to find out if one following.
+// If an exception, it will be included inside the Response Header.
+//
+// Any time we write a pb, we do it with Message#writeDelimitedTo EXCEPT when
+// the connection header is sent; this is prefaced by an int with its length
+// and the pb connection header is then written with Message#writeTo.
+//
+
+// User Information proto. Included in ConnectionHeader on connection setup
+message UserInformation {
+ required string effective_user = 1;
+ optional string real_user = 2;
+}
+
+// This is sent on connection setup after the connection preamble is sent.
+message ConnectionHeader {
+ optional UserInformation user_info = 1;
+ optional string service_name = 2;
+ // Cell block codec we will use sending over optional cell blocks. Server throws exception
+ // if cannot deal. Null means no codec'ing going on so we are pb all the time (SLOW!!!)
+ optional string cell_block_codec_class = 3;
+ // Compressor we will use if cell block is compressed. Server will throw exception if not supported.
+ // Class must implement hadoop's CompressionCodec Interface. Can't compress if no codec.
+ optional string cell_block_compressor_class = 4;
+ optional VersionInfo version_info = 5;
+}
+
+// Optional Cell block Message. Included in client RequestHeader
+message CellBlockMeta {
+ // Length of the following cell block. Could calculate it but convenient having it too hand.
+ optional uint32 length = 1;
+}
+
+// At the RPC layer, this message is used to carry
+// the server side exception to the RPC client.
+message ExceptionResponse {
+ // Class name of the exception thrown from the server
+ optional string exception_class_name = 1;
+ // Exception stack trace from the server side
+ optional string stack_trace = 2;
+ // Optional hostname. Filled in for some exceptions such as region moved
+ // where exception gives clue on where the region may have moved.
+ optional string hostname = 3;
+ optional int32 port = 4;
+ // Set if we are NOT to retry on receipt of this exception
+ optional bool do_not_retry = 5;
+}
+
+// Header sent making a request.
+message RequestHeader {
+ // Monotonically increasing call_id to keep track of RPC requests and their response
+ optional uint32 call_id = 1;
+ optional RPCTInfo trace_info = 2;
+ optional string method_name = 3;
+ // If true, then a pb Message param follows.
+ optional bool request_param = 4;
+ // If present, then an encoded data block follows.
+ optional CellBlockMeta cell_block_meta = 5;
+ // 0 is NORMAL priority. 200 is HIGH. If no priority, treat it as NORMAL.
+ // See HConstants.
+ optional uint32 priority = 6;
+}
+
+message ResponseHeader {
+ optional uint32 call_id = 1;
+ // If present, then request threw an exception and no response message (else we presume one)
+ optional ExceptionResponse exception = 2;
+ // If present, then an encoded data block follows.
+ optional CellBlockMeta cell_block_meta = 3;
+}
diff --git a/core/client.h b/if/RegionNormalizer.proto
similarity index 69%
copy from core/client.h
copy to if/RegionNormalizer.proto
index 921cecc..e5305d6 100644
--- a/core/client.h
+++ b/if/RegionNormalizer.proto
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -14,11 +14,15 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
- *
*/
-#pragma once
+// This file contains protocol buffers to represent the state of the load balancer.
-#include <folly/io/IOBuf.h>
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "RegionNormalizerProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
-class Client {};
+message RegionNormalizerState {
+ optional bool normalizer_on = 1;
+}
diff --git a/if/RegionServerStatus.proto b/if/RegionServerStatus.proto
new file mode 100644
index 0000000..fda9de2
--- /dev/null
+++ b/if/RegionServerStatus.proto
@@ -0,0 +1,158 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers that are used for RegionServerStatusProtocol.
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "RegionServerStatusProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "ClusterStatus.proto";
+
+message RegionServerStartupRequest {
+ /** Port number this regionserver is up on */
+ required uint32 port = 1;
+
+ /** This servers' startcode */
+ required uint64 server_start_code = 2;
+
+ /** Current time of the region server in ms */
+ required uint64 server_current_time = 3;
+
+ /** hostname for region server, optional */
+ optional string use_this_hostname_instead = 4;
+}
+
+message RegionServerStartupResponse {
+ /**
+ * Configuration for the regionserver to use: e.g. filesystem,
+ * hbase rootdir, the hostname to use creating the RegionServer ServerName,
+ * etc
+ */
+ repeated NameStringPair map_entries = 1;
+}
+
+message RegionServerReportRequest {
+ required ServerName server = 1;
+
+ /** load the server is under */
+ optional ServerLoad load = 2;
+}
+
+message RegionServerReportResponse {
+}
+
+message ReportRSFatalErrorRequest {
+ /** name of the server experiencing the error */
+ required ServerName server = 1;
+
+ /** informative text to expose in the master logs and UI */
+ required string error_message = 2;
+}
+
+message ReportRSFatalErrorResponse {
+}
+
+message GetLastFlushedSequenceIdRequest {
+ /** region name */
+ required bytes region_name = 1;
+}
+
+message GetLastFlushedSequenceIdResponse {
+ /** the last WAL sequence id flushed from MemStore to HFile for the region */
+ required uint64 last_flushed_sequence_id = 1;
+
+ /** the last WAL sequence id flushed from MemStore to HFile for stores of the region */
+ repeated StoreSequenceId store_last_flushed_sequence_id = 2;
+}
+
+message RegionStateTransition {
+ required TransitionCode transition_code = 1;
+
+ /** Mutliple regions are involved during merging/splitting */
+ repeated RegionInfo region_info = 2;
+
+ /** For newly opened region, the open seq num is needed */
+ optional uint64 open_seq_num = 3;
+
+ enum TransitionCode {
+ OPENED = 0;
+ FAILED_OPEN = 1;
+ /** No failed_close, in which case region server will abort */
+ CLOSED = 2;
+
+ /** Ask master for ok to split/merge region(s) */
+ READY_TO_SPLIT = 3;
+ READY_TO_MERGE = 4;
+
+ SPLIT_PONR = 5;
+ MERGE_PONR = 6;
+
+ SPLIT = 7;
+ MERGED = 8;
+ SPLIT_REVERTED = 9;
+ MERGE_REVERTED = 10;
+ }
+}
+
+message ReportRegionStateTransitionRequest {
+ /** This region server's server name */
+ required ServerName server = 1;
+
+ repeated RegionStateTransition transition = 2;
+}
+
+message ReportRegionStateTransitionResponse {
+ /** Error message if failed to update the region state */
+ optional string error_message = 1;
+}
+
+service RegionServerStatusService {
+ /** Called when a region server first starts. */
+ rpc RegionServerStartup(RegionServerStartupRequest)
+ returns(RegionServerStartupResponse);
+
+ /** Called to report the load the RegionServer is under. */
+ rpc RegionServerReport(RegionServerReportRequest)
+ returns(RegionServerReportResponse);
+
+ /**
+ * Called by a region server to report a fatal error that is causing it to
+ * abort.
+ */
+ rpc ReportRSFatalError(ReportRSFatalErrorRequest)
+ returns(ReportRSFatalErrorResponse);
+
+ /** Called to get the sequence id of the last MemStore entry flushed to an
+ * HFile for a specified region. Used by the region server to speed up
+ * log splitting. */
+ rpc GetLastFlushedSequenceId(GetLastFlushedSequenceIdRequest)
+ returns(GetLastFlushedSequenceIdResponse);
+
+ /**
+ * Called by a region server to report the progress of a region
+ * transition. If the request fails, the transition should
+ * be aborted.
+ */
+ rpc ReportRegionStateTransition(ReportRegionStateTransitionRequest)
+ returns(ReportRegionStateTransitionResponse);
+}
diff --git a/if/RowProcessor.proto b/if/RowProcessor.proto
new file mode 100644
index 0000000..cf2f30f
--- /dev/null
+++ b/if/RowProcessor.proto
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Defines a protocol to perform multi row transactions.
+ * See BaseRowProcessorEndpoint for the implementation.
+ * See HRegion#processRowsWithLocks() for details.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "RowProcessorProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+message ProcessRequest {
+ required string row_processor_class_name = 1;
+ optional string row_processor_initializer_message_name = 2;
+ optional bytes row_processor_initializer_message = 3;
+ optional uint64 nonce_group = 4;
+ optional uint64 nonce = 5;
+}
+
+message ProcessResponse {
+ required bytes row_processor_result = 1;
+}
+
+service RowProcessorService {
+ rpc Process(ProcessRequest) returns (ProcessResponse);
+}
diff --git a/if/SecureBulkLoad.proto b/if/SecureBulkLoad.proto
new file mode 100644
index 0000000..814735b
--- /dev/null
+++ b/if/SecureBulkLoad.proto
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "SecureBulkLoadProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import 'HBase.proto';
+import 'Client.proto';
+
+message SecureBulkLoadHFilesRequest {
+ repeated BulkLoadHFileRequest.FamilyPath family_path = 1;
+ optional bool assign_seq_num = 2;
+ required DelegationToken fs_token = 3;
+ required string bulk_token = 4;
+}
+
+message SecureBulkLoadHFilesResponse {
+ required bool loaded = 1;
+}
+
+message DelegationToken {
+ optional bytes identifier = 1;
+ optional bytes password = 2;
+ optional string kind = 3;
+ optional string service = 4;
+}
+
+message PrepareBulkLoadRequest {
+ required TableName table_name = 1;
+}
+
+message PrepareBulkLoadResponse {
+ required string bulk_token = 1;
+}
+
+message CleanupBulkLoadRequest {
+ required string bulk_token = 1;
+
+}
+
+message CleanupBulkLoadResponse {
+}
+
+service SecureBulkLoadService {
+ rpc PrepareBulkLoad(PrepareBulkLoadRequest)
+ returns (PrepareBulkLoadResponse);
+
+ rpc SecureBulkLoadHFiles(SecureBulkLoadHFilesRequest)
+ returns (SecureBulkLoadHFilesResponse);
+
+ rpc CleanupBulkLoad(CleanupBulkLoadRequest)
+ returns (CleanupBulkLoadResponse);
+}
diff --git a/if/Snapshot.proto b/if/Snapshot.proto
new file mode 100644
index 0000000..ae1a1e6
--- /dev/null
+++ b/if/Snapshot.proto
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "SnapshotProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "FS.proto";
+import "HBase.proto";
+
+message SnapshotFileInfo {
+ enum Type {
+ HFILE = 1;
+ WAL = 2;
+ }
+
+ required Type type = 1;
+
+ optional string hfile = 3;
+
+ optional string wal_server = 4;
+ optional string wal_name = 5;
+}
+
+message SnapshotRegionManifest {
+ optional int32 version = 1;
+
+ required RegionInfo region_info = 2;
+ repeated FamilyFiles family_files = 3;
+
+ message StoreFile {
+ required string name = 1;
+ optional Reference reference = 2;
+
+ // TODO: Add checksums or other fields to verify the file
+ optional uint64 file_size = 3;
+ }
+
+ message FamilyFiles {
+ required bytes family_name = 1;
+ repeated StoreFile store_files = 2;
+ }
+}
+
+message SnapshotDataManifest {
+ required TableSchema table_schema = 1;
+ repeated SnapshotRegionManifest region_manifests = 2;
+}
diff --git a/core/client.cc b/if/Tracing.proto
similarity index 56%
copy from core/client.cc
copy to if/Tracing.proto
index 98cf38a..5a64cfc 100644
--- a/core/client.cc
+++ b/if/Tracing.proto
@@ -1,4 +1,4 @@
-/*
+/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -14,25 +14,20 @@
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
- *
*/
+package hbase.pb;
-#include "core/client.h"
-
-#include <folly/Logging.h>
-#include <folly/Random.h>
-#include <glog/logging.h>
-#include <gflags/gflags.h>
-
-using namespace folly;
-
-int main(int argc, char *argv[]) {
- google::ParseCommandLineFlags(&argc, &argv, true);
- google::InitGoogleLogging(argv[0]);
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "TracingProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
- FB_LOG_EVERY_MS(INFO, 10000) << "Hello";
- for (long i = 0; i < 10000000; i++) {
- FB_LOG_EVERY_MS(INFO, 1) << Random::rand32();
- }
- return 0;
+//Used to pass through the information necessary to continue
+//a trace after an RPC is made. All we need is the traceid
+//(so we know the overarching trace this message is a part of), and
+//the id of the current span when this message was sent, so we know
+//what span caused the new span we will create when this message is received.
+message RPCTInfo {
+ optional int64 trace_id = 1;
+ optional int64 parent_id = 2;
}
diff --git a/if/VisibilityLabels.proto b/if/VisibilityLabels.proto
new file mode 100644
index 0000000..d2dc44d
--- /dev/null
+++ b/if/VisibilityLabels.proto
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "VisibilityLabelsProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "Client.proto";
+
+message VisibilityLabelsRequest {
+ repeated VisibilityLabel visLabel = 1;
+}
+
+message VisibilityLabel {
+ required bytes label = 1;
+ optional uint32 ordinal = 2;
+}
+
+message VisibilityLabelsResponse {
+ repeated RegionActionResult result = 1;
+}
+
+message SetAuthsRequest {
+ required bytes user = 1;
+ repeated bytes auth = 2;
+}
+
+message UserAuthorizations {
+ required bytes user = 1;
+ repeated uint32 auth = 2;
+}
+
+message MultiUserAuthorizations {
+ repeated UserAuthorizations userAuths = 1;
+}
+
+message GetAuthsRequest {
+ required bytes user = 1;
+}
+
+message GetAuthsResponse {
+ required bytes user = 1;
+ repeated bytes auth = 2;
+}
+
+message ListLabelsRequest {
+ optional string regex = 1;
+}
+
+message ListLabelsResponse {
+ repeated bytes label = 1;
+}
+
+service VisibilityLabelsService {
+ rpc addLabels(VisibilityLabelsRequest)
+ returns (VisibilityLabelsResponse);
+ rpc setAuths(SetAuthsRequest)
+ returns (VisibilityLabelsResponse);
+ rpc clearAuths(SetAuthsRequest)
+ returns (VisibilityLabelsResponse);
+ rpc getAuths(GetAuthsRequest)
+ returns (GetAuthsResponse);
+ rpc listLabels(ListLabelsRequest)
+ returns (ListLabelsResponse);
+}
\ No newline at end of file
diff --git a/if/WAL.proto b/if/WAL.proto
new file mode 100644
index 0000000..cb9bd8f
--- /dev/null
+++ b/if/WAL.proto
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "WALProtos";
+option java_generic_services = false;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "Client.proto";
+
+message WALHeader {
+ optional bool has_compression = 1;
+ optional bytes encryption_key = 2;
+ optional bool has_tag_compression = 3;
+ optional string writer_cls_name = 4;
+ optional string cell_codec_cls_name = 5;
+}
+
+/*
+ * Protocol buffer version of WALKey; see WALKey comment, not really a key but WALEdit header
+ * for some KVs
+ */
+message WALKey {
+ required bytes encoded_region_name = 1;
+ required bytes table_name = 2;
+ required uint64 log_sequence_number = 3;
+ required uint64 write_time = 4;
+ /*
+ This parameter is deprecated in favor of clusters which
+ contains the list of clusters that have consumed the change.
+ It is retained so that the log created by earlier releases (0.94)
+ can be read by the newer releases.
+ */
+ optional UUID cluster_id = 5 [deprecated=true];
+
+ repeated FamilyScope scopes = 6;
+ optional uint32 following_kv_count = 7;
+
+ /*
+ This field contains the list of clusters that have
+ consumed the change
+ */
+ repeated UUID cluster_ids = 8;
+
+ optional uint64 nonceGroup = 9;
+ optional uint64 nonce = 10;
+ optional uint64 orig_sequence_number = 11;
+
+/*
+ optional CustomEntryType custom_entry_type = 9;
+
+ enum CustomEntryType {
+ COMPACTION = 0;
+ }
+*/
+}
+
+enum ScopeType {
+ REPLICATION_SCOPE_LOCAL = 0;
+ REPLICATION_SCOPE_GLOBAL = 1;
+}
+
+message FamilyScope {
+ required bytes family = 1;
+ required ScopeType scope_type = 2;
+}
+
+/**
+ * Custom WAL entries
+ */
+
+/**
+ * Special WAL entry to hold all related to a compaction.
+ * Written to WAL before completing compaction. There is
+ * sufficient info in the below message to complete later
+ * the * compaction should we fail the WAL write.
+ */
+message CompactionDescriptor {
+ required bytes table_name = 1; // TODO: WALKey already stores these, might remove
+ required bytes encoded_region_name = 2;
+ required bytes family_name = 3;
+ repeated string compaction_input = 4; // relative to store dir
+ repeated string compaction_output = 5;
+ required string store_home_dir = 6; // relative to region dir
+ optional bytes region_name = 7; // full region name
+}
+
+/**
+ * Special WAL entry to hold all related to a flush.
+ */
+message FlushDescriptor {
+ enum FlushAction {
+ START_FLUSH = 0;
+ COMMIT_FLUSH = 1;
+ ABORT_FLUSH = 2;
+ CANNOT_FLUSH = 3; // marker for indicating that a flush has been requested but cannot complete
+ }
+
+ message StoreFlushDescriptor {
+ required bytes family_name = 1;
+ required string store_home_dir = 2; //relative to region dir
+ repeated string flush_output = 3; // relative to store dir (if this is a COMMIT_FLUSH)
+ }
+
+ required FlushAction action = 1;
+ required bytes table_name = 2;
+ required bytes encoded_region_name = 3;
+ optional uint64 flush_sequence_number = 4;
+ repeated StoreFlushDescriptor store_flushes = 5;
+ optional bytes region_name = 6; // full region name
+}
+
+message StoreDescriptor {
+ required bytes family_name = 1;
+ required string store_home_dir = 2; //relative to region dir
+ repeated string store_file = 3; // relative to store dir
+}
+
+/**
+ * Special WAL entry used for writing bulk load events to WAL
+ */
+message BulkLoadDescriptor {
+ required TableName table_name = 1;
+ required bytes encoded_region_name = 2;
+ repeated StoreDescriptor stores = 3;
+ required int64 bulkload_seq_num = 4;
+}
+
+/**
+ * Special WAL entry to hold all related to a region event (open/close).
+ */
+message RegionEventDescriptor {
+ enum EventType {
+ REGION_OPEN = 0;
+ REGION_CLOSE = 1;
+ }
+
+ required EventType event_type = 1;
+ required bytes table_name = 2;
+ required bytes encoded_region_name = 3;
+ optional uint64 log_sequence_number = 4;
+ repeated StoreDescriptor stores = 5;
+ optional ServerName server = 6; // Server who opened the region
+ optional bytes region_name = 7; // full region name
+}
+
+/**
+ * A trailer that is appended to the end of a properly closed WAL file.
+ * If missing, this is either a legacy or a corrupted WAL file.
+ * N.B. This trailer currently doesn't contain any information and we
+ * purposefully don't expose it in the WAL APIs. It's for future growth.
+ */
+message WALTrailer {
+}
diff --git a/if/ZooKeeper.proto b/if/ZooKeeper.proto
new file mode 100644
index 0000000..54652af
--- /dev/null
+++ b/if/ZooKeeper.proto
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// ZNode data in hbase are serialized protobufs with a four byte
+// 'magic' 'PBUF' prefix.
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "ZooKeeperProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "ClusterStatus.proto";
+
+/**
+ * Content of the meta-region-server znode.
+ */
+message MetaRegionServer {
+ // The ServerName hosting the meta region currently, or destination server,
+ // if meta region is in transition.
+ required ServerName server = 1;
+ // The major version of the rpc the server speaks. This is used so that
+ // clients connecting to the cluster can have prior knowledge of what version
+ // to send to a RegionServer. AsyncHBase will use this to detect versions.
+ optional uint32 rpc_version = 2;
+
+ // State of the region transition. OPEN means fully operational 'hbase:meta'
+ optional RegionState.State state = 3;
+}
+
+/**
+ * Content of the master znode.
+ */
+message Master {
+ // The ServerName of the current Master
+ required ServerName master = 1;
+ // Major RPC version so that clients can know what version the master can accept.
+ optional uint32 rpc_version = 2;
+ optional uint32 info_port = 3;
+}
+
+/**
+ * Content of the '/hbase/running', cluster state, znode.
+ */
+message ClusterUp {
+ // If this znode is present, cluster is up. Currently
+ // the data is cluster start_date.
+ required string start_date = 1;
+}
+
+/**
+ * WAL SplitLog directory znodes have this for content. Used doing distributed
+ * WAL splitting. Holds current state and name of server that originated split.
+ */
+message SplitLogTask {
+ enum State {
+ UNASSIGNED = 0;
+ OWNED = 1;
+ RESIGNED = 2;
+ DONE = 3;
+ ERR = 4;
+ }
+ enum RecoveryMode {
+ UNKNOWN = 0;
+ LOG_SPLITTING = 1;
+ LOG_REPLAY = 2;
+ }
+ required State state = 1;
+ required ServerName server_name = 2;
+ optional RecoveryMode mode = 3 [default = UNKNOWN];
+}
+
+/**
+ * The znode that holds state of table.
+ * Deprected, table state is stored in table descriptor on HDFS.
+ */
+message DeprecatedTableState {
+ // Table's current state
+ enum State {
+ ENABLED = 0;
+ DISABLED = 1;
+ DISABLING = 2;
+ ENABLING = 3;
+ }
+ // This is the table's state. If no znode for a table,
+ // its state is presumed enabled. See o.a.h.h.zookeeper.ZKTable class
+ // for more.
+ required State state = 1 [default = ENABLED];
+}
+
+/**
+ * Used by replication. Holds a replication peer key.
+ */
+message ReplicationPeer {
+ // clusterkey is the concatenation of the slave cluster's
+ // hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent
+ required string clusterkey = 1;
+ optional string replicationEndpointImpl = 2;
+ repeated BytesBytesPair data = 3;
+ repeated NameStringPair configuration = 4;
+}
+
+/**
+ * Used by replication. Holds whether enabled or disabled
+ */
+message ReplicationState {
+ enum State {
+ ENABLED = 0;
+ DISABLED = 1;
+ }
+ required State state = 1;
+}
+
+/**
+ * Used by replication. Holds the current position in an WAL file.
+ */
+message ReplicationHLogPosition {
+ required int64 position = 1;
+}
+
+/**
+ * Used by replication. Used to lock a region server during failover.
+ */
+message ReplicationLock {
+ required string lock_owner = 1;
+}
+
+/**
+ * Metadata associated with a table lock in zookeeper
+ */
+message TableLock {
+ optional TableName table_name = 1;
+ optional ServerName lock_owner = 2;
+ optional int64 thread_id = 3;
+ optional bool is_shared = 4;
+ optional string purpose = 5;
+ optional int64 create_time = 6;
+}
diff --git a/rpc/CMakeLists.txt b/rpc/CMakeLists.txt
deleted file mode 100644
index 2456923..0000000
--- a/rpc/CMakeLists.txt
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
diff --git a/third-party/BUCK b/third-party/BUCK
index 89d2c93..b7baa86 100644
--- a/third-party/BUCK
+++ b/third-party/BUCK
@@ -21,7 +21,8 @@ def add_system_libs(names = []):
prebuilt_cxx_library(
name = name,
lib_name = name,
- lib_dir = "/usr/lib/x86_64-linux-gnu"
+ lib_dir = "/usr/lib/x86_64-linux-gnu",
+ visibility = [ 'PUBLIC', ],
)
rules.append(":" + name)
@@ -30,6 +31,7 @@ def add_system_libs(names = []):
system_libs = [
"double-conversion",
"glog",
+ "protobuf",
"gflags",
"unwind",
"lzma",
[hbase] 51/137: HBASE-17051 [C++] implement RPC client and
connection management (Xiaobing Zhou)
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 062c1462b834f4696a1fafef4ba60106f9bc3760
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Mon Dec 19 18:38:51 2016 -0800
HBASE-17051 [C++] implement RPC client and connection management (Xiaobing Zhou)
---
Makefile | 2 +-
connection/BUCK | 3 +
connection/connection-id.h | 88 +++++++++++++++++++++++++++
connection/connection-pool-test.cc | 35 ++++++-----
connection/connection-pool.cc | 47 ++++++++-------
connection/connection-pool.h | 50 +++++++---------
connection/rpc-client.cc | 119 +++++++++++++++++++++++++++++++++++++
connection/rpc-client.h | 116 ++++++++++++++++++++++++++++++++++++
connection/rpc-connection.h | 58 ++++++++++++++++++
core/location-cache.cc | 17 ++++--
{connection => security}/BUCK | 28 +--------
security/user.h | 39 ++++++++++++
12 files changed, 504 insertions(+), 98 deletions(-)
diff --git a/Makefile b/Makefile
index be5d461..96c275e 100644
--- a/Makefile
+++ b/Makefile
@@ -22,7 +22,7 @@ LD:=g++
DEBUG_PATH = build/debug
RELEASE_PATH = build/release
PROTO_SRC_DIR = build/if
-MODULES = connection core serde test-util utils
+MODULES = connection core serde test-util utils security
SRC_DIR = $(MODULES)
DEBUG_BUILD_DIR = $(addprefix $(DEBUG_PATH)/,$(MODULES))
RELEASE_BUILD_DIR = $(addprefix $(RELEASE_PATH)/,$(MODULES))
diff --git a/connection/BUCK b/connection/BUCK
index f093d5a..c22cc89 100644
--- a/connection/BUCK
+++ b/connection/BUCK
@@ -24,8 +24,10 @@ cxx_library(
"client-handler.h",
"connection-factory.h",
"connection-pool.h",
+ "connection-id.h",
"pipeline.h",
"request.h",
+ "rpc-connection.h",
"response.h",
"service.h",
],
@@ -41,6 +43,7 @@ cxx_library(
"//if:if",
"//utils:utils",
"//serde:serde",
+ "//security:security",
"//third-party:folly",
"//third-party:wangle",
],
diff --git a/connection/connection-id.h b/connection/connection-id.h
new file mode 100644
index 0000000..62fe222
--- /dev/null
+++ b/connection/connection-id.h
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include "if/HBase.pb.h"
+#include "security/user.h"
+
+#include <memory>
+#include <utility>
+#include <boost/functional/hash.hpp>
+
+using hbase::pb::ServerName;
+using hbase::security::User;
+
+namespace hbase {
+class ConnectionId {
+ public:
+ ConnectionId(const std::string &host, uint16_t port)
+ : ConnectionId(host, port, User::defaultUser(), "") {}
+
+ ConnectionId(const std::string &host, uint16_t port,
+ std::shared_ptr<User> user)
+ : ConnectionId(host, port, user, "") {}
+
+ ConnectionId(const std::string &host, uint16_t port,
+ std::shared_ptr<User> user, const std::string &service_name)
+ : user_(user), service_name_(service_name), host_(host), port_(port) {}
+
+ virtual ~ConnectionId() = default;
+
+ std::shared_ptr<User> user() const { return user_; }
+ std::string service_name() const { return service_name_; }
+ std::string host() { return host_; }
+ uint16_t port() { return port_; }
+
+ private:
+ std::shared_ptr<User> user_;
+ std::string service_name_;
+ std::string host_;
+ uint16_t port_;
+};
+
+/* Equals function for ConnectionId */
+struct ConnectionIdEquals {
+ /** equals */
+ bool operator()(const std::shared_ptr<ConnectionId> &lhs,
+ const std::shared_ptr<ConnectionId> &rhs) const {
+ return userEquals(lhs->user(), rhs->user()) && lhs->host() == rhs->host() &&
+ lhs->port() == rhs->port();
+ }
+
+ private:
+ bool userEquals(const std::shared_ptr<User> &lhs,
+ const std::shared_ptr<User> &rhs) const {
+ return lhs == nullptr ? rhs == nullptr
+ : (rhs == nullptr ? false : lhs->user_name() ==
+ rhs->user_name());
+ }
+};
+
+/** Hash for ConnectionId. */
+struct ConnectionIdHash {
+ /** hash */
+ std::size_t operator()(const std::shared_ptr<ConnectionId> &ci) const {
+ std::size_t h = 0;
+ boost::hash_combine(h, ci->user() == nullptr ? 0 : ci->user()->user_name());
+ boost::hash_combine(h, ci->host());
+ boost::hash_combine(h, ci->port());
+ return h;
+ }
+};
+} // namespace hbase
diff --git a/connection/connection-pool-test.cc b/connection/connection-pool-test.cc
index e17c16c..4547b30 100644
--- a/connection/connection-pool-test.cc
+++ b/connection/connection-pool-test.cc
@@ -17,20 +17,22 @@
*
*/
+#include "connection/connection-id.h"
#include "connection/connection-pool.h"
-
-#include <folly/Logging.h>
-#include <gmock/gmock.h>
-
#include "connection/connection-factory.h"
+
#include "if/HBase.pb.h"
#include "serde/server-name.h"
+#include <folly/Logging.h>
+#include <gmock/gmock.h>
+
using namespace hbase;
using hbase::pb::ServerName;
using ::testing::Return;
using ::testing::_;
+using hbase::ConnectionId;
class MockConnectionFactory : public ConnectionFactory {
public:
@@ -75,13 +77,10 @@ TEST(TestConnectionPool, TestOnlyCreateOnce) {
.WillRepeatedly(Return(mock_boot));
ConnectionPool cp{mock_cf};
- ServerName sn;
- sn.set_host_name(hostname);
- sn.set_port(port);
-
- auto result = cp.Get(sn);
+ auto remote_id = std::make_shared<ConnectionId>(hostname, port);
+ auto result = cp.GetConnection(remote_id);
ASSERT_TRUE(result != nullptr);
- result = cp.Get(sn);
+ result = cp.GetConnection(remote_id);
}
TEST(TestConnectionPool, TestOnlyCreateMultipleDispose) {
@@ -102,13 +101,13 @@ TEST(TestConnectionPool, TestOnlyCreateMultipleDispose) {
ConnectionPool cp{mock_cf};
{
- auto result_one = cp.Get(folly::to<ServerName>(
- hostname_one + ":" + folly::to<std::string>(port)));
- auto result_two = cp.Get(folly::to<ServerName>(
- hostname_two + ":" + folly::to<std::string>(port)));
+ auto remote_id = std::make_shared<ConnectionId>(hostname_one, port);
+ auto result_one = cp.GetConnection(remote_id);
+ auto remote_id2 = std::make_shared<ConnectionId>(hostname_two, port);
+ auto result_two = cp.GetConnection(remote_id2);
}
- auto result_one = cp.Get(
- folly::to<ServerName>(hostname_one + ":" + folly::to<std::string>(port)));
- auto result_two = cp.Get(
- folly::to<ServerName>(hostname_two + ":" + folly::to<std::string>(port)));
+ auto remote_id = std::make_shared<ConnectionId>(hostname_one, port);
+ auto result_one = cp.GetConnection(remote_id);
+ auto remote_id2 = std::make_shared<ConnectionId>(hostname_two, port);
+ auto result_two = cp.GetConnection(remote_id2);
}
diff --git a/connection/connection-pool.cc b/connection/connection-pool.cc
index c216d0b..e022f9e 100644
--- a/connection/connection-pool.cc
+++ b/connection/connection-pool.cc
@@ -28,7 +28,6 @@
using std::mutex;
using std::unique_ptr;
using std::shared_ptr;
-using hbase::pb::ServerName;
using hbase::ConnectionPool;
using hbase::HBaseService;
using folly::SharedMutexWritePriority;
@@ -47,33 +46,36 @@ ConnectionPool::~ConnectionPool() {
SharedMutexWritePriority::WriteHolder holder(map_mutex_);
for (auto &item : connections_) {
auto &con = item.second;
- con->close();
+ con->Close();
}
connections_.clear();
clients_.clear();
}
-std::shared_ptr<HBaseService> ConnectionPool::Get(const ServerName &sn) {
+std::shared_ptr<RpcConnection> ConnectionPool::GetConnection(
+ std::shared_ptr<ConnectionId> remote_id) {
// Try and get th cached connection.
- auto found_ptr = GetCached(sn);
+ auto found_ptr = GetCachedConnection(remote_id);
// If there's no connection then create it.
if (found_ptr == nullptr) {
- found_ptr = GetNew(sn);
+ found_ptr = GetNewConnection(remote_id);
}
return found_ptr;
}
-std::shared_ptr<HBaseService> ConnectionPool::GetCached(const ServerName &sn) {
+std::shared_ptr<RpcConnection> ConnectionPool::GetCachedConnection(
+ std::shared_ptr<ConnectionId> remote_id) {
SharedMutexWritePriority::ReadHolder holder(map_mutex_);
- auto found = connections_.find(sn);
+ auto found = connections_.find(remote_id);
if (found == connections_.end()) {
return nullptr;
}
return found->second;
}
-std::shared_ptr<HBaseService> ConnectionPool::GetNew(const ServerName &sn) {
+std::shared_ptr<RpcConnection> ConnectionPool::GetNewConnection(
+ std::shared_ptr<ConnectionId> remote_id) {
// Grab the upgrade lock. While we are double checking other readers can
// continue on
SharedMutexWritePriority::UpgradeHolder u_holder{map_mutex_};
@@ -81,7 +83,7 @@ std::shared_ptr<HBaseService> ConnectionPool::GetNew(const ServerName &sn) {
// Now check if someone else created the connection before we got the lock
// This is safe since we hold the upgrade lock.
// upgrade lock is more power than the reader lock.
- auto found = connections_.find(sn);
+ auto found = connections_.find(remote_id);
if (found != connections_.end() && found->second != nullptr) {
return found->second;
} else {
@@ -89,24 +91,29 @@ std::shared_ptr<HBaseService> ConnectionPool::GetNew(const ServerName &sn) {
SharedMutexWritePriority::WriteHolder w_holder{std::move(u_holder)};
// Make double sure there are not stale connections hanging around.
- connections_.erase(sn);
-
- // Nope we are the ones who should create the new connection.
- auto client = cf_->MakeBootstrap();
- auto dispatcher = cf_->Connect(client, sn.host_name(), sn.port());
- clients_.insert(std::make_pair(sn, client));
- connections_.insert(std::make_pair(sn, dispatcher));
- return dispatcher;
+ connections_.erase(remote_id);
+
+ /* create new connection */
+ auto clientBootstrap = cf_->MakeBootstrap();
+ auto dispatcher =
+ cf_->Connect(clientBootstrap, remote_id->host(), remote_id->port());
+
+ auto conneciton = std::make_shared<RpcConnection>(remote_id, dispatcher);
+
+ connections_.insert(std::make_pair(remote_id, conneciton));
+ clients_.insert(std::make_pair(remote_id, clientBootstrap));
+
+ return conneciton;
}
}
-void ConnectionPool::Close(const ServerName &sn) {
+void ConnectionPool::Close(std::shared_ptr<ConnectionId> remote_id) {
SharedMutexWritePriority::WriteHolder holder{map_mutex_};
- auto found = connections_.find(sn);
+ auto found = connections_.find(remote_id);
if (found == connections_.end() || found->second == nullptr) {
return;
}
- auto service = found->second;
+ found->second->Close();
connections_.erase(found);
}
diff --git a/connection/connection-pool.h b/connection/connection-pool.h
index 2624336..96d89ac 100644
--- a/connection/connection-pool.h
+++ b/connection/connection-pool.h
@@ -25,38 +25,24 @@
#include <unordered_map>
#include "connection/connection-factory.h"
+#include "connection/connection-id.h"
+#include "connection/rpc-connection.h"
#include "connection/service.h"
#include "if/HBase.pb.h"
-namespace hbase {
-/** Equals function for server name that ignores start time */
-struct ServerNameEquals {
- /** equals */
- bool operator()(const hbase::pb::ServerName &lhs,
- const hbase::pb::ServerName &rhs) const {
- return lhs.host_name() == rhs.host_name() && lhs.port() == rhs.port();
- }
-};
+using hbase::ConnectionId;
+using hbase::ConnectionIdEquals;
+using hbase::ConnectionIdHash;
+using hbase::RpcConnection;
-/** Hash for ServerName that ignores the start time. */
-struct ServerNameHash {
- /** hash */
- std::size_t operator()(hbase::pb::ServerName const &s) const {
- std::size_t h = 0;
- boost::hash_combine(h, s.host_name());
- boost::hash_combine(h, s.port());
- return h;
- }
-};
+namespace hbase {
/**
* @brief Connection pooling for HBase rpc connection.
*
* This is a thread safe connection pool. It allows getting
- * a shared connection to HBase by server name. This is
- * useful for keeping a single connection no matter how many regions a
- * regionserver has on it.
+ * a shared rpc connection to HBase servers by connection id.
*/
class ConnectionPool {
public:
@@ -81,23 +67,27 @@ class ConnectionPool {
* Get a connection to the server name. Start time is ignored.
* This can be a blocking operation for a short time.
*/
- std::shared_ptr<HBaseService> Get(const hbase::pb::ServerName &sn);
+ std::shared_ptr<RpcConnection> GetConnection(
+ std::shared_ptr<ConnectionId> remote_id);
/**
* Close/remove a connection.
*/
- void Close(const hbase::pb::ServerName &sn);
+ void Close(std::shared_ptr<ConnectionId> remote_id);
private:
- std::shared_ptr<HBaseService> GetCached(const hbase::pb::ServerName &sn);
- std::shared_ptr<HBaseService> GetNew(const hbase::pb::ServerName &sn);
- std::unordered_map<hbase::pb::ServerName, std::shared_ptr<HBaseService>,
- ServerNameHash, ServerNameEquals>
+ std::shared_ptr<RpcConnection> GetCachedConnection(
+ std::shared_ptr<ConnectionId> remote_id);
+ std::shared_ptr<RpcConnection> GetNewConnection(
+ std::shared_ptr<ConnectionId> remote_id);
+ std::unordered_map<std::shared_ptr<ConnectionId>,
+ std::shared_ptr<RpcConnection>, ConnectionIdHash,
+ ConnectionIdEquals>
connections_;
std::unordered_map<
- hbase::pb::ServerName,
+ std::shared_ptr<ConnectionId>,
std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>,
- ServerNameHash, ServerNameEquals>
+ ConnectionIdHash, ConnectionIdEquals>
clients_;
folly::SharedMutexWritePriority map_mutex_;
std::shared_ptr<ConnectionFactory> cf_;
diff --git a/connection/rpc-client.cc b/connection/rpc-client.cc
new file mode 100644
index 0000000..66ec231
--- /dev/null
+++ b/connection/rpc-client.cc
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "connection/rpc-client.h"
+#include <unistd.h>
+#include <wangle/concurrent/IOThreadPoolExecutor.h>
+
+using hbase::RpcClient;
+using hbase::AbstractRpcChannel;
+
+namespace hbase {
+
+class RpcChannelImplementation : public AbstractRpcChannel {
+ public:
+ RpcChannelImplementation(std::shared_ptr<RpcClient> rpc_client,
+ const std::string& host, uint16_t port,
+ std::shared_ptr<User> ticket, int rpc_timeout)
+ : AbstractRpcChannel(rpc_client, host, port, ticket, rpc_timeout) {}
+
+ void CallMethod(const MethodDescriptor* method, RpcController* controller,
+ const Message* request, Message* response,
+ Closure* done) override {
+ rpc_client_->CallMethod(method, controller, request, response, done, host_,
+ port_, ticket_);
+ }
+};
+} // namespace hbase
+
+RpcClient::RpcClient() {
+ auto io_executor = std::make_shared<wangle::IOThreadPoolExecutor>(
+ sysconf(_SC_NPROCESSORS_ONLN));
+
+ cp_ = std::make_shared<ConnectionPool>(io_executor);
+}
+
+void RpcClient::Close() {}
+
+std::shared_ptr<Response> RpcClient::SyncCall(const std::string& host,
+ uint16_t port,
+ std::unique_ptr<Request> req,
+ std::shared_ptr<User> ticket) {
+ return std::make_shared<Response>(
+ AsyncCall(host, port, std::move(req), ticket).get());
+}
+
+std::shared_ptr<Response> RpcClient::SyncCall(const std::string& host,
+ uint16_t port,
+ std::unique_ptr<Request> req,
+ std::shared_ptr<User> ticket,
+ const std::string& service_name) {
+ return std::make_shared<Response>(
+ AsyncCall(host, port, std::move(req), ticket, service_name).get());
+}
+
+folly::Future<Response> RpcClient::AsyncCall(const std::string& host,
+ uint16_t port,
+ std::unique_ptr<Request> req,
+ std::shared_ptr<User> ticket) {
+ auto remote_id = std::make_shared<ConnectionId>(host, port, ticket);
+ return GetConnection(remote_id)->SendRequest(std::move(req));
+}
+
+folly::Future<Response> RpcClient::AsyncCall(const std::string& host,
+ uint16_t port,
+ std::unique_ptr<Request> req,
+ std::shared_ptr<User> ticket,
+ const std::string& service_name) {
+ auto remote_id =
+ std::make_shared<ConnectionId>(host, port, ticket, service_name);
+ return GetConnection(remote_id)->SendRequest(std::move(req));
+}
+
+std::shared_ptr<RpcConnection> RpcClient::GetConnection(
+ std::shared_ptr<ConnectionId> remote_id) {
+ return cp_->GetConnection(remote_id);
+}
+
+std::shared_ptr<RpcChannel> RpcClient::CreateRpcChannel(
+ const std::string& host, uint16_t port, std::shared_ptr<User> ticket,
+ int rpc_timeout) {
+ std::shared_ptr<RpcChannelImplementation> channel =
+ std::make_shared<RpcChannelImplementation>(shared_from_this(), host, port,
+ ticket, rpc_timeout);
+
+ /* static_pointer_cast is safe since RpcChannelImplementation derives
+ * from RpcChannel, otherwise, dynamic_pointer_cast should be used. */
+ return std::static_pointer_cast<RpcChannel>(channel);
+}
+
+void RpcClient::CallMethod(const MethodDescriptor* method,
+ RpcController* controller, const Message* req_msg,
+ Message* resp_msg, Closure* done,
+ const std::string& host, uint16_t port,
+ std::shared_ptr<User> ticket) {
+ std::shared_ptr<Message> shared_req(const_cast<Message*>(req_msg));
+ std::shared_ptr<Message> shared_resp(resp_msg);
+
+ std::unique_ptr<Request> req =
+ std::make_unique<Request>(shared_req, shared_resp, method->name());
+
+ AsyncCall(host, port, std::move(req), ticket)
+ .then([done, this](Response resp) { done->Run(); });
+}
diff --git a/connection/rpc-client.h b/connection/rpc-client.h
new file mode 100644
index 0000000..c24db9d
--- /dev/null
+++ b/connection/rpc-client.h
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include "connection/connection-id.h"
+#include "connection/connection-pool.h"
+#include "connection/request.h"
+#include "connection/response.h"
+#include "security/user.h"
+
+#include <google/protobuf/service.h>
+
+using hbase::security::User;
+using hbase::pb::ServerName;
+using hbase::Request;
+using hbase::Response;
+using hbase::ConnectionId;
+using hbase::ConnectionPool;
+using hbase::RpcConnection;
+using hbase::security::User;
+
+using google::protobuf::MethodDescriptor;
+using google::protobuf::RpcChannel;
+using google::protobuf::Message;
+using google::protobuf::RpcController;
+using google::protobuf::Closure;
+
+class RpcChannelImplementation;
+
+namespace hbase {
+
+class RpcClient : public std::enable_shared_from_this<RpcClient> {
+ friend class RpcChannelImplementation;
+
+ public:
+ RpcClient();
+
+ virtual ~RpcClient() { Close(); }
+
+ virtual std::shared_ptr<Response> SyncCall(const std::string &host,
+ uint16_t port,
+ std::unique_ptr<Request> req,
+ std::shared_ptr<User> ticket);
+
+ virtual std::shared_ptr<Response> SyncCall(const std::string &host,
+ uint16_t port,
+ std::unique_ptr<Request> req,
+ std::shared_ptr<User> ticket,
+ const std::string &service_name);
+
+ virtual folly::Future<Response> AsyncCall(const std::string &host,
+ uint16_t port,
+ std::unique_ptr<Request> req,
+ std::shared_ptr<User> ticket);
+
+ virtual folly::Future<Response> AsyncCall(const std::string &host,
+ uint16_t port,
+ std::unique_ptr<Request> req,
+ std::shared_ptr<User> ticket,
+ const std::string &service_name);
+
+ virtual void Close();
+
+ virtual std::shared_ptr<RpcChannel> CreateRpcChannel(
+ const std::string &host, uint16_t port, std::shared_ptr<User> ticket,
+ int rpc_timeout);
+
+ private:
+ void CallMethod(const MethodDescriptor *method, RpcController *controller,
+ const Message *req_msg, Message *resp_msg, Closure *done,
+ const std::string &host, uint16_t port,
+ std::shared_ptr<User> ticket);
+ std::shared_ptr<RpcConnection> GetConnection(
+ std::shared_ptr<ConnectionId> remote_id);
+
+ private:
+ std::shared_ptr<ConnectionPool> cp_;
+};
+
+class AbstractRpcChannel : public RpcChannel {
+ public:
+ AbstractRpcChannel(std::shared_ptr<RpcClient> rpc_client,
+ const std::string &host, uint16_t port,
+ std::shared_ptr<User> ticket, int rpc_timeout)
+ : rpc_client_(rpc_client),
+ host_(host),
+ port_(port),
+ ticket_(ticket),
+ rpc_timeout_(rpc_timeout) {}
+
+ virtual ~AbstractRpcChannel() = default;
+
+ protected:
+ std::shared_ptr<RpcClient> rpc_client_;
+ std::string host_;
+ uint16_t port_;
+ std::shared_ptr<User> ticket_;
+ int rpc_timeout_;
+};
+} // namespace hbase
diff --git a/connection/rpc-connection.h b/connection/rpc-connection.h
new file mode 100644
index 0000000..2e06ec3
--- /dev/null
+++ b/connection/rpc-connection.h
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include "connection/connection-id.h"
+#include "connection/request.h"
+#include "connection/response.h"
+#include "connection/service.h"
+
+#include <memory>
+#include <utility>
+
+using hbase::HBaseService;
+
+namespace hbase {
+class RpcConnection {
+ public:
+ RpcConnection(std::shared_ptr<ConnectionId> connection_id,
+ std::shared_ptr<HBaseService> hbase_service)
+ : connection_id_(connection_id), hbase_service_(hbase_service) {}
+
+ virtual ~RpcConnection() { Close(); }
+
+ virtual std::shared_ptr<ConnectionId> remote_id() const {
+ return connection_id_;
+ }
+
+ virtual std::shared_ptr<HBaseService> get_service() const {
+ return hbase_service_;
+ }
+
+ virtual folly::Future<Response> SendRequest(std::unique_ptr<Request> req) {
+ return (*hbase_service_)(std::move(req));
+ }
+
+ virtual void Close() { hbase_service_->close(); }
+
+ private:
+ std::shared_ptr<ConnectionId> connection_id_;
+ std::shared_ptr<HBaseService> hbase_service_;
+};
+} // namespace hbase
diff --git a/core/location-cache.cc b/core/location-cache.cc
index a0ca5ca..4c29a61 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -26,6 +26,7 @@
#include <utility>
#include "connection/response.h"
+#include "connection/rpc-connection.h"
#include "if/Client.pb.h"
#include "if/ZooKeeper.pb.h"
#include "serde/region-info.h"
@@ -34,6 +35,7 @@
using namespace std;
using namespace folly;
+using hbase::RpcConnection;
using wangle::ServiceFilter;
using hbase::Request;
@@ -121,9 +123,14 @@ Future<std::shared_ptr<RegionLocation>> LocationCache::LocateFromMeta(
const TableName &tn, const string &row) {
return this->LocateMeta()
.via(cpu_executor_.get())
- .then([this](ServerName sn) { return this->cp_.Get(sn); })
- .then([tn, row, this](std::shared_ptr<HBaseService> service) {
- return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
+ .then([this](ServerName sn) {
+ auto remote_id =
+ std::make_shared<ConnectionId>(sn.host_name(), sn.port());
+ return this->cp_.GetConnection(remote_id);
+ })
+ .then([tn, row, this](std::shared_ptr<RpcConnection> rpc_connection) {
+ return (*rpc_connection->get_service())(
+ std::move(meta_util_.MetaRequest(tn, row)));
})
.then([this](Response resp) {
// take the protobuf response and make it into
@@ -139,8 +146,10 @@ Future<std::shared_ptr<RegionLocation>> LocationCache::LocateFromMeta(
return rl;
})
.then([this](std::shared_ptr<RegionLocation> rl) {
+ auto remote_id = std::make_shared<ConnectionId>(
+ rl->server_name().host_name(), rl->server_name().port());
// Now fill out the connection.
- rl->set_service(cp_.Get(rl->server_name()));
+ rl->set_service(cp_.GetConnection(remote_id)->get_service());
return rl;
});
}
diff --git a/connection/BUCK b/security/BUCK
similarity index 59%
copy from connection/BUCK
copy to security/BUCK
index f093d5a..5b935d3 100644
--- a/connection/BUCK
+++ b/security/BUCK
@@ -18,35 +18,13 @@
# This is the library dealing with a single connection
# to a single server.
cxx_library(
- name="connection",
+ name="security",
exported_headers=[
- "client-dispatcher.h",
- "client-handler.h",
- "connection-factory.h",
- "connection-pool.h",
- "pipeline.h",
- "request.h",
- "response.h",
- "service.h",
+ "user.h",
],
srcs=[
- "client-dispatcher.cc",
- "client-handler.cc",
- "connection-factory.cc",
- "connection-pool.cc",
- "pipeline.cc",
- "request.cc",
],
deps=[
- "//if:if",
- "//utils:utils",
- "//serde:serde",
- "//third-party:folly",
- "//third-party:wangle",
],
compiler_flags=['-Weffc++'],
- visibility=['//core/...',],)
-cxx_test(
- name="connection-pool-test",
- srcs=["connection-pool-test.cc",],
- deps=[":connection",],)
+ visibility=['//core/...','//connection/...'],)
diff --git a/security/user.h b/security/user.h
new file mode 100644
index 0000000..795f5ac
--- /dev/null
+++ b/security/user.h
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <string>
+
+namespace hbase {
+namespace security {
+class User {
+public:
+ explicit User(const std::string& user_name) : user_name_(user_name) {}
+ virtual ~User() = default;
+
+ std::string user_name() {return user_name_;}
+
+ static std::shared_ptr<User> defaultUser() {
+ return std::make_shared<User>("__drwho");
+ }
+private:
+ std::string user_name_;
+};
+}
+}
[hbase] 41/137: HBASE-16216 Clean up source code.
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit f2d74a34451ee7d432a226fc468e820229874dc0
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Tue Jul 12 12:21:11 2016 -0700
HBASE-16216 Clean up source code.
---
core/cell-test.cc | 8 ++++----
core/cell.cc | 40 ++++++++++------------------------------
core/cell.h | 4 ++--
if/HBase.proto | 5 -----
test-util/test-util.h | 2 +-
third-party/BUCK | 6 +++---
utils/user-util.h | 2 +-
7 files changed, 21 insertions(+), 46 deletions(-)
diff --git a/core/cell-test.cc b/core/cell-test.cc
index 49f567b..1d5c0eb 100644
--- a/core/cell-test.cc
+++ b/core/cell-test.cc
@@ -19,9 +19,9 @@
#include "core/cell.h"
-#include <memory>
-#include <gtest/gtest.h>
#include <glog/logging.h>
+#include <gtest/gtest.h>
+#include <memory>
using namespace hbase;
TEST(CellTest, CellFailureTest) {
@@ -74,8 +74,8 @@ TEST(CellTest, MultipleCellsTest) {
row += std::to_string(i);
value += std::to_string(i);
CellType cell_type = CellType::PUT;
- const Cell *cell = new Cell(row, family, column, timestamp, value,
- cell_type);
+ const Cell *cell =
+ new Cell(row, family, column, timestamp, value, cell_type);
cells.push_back(cell);
}
int i = 0;
diff --git a/core/cell.cc b/core/cell.cc
index f214479..5129bc9 100644
--- a/core/cell.cc
+++ b/core/cell.cc
@@ -25,13 +25,8 @@ namespace hbase {
Cell::Cell(const std::string &row, const std::string &family,
const std::string &qualifier, const long ×tamp,
const std::string &value, const hbase::CellType &cell_type)
- : row_(row),
- family_(family),
- qualifier_(qualifier),
- timestamp_(timestamp),
- cell_type_(cell_type),
- value_(value),
- sequence_id_(0) {
+ : row_(row), family_(family), qualifier_(qualifier), timestamp_(timestamp),
+ cell_type_(cell_type), value_(value), sequence_id_(0) {
if (0 == row.size())
throw std::runtime_error("Row size should be greater than 0");
@@ -43,35 +38,20 @@ Cell::Cell(const std::string &row, const std::string &family,
throw std::runtime_error("Timestamp should be greater than 0");
}
-Cell::~Cell() {
-}
+Cell::~Cell() {}
-const std::string &Cell::Row() const {
- return row_;
-}
+const std::string &Cell::Row() const { return row_; }
-const std::string &Cell::Family() const {
- return family_;
-}
+const std::string &Cell::Family() const { return family_; }
-const std::string &Cell::Qualifier() const {
- return qualifier_;
-}
+const std::string &Cell::Qualifier() const { return qualifier_; }
-unsigned long Cell::Timestamp() const {
- return timestamp_;
-}
+unsigned long Cell::Timestamp() const { return timestamp_; }
-const std::string &Cell::Value() const {
- return value_;
-}
+const std::string &Cell::Value() const { return value_; }
-hbase::CellType Cell::Type() const {
- return cell_type_;
-}
+hbase::CellType Cell::Type() const { return cell_type_; }
-long Cell::SequenceId() const {
- return sequence_id_;
-}
+long Cell::SequenceId() const { return sequence_id_; }
} /* namespace hbase */
diff --git a/core/cell.h b/core/cell.h
index 16ed280..2b15ad6 100644
--- a/core/cell.h
+++ b/core/cell.h
@@ -34,7 +34,7 @@ enum CellType {
};
class Cell {
- public:
+public:
Cell(const std::string &row, const std::string &family,
const std::string &qualifier, const long ×tamp,
const std::string &value, const hbase::CellType &cell_type);
@@ -47,7 +47,7 @@ class Cell {
CellType Type() const;
long SequenceId() const;
- private:
+private:
std::string row_;
std::string family_;
std::string qualifier_;
diff --git a/if/HBase.proto b/if/HBase.proto
index e5c967a..c36b214 100644
--- a/if/HBase.proto
+++ b/if/HBase.proto
@@ -58,11 +58,6 @@ message TableState {
required State state = 1;
}
-/** On HDFS representation of table state. */
-message TableDescriptor {
- required TableSchema schema = 1;
-}
-
/**
* Column Family Schema
* Inspired by the rest ColumSchemaMessage
diff --git a/test-util/test-util.h b/test-util/test-util.h
index 20e4981..9050961 100644
--- a/test-util/test-util.h
+++ b/test-util/test-util.h
@@ -18,8 +18,8 @@
*/
#pragma once
-#include <folly/experimental/TestUtil.h>
#include <folly/Random.h>
+#include <folly/experimental/TestUtil.h>
#include <cstdlib>
#include <string>
diff --git a/third-party/BUCK b/third-party/BUCK
index 6561b2a..8c0b5f9 100644
--- a/third-party/BUCK
+++ b/third-party/BUCK
@@ -51,7 +51,8 @@ def add_dynamic_libs(names=[]):
return rules
-system_libs = ["lzma", "event", ]
+system_libs = ["lzma",
+ "event", ]
local_libs = [
"double-conversion",
"boost_regex",
@@ -85,8 +86,7 @@ wangle = add_system_libs(['wangle'],
genrule(
name="gen_zk",
out="gen_zk",
- bash=
- "mkdir -p $OUT && wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && tar zxf zookeeper-3.4.8.tar.gz && rm -rf zookeeper-3.4.8.tar.gz && cd zookeeper-3.4.8 && cd src/c && ./configure --prefix=$OUT && make && make install && cd $OUT && rm -rf zookeeper-3.4.8*")
+ bash="mkdir -p $OUT && wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && tar zxf zookeeper-3.4.8.tar.gz && rm -rf zookeeper-3.4.8.tar.gz && cd zookeeper-3.4.8 && cd src/c && ./configure --prefix=$OUT && make && make install && cd $OUT && rm -rf zookeeper-3.4.8*")
cxx_library(name='google-test',
srcs=[
'googletest/googletest/src/gtest-all.cc',
diff --git a/utils/user-util.h b/utils/user-util.h
index fdfc0c8..0704bc6 100644
--- a/utils/user-util.h
+++ b/utils/user-util.h
@@ -19,8 +19,8 @@
#pragma once
-#include <string>
#include <mutex>
+#include <string>
namespace hbase {
[hbase] 46/137: HBASE-17218 [C++] Use Google Style guide and cpplint
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit a9f76a224e851568fa943f1e1ef499aeada9d227
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Mon Dec 5 15:10:28 2016 -0800
HBASE-17218 [C++] Use Google Style guide and cpplint
---
Makefile | 3 +++
bin/{format-code.sh => cpplint.sh} | 10 +++++++---
bin/format-code.sh | 2 +-
3 files changed, 11 insertions(+), 4 deletions(-)
diff --git a/Makefile b/Makefile
index 64cef06..be5d461 100644
--- a/Makefile
+++ b/Makefile
@@ -116,6 +116,9 @@ $(foreach bdir,$(RELEASE_BUILD_DIR),$(eval $(call make-goal-rel,$(bdir))))
check:
$(shell buck test --all --no-results-cache)
+lint:
+ bin/cpplint.sh
+
doc:
$(shell doxygen hbase.doxygen > /dev/null)
diff --git a/bin/format-code.sh b/bin/cpplint.sh
old mode 100755
new mode 100644
similarity index 78%
copy from bin/format-code.sh
copy to bin/cpplint.sh
index cc8b368..52ca581
--- a/bin/format-code.sh
+++ b/bin/cpplint.sh
@@ -15,9 +15,13 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-set -euo pipefail
IFS=$'\n\t'
+CPPLINT_LOC=https://raw.githubusercontent.com/google/styleguide/gh-pages/cpplint/cpplint.py
+OUTPUT=build/cpplint.py
-find core connection serde utils test-util -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i
-find core connection serde utils third-party -name "BUCK" | xargs -P8 yapf -i
+# Download if not already there
+wget -nc $CPPLINT_LOC -O $OUTPUT
+
+# Execute the script
+find core connection serde utils test-util -name "*.h" -or -name "*.cc" | xargs -P8 python $OUTPUT
diff --git a/bin/format-code.sh b/bin/format-code.sh
index cc8b368..331717a 100755
--- a/bin/format-code.sh
+++ b/bin/format-code.sh
@@ -19,5 +19,5 @@ set -euo pipefail
IFS=$'\n\t'
-find core connection serde utils test-util -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i
+find core connection serde utils test-util -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i --style=Google
find core connection serde utils third-party -name "BUCK" | xargs -P8 yapf -i
[hbase] 18/137: HBASE-15724 Use explicit docker image
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit c87f191f18ed8205af16a9e1efd93ed9c05b909f
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Wed Apr 27 10:51:05 2016 -0700
HBASE-15724 Use explicit docker image
Summary:
Set the tag for cpp docker image
Don't set the env on command line
Test Plan: Build it.
Differential Revision: https://reviews.facebook.net/D57315
---
Dockerfile | 4 +++-
bin/start-docker.sh | 1 -
2 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 1524c56..8e33e74 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -15,13 +15,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM pjameson/buck-folly-watchman
+FROM pjameson/buck-folly-watchman:20160425
ARG CC=/usr/bin/gcc-5
ARG CXX=/usr/bin/g++-5
ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -pthread"
ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -pthread"
+ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
+
RUN apt-get install -y vim maven inetutils-ping python-pip && \
pip install yapf && \
apt-get -qq clean && \
diff --git a/bin/start-docker.sh b/bin/start-docker.sh
index 9cbd8b7..5c03eb9 100755
--- a/bin/start-docker.sh
+++ b/bin/start-docker.sh
@@ -53,7 +53,6 @@ docker build -t hbase_native .
# After the image is built run the thing
docker run -p 16010:16010/tcp \
- -e "JAVA_HOME=/usr/lib/jvm/java-8-oracle" \
-v ${BASE_DIR}/..:/usr/src/hbase \
-v ~/.m2:/root/.m2 \
-it hbase_native /bin/bash
[hbase] 28/137: HBASE-15771 Document all the public classes
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 33806c31440cd262f1b683131448c0bfdb24975e
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Thu May 5 13:14:20 2016 -0700
HBASE-15771 Document all the public classes
Summary: Add on a bunch of documentation around implementation so far.
Test Plan:
make doc
Doxygen has no warnings
Differential Revision: https://reviews.facebook.net/D57753
---
BUILDING.md | 18 ++++++++++--
connection/client-dispatcher.h | 11 ++++++--
connection/client-handler.cc | 37 +++++++++----------------
connection/client-handler.h | 36 ++++++++++++++----------
connection/connection-factory.h | 18 ++++++++++++
connection/connection-pool.cc | 5 ++--
connection/connection-pool.h | 44 +++++++++++++++++++++++++++--
connection/pipeline.h | 18 ++++++++++++
connection/request.h | 32 ++++++++++++++-------
connection/response.h | 33 ++++++++++++++++++----
core/BUCK | 1 -
core/client.h | 12 ++++++++
core/connection.cc | 20 --------------
core/connection.h | 26 ------------------
core/location-cache.cc | 14 +++++++++-
core/location-cache.h | 36 ++++++++++++++++++++----
core/meta-utils.h | 12 ++++++++
core/put.cc | 21 --------------
core/put.h | 27 ------------------
core/region-location.h | 41 +++++++++++++++++++++++++++
serde/rpc.cc | 1 -
serde/rpc.h | 61 ++++++++++++++++++++++++++++++++++++++++-
serde/zk.h | 14 ++++++++++
utils/user-util.h | 18 ++++++++++++
24 files changed, 387 insertions(+), 169 deletions(-)
diff --git a/BUILDING.md b/BUILDING.md
index 20ef2a0..4c06776 100644
--- a/BUILDING.md
+++ b/BUILDING.md
@@ -17,7 +17,7 @@ specific language governing permissions and limitations
under the License.
-->
-#Building HBase native client
+# Building HBase native client
The HBase native client build using buck and produces a linux library.
@@ -58,6 +58,20 @@ buck build //core:simple-client
```
That will build the library, then build and test everything, then build
-the simple-client binary. Buck will find all modules used, and compile
+the simple-client binary. Buck will find all modules used, and compile
them in parallel, caching the results. Output from buck is in the buck-out
foulder. Generated binaries are in buck-out/gen logs are in buck-out/logs
+
+
+# Make
+
+If learning buck isn't your thing there is a Makefile wrapper for your
+convenience.
+
+```
+make help
+make check
+make clean
+make all
+make build
+```
diff --git a/connection/client-dispatcher.h b/connection/client-dispatcher.h
index 4435a1b..4bfb35d 100644
--- a/connection/client-dispatcher.h
+++ b/connection/client-dispatcher.h
@@ -30,17 +30,22 @@
#include "connection/response.h"
namespace hbase {
+/**
+ * Dispatcher that assigns a call_id and then routes the response back to the future.
+ */
class ClientDispatcher
: public wangle::ClientDispatcherBase<SerializePipeline,
std::unique_ptr<Request>, Response> {
public:
+ /** Create a new ClientDispatcher */
ClientDispatcher();
- ~ClientDispatcher() {
- LOG(ERROR) << "Killing ClientDispatcher call_id = " << current_call_id_;
- }
+ /** Read a response off the pipeline. */
void read(Context *ctx, Response in) override;
+ /** Take a request as a call and send it down the pipeline. */
folly::Future<Response> operator()(std::unique_ptr<Request> arg) override;
+ /** Close the dispatcher and the associated pipeline. */
folly::Future<folly::Unit> close(Context *ctx) override;
+ /** Close the dispatcher and the associated pipeline. */
folly::Future<folly::Unit> close() override;
private:
diff --git a/connection/client-handler.cc b/connection/client-handler.cc
index 4fdb7ae..cae03c7 100644
--- a/connection/client-handler.cc
+++ b/connection/client-handler.cc
@@ -37,7 +37,7 @@ using hbase::pb::GetResponse;
using google::protobuf::Message;
ClientHandler::ClientHandler(std::string user_name)
- : user_name_(user_name), serde_(), header_info_(),
+ : user_name_(user_name), serde_(), once_flag_(),
resp_msgs_(
make_unique<folly::AtomicHashMap<
uint32_t, std::shared_ptr<google::protobuf::Message>>>(5000)) {}
@@ -75,36 +75,25 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
used_bytes = serde_.ParseDelimited(buf.get(), resp_msg.get());
// Make sure that bytes were parsed.
CHECK(used_bytes == buf->length());
- received.set_response(resp_msg);
+ received.set_resp_msg(resp_msg);
}
ctx->fireRead(std::move(received));
}
}
Future<Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
- // Keep track of if we have sent the header.
- //
- // even though the bool is atomic we can load it lazily here.
- if (UNLIKELY(header_info_->need_.load(std::memory_order_relaxed))) {
-
- // Grab the lock.
- // We need to make sure that no one gets past here without there being a
- // hearder sent.
- std::lock_guard<std::mutex> lock(header_info_->mutex_);
-
- // Now see if we are the first thread to get through.
- //
- // If this is the first thread to get through then the
- // need_send_header will have been true before this.
- if (header_info_->need_.exchange(false)) {
- auto pre = serde_.Preamble();
- auto header = serde_.Header(user_name_);
- pre->appendChain(std::move(header));
- ctx->fireWrite(std::move(pre));
- }
- }
-
+ // We need to send the header once.
+ // So use call_once to make sure that only one thread wins this.
+ std::call_once((*once_flag_), [ctx, this]() {
+ auto pre = serde_.Preamble();
+ auto header = serde_.Header(user_name_);
+ pre->appendChain(std::move(header));
+ ctx->fireWrite(std::move(pre));
+ });
+
+ // Now store the call id to response.
resp_msgs_->insert(r->call_id(), r->resp_msg());
+ // Send the data down the pipeline.
return ctx->fireWrite(
serde_.Request(r->call_id(), r->method(), r->req_msg().get()));
}
diff --git a/connection/client-handler.h b/connection/client-handler.h
index 1a4275f..984a947 100644
--- a/connection/client-handler.h
+++ b/connection/client-handler.h
@@ -41,17 +41,38 @@ class Message;
namespace hbase {
+/**
+ * wangle::Handler implementation to convert hbase::Request to IOBuf and
+ * convert IOBuf to hbase::Response.
+ *
+ * This class deals with sending the connection header and preamble
+ * on first request.
+ */
class ClientHandler : public wangle::Handler<std::unique_ptr<folly::IOBuf>,
Response, std::unique_ptr<Request>,
std::unique_ptr<folly::IOBuf>> {
public:
+ /**
+ * Create the handler
+ * @param user_name the user name of the user running this process.
+ */
ClientHandler(std::string user_name);
+
+ /**
+ * Get bytes from the wire.
+ * This should be the full message as the length field decoder should be
+ * in the pipeline before this.
+ */
void read(Context *ctx, std::unique_ptr<folly::IOBuf> msg) override;
+
+ /**
+ * Write the data down the wire.
+ */
folly::Future<folly::Unit> write(Context *ctx,
std::unique_ptr<Request> r) override;
private:
- std::unique_ptr<HeaderInfo> header_info_;
+ std::unique_ptr<std::once_flag> once_flag_;
std::string user_name_;
RpcSerde serde_;
@@ -60,17 +81,4 @@ private:
uint32_t, std::shared_ptr<google::protobuf::Message>>>
resp_msgs_;
};
-
-/**
- * Class to contain the info about if the connection header and preamble has
- * been sent.
- *
- * We use a serperate class here so that ClientHandler is relocatable.
- */
-class HeaderInfo {
-public:
- HeaderInfo() : need_(true), mutex_() {}
- std::atomic<bool> need_;
- std::mutex mutex_;
-};
} // namespace hbase
diff --git a/connection/connection-factory.h b/connection/connection-factory.h
index 2284a7c..8b6d8d8 100644
--- a/connection/connection-factory.h
+++ b/connection/connection-factory.h
@@ -28,14 +28,32 @@
#include "connection/service.h"
namespace hbase {
+
+/**
+ * Class to create a ClientBootstrap and turn it into a connected
+ * pipeline.
+ */
class ConnectionFactory {
public:
+ /**
+ * Constructor.
+ * There should only be one ConnectionFactory per client.
+ */
ConnectionFactory();
+ /** Default Desctructor */
virtual ~ConnectionFactory() = default;
+ /**
+ * Create a BootStrap from which a connection can be made.
+ */
virtual std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>
MakeBootstrap();
+ /**
+ * Connect a ClientBootstrap to a server and return the pipeline.
+ *
+ * This is mostly visible so that mocks can override socket connections.
+ */
virtual std::shared_ptr<HBaseService>
Connect(std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> client,
const std::string &hostname, int port);
diff --git a/connection/connection-pool.cc b/connection/connection-pool.cc
index 6ed5ad9..75f343e 100644
--- a/connection/connection-pool.cc
+++ b/connection/connection-pool.cc
@@ -26,11 +26,11 @@ using std::mutex;
using std::unique_ptr;
using std::shared_ptr;
using hbase::pb::ServerName;
+using hbase::ConnectionPool;
+using hbase::HBaseService;
using folly::SharedMutexWritePriority;
using folly::SocketAddress;
-namespace hbase {
-
ConnectionPool::ConnectionPool()
: cf_(std::make_shared<ConnectionFactory>()), clients_(), connections_(),
map_mutex_() {}
@@ -75,4 +75,3 @@ void ConnectionPool::close(const ServerName &sn) {
auto service = found->second;
connections_.erase(found);
}
-}
diff --git a/connection/connection-pool.h b/connection/connection-pool.h
index 907afdb..5edd407 100644
--- a/connection/connection-pool.h
+++ b/connection/connection-pool.h
@@ -18,6 +18,7 @@
*/
#pragma once
+#include <boost/functional/hash.hpp>
#include <folly/SharedMutex.h>
#include <mutex>
#include <unordered_map>
@@ -27,26 +28,63 @@
#include "if/HBase.pb.h"
namespace hbase {
+
+/** Equals function for server name that ignores start time */
struct ServerNameEquals {
+
+ /** equals */
bool operator()(const hbase::pb::ServerName &lhs,
const hbase::pb::ServerName &rhs) const {
return lhs.host_name() == rhs.host_name() && lhs.port() == rhs.port();
}
};
+
+/** Hash for ServerName that ignores the start time. */
struct ServerNameHash {
+ /** hash */
std::size_t operator()(hbase::pb::ServerName const &s) const {
- std::size_t h1 = std::hash<std::string>()(s.host_name());
- std::size_t h2 = std::hash<uint32_t>()(s.port());
- return h1 ^ (h2 << 2);
+ std::size_t h = 0;
+ boost::hash_combine(h, s.host_name());
+ boost::hash_combine(h, s.port());
+ return h;
}
};
+/**
+ * @brief Connection pooling for HBase rpc connection.
+ *
+ * This is a thread safe connection pool. It allows getting
+ * a shared connection to HBase by server name. This is
+ * useful for keeping a single connection no matter how many regions a
+ * regionserver has on it.
+ */
class ConnectionPool {
public:
+ /** Create connection pool wit default connection factory */
ConnectionPool();
+
+ /**
+ * Desctructor.
+ * All connections will be close.
+ * All connections will be released
+ */
~ConnectionPool();
+
+ /**
+ * Constructor that allows specifiying the connetion factory.
+ * This is useful for testing.
+ */
explicit ConnectionPool(std::shared_ptr<ConnectionFactory> cf);
+
+ /**
+ * Get a connection to the server name. Start time is ignored.
+ * This can be a blocking operation for a short time.
+ */
std::shared_ptr<HBaseService> get(const hbase::pb::ServerName &sn);
+
+ /**
+ * Close/remove a connection.
+ */
void close(const hbase::pb::ServerName &sn);
private:
diff --git a/connection/pipeline.h b/connection/pipeline.h
index e90cb93..88d7845 100644
--- a/connection/pipeline.h
+++ b/connection/pipeline.h
@@ -26,12 +26,30 @@
#include "utils/user-util.h"
namespace hbase {
+
+/** Pipeline to turn IOBuf into requests */
using SerializePipeline =
wangle::Pipeline<folly::IOBufQueue &, std::unique_ptr<Request>>;
+/**
+ * Factory to create new pipelines for HBase RPC's.
+ */
class RpcPipelineFactory : public wangle::PipelineFactory<SerializePipeline> {
public:
+ /**
+ * Constructor. This will create user util.
+ */
RpcPipelineFactory();
+
+ /**
+ * Create a new pipeline.
+ * The pipeline will be:
+ *
+ * - Async Socke Handler
+ * - Event Base Handler
+ * - Length Field Based Frame Decoder
+ * - Client Handler
+ */
SerializePipeline::Ptr
newPipeline(std::shared_ptr<folly::AsyncTransportWrapper> sock) override;
diff --git a/connection/request.h b/connection/request.h
index 743c469..634d1ba 100644
--- a/connection/request.h
+++ b/connection/request.h
@@ -25,30 +25,42 @@
#include <string>
namespace hbase {
+
+/**
+ * Main request class.
+ * This holds the request object and the un-filled in approriatley typed
+ * response object.
+ */
class Request {
public:
+ /** Create a request object for a get */
static std::unique_ptr<Request> get();
+ /** Create a request object for a mutate */
static std::unique_ptr<Request> mutate();
+ /** Create a request object for a scan */
static std::unique_ptr<Request> scan();
+ /**
+ * This should be private. Do not use this.
+ *
+ *
+ * Constructor that's public for make_unique. This sets all the messages and
+ * method name.
+ */
Request(std::shared_ptr<google::protobuf::Message> req,
std::shared_ptr<google::protobuf::Message> resp, std::string method);
+ /** Get the call id. */
uint32_t call_id() { return call_id_; }
+ /** Set the call id. This should only be set once. */
void set_call_id(uint32_t call_id) { call_id_ = call_id; }
-
+ /** Get the backing request protobuf message. */
std::shared_ptr<google::protobuf::Message> req_msg() { return req_msg_; }
+ /** Get the backing response protobuf message. */
std::shared_ptr<google::protobuf::Message> resp_msg() { return resp_msg_; }
-
- void set_req_msg(std::shared_ptr<google::protobuf::Message> msg) {
- req_msg_ = msg;
- }
- void set_resp_msg(std::shared_ptr<google::protobuf::Message> msg) {
- resp_msg_ = msg;
- }
-
+ /** Get the method name. This is used to the the receiving rpc server what
+ * method type to decode. */
std::string method() { return method_; }
- void set_method(std::string method) { method_ = method; }
private:
uint32_t call_id_;
diff --git a/connection/response.h b/connection/response.h
index d2eb19a..aac57c2 100644
--- a/connection/response.h
+++ b/connection/response.h
@@ -29,20 +29,41 @@ class Message;
namespace hbase {
+/**
+ * @brief Class representing a rpc response
+ *
+ * This is the class sent to a service.
+ */
class Response {
public:
- Response() : call_id_(0), response_(nullptr) {}
+ /**
+ * Constructor.
+ * Initinalizes the call id to 0. 0 should never be a valid call id.
+ */
+ Response() : call_id_(0), resp_msg_(nullptr) {}
+
+ /** Get the call_id */
uint32_t call_id() { return call_id_; }
+
+ /** Set the call_id */
void set_call_id(uint32_t call_id) { call_id_ = call_id; }
- std::shared_ptr<google::protobuf::Message> response() const {
- return response_;
+
+ /**
+ * Get the response message.
+ * The caller is reponsible for knowing the type. In practice the call id is
+ * used to figure out the type.
+ */
+ std::shared_ptr<google::protobuf::Message> resp_msg() const {
+ return resp_msg_;
}
- void set_response(std::shared_ptr<google::protobuf::Message> response) {
- response_ = std::move(response);
+
+ /** Set the response message. */
+ void set_resp_msg(std::shared_ptr<google::protobuf::Message> response) {
+ resp_msg_ = std::move(response);
}
private:
uint32_t call_id_;
- std::shared_ptr<google::protobuf::Message> response_;
+ std::shared_ptr<google::protobuf::Message> resp_msg_;
};
} // namespace hbase
diff --git a/core/BUCK b/core/BUCK
index ef8c2f8..485f9ba 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -20,7 +20,6 @@ cxx_library(
name="core",
exported_headers=[
"client.h",
- "connection.h",
"hbase_macros.h",
"region-location.h",
"location-cache.h",
diff --git a/core/client.h b/core/client.h
index 4bed751..4a6d23b 100644
--- a/core/client.h
+++ b/core/client.h
@@ -28,8 +28,20 @@
#include "if/Cell.pb.h"
namespace hbase {
+
+/**
+ * Client.
+ *
+ * This is the class that provides access to an HBase cluster.
+ * It is thread safe and does connection pooling. Current recommendations are to have only one Client per cluster around.
+ */
class Client {
public:
+
+ /**
+ * Create a new client.
+ * @param quorum_spec Where to connect to get Zookeeper bootstrap information.
+ */
explicit Client(std::string quorum_spec);
private:
diff --git a/core/connection.cc b/core/connection.cc
deleted file mode 100644
index e9a28eb..0000000
--- a/core/connection.cc
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include "core/connection.h"
-
-void Connection::set_zk_quorum(char *zk_q) { this->zk_quorum = zk_q; }
diff --git a/core/connection.h b/core/connection.h
deleted file mode 100644
index 5c9d6b8..0000000
--- a/core/connection.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-class Connection {
- char *zk_quorum;
-
-public:
- void set_zk_quorum(char *zk_q);
-};
diff --git a/core/location-cache.cc b/core/location-cache.cc
index e2a6251..583d305 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -128,15 +128,24 @@ LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
});
}
+/**
+ * Filter to remove a service from the location cache and the connection cache on errors
+ * or on cloase.
+ */
class RemoveServiceFilter
: public ServiceFilter<std::unique_ptr<Request>, Response> {
public:
+
+ /** Create a new filter. */
RemoveServiceFilter(std::shared_ptr<HBaseService> service, ServerName sn,
ConnectionPool &cp)
: ServiceFilter<unique_ptr<Request>, Response>(service), sn_(sn),
cp_(cp) {}
+ /**
+ * Close will remove the connection from all caches.
+ */
folly::Future<folly::Unit> close() override {
if (!released.exchange(true)) {
return this->service_->close().then([this]() {
@@ -148,10 +157,13 @@ public:
}
}
+
+ /** Has this been closed */
virtual bool isAvailable() override {
return !released && service_->isAvailable();
}
+ /** Send the message. */
folly::Future<Response> operator()(unique_ptr<Request> req) override {
// TODO(eclark): add in an on error handler that will
// remove the region location from the cache if needed.
@@ -168,7 +180,7 @@ private:
std::shared_ptr<RegionLocation>
LocationCache::CreateLocation(const Response &resp) {
- auto resp_msg = static_pointer_cast<ScanResponse>(resp.response());
+ auto resp_msg = static_pointer_cast<ScanResponse>(resp.resp_msg());
auto &results = resp_msg->results().Get(0);
auto &cells = results.cell();
auto ri = folly::to<RegionInfo>(cells.Get(0).value());
diff --git a/core/location-cache.h b/core/location-cache.h
index 7f76428..88bec18 100644
--- a/core/location-cache.h
+++ b/core/location-cache.h
@@ -33,25 +33,48 @@
#include "serde/table-name.h"
namespace hbase {
-
+// Forward
class Request;
class Response;
namespace pb {
class ServerName;
}
+/**
+ * Class that can look up and cache locations.
+ */
class LocationCache {
public:
- explicit LocationCache(std::string quorum_spec,
- std::shared_ptr<folly::Executor> executor);
+ /**
+ * Constructor.
+ * @param quorum_spec Where to connect for Zookeeper.
+ * @param executor The cpu executor to run on.
+ */
+ LocationCache(std::string quorum_spec,
+ std::shared_ptr<folly::Executor> executor);
+ /**
+ * Destructor.
+ * This will clean up the zookeeper connections.
+ */
~LocationCache();
- // Meta Related Methods.
- // These are only public until testing is complete
+
+ /**
+ * Where is meta hosted.
+ *
+ * TODO: This should be a RegionLocation.
+ */
folly::Future<hbase::pb::ServerName> LocateMeta();
+
+ /**
+ * Go read meta and find out where a region is located.
+ */
folly::Future<std::shared_ptr<RegionLocation>>
LocateFromMeta(const hbase::pb::TableName &tn, const std::string &row);
+
+ /**
+ * Remove the cached location of meta.
+ */
void InvalidateMeta();
- ConnectionPool cp_;
private:
void RefreshMetaLocation();
@@ -63,6 +86,7 @@ private:
std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> meta_promise_;
std::mutex meta_lock_;
MetaUtil meta_util_;
+ ConnectionPool cp_;
// TODO: migrate this to a smart pointer with a deleter.
zhandle_t *zk_;
diff --git a/core/meta-utils.h b/core/meta-utils.h
index dfef065..f6cc84f 100644
--- a/core/meta-utils.h
+++ b/core/meta-utils.h
@@ -26,10 +26,22 @@
namespace hbase {
+/**
+ * @brief Utility for meta operations.
+ */
class MetaUtil {
public:
+ /**
+ * Given a table and a row give the row key from which to start a scan to find
+ * region locations.
+ */
std::string RegionLookupRowkey(const hbase::pb::TableName &tn,
const std::string &row) const;
+
+ /**
+ * Given a row we're trying to access create a request to look up the
+ * location.
+ */
std::unique_ptr<Request> MetaRequest(const hbase::pb::TableName tn,
const std::string &row) const;
};
diff --git a/core/put.cc b/core/put.cc
deleted file mode 100644
index 806a478..0000000
--- a/core/put.cc
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "core/put.h"
-
-Put::~Put() {}
diff --git a/core/put.h b/core/put.h
deleted file mode 100644
index d34aa0d..0000000
--- a/core/put.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#pragma once
-
-#include "core/mutation.h"
-
-class Put : public Mutation {
-public:
- ~Put();
-};
diff --git a/core/region-location.h b/core/region-location.h
index 7887526..a3d832e 100644
--- a/core/region-location.h
+++ b/core/region-location.h
@@ -25,19 +25,60 @@
namespace hbase {
+/**
+ * @brief class to hold where a region is located.
+ *
+ * This class holds where a region is located, the information about it, the
+ * region name, and a connection to the service used for connecting to it.
+ */
class RegionLocation {
public:
+ /**
+ * Constructor.
+ * @param region_name The region name of this region.
+ * @param ri The decoded RegionInfo of this region.
+ * @param sn The server name of the HBase regionserver thought to be hosting
+ * this region.
+ * @param service the connected service to the regionserver.
+ */
RegionLocation(std::string region_name, hbase::pb::RegionInfo ri,
hbase::pb::ServerName sn,
std::shared_ptr<HBaseService> service)
: region_name_(region_name), ri_(ri), sn_(sn), service_(service) {}
+ /**
+ * Get a reference to the regio info
+ */
const hbase::pb::RegionInfo ®ion_info() { return ri_; }
+
+ /**
+ * Get a reference to the server name
+ */
const hbase::pb::ServerName &server_name() { return sn_; }
+
+ /**
+ * Get a reference to the region name.
+ */
const std::string ®ion_name() { return region_name_; }
+
+ /**
+ * Get a service. This could be closed or null. It's the caller's
+ * responsibility to check.
+ */
std::shared_ptr<HBaseService> service() { return service_; }
+
+ /**
+ * Set the service.
+ * This should be used if the region moved or if the connection is thought to
+ * be bad and a new tcp connection needs to be made.
+ */
void set_service(std::shared_ptr<HBaseService> s) { service_ = s; }
+ /**
+ * Set the servername if the region has moved.
+ */
+ void set_server_name(hbase::pb::ServerName sn) { sn_ = sn; }
+
private:
std::string region_name_;
hbase::pb::RegionInfo ri_;
diff --git a/serde/rpc.cc b/serde/rpc.cc
index 4c3c999..b573738 100644
--- a/serde/rpc.cc
+++ b/serde/rpc.cc
@@ -85,7 +85,6 @@ int RpcSerde::ParseDelimited(const IOBuf *buf, Message *msg) {
}
RpcSerde::RpcSerde() : auth_type_(DEFAULT_AUTH_TYPE) {}
-RpcSerde::~RpcSerde() {}
unique_ptr<IOBuf> RpcSerde::Preamble() {
auto magic = IOBuf::copyBuffer(PREAMBLE, 0, 2);
diff --git a/serde/rpc.h b/serde/rpc.h
index cefb583..b28b4a3 100644
--- a/serde/rpc.h
+++ b/serde/rpc.h
@@ -32,22 +32,81 @@ class Message;
}
namespace hbase {
+
+/**
+ * @brief Class for serializing a deserializing rpc formatted data.
+ *
+ * RpcSerde is the one stop shop for reading/writing data to HBase daemons.
+ * It should throw exceptions if anything goes wrong.
+ */
class RpcSerde {
public:
+ /**
+ * Constructor assumes the default auth type.
+ */
RpcSerde();
- virtual ~RpcSerde();
+
+ /**
+ * Destructor. This is provided just for testing purposes.
+ */
+ virtual ~RpcSerde() = default;
+
+ /**
+ * Pase a message in the delimited format.
+ *
+ * A message in delimited format consists of the following:
+ *
+ * - a protobuf var int32.
+ * - A protobuf object serialized.
+ */
int ParseDelimited(const folly::IOBuf *buf, google::protobuf::Message *msg);
+
+ /**
+ * Create a new connection preamble in a new IOBuf.
+ */
std::unique_ptr<folly::IOBuf> Preamble();
+
+ /**
+ * Create the header protobuf object and serialize it to a new IOBuf.
+ * Header is in the following format:
+ *
+ * - Big endian length
+ * - ConnectionHeader object serialized out.
+ */
std::unique_ptr<folly::IOBuf> Header(const std::string &user);
+
+ /**
+ * Serialize a request message into a protobuf.
+ * Request consists of:
+ *
+ * - Big endian length
+ * - RequestHeader object
+ * - The passed in Message object
+ */
std::unique_ptr<folly::IOBuf> Request(const uint32_t call_id,
const std::string &method,
const google::protobuf::Message *msg);
+
+ /**
+ * Serialize a message in the delimited format.
+ * Delimited format consists of the following:
+ *
+ * - A protobuf var int32
+ * - The message object seriailized after that.
+ */
std::unique_ptr<folly::IOBuf>
SerializeDelimited(const google::protobuf::Message &msg);
+ /**
+ * Serilalize a message. This does not add any length prepend.
+ */
std::unique_ptr<folly::IOBuf>
SerializeMessage(const google::protobuf::Message &msg);
+ /**
+ * Prepend a length IOBuf to the given IOBuf chain.
+ * This involves no copies or moves of the passed in data.
+ */
std::unique_ptr<folly::IOBuf>
PrependLength(std::unique_ptr<folly::IOBuf> msg);
diff --git a/serde/zk.h b/serde/zk.h
index b672bf4..fa3de5b 100644
--- a/serde/zk.h
+++ b/serde/zk.h
@@ -28,8 +28,22 @@ class IOBuf;
}
namespace hbase {
+
+/** @brief A class to convert data from ZooKeeper to other formats.
+ *
+ * This class will convert data to and from Zookeeper into protobuf objects.
+ *
+ */
class ZkDeserializer {
public:
+ /**
+ * Merge the data from a buffer into a given message.
+ *
+ * @param buf Naked pointer to iobuf containing data read from zookeeper.
+ * @param out Naked pointer into which the data will be merged. The message
+ * should be the correct type.
+ * @return returns true if the parsing was successful.
+ */
bool Parse(folly::IOBuf *buf, google::protobuf::Message *out);
};
} // namespace hbase
diff --git a/utils/user-util.h b/utils/user-util.h
index da95d78..0b4cc73 100644
--- a/utils/user-util.h
+++ b/utils/user-util.h
@@ -24,12 +24,30 @@
#include <mutex>
namespace hbase {
+
+/**
+ * @brief Class to help with user/group information.
+ *
+ * This class will get the current user, and information about them. It caches
+ * the user information after the first invocation.
+ */
class UserUtil {
public:
+ /**
+ * Constructor.
+ */
UserUtil();
+
+ /**
+ * Get the username of the user owning this process. This is thread safe and
+ * lockless for every invocation other than the first one.
+ */
std::string user_name();
private:
+ /**
+ * Compute the username. This will block.
+ */
void compute_user_name();
std::atomic<bool> init_;
std::string user_name_;
[hbase] 38/137: HBASE-15926 Added ASF headers which were missed in
last commit
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 693849dec21ca55216d207e89d198d368a8d0554
Author: Sudeep Sunthankar <su...@hashmapinc.com>
AuthorDate: Thu Jun 2 20:22:37 2016 +1000
HBASE-15926 Added ASF headers which were missed in last commit
Signed-off-by: Elliott Clark <ec...@apache.org>
---
Makefile | 17 +++++++++++++++++
Makefile.protos | 17 +++++++++++++++++
2 files changed, 34 insertions(+)
diff --git a/Makefile b/Makefile
index 7e68b6a..64cef06 100644
--- a/Makefile
+++ b/Makefile
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
#use "gcc" to compile source files
CC:=g++
LD:=g++
diff --git a/Makefile.protos b/Makefile.protos
index 4f1727a..2c6316b 100644
--- a/Makefile.protos
+++ b/Makefile.protos
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
#use "gcc" to compile source files
CC:=g++
LD:=g++
[hbase] 104/137: HBASE-18400 [C++] ConnectionId Equals/Hash should
consider service_name (Xiaobing Zhou) - ADDENDUM to reformat the code.
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit f212486b8fd011de7863ff452de26bc4ec49e557
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Wed Jul 19 11:45:45 2017 -0700
HBASE-18400 [C++] ConnectionId Equals/Hash should consider service_name (Xiaobing Zhou) - ADDENDUM to reformat the code.
---
connection/connection-id.h | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/connection/connection-id.h b/connection/connection-id.h
index 059469f..4f84bf8 100644
--- a/connection/connection-id.h
+++ b/connection/connection-id.h
@@ -39,9 +39,11 @@ class ConnectionId {
const std::string &service_name)
: user_(user), service_name_(service_name), host_(host), port_(port) {}
- ConnectionId(const std::string &host, uint16_t port,
- const std::string &service_name)
- : user_(security::User::defaultUser()), service_name_(service_name), host_(host), port_(port) {}
+ ConnectionId(const std::string &host, uint16_t port, const std::string &service_name)
+ : user_(security::User::defaultUser()),
+ service_name_(service_name),
+ host_(host),
+ port_(port) {}
virtual ~ConnectionId() = default;
[hbase] 110/137: HBASE-18338 [C++] Implement RpcTestServer -
ADDENDUM to fix rsync for new proto files
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 16f45a0a111fe2f2bf455bd4bb33a8eefd4220d0
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Wed Jul 26 14:01:01 2017 -0700
HBASE-18338 [C++] Implement RpcTestServer - ADDENDUM to fix rsync for new proto files
---
bin/copy-protobuf.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/bin/copy-protobuf.sh b/bin/copy-protobuf.sh
index 36541cf..f727800 100755
--- a/bin/copy-protobuf.sh
+++ b/bin/copy-protobuf.sh
@@ -21,4 +21,4 @@ IFS=$'\n\t'
BIN_DIR=$(dirname "$0")
PB_SOURCE_DIR="${BIN_DIR}/../../hbase-protocol/src/main/protobuf/"
PB_DEST_DIR="${BIN_DIR}/../if/"
-rsync -r --delete --exclude BUCK ${PB_SOURCE_DIR} ${PB_DEST_DIR}
+rsync -r --exclude BUCK ${PB_SOURCE_DIR} ${PB_DEST_DIR}
[hbase] 101/137: HBASE-18236 [C++] Add batching and reporting to
simple-client
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 7d796134b0657f0acbf45663334224e82f91a71e
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Mon Jun 26 12:28:42 2017 -0700
HBASE-18236 [C++] Add batching and reporting to simple-client
---
core/client-test.cc | 3 +--
core/simple-client.cc | 53 ++++++++++++++++++++++++++++++++++++++++-----------
2 files changed, 43 insertions(+), 13 deletions(-)
diff --git a/core/client-test.cc b/core/client-test.cc
index 1a9fe11..ba213bd 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -298,8 +298,7 @@ TEST_F(ClientTest, CheckAndDelete) {
// Perform Puts
table->Put(Put{row}.AddColumn("d", "1", val1));
table->Put(Put{row}.AddColumn("d", "2", "value2"));
- auto result =
- table->CheckAndDelete(row, "d", "1", val1, hbase::Delete{row}.AddColumn("d", "2"));
+ auto result = table->CheckAndDelete(row, "d", "1", val1, hbase::Delete{row}.AddColumn("d", "2"));
ASSERT_TRUE(result) << "CheckAndDelete didn't replace value";
// Perform the Get
diff --git a/core/simple-client.cc b/core/simple-client.cc
index 2fd7108..d36689e 100644
--- a/core/simple-client.cc
+++ b/core/simple-client.cc
@@ -43,6 +43,7 @@ using hbase::Get;
using hbase::HBaseConfigurationLoader;
using hbase::Scan;
using hbase::Put;
+using hbase::Result;
using hbase::Table;
using hbase::pb::TableName;
using hbase::pb::ServerName;
@@ -53,6 +54,8 @@ DEFINE_string(row, "row_", "row prefix");
DEFINE_string(zookeeper, "localhost:2181", "What zk quorum to talk to");
DEFINE_string(conf, "", "Conf directory to read the config from (optional)");
DEFINE_uint64(num_rows, 10000, "How many rows to write and read");
+DEFINE_uint64(batch_num_rows, 10000, "How many rows batch for multi-gets and multi-puts");
+DEFINE_uint64(report_num_rows, 10000, "How frequent we should report the progress");
DEFINE_bool(puts, true, "Whether to perform puts");
DEFINE_bool(gets, true, "Whether to perform gets");
DEFINE_bool(multigets, true, "Whether to perform multi-gets");
@@ -71,6 +74,13 @@ std::string Row(const std::string &prefix, uint64_t i) {
return prefix + suf;
}
+void ValidateResult(const Result &result, const std::string &row) {
+ CHECK(!result.IsEmpty());
+ CHECK_EQ(result.Row(), row);
+ CHECK_EQ(result.Size(), 1);
+ CHECK_EQ(result.Value("f", "q").value(), row);
+}
+
int main(int argc, char *argv[]) {
google::SetUsageMessage("Simple client to get a single row from HBase on the comamnd line");
google::ParseCommandLineFlags(&argc, &argv, true);
@@ -106,6 +116,10 @@ int main(int argc, char *argv[]) {
LOG(INFO) << "Sending put requests";
for (uint64_t i = 0; i < num_puts; i++) {
table->Put(*MakePut(Row(FLAGS_row, i)));
+ if (i != 0 && i % FLAGS_report_num_rows == 0) {
+ LOG(INFO) << "Sent " << i << " Put requests in " << TimeUtil::ElapsedMillis(start_ns)
+ << " ms.";
+ }
}
LOG(INFO) << "Successfully sent " << num_puts << " Put requests in "
@@ -117,10 +131,15 @@ int main(int argc, char *argv[]) {
LOG(INFO) << "Sending get requests";
start_ns = TimeUtil::GetNowNanos();
for (uint64_t i = 0; i < num_puts; i++) {
- auto result = table->Get(Get{Row(FLAGS_row, i)});
+ auto row = Row(FLAGS_row, i);
+ auto result = table->Get(Get{row});
if (FLAGS_display_results) {
LOG(INFO) << result->DebugString();
+ } else if (i != 0 && i % FLAGS_report_num_rows == 0) {
+ LOG(INFO) << "Sent " << i << " Get requests in " << TimeUtil::ElapsedMillis(start_ns)
+ << " ms.";
}
+ ValidateResult(*result, row);
}
LOG(INFO) << "Successfully sent " << num_puts << " Get requests in "
@@ -129,21 +148,29 @@ int main(int argc, char *argv[]) {
// Do the Multi-Gets
if (FLAGS_multigets) {
- std::vector<hbase::Get> gets;
- for (uint64_t i = 0; i < num_puts; ++i) {
- hbase::Get get(Row(FLAGS_row, i));
- gets.push_back(get);
- }
-
LOG(INFO) << "Sending multi-get requests";
start_ns = TimeUtil::GetNowNanos();
- auto results = table->Get(gets);
+ std::vector<hbase::Get> gets;
+
+ for (uint64_t i = 0; i < num_puts;) {
+ gets.clear();
+ // accumulate batch_num_rows at a time
+ for (uint64_t j = 0; j < FLAGS_batch_num_rows && i < num_puts; ++j) {
+ hbase::Get get(Row(FLAGS_row, i));
+ gets.push_back(get);
+ i++;
+ }
+ auto results = table->Get(gets);
- if (FLAGS_display_results) {
- for (const auto &result : results) LOG(INFO) << result->DebugString();
+ if (FLAGS_display_results) {
+ for (const auto &result : results) LOG(INFO) << result->DebugString();
+ } else if (i != 0 && i % FLAGS_report_num_rows == 0) {
+ LOG(INFO) << "Sent " << i << " Multi-Get requests in " << TimeUtil::ElapsedMillis(start_ns)
+ << " ms.";
+ }
}
- LOG(INFO) << "Successfully sent " << gets.size() << " Multi-Get requests in "
+ LOG(INFO) << "Successfully sent " << num_puts << " Multi-Get requests in "
<< TimeUtil::ElapsedMillis(start_ns) << " ms.";
}
@@ -162,6 +189,10 @@ int main(int argc, char *argv[]) {
}
r = scanner->Next();
i++;
+ if (!FLAGS_display_results && i != 0 && i % FLAGS_report_num_rows == 0) {
+ LOG(INFO) << "Scan iterated over " << i << " results " << TimeUtil::ElapsedMillis(start_ns)
+ << " ms.";
+ }
}
LOG(INFO) << "Successfully iterated over " << i << " Scan results in "
[hbase] 01/137: HBASE-9977 Define C interface of HBase Client
Asynchronous APIs
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 920676ab44d296a8bd10106f0975477bf3af5a3a
Author: eclark <ec...@unknown>
AuthorDate: Fri Jan 3 22:52:41 2014 +0000
HBASE-9977 Define C interface of HBase Client Asynchronous APIs
git-svn-id: https://svn.apache.org/repos/asf/hbase/trunk@1555272 13f79535-47bb-0310-9956-ffa450edef68
---
.gitignore | 38 ++++++++++
CMakeLists.txt | 153 ++++++++++++++++++++++++++++++++++++++
README.md | 34 +++++++++
bin/build-all.sh | 41 ++++++++++
bin/build-thirdparty.sh | 64 ++++++++++++++++
bin/download-thirdparty.sh | 70 +++++++++++++++++
bin/hbase-client-env.sh | 47 ++++++++++++
cmake_modules/FindGTest.cmake | 36 +++++++++
cmake_modules/FindLibEv.cmake | 30 ++++++++
src/async/CMakeLists.txt | 27 +++++++
src/async/get-test.cc | 59 +++++++++++++++
src/async/hbase_admin.cc | 57 ++++++++++++++
src/async/hbase_admin.h | 69 +++++++++++++++++
src/async/hbase_client.cc | 47 ++++++++++++
src/async/hbase_client.h | 60 +++++++++++++++
src/async/hbase_connection.cc | 37 +++++++++
src/async/hbase_connection.h | 52 +++++++++++++
src/async/hbase_errno.h | 23 ++++++
src/async/hbase_get.cc | 61 +++++++++++++++
src/async/hbase_get.h | 73 ++++++++++++++++++
src/async/hbase_mutations.cc | 111 +++++++++++++++++++++++++++
src/async/hbase_mutations.h | 119 +++++++++++++++++++++++++++++
src/async/hbase_result.cc | 37 +++++++++
src/async/hbase_result.h | 44 +++++++++++
src/async/hbase_scanner.cc | 59 +++++++++++++++
src/async/hbase_scanner.h | 80 ++++++++++++++++++++
src/async/mutations-test.cc | 102 +++++++++++++++++++++++++
src/core/CMakeLists.txt | 26 +++++++
src/core/admin.cc | 20 +++++
src/core/admin.h | 25 +++++++
src/core/client.cc | 20 +++++
src/core/client.h | 25 +++++++
src/core/connection.cc | 22 ++++++
src/core/connection.h | 26 +++++++
src/core/connection_attr.h | 30 ++++++++
src/core/delete.cc | 22 ++++++
src/core/delete.h | 29 ++++++++
src/core/get.cc | 20 +++++
src/core/get.h | 26 +++++++
src/core/hbase_connection_attr.cc | 41 ++++++++++
src/core/hbase_connection_attr.h | 51 +++++++++++++
src/core/hbase_macros.h | 60 +++++++++++++++
src/core/hbase_types.h | 83 +++++++++++++++++++++
src/core/mutation.cc | 42 +++++++++++
src/core/mutation.h | 48 ++++++++++++
src/core/put.cc | 22 ++++++
src/core/put.h | 29 ++++++++
src/core/scanner.cc | 20 +++++
src/core/scanner.h | 25 +++++++
src/rpc/CMakeLists.txt | 0
src/sync/CMakeLists.txt | 19 +++++
src/sync/hbase_admin.cc | 51 +++++++++++++
src/sync/hbase_admin.h | 61 +++++++++++++++
src/sync/hbase_connection.cc | 37 +++++++++
src/sync/hbase_connection.h | 52 +++++++++++++
55 files changed, 2562 insertions(+)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..f14b3be
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,38 @@
+# Compiled Object files
+*.slo
+*.lo
+*.o
+
+# Compiled Dynamic libraries
+*.so
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+
+#python
+*.pyc
+
+# CMake Generated Files
+CMakeCache.txt
+CMakeFiles
+Makefile
+cmake_install.CMakeCache
+cmake_install.cmake
+install_manifest.txt
+CTestTestfile.cmake
+build
+Testing
+
+
+
+# Thirdparty dirs
+
+thirdparty/glog-*
+thirdparty/gtest-*
+thirdparty/gtest
+thirdparty/protobuf-*
+thirdparty/libevent-*
+thirdparty/libev-*
+thirdparty/installed
diff --git a/CMakeLists.txt b/CMakeLists.txt
new file mode 100644
index 0000000..be872df
--- /dev/null
+++ b/CMakeLists.txt
@@ -0,0 +1,153 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+cmake_minimum_required(VERSION 2.6)
+
+# generate CTest input files
+enable_testing()
+
+# where to find cmake modules
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
+
+# if no build build type is specified, default to debug builds
+if (NOT CMAKE_BUILD_TYPE)
+ set(CMAKE_BUILD_TYPE Debug)
+endif(NOT CMAKE_BUILD_TYPE)
+
+STRING(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE)
+
+set(CXX_COMMON_FLAGS "-Wall -Wextra -std=c++11")
+set(C_COMMON_FLAGS "-Wall -Wextra -std=c11")
+
+set(CXX_FLAGS_DEBUG "-ggdb -g")
+set(C_FLAGS_DEBUG "-ggdb")
+
+set(CXX_FLAGS_FASTDEBUG "-ggdb -O1")
+set(C_FLAGS_FASTDEBUG "-ggdb -O1")
+
+set(CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG -Wno-strict-aliasing")
+set(C_FLAGS_RELEASE "-O3 -g -DNDEBUG -Wno-strict-aliasing")
+
+# if no build build type is specified, default to debug builds
+if (NOT CMAKE_BUILD_TYPE)
+ set(CMAKE_BUILD_TYPE Debug)
+endif(NOT CMAKE_BUILD_TYPE)
+
+string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE)
+
+# Set compile flags based on the build type.
+message("Configured for ${CMAKE_BUILD_TYPE} build (set with cmake -DCMAKE_BUILD_TYPE={release,debug,...})")
+if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
+ set(CMAKE_CXX_FLAGS ${CXX_FLAGS_DEBUG})
+ set(CMAKE_C_FLAGS ${C_FLAGS_DEBUG})
+elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "FASTDEBUG")
+ set(CMAKE_CXX_FLAGS ${CXX_FLAGS_FASTDEBUG})
+ set(CMAKE_C_FLAGS ${C_FLAGS_FASTDEBUG})
+elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")
+ set(CMAKE_CXX_FLAGS ${CXX_FLAGS_RELEASE})
+ set(CMAKE_C_FLAGS ${C_FLAGS_RELEASE})
+else()
+ message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}")
+endif ()
+
+set(CMAKE_CXX_FLAGS "${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}")
+set(CMAKE_C_FLAGS "${C_COMMON_FLAGS} ${CMAKE_C_FLAGS}")
+
+
+# set compile output directory
+string(TOLOWER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_LOWERCASE)
+set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build/${CMAKE_BUILD_TYPE_LOWERCASE}/")
+
+# Link build/latest to the current build directory, to avoid developers
+# accidentally running the latest debug build when in fact they're building
+# release builds.
+file(MAKE_DIRECTORY ${BUILD_OUTPUT_ROOT_DIRECTORY})
+execute_process(COMMAND rm -f ${CMAKE_CURRENT_SOURCE_DIR}/build/latest)
+execute_process(COMMAND ln -sf ${BUILD_OUTPUT_ROOT_DIRECTORY}
+ ${CMAKE_CURRENT_SOURCE_DIR}/build/latest)
+
+# where to put generated libraries
+set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+set(ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+
+# where to put generated binaries
+set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}")
+
+include_directories(src)
+
+#####################
+# Third Party
+#####################
+
+# find boost headers and libs
+set(Boost_DEBUG TRUE)
+set(Boost_USE_MULTITHREADED ON)
+find_package(Boost REQUIRED COMPONENTS thread system-mt)
+include_directories(${Boost_INCLUDE_DIRS})
+
+find_package(Protobuf REQUIRED)
+include_directories(${PROTOBUF_INCLUDE_DIRS})
+add_library(protobuf STATIC IMPORTED)
+set_target_properties(protobuf PROPERTIES IMPORTED_LOCATION "${PROTOBUF_STATIC_LIBRARY}")
+
+find_package(LibEv REQUIRED)
+include_directories(${LIBEV_INCLUDE_DIR})
+add_library(libev STATIC IMPORTED)
+
+
+set(HBASE_LIBS
+ ${PROTOBUF}
+ ${LIBEV}
+ ${Boost_LIBRARIES}
+)
+#########
+# Testing Config
+#########
+
+# find GTest headers and libs
+find_package(GTest REQUIRED)
+include_directories(${GTEST_INCLUDE_DIR})
+add_library(gtest STATIC IMPORTED)
+set_target_properties(gtest PROPERTIES IMPORTED_LOCATION "${GTEST_LIBS}")
+
+set(HBASE_TEST_LIBS ${HBASE_LIBS} ${GTEST_LIBS})
+set(HBASE_ASYNC_TEST_LIBS ${HBASE_LIBS} ${GTEST_LIBS} hbase-async)
+set(HBASE_SYNC_TEST_LIBS ${HBASE_LIBS} ${GTEST_LIBS} hbase-sync)
+
+function(ADD_HBASE_ASYNC_TEST TEST_NAME)
+ add_executable(${TEST_NAME} ${TEST_NAME}.cc)
+ target_link_libraries(${TEST_NAME} ${HBASE_ASYNC_TEST_LIBS})
+ add_test(${TEST_NAME} "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
+endfunction()
+
+
+function(ADD_HBASE_SYNC_TEST TEST_NAME)
+ add_executable(${TEST_NAME} ${TEST_NAME}.cc)
+ target_link_libraries(${TEST_NAME} ${HBASE_ASYNC_TEST_LIBS})
+ add_test(${TEST_NAME} "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
+endfunction()
+
+
+#####
+# Actual project definition
+#####
+add_subdirectory(src/async)
+add_subdirectory(src/sync)
+
+add_subdirectory(src/core)
+add_subdirectory(src/rpc)
+
+
+add_library(hbase-async SHARED $<TARGET_OBJECTS:hcore> $<TARGET_OBJECTS:hasync>)
+add_library(hbase-sync SHARED $<TARGET_OBJECTS:hcore> $<TARGET_OBJECTS:hsync>)
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..a959eca
--- /dev/null
+++ b/README.md
@@ -0,0 +1,34 @@
+# hbase-native-client
+
+Native client for HBase 0.96
+
+This is a C library that implements a
+HBase client. It's thread safe and libEv
+based.
+
+
+## Design Philosphy
+
+Synchronous and Async versions will both be built
+on the same foundation. The core foundation will
+be C++. External users wanting a C library will
+have to choose either async or sync. These
+libraries will be thin veneers ontop of the C++.
+
+We should try and follow pthreads example as much
+as possible:
+
+* Consistent naming.
+* Opaque pointers as types so that binary compat is easy.
+* Simple setup when the defaults are good.
+* Attr structs when lots of paramters could be needed.
+
+
+## Naming
+All public C files will start with hbase_*.{h, cc}. This
+is to keep naming conflicts to a minimum. Anything without
+the hbase_ prefix is assumed to be implementation private.
+
+All C apis and typedefs will be prefixed with hb_.
+
+All typedefs end with _t.
diff --git a/bin/build-all.sh b/bin/build-all.sh
new file mode 100755
index 0000000..a3e45de
--- /dev/null
+++ b/bin/build-all.sh
@@ -0,0 +1,41 @@
+#! /usr/bin/env bash
+#
+#/**
+# * Copyright The Apache Software Foundation
+# *
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+set -e
+set -x
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+ DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+ SOURCE="$(readlink "$SOURCE")"
+ [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+source ${DIR}/hbase-client-env.sh
+
+${DIR}/download-thirdparty.sh
+${DIR}/build-thirdparty.sh
+cd ${HBASE_DIR}
+cmake .
+make clean all
+make test
diff --git a/bin/build-thirdparty.sh b/bin/build-thirdparty.sh
new file mode 100755
index 0000000..097c1a4
--- /dev/null
+++ b/bin/build-thirdparty.sh
@@ -0,0 +1,64 @@
+#! /usr/bin/env bash
+#
+#/**
+# * Copyright The Apache Software Foundation
+# *
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+set -x
+set -e
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+ DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+ SOURCE="$(readlink "$SOURCE")"
+ [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+source ${DIR}/../bin/hbase-client-env.sh
+
+# On some systems, autotools installs libraries to lib64 rather than lib. Fix
+# this by setting up lib64 as a symlink to lib. We have to do this step first
+# to handle cases where one third-party library depends on another.
+mkdir -p "${HBASE_PREFIX}/lib"
+cd ${HBASE_PREFIX}
+ln -sf lib "${HBASE_PREFIX}/lib64"
+
+if [ ! -f gtest ]; then
+ cd ${HBASE_GTEST_DIR}
+ cmake .
+ make -j4
+ cd ..
+ ln -sf ${HBASE_GTEST_DIR} gtest
+fi
+
+if [ ! -f ${HBASE_PREFIX}/lib/libprotobuf.a ]; then
+ cd ${HBASE_PROTOBUF_DIR}
+ ./configure --with-pic --disable-shared --prefix=${HBASE_PREFIX}
+ make -j4 install
+fi
+
+if [ ! -f ${HBASE_PREFIX}/lib/libev.a ]; then
+ cd ${HBASE_LIBEV_DIR}
+ ./configure --with-pic --disable-shared --prefix=${HBASE_PREFIX}
+ make -j4 install
+fi
+
+echo "---------------------"
+echo "Thirdparty dependencies built and installed into $HBASE_PREFIX successfully"
diff --git a/bin/download-thirdparty.sh b/bin/download-thirdparty.sh
new file mode 100755
index 0000000..db98aae
--- /dev/null
+++ b/bin/download-thirdparty.sh
@@ -0,0 +1,70 @@
+#! /usr/bin/env bash
+#
+#/**
+# * Copyright The Apache Software Foundation
+# *
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+set -x
+set -e
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+ DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+ SOURCE="$(readlink "$SOURCE")"
+ [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+source ${DIR}/../bin/hbase-client-env.sh
+
+mkdir -p ${HBASE_TP_DIR}
+cd ${HBASE_TP_DIR}
+
+
+if [ ! -d ${HBASE_GTEST_DIR} ]; then
+ echo "Fetching gtest"
+ wget -c http://googletest.googlecode.com/files/gtest-${HBASE_GTEST_VERSION}.zip
+ unzip gtest-${HBASE_GTEST_VERSION}.zip
+ rm gtest-${HBASE_GTEST_VERSION}.zip
+fi
+
+if [ ! -d ${HBASE_PROTOBUF_DIR} ]; then
+ echo "Fetching protobuf"
+ wget -c http://protobuf.googlecode.com/files/protobuf-${HBASE_PROTOBUF_VERSION}.tar.gz
+ tar xzf protobuf-${HBASE_PROTOBUF_VERSION}.tar.gz
+ rm protobuf-${HBASE_PROTOBUF_VERSION}.tar.gz
+fi
+
+if [ ! -d $HBASE_LIBEV_DIR ]; then
+ echo "Fetching libev"
+ wget -c http://dist.schmorp.de/libev/libev-${HBASE_LIBEV_VERSION}.tar.gz
+ tar zxf libev-${HBASE_LIBEV_VERSION}.tar.gz
+ rm -rf libev-${HBASE_LIBEV_VERSION}.tar.gz
+fi
+
+if [ ! -d $HBASE_PREFIX/bin/cpplint.py ]; then
+ echo "Fetching cpplint"
+ mkdir -p $HBASE_PREFIX/bin/
+ wget -O $HBASE_PREFIX/bin/cpplint.py http://google-styleguide.googlecode.com/svn/trunk/cpplint/cpplint.py
+ chmod +x $HBASE_PREFIX/bin/cpplint.py
+fi
+
+
+echo "---------------"
+echo "Thirdparty dependencies downloaded successfully"
diff --git a/bin/hbase-client-env.sh b/bin/hbase-client-env.sh
new file mode 100644
index 0000000..96763da
--- /dev/null
+++ b/bin/hbase-client-env.sh
@@ -0,0 +1,47 @@
+#! /usr/bin/env bash
+#
+#/**
+# * Copyright The Apache Software Foundation
+# *
+# * Licensed to the Apache Software Foundation (ASF) under one
+# * or more contributor license agreements. See the NOTICE file
+# * distributed with this work for additional information
+# * regarding copyright ownership. The ASF licenses this file
+# * to you under the Apache License, Version 2.0 (the
+# * "License"); you may not use this file except in compliance
+# * with the License. You may obtain a copy of the License at
+# *
+# * http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+ DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+ SOURCE="$(readlink "$SOURCE")"
+ [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to
+ # resolve it relative to the path where
+ # the symlink file was located
+done
+DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+HBASE_DIR="${DIR}/../"
+HBASE_TP_DIR="${HBASE_DIR}/thirdparty/"
+
+HBASE_PREFIX=$HBASE_TP_DIR/installed
+
+HBASE_GTEST_VERSION=1.7.0
+HBASE_GTEST_DIR=$HBASE_TP_DIR/gtest-$HBASE_GTEST_VERSION
+
+HBASE_PROTOBUF_VERSION=2.5.0
+HBASE_PROTOBUF_DIR=$HBASE_TP_DIR/protobuf-$HBASE_PROTOBUF_VERSION
+
+HBASE_LIBEV_VERSION=4.15
+HBASE_LIBEV_DIR=$HBASE_TP_DIR/libev-$HBASE_LIBEV_VERSION
+
+# use the compiled tools
+export PATH=$HBASE_PREFIX/bin:$PATH
diff --git a/cmake_modules/FindGTest.cmake b/cmake_modules/FindGTest.cmake
new file mode 100644
index 0000000..2d7e3e9
--- /dev/null
+++ b/cmake_modules/FindGTest.cmake
@@ -0,0 +1,36 @@
+set( GTEST_INCLUDE_SEARCH
+ ${CMAKE_SOURCE_DIR}/thirdparty/gtest/include
+)
+
+set( GTEST_LIB_SEARCH
+ ${CMAKE_SOURCE_DIR}/thirdparty/gtest
+)
+
+find_path(GTEST_INCLUDE
+ NAMES gtest/gtest.h
+ PATHS ${GTEST_INCLUDE_SEARCH}
+ NO_DEFAULT_PATH
+)
+
+find_library(GTEST_LIBRARY
+ NAMES gtest
+ PATHS ${GTEST_LIB_SEARCH}
+ NO_DEFAULT_PATH
+)
+
+find_library(GTEST_LIBRARY_MAIN
+ NAMES gtest_main
+ PATHS ${GTEST_LIB_SEARCH}
+ NO_DEFAULT_PATH
+)
+
+if (GTEST_INCLUDE AND GTEST_LIBRARY AND GTEST_LIBRARY_MAIN)
+ set(GTEST_LIBS ${GTEST_LIBRARY} ${GTEST_LIBRARY_MAIN})
+ set(GTEST_INCLUDE_DIR ${GTEST_INCLUDE})
+ set(GTEST_FOUND TRUE)
+endif()
+
+mark_as_advanced(
+ GTEST_INCLUDE_DIR
+ GTEST_LIBS
+)
diff --git a/cmake_modules/FindLibEv.cmake b/cmake_modules/FindLibEv.cmake
new file mode 100644
index 0000000..f61a2a5
--- /dev/null
+++ b/cmake_modules/FindLibEv.cmake
@@ -0,0 +1,30 @@
+set( LIBEV_INCLUDE_SEARCH
+ ${CMAKE_SOURCE_DIR}/thirdparty/installed/include
+)
+
+set( LIBEV_LIB_SEARCH
+ ${CMAKE_SOURCE_DIR}/thirdparty/installed/include
+)
+
+find_path(LIBEV_INCLUDE
+ NAMES ev++.h
+ PATHS ${LIBEV_INCLUDE_SEARCH}
+ NO_DEFAULT_PATH
+)
+
+find_library(LIBEV_LIBRARY
+ NAMES ev
+ PATHS ${LIBEV_LIB_SEARCH}
+ NO_DEFAULT_PATH
+)
+
+if (LIBEV_INCLUDE_PATH AND LIBEV_LIBRARY)
+ set(LIBEV_LIBS ${LIBEV_LIBRARY})
+ set(LIBEV_INCLUDE_DIR ${LIBEV_INCLUDE})
+ set(LIBEV_FOUND TRUE)
+endif()
+
+mark_as_advanced(
+ LIBEV_INCLUDE_DIR
+ LIBEV_LIBS
+)
diff --git a/src/async/CMakeLists.txt b/src/async/CMakeLists.txt
new file mode 100644
index 0000000..c11537c
--- /dev/null
+++ b/src/async/CMakeLists.txt
@@ -0,0 +1,27 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set( ASYNC_SRC
+ hbase_admin.cc
+ hbase_client.cc
+ hbase_connection.cc
+ hbase_get.cc
+ hbase_mutations.cc
+ hbase_result.cc
+ hbase_scanner.cc
+)
+
+
+add_library(hasync OBJECT ${ASYNC_SRC})
+
+ADD_HBASE_ASYNC_TEST(mutations-test)
+ADD_HBASE_ASYNC_TEST(get-test)
diff --git a/src/async/get-test.cc b/src/async/get-test.cc
new file mode 100644
index 0000000..df94ce2
--- /dev/null
+++ b/src/async/get-test.cc
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <pthread.h>
+
+#include "gtest/gtest.h"
+#include "async/hbase_get.h"
+#include "async/hbase_client.h"
+
+
+pthread_cond_t cv;
+pthread_mutex_t mutex;
+
+TEST(GetTest, TestPut) {
+ char tn[] = "T1";
+ hb_byte_t row[] = "ROW";
+
+ hb_client_t client = NULL;
+ hb_get_t get = NULL;
+ int32_t s1 = -1;
+
+ pthread_cond_init(&cv, NULL);
+ pthread_mutex_init(&mutex, NULL);
+
+ s1 = hb_client_create(&client, NULL);
+ EXPECT_EQ(0, s1);
+
+ s1 = hb_get_create(&get);
+ EXPECT_EQ(0, s1);
+
+ hb_get_set_table(get, tn, 2);
+ hb_get_set_row(get, row, 3);
+
+ /*
+ * TODO:
+ * This is currently a NO-OP as there is no CB.
+ */
+ hb_get_send(client, get, NULL, NULL);
+
+ hb_client_destroy(client, NULL, NULL);
+
+ EXPECT_EQ(0, s1);
+}
diff --git a/src/async/hbase_admin.cc b/src/async/hbase_admin.cc
new file mode 100644
index 0000000..17ff2a7
--- /dev/null
+++ b/src/async/hbase_admin.cc
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "async/hbase_admin.h"
+
+#include <stdlib.h>
+#include <stdbool.h>
+
+#include "core/admin.h"
+#include "async/hbase_connection.h"
+
+int32_t hb_admin_create(hb_admin_t* admin_ptr,
+ hb_connection_t connection) {
+ (*admin_ptr) = reinterpret_cast<hb_admin_t>(new Admin());
+ return 0;
+}
+
+/*
+ * Disconnect the admin releasing any internal objects
+ * or connections created in the background.
+ */
+int32_t hb_admin_destroy(hb_admin_t admin,
+ hb_admin_disconnection_cb cb, void * extra) {
+ if (cb)
+ cb(0, admin, extra);
+ free(admin);
+ return 0;
+}
+
+/*
+ * See if a table exists.
+ */
+int32_t hb_admin_table_exists(hb_admin_t admin,
+ char * name_space, size_t name_space_length,
+ char * table, size_t table_length,
+ hb_admin_table_exists_cb cb, void * extra) {
+ if (cb)
+ cb(0, admin, name_space, name_space_length,
+ table, table_length, true, extra);
+ return 0;
+}
diff --git a/src/async/hbase_admin.h b/src/async/hbase_admin.h
new file mode 100644
index 0000000..f03a83f
--- /dev/null
+++ b/src/async/hbase_admin.h
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef ASYNC_HBASE_ADMIN_H_
+#define ASYNC_HBASE_ADMIN_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdlib.h>
+#include <stdbool.h>
+
+#include "core/hbase_macros.h"
+#include "async/hbase_connection.h"
+
+
+typedef void (* hb_admin_disconnection_cb)( int32_t status,
+ hb_admin_t admin, void * extra);
+typedef void (* hb_admin_table_exists_cb)(int32_t status, hb_admin_t admin,
+ char * name_space, size_t name_space_length,
+ char * table, size_t table_length, bool exsists, void * extra);
+
+/**
+ * Create a new hb_admin.
+ * All fields are initialized to the defaults. If you want to set
+ * connection or other properties, set those before calling any
+ * RPC functions.
+ */
+HBASE_API int32_t hb_admin_create(hb_admin_t* admin_ptr,
+ hb_connection_t connection);
+
+/*
+ * Disconnect the admin releasing any internal objects
+ * or connections created in the background.
+ */
+HBASE_API int32_t hb_admin_destroy(hb_admin_t admin,
+ hb_admin_disconnection_cb cb, void * extra);
+
+/*
+ * See if a table exists.
+ */
+HBASE_API int32_t hb_admin_table_exists(hb_admin_t admin,
+ char * name_space, size_t name_space_length,
+ char * table, size_t table_length,
+ hb_admin_table_exists_cb cb, void * extra);
+
+
+#ifdef __cplusplus
+} // extern "C"
+#endif // __cplusplus
+
+#endif // ASYNC_HBASE_ADMIN_H_
diff --git a/src/async/hbase_client.cc b/src/async/hbase_client.cc
new file mode 100644
index 0000000..98e3dbf
--- /dev/null
+++ b/src/async/hbase_client.cc
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "async/hbase_client.h"
+
+#include <pthread.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "core/client.h"
+#include "async/hbase_connection.h"
+
+int32_t hb_client_create(hb_client_t* client_ptr,
+ hb_connection_t connection) {
+ (*client_ptr) = reinterpret_cast<hb_client_t>(new Client());
+ if (client_ptr == NULL)
+ return -1; // TODO(eclark): setup the errno file.
+ return 0;
+}
+
+int32_t hb_client_destroy(hb_client_t client,
+ hb_client_disconnection_cb cb, void * extra) {
+ if (client == NULL)
+ return -2;
+ if (cb) {
+ cb(0, client, extra);
+ }
+ free(client);
+ return 0;
+}
+
diff --git a/src/async/hbase_client.h b/src/async/hbase_client.h
new file mode 100644
index 0000000..34c3d98
--- /dev/null
+++ b/src/async/hbase_client.h
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef ASYNC_HBASE_CLIENT_H_
+#define ASYNC_HBASE_CLIENT_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "core/hbase_macros.h"
+#include "core/hbase_types.h"
+
+/*
+ * Client disconnection callback typedef
+ *
+ * This is called after the connections are closed, but just
+ * before the client is freed.
+ */
+typedef void (* hb_client_disconnection_cb)( int32_t status,
+ hb_client_t client, void * extra);
+
+/**
+ * Create an hb_client_t.
+ *
+ * If connection is null then all defaults will be used.
+ */
+HBASE_API int32_t hb_client_create(hb_client_t * client_ptr,
+ hb_connection_t connection);
+
+/*
+ * Disconnect the client releasing any internal objects
+ * or connections created in the background.
+ */
+HBASE_API int32_t hb_client_destroy(hb_client_t client,
+ hb_client_disconnection_cb cb, void * extra);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif // __cplusplus
+
+#endif // ASYNC_HBASE_CLIENT_H_
+
+
diff --git a/src/async/hbase_connection.cc b/src/async/hbase_connection.cc
new file mode 100644
index 0000000..57fa4f5
--- /dev/null
+++ b/src/async/hbase_connection.cc
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "async/hbase_connection.h"
+
+#include "core/connection.h"
+#include "core/hbase_types.h"
+
+extern "C" {
+int32_t hb_connection_create(hb_connection_t * connection_ptr,
+ hb_connection_attr_t connection_attr) {
+ (*connection_ptr) = reinterpret_cast<hb_connection_t>(new Connection());
+ if ((*connection_ptr) == NULL)
+ return -1;
+ return 0;
+}
+int32_t hb_connection_destroy(hb_connection_t connection) {
+ free(connection);
+ return 0;
+}
+} // extern "C"
diff --git a/src/async/hbase_connection.h b/src/async/hbase_connection.h
new file mode 100644
index 0000000..343f36c
--- /dev/null
+++ b/src/async/hbase_connection.h
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef ASYNC_HBASE_CONNECTION_H_
+#define ASYNC_HBASE_CONNECTION_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "core/hbase_macros.h"
+#include "core/hbase_types.h"
+#include "core/hbase_connection_attr.h"
+
+#include <stdlib.h>
+
+/**
+ * Create an hb_connection.
+ *
+ * if connection_attr is null everything will be left as default
+ */
+HBASE_API int32_t hb_connection_create(hb_connection_t * connection_ptr,
+ hb_connection_attr_t connection_attr);
+
+/**
+ * Destroy the connection and free all resources allocated at creation
+ * time.
+ */
+HBASE_API int32_t hb_connection_destroy(hb_connection_t connection);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif // __cplusplus
+
+#endif // ASYNC_HBASE_CONNECTION_H_
+
diff --git a/src/async/hbase_errno.h b/src/async/hbase_errno.h
new file mode 100644
index 0000000..1698bbd
--- /dev/null
+++ b/src/async/hbase_errno.h
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef ASYNC_HBASE_ERRNO_H_
+#define ASYNC_HBASE_ERRNO_H_
+
+#endif // ASYNC_HBASE_ERRNO_H_
diff --git a/src/async/hbase_get.cc b/src/async/hbase_get.cc
new file mode 100644
index 0000000..32048d8
--- /dev/null
+++ b/src/async/hbase_get.cc
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "async/hbase_get.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "core/get.h"
+
+int32_t hb_get_create(hb_get_t * get_ptr) {
+ (*get_ptr) = reinterpret_cast<hb_get_t>(new Get());
+ if ((*get_ptr) == NULL) {
+ return -1;
+ }
+ return 0;
+}
+
+int32_t hb_get_destroy(hb_get_t get) {
+ free(get);
+ return 0;
+}
+
+int32_t hb_get_set_row(hb_get_t get, unsigned char * row,
+ size_t row_length) {
+ return 0;
+}
+
+int32_t hb_get_set_table(hb_get_t get,
+ char * table, size_t table_length) {
+ return 0;
+}
+
+int32_t hb_get_set_namespace(hb_get_t get,
+ char * name_space, size_t name_space_length) {
+ return 0;
+}
+
+int32_t hb_get_send(hb_client_t client,
+ hb_get_t get, hb_get_cb cb, void * extra) {
+ if (cb) {
+ cb(0, client, get, NULL, extra);
+ }
+ return 0;
+}
diff --git a/src/async/hbase_get.h b/src/async/hbase_get.h
new file mode 100644
index 0000000..7d91c08
--- /dev/null
+++ b/src/async/hbase_get.h
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef ASYNC_HBASE_GET_H_
+#define ASYNC_HBASE_GET_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "core/hbase_macros.h"
+#include "core/hbase_types.h"
+
+/**
+ * Allocate a new get structure.
+ * Ownership passes to the caller.
+ */
+HBASE_API int32_t hb_get_create(hb_get_t * get_ptr);
+
+/**
+ * Destroy and free a get structure.
+ */
+HBASE_API int32_t hb_get_destroy(hb_get_t get);
+
+/**
+ * set the row of this get.
+ */
+HBASE_API int32_t hb_get_set_row(hb_get_t get, hb_byte_t * row,
+ size_t row_length);
+
+/**
+ * Set the table.
+ */
+HBASE_API int32_t hb_get_set_table(hb_get_t get,
+ char * table, size_t table_length);
+
+/**
+ * Set the namespace this get is targeting.
+ */
+HBASE_API int32_t hb_get_set_namespace(hb_get_t get,
+ char * name_space, size_t name_space_length);
+
+/*
+ * get call back typedef.
+ */
+typedef void (* hb_get_cb)(int32_t status, hb_client_t client,
+ hb_get_t get, hb_result_t results, void * extra);
+
+HBASE_API int32_t hb_get_send(hb_client_t client,
+ hb_get_t get, hb_get_cb cb, void * extra);
+
+
+#ifdef __cplusplus
+} // extern "C"
+#endif // __cplusplus
+
+#endif // ASYNC_HBASE_GET_H_
diff --git a/src/async/hbase_mutations.cc b/src/async/hbase_mutations.cc
new file mode 100644
index 0000000..2456dc0
--- /dev/null
+++ b/src/async/hbase_mutations.cc
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "async/hbase_mutations.h"
+
+#include <stdlib.h>
+
+#include "core/hbase_types.h"
+#include "core/mutation.h"
+#include "core/put.h"
+#include "core/delete.h"
+
+#include "async/hbase_result.h"
+
+extern "C" {
+int32_t hb_put_create(hb_put_t* put_ptr) {
+ (*put_ptr) = reinterpret_cast<hb_put_t>(new Put());
+ return 0;
+}
+
+int32_t hb_delete_create(hb_delete_t * delete_ptr) {
+ (*delete_ptr) = reinterpret_cast<hb_delete_t>(new Delete());
+ return 0;
+}
+
+int32_t hb_increment_create(hb_increment_t * increment_ptr) {
+ return 0;
+}
+
+int32_t hb_append_create(hb_append_t * append_ptr) {
+ return 0;
+}
+
+int32_t hb_mutation_destroy(hb_mutation_t mutation) {
+ return 0;
+}
+
+HBASE_API int32_t hb_mutation_set_namespace(hb_mutation_t mutation,
+ char * name_space, size_t name_space_length) {
+ Mutation * m = reinterpret_cast<Mutation *>(mutation);
+ m->set_namespace(name_space, name_space_length);
+ return 0;
+}
+
+HBASE_API int32_t hb_mutation_set_table(hb_mutation_t mutation,
+ char * table, size_t table_length) {
+ Mutation * m = reinterpret_cast<Mutation *>(mutation);
+ m->set_namespace(table, table_length);
+ return 0;
+}
+
+int32_t hb_mutation_set_row(hb_mutation_t mutation,
+ unsigned char * rk, size_t row_length) {
+ Mutation * m = reinterpret_cast<Mutation *>(mutation);
+ m->set_row(rk, row_length);
+ return 0;
+}
+
+int32_t hb_mutation_set_durability(hb_mutation_t mutation,
+ hb_durability_type durability) {
+ Mutation * m = reinterpret_cast<Mutation *>(mutation);
+ m->set_durability(durability);
+ return 0;
+}
+
+int32_t hb_put_add_cell(hb_put_t put, hb_cell_t * cell) {
+ return 0;
+}
+
+int32_t hb_delete_add_col(hb_increment_t incr,
+ unsigned char * family, size_t family_length,
+ unsigned char * qual, size_t qual_length) {
+ return 0;
+}
+
+int32_t hb_increment_add_value(hb_increment_t incr,
+ unsigned char * family, size_t family_length,
+ unsigned char * qual, size_t qual_length,
+ int64_t ammount) {
+ return 0;
+}
+
+int32_t hb_append_add_cell(hb_append_t put, hb_cell_t * cell) {
+ return 0;
+}
+
+int32_t hb_mutation_send(hb_client_t client,
+ hb_mutation_t mutation, hb_mutation_cb cb,
+ void * extra) {
+ if (cb) {
+ cb(0, client, mutation, NULL, extra);
+ }
+ return 0;
+}
+} // extern "C"
diff --git a/src/async/hbase_mutations.h b/src/async/hbase_mutations.h
new file mode 100644
index 0000000..4000390
--- /dev/null
+++ b/src/async/hbase_mutations.h
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef ASYNC_HBASE_MUTATIONS_H_
+#define ASYNC_HBASE_MUTATIONS_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "core/hbase_types.h"
+#include "async/hbase_result.h"
+
+// Creation methods
+
+/**
+ * Create a put.
+ * Ownership passes to the caller.
+ */
+HBASE_API int32_t hb_put_create(hb_put_t * put_ptr);
+
+/**
+ * Create a delete
+ * Ownership passes to the caller.
+ */
+HBASE_API int32_t hb_delete_create(hb_delete_t * delete_ptr);
+
+/**
+ * Create an increment
+ * Ownership passes to the caller.
+ */
+HBASE_API int32_t hb_increment_create(hb_increment_t * increment_ptr);
+
+/**
+ * Create an append
+ * Ownership passes to the caller.
+ */
+HBASE_API int32_t hb_append_create(hb_append_t * append_ptr);
+
+/**
+ * Destroy the mutation.
+ * All internal structures are cleaned up. However any backing
+ * data structures passed in by the user are not cleaned up.
+ */
+HBASE_API int32_t hb_mutation_destroy(hb_mutation_t mutation);
+
+// Shared setters.
+HBASE_API int32_t hb_mutation_set_namespace(hb_mutation_t mutation,
+ char * name_space, size_t name_space_length);
+HBASE_API int32_t hb_mutation_set_table(hb_mutation_t mutation,
+ char * table, size_t table_length);
+HBASE_API int32_t hb_mutation_set_row(hb_mutation_t mutation,
+ unsigned char * rk, size_t row_length);
+HBASE_API int32_t hb_mutation_set_durability(hb_mutation_t mutation,
+ hb_durability_type durability);
+
+// Put Setters etc.
+HBASE_API int32_t hb_put_add_cell(hb_put_t put, hb_cell_t * cell);
+
+// Delete
+HBASE_API int32_t hb_delete_add_col(hb_increment_t incr,
+ unsigned char * family, size_t family_length,
+ unsigned char * qual, size_t qual_length);
+
+// Increment
+HBASE_API int32_t hb_increment_add_value(hb_increment_t incr,
+ unsigned char * family, size_t family_length,
+ unsigned char * qual, size_t qual_length,
+ int64_t ammount);
+
+// Append
+HBASE_API int32_t hb_append_add_cell(hb_append_t put, hb_cell_t * cell);
+
+// Now that the mutations are created and populated
+// The real meat of the client is below.
+
+/*
+ * mutation call back typedef
+ */
+typedef void (* hb_mutation_cb)(int32_t status,
+ hb_client_t client, hb_mutation_t mutation,
+ hb_result_t result, void * extra);
+
+/*
+ * Queue a single mutation. This mutation will be
+ * sent out in the background and can be batched with
+ * other requests destined for the same server.
+ *
+ * The call back will be executed after the response
+ * is received from the RegionServer. Even if the
+ * mutation was batched with other requests,
+ * the call back will be invoked for every mutation
+ * individually.
+ */
+HBASE_API int32_t hb_mutation_send(hb_client_t client,
+ hb_mutation_t mutation, hb_mutation_cb cb,
+ void * extra);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif // __cplusplus
+
+#endif // ASYNC_HBASE_MUTATIONS_H_
diff --git a/src/async/hbase_result.cc b/src/async/hbase_result.cc
new file mode 100644
index 0000000..9351270
--- /dev/null
+++ b/src/async/hbase_result.cc
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "async/hbase_result.h"
+
+#include "core/hbase_types.h"
+
+int32_t hb_result_get_cells(hb_result_t result,
+ hb_cell_t ** cell_ptr, size_t * num_cells) {
+ return 0;
+}
+
+int32_t hb_result_get_table(hb_result_t result,
+ char ** table, size_t * table_length) {
+ return 0;
+}
+
+int32_t hb_result_get_namespace(hb_result_t result,
+ char ** name_space, size_t * name_space_length) {
+ return 0;
+}
diff --git a/src/async/hbase_result.h b/src/async/hbase_result.h
new file mode 100644
index 0000000..eecbbb3
--- /dev/null
+++ b/src/async/hbase_result.h
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef ASYNC_HBASE_RESULT_H_
+#define ASYNC_HBASE_RESULT_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "core/hbase_macros.h"
+#include "core/hbase_types.h"
+
+HBASE_API int32_t hb_result_destroy(hb_result_t result);
+
+HBASE_API int32_t hb_result_get_cells(hb_result_t result,
+ hb_cell_t ** cell_ptr, size_t * num_cells);
+
+HBASE_API int32_t hb_result_get_table(hb_result_t result,
+ char ** table, size_t * table_length);
+HBASE_API int32_t hb_result_get_namespace(hb_result_t result,
+ char ** name_space, size_t * name_space_length);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif // __cplusplus
+
+#endif // ASYNC_HBASE_RESULT_H_
diff --git a/src/async/hbase_scanner.cc b/src/async/hbase_scanner.cc
new file mode 100644
index 0000000..5a8e555
--- /dev/null
+++ b/src/async/hbase_scanner.cc
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include "async/hbase_scanner.h"
+
+#include <stdlib.h>
+
+#include "core/hbase_types.h"
+#include "core/scanner.h"
+
+int32_t hb_scanner_create(hb_scanner_t * scanner_ptr) {
+ (*scanner_ptr) = reinterpret_cast<hb_scanner_t>(new Scanner());
+ return (*scanner_ptr != NULL)?0:1;
+}
+
+HBASE_API int32_t hb_scanner_set_table(hb_scanner_t scanner,
+ char * table, size_t table_length) {
+ return 0;
+}
+
+HBASE_API int32_t hb_scanner_set_namespace(hb_scanner_t scanner,
+ char * name_space, size_t name_space_length) {
+ return 0;
+}
+
+int32_t hb_scanner_set_start_row(hb_scanner_t scanner,
+ unsigned char * start_row, size_t start_row_length) {
+ return 0;
+}
+
+int32_t hb_scanner_set_end_row(hb_scanner_t scanner,
+ unsigned char * end_row, size_t end_row_length) {
+ return 0;
+}
+
+int32_t hb_scanner_set_cache_size(hb_scanner_t scanner,
+ size_t cache_size) {
+ return 0;
+}
+
+int32_t hb_scanner_set_num_versions(hb_scanner_t scanner,
+ int8_t num_versions) {
+ return 0;
+}
diff --git a/src/async/hbase_scanner.h b/src/async/hbase_scanner.h
new file mode 100644
index 0000000..cd3f544
--- /dev/null
+++ b/src/async/hbase_scanner.h
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef ASYNC_HBASE_SCANNER_H_
+#define ASYNC_HBASE_SCANNER_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "async/hbase_result.h"
+#include "core/hbase_types.h"
+
+HBASE_API int32_t hb_scanner_create(hb_scanner_t * scanner_ptr);
+
+HBASE_API int32_t hb_scanner_set_table(hb_scanner_t scanner,
+ char * table, size_t table_length);
+HBASE_API int32_t hb_scanner_set_namespace(hb_scanner_t scanner,
+ char * name_space, size_t name_space_length);
+
+HBASE_API int32_t hb_scanner_set_start_row(hb_scanner_t scanner,
+ unsigned char * start_row, size_t start_row_length);
+HBASE_API int32_t hb_scanner_set_end_row(hb_scanner_t scanner,
+ unsigned char * end_row, size_t end_row_length);
+
+HBASE_API int32_t hb_scanner_set_cache_size(hb_scanner_t scanner,
+ size_t cache_size);
+HBASE_API int32_t hb_scanner_set_batch_size(hb_scanner_t scanner,
+ size_t batch_size);
+HBASE_API int32_t hb_scanner_set_num_versions(hb_scanner_t scanner,
+ int8_t num_versions);
+
+/*
+ * Scanner call back typedef.
+ *
+ * This will be called when initinalization of the scanner
+ * is complete. It will also be called when scanner next
+ * returns results.
+ */
+typedef void (* hb_scanner_cb)(int32_t status,
+ hb_client_t client,
+ hb_scanner_t scanner,
+ hb_result_t results,
+ size_t num_results,
+ void * extra);
+/*
+ * Get the next results from the scanner
+ */
+HBASE_API int32_t hb_scanner_next(hb_client_t client,
+ hb_scanner_t scanner, hb_scanner_cb cb, void * extra);
+
+/*
+ * Close the scanner releasing any local and server side
+ * resources held. The call back is fired just before the
+ * scanner's memory is freed.
+ */
+HBASE_API int32_t hb_scanner_destroy(hb_client_t client,
+ hb_scanner_t scanner, hb_scanner_cb cb, void * extra);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif // __cplusplus
+
+#endif // ASYNC_HBASE_SCANNER_H_
diff --git a/src/async/mutations-test.cc b/src/async/mutations-test.cc
new file mode 100644
index 0000000..be5898e
--- /dev/null
+++ b/src/async/mutations-test.cc
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <pthread.h>
+
+#include "gtest/gtest.h"
+#include "async/hbase_mutations.h"
+#include "async/hbase_client.h"
+
+pthread_cond_t cv;
+pthread_mutex_t mutex;
+
+bool sent = false;
+
+TEST(ClientTest, EasyTest) {
+ EXPECT_EQ(1, 1);
+}
+
+void mutate_cb(int32_t status,
+ hb_client_t client, hb_mutation_t mutation,
+ hb_result_t result, void * extra) {
+
+ // Test Stuff.
+ EXPECT_EQ(status, 0);
+ EXPECT_TRUE(client != NULL);
+ EXPECT_TRUE(mutation != NULL);
+
+ pthread_mutex_lock(&mutex);
+ sent = true;
+ pthread_cond_signal(&cv);
+ pthread_mutex_unlock(&mutex);
+}
+
+void wait_send() {
+ pthread_mutex_lock(&mutex);
+ while (!sent) {
+ pthread_cond_wait(&cv, &mutex);
+ }
+ pthread_mutex_unlock(&mutex);
+}
+
+TEST(MutationTest, TestPut) {
+ char tn[] = "T1";
+ hb_byte_t row[] = "ROW";
+ char fam[] = "D";
+ hb_byte_t qual[] = "QUAL";
+ hb_byte_t data[] = "Z";
+
+ hb_client_t client = NULL;
+ hb_put_t put = NULL;
+ hb_cell_t cell;
+
+ cell.family = fam;
+ cell.family_length = 1;
+
+ cell.qual = qual;
+ cell.qual_length = 4;
+
+ cell.value = data;
+ cell.value_length = 1;
+
+ int32_t status = -1;
+
+ status = hb_client_create(&client, NULL);
+ EXPECT_EQ(0, status);
+
+ hb_put_create(&put);
+ hb_mutation_set_table((hb_mutation_t) put, tn, 2);
+ hb_mutation_set_row((hb_mutation_t) put, row, 3);
+ hb_put_add_cell(put, &cell);
+
+ pthread_cond_init(&cv, NULL);
+ pthread_mutex_init(&mutex, NULL);
+
+ status = hb_mutation_send(client, (hb_mutation_t) put, &mutate_cb, NULL);
+ EXPECT_EQ(0, status);
+
+ // Now wait a while for things to send.
+ wait_send();
+ EXPECT_EQ(true, sent);
+
+ hb_mutation_destroy((hb_mutation_t *) put);
+ hb_client_destroy(client, NULL, NULL);
+
+ EXPECT_EQ(0, status);
+}
diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt
new file mode 100644
index 0000000..c15f649
--- /dev/null
+++ b/src/core/CMakeLists.txt
@@ -0,0 +1,26 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set( CORE_SRC
+ admin.cc
+ client.cc
+ connection.cc
+ get.cc
+ mutation.cc
+ put.cc
+ delete.cc
+ scanner.cc
+ hbase_connection_attr.cc
+)
+
+
+add_library(hcore OBJECT ${CORE_SRC})
diff --git a/src/core/admin.cc b/src/core/admin.cc
new file mode 100644
index 0000000..897e6bf
--- /dev/null
+++ b/src/core/admin.cc
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/admin.h"
diff --git a/src/core/admin.h b/src/core/admin.h
new file mode 100644
index 0000000..624c4ac
--- /dev/null
+++ b/src/core/admin.h
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef CORE_ADMIN_H_
+#define CORE_ADMIN_H_
+
+class Admin {
+};
+#endif // CORE_ADMIN_H_
diff --git a/src/core/client.cc b/src/core/client.cc
new file mode 100644
index 0000000..91c235c
--- /dev/null
+++ b/src/core/client.cc
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/client.h"
diff --git a/src/core/client.h b/src/core/client.h
new file mode 100644
index 0000000..68348a8
--- /dev/null
+++ b/src/core/client.h
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef CORE_CLIENT_H_
+#define CORE_CLIENT_H_
+
+class Client {
+};
+#endif // CORE_CLIENT_H_
diff --git a/src/core/connection.cc b/src/core/connection.cc
new file mode 100644
index 0000000..099f31d
--- /dev/null
+++ b/src/core/connection.cc
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "core/connection.h"
+
+void Connection::set_zk_quorum(char * zk_q) {
+ this->zk_quorum = zk_q;
+}
diff --git a/src/core/connection.h b/src/core/connection.h
new file mode 100644
index 0000000..463b7b5
--- /dev/null
+++ b/src/core/connection.h
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef CORE_CONNECTION_H_
+#define CORE_CONNECTION_H_
+
+class Connection {
+ char * zk_quorum;
+ public:
+ void set_zk_quorum(char * zk_q);
+};
+#endif // CORE_CONNECTION_H_
diff --git a/src/core/connection_attr.h b/src/core/connection_attr.h
new file mode 100644
index 0000000..a7c229e
--- /dev/null
+++ b/src/core/connection_attr.h
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef CORE_CONNECTION_ATTR_H_
+#define CORE_CONNECTION_ATTR_H_
+
+#include "core/hbase_macros.h"
+#include "core/hbase_types.h"
+
+class ConnectionAttr {
+};
+
+#endif // CORE_CONNECTION_ATTR_H_
+
diff --git a/src/core/delete.cc b/src/core/delete.cc
new file mode 100644
index 0000000..695f5bf
--- /dev/null
+++ b/src/core/delete.cc
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "core/delete.h"
+
+Delete::~Delete() {
+}
diff --git a/src/core/delete.h b/src/core/delete.h
new file mode 100644
index 0000000..5740fd9
--- /dev/null
+++ b/src/core/delete.h
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef CORE_DELETE_H_
+#define CORE_DELETE_H_
+
+#include "core/mutation.h"
+
+class Delete: public Mutation {
+ public:
+ ~Delete();
+};
+#endif // CORE_DELETE_H_
diff --git a/src/core/get.cc b/src/core/get.cc
new file mode 100644
index 0000000..9e11332
--- /dev/null
+++ b/src/core/get.cc
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/get.h"
diff --git a/src/core/get.h b/src/core/get.h
new file mode 100644
index 0000000..3a9fd28
--- /dev/null
+++ b/src/core/get.h
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef CORE_GET_H_
+#define CORE_GET_H_
+
+class Get {
+};
+
+#endif // CORE_GET_H_
diff --git a/src/core/hbase_connection_attr.cc b/src/core/hbase_connection_attr.cc
new file mode 100644
index 0000000..61940e8
--- /dev/null
+++ b/src/core/hbase_connection_attr.cc
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/hbase_connection_attr.h"
+
+#include "core/hbase_macros.h"
+#include "core/hbase_types.h"
+#include "core/connection_attr.h"
+
+extern "C" {
+int32_t hb_connection_attr_create(hb_connection_attr_t * attr_ptr) {
+ (*attr_ptr) = new ConnectionAttr();
+ return (attr_ptr == NULL)?-1:0;
+}
+
+int32_t hb_connection_attr_set_zk_quorum(hb_connection_t connection,
+ char * zk_quorum) {
+ return 0;
+}
+
+int32_t hb_connection_attr_set_zk_root(hb_connection_t connection,
+ char * zk_root) {
+ return 0;
+}
+}
diff --git a/src/core/hbase_connection_attr.h b/src/core/hbase_connection_attr.h
new file mode 100644
index 0000000..c73c818
--- /dev/null
+++ b/src/core/hbase_connection_attr.h
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef CORE_HBASE_CONNECTION_ATTR_H_
+#define CORE_HBASE_CONNECTION_ATTR_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "core/hbase_macros.h"
+#include "core/hbase_types.h"
+
+#include <stdlib.h>
+
+HBASE_API int32_t hb_connection_attr_create(hb_connection_attr_t * attr_ptr);
+
+/**
+ * Set the zk quorum of a connection that will be created.
+ */
+HBASE_API int32_t hb_connection_attr_set_zk_quorum(hb_connection_t connection,
+ char * zk_quorum);
+
+/**
+ * Set the zk root of a connection that will be created.
+ */
+HBASE_API int32_t hb_connection_attr_set_zk_root(hb_connection_t connection,
+ char * zk_root);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif // __cplusplus
+
+#endif // CORE_HBASE_CONNECTION_ATTR_H_
+
diff --git a/src/core/hbase_macros.h b/src/core/hbase_macros.h
new file mode 100644
index 0000000..71765c8
--- /dev/null
+++ b/src/core/hbase_macros.h
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef CORE_HBASE_MACROS_H_
+#define CORE_HBASE_MACROS_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * The following code block define API as the tag for exported
+ * functions. The library should be compiled with symbols visibility
+ * set to hidden by default and only the exported functions should be
+ * tagged as HBASE_API.
+ *
+ * When building the library on Windows, compile with compiler flag
+ * "-D_LIBHBASE_IMPLEMENTATION_", whereas when linking application with
+ * this library, this compiler flag should not be used.
+ */
+#if defined _WIN32 || defined __CYGWIN__
+ #ifdef _LIBHBASE_IMPLEMENTATION_
+ #define API __declspec(dllexport)
+ #else
+ #ifdef _LIBHBASE_TEST_
+ #define HBASE_API
+ #else
+ #define HBASE_API __declspec(dllimport)
+ #endif
+ #endif
+#else
+ #if __GNUC__ >= 4
+ #define HBASE_API __attribute__ ((visibility ("default")))
+ #else
+ #define HBASE_API
+ #endif
+#endif
+
+#ifdef __cplusplus
+} // extern "C"
+#endif // __cplusplus
+
+#endif // CORE_HBASE_MACROS_H_
+
diff --git a/src/core/hbase_types.h b/src/core/hbase_types.h
new file mode 100644
index 0000000..8889b92
--- /dev/null
+++ b/src/core/hbase_types.h
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef CORE_HBASE_TYPES_H_
+#define CORE_HBASE_TYPES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdint.h>
+#include <stddef.h>
+
+typedef unsigned char hb_byte_t;
+
+/*
+ * Base kv type.
+ */
+typedef struct {
+ hb_byte_t* row;
+ size_t row_length;
+
+ char * family;
+ size_t family_length;
+
+ hb_byte_t* qual;
+ size_t qual_length;
+
+ hb_byte_t* value;
+ size_t value_length;
+
+ uint64_t timestamp;
+} hb_cell_t;
+
+typedef enum {
+ DELETE_ONE_VERSION,
+ DELETE_MULTIPLE_VERSIONS,
+ DELETE_FAMILY,
+ DELETE_FAMILY_VERSION
+} hb_delete_type;
+
+typedef enum {
+ USE_DEFAULT,
+ SKIP_WAL,
+ ASYNC_WAL,
+ SYNC_WAL,
+ HSYNC_WAL
+} hb_durability_type;
+
+typedef void* hb_admin_t;
+typedef void* hb_client_t;
+typedef void* hb_connection_attr_t;
+typedef void* hb_connection_t;
+typedef void* hb_get_t;
+typedef void* hb_mutation_t;
+typedef void* hb_put_t;
+typedef void* hb_delete_t;
+typedef void* hb_increment_t;
+typedef void* hb_append_t;
+typedef void* hb_result_t;
+typedef void* hb_scanner_t;
+
+#ifdef __cplusplus
+} // extern "C"
+#endif // __cplusplus
+
+#endif // CORE_HBASE_TYPES_H_
diff --git a/src/core/mutation.cc b/src/core/mutation.cc
new file mode 100644
index 0000000..4855076
--- /dev/null
+++ b/src/core/mutation.cc
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/mutation.h"
+
+void Mutation::set_namespace(char * name_space, size_t name_space_length) {
+ this->name_space = name_space;
+ this->name_space_length = name_space_length;
+}
+
+void Mutation::set_table(char * table, size_t table_length) {
+ this->table = table;
+ this->table_length = table_length;
+}
+
+void Mutation::set_row(unsigned char * row, size_t row_length) {
+ this->row = row;
+ this->row_length = row_length;
+}
+
+void Mutation::set_durability(hb_durability_type durability) {
+ this->durability = durability;
+}
+
+Mutation::~Mutation() {
+}
diff --git a/src/core/mutation.h b/src/core/mutation.h
new file mode 100644
index 0000000..e35d595
--- /dev/null
+++ b/src/core/mutation.h
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef CORE_MUTATION_H_
+#define CORE_MUTATION_H_
+
+#include <stdlib.h>
+
+#include "core/hbase_types.h"
+
+class Mutation {
+ char * name_space;
+ size_t name_space_length;
+
+ char * table;
+ size_t table_length;
+
+ unsigned char * row;
+ size_t row_length;
+
+ hb_durability_type durability;
+ public:
+ void set_namespace(char * name_space, size_t name_space_length);
+ void set_table(char * table, size_t table_length);
+ void set_row(unsigned char * row, size_t row_length);
+ void set_durability(hb_durability_type durability);
+
+ virtual ~Mutation();
+};
+#endif // CORE_MUTATION_H_
+
+
diff --git a/src/core/put.cc b/src/core/put.cc
new file mode 100644
index 0000000..8bedc8b
--- /dev/null
+++ b/src/core/put.cc
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "core/put.h"
+
+Put::~Put() {
+}
diff --git a/src/core/put.h b/src/core/put.h
new file mode 100644
index 0000000..7bca3e3
--- /dev/null
+++ b/src/core/put.h
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef CORE_PUT_H_
+#define CORE_PUT_H_
+
+#include "core/mutation.h"
+
+class Put: public Mutation {
+ public:
+ ~Put();
+};
+#endif // CORE_PUT_H_
diff --git a/src/core/scanner.cc b/src/core/scanner.cc
new file mode 100644
index 0000000..a10e444
--- /dev/null
+++ b/src/core/scanner.cc
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/scanner.h"
diff --git a/src/core/scanner.h b/src/core/scanner.h
new file mode 100644
index 0000000..257583b
--- /dev/null
+++ b/src/core/scanner.h
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef CORE_SCANNER_H_
+#define CORE_SCANNER_H_
+
+class Scanner {
+};
+#endif // CORE_SCANNER_H_
diff --git a/src/rpc/CMakeLists.txt b/src/rpc/CMakeLists.txt
new file mode 100644
index 0000000..e69de29
diff --git a/src/sync/CMakeLists.txt b/src/sync/CMakeLists.txt
new file mode 100644
index 0000000..ddd2212
--- /dev/null
+++ b/src/sync/CMakeLists.txt
@@ -0,0 +1,19 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set( SYNC_SRC
+ hbase_connection.cc
+ hbase_admin.cc
+)
+
+
+add_library(hsync OBJECT ${SYNC_SRC})
diff --git a/src/sync/hbase_admin.cc b/src/sync/hbase_admin.cc
new file mode 100644
index 0000000..d43c8ec
--- /dev/null
+++ b/src/sync/hbase_admin.cc
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "sync/hbase_admin.h"
+
+#include <stdlib.h>
+#include <stdbool.h>
+
+#include "core/admin.h"
+
+int32_t hb_admin_create(hb_admin_t** admin_ptr) {
+ (*admin_ptr) = reinterpret_cast<hb_admin_t *>(new Admin());
+ return 0;
+}
+
+/*
+ * Disconnect the admin releasing any internal objects
+ * or connections created in the background.
+ */
+int32_t hb_admin_destroy(hb_admin_t * admin) {
+ Admin * adm = reinterpret_cast<Admin *>(admin);
+ delete adm;
+ return 0;
+}
+
+/*
+ * See if a table exists.
+ */
+int32_t hb_admin_table_exists(hb_admin_t * admin,
+ char * name_space, size_t name_space_length,
+ char * table, size_t table_length,
+ bool * exists) {
+ *exists = true;
+ return 0;
+}
diff --git a/src/sync/hbase_admin.h b/src/sync/hbase_admin.h
new file mode 100644
index 0000000..860bfba
--- /dev/null
+++ b/src/sync/hbase_admin.h
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef SYNC_HBASE_ADMIN_H_
+#define SYNC_HBASE_ADMIN_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdlib.h>
+#include <stdbool.h>
+
+#include "core/hbase_macros.h"
+#include "core/hbase_types.h"
+#include "sync/hbase_connection.h"
+
+
+/**
+ * Create a new hb_admin.
+ * All fields are initialized to the defaults. If you want to set
+ * connection or other properties, set those before calling any
+ * RPC functions.
+ */
+HBASE_API int32_t hb_admin_create(hb_admin_t* admin_ptr,
+ hb_connection_t connection);
+
+/*
+ * Disconnect the admin releasing any internal objects
+ * or connections created in the background.
+ */
+HBASE_API int32_t hb_admin_destroy(hb_admin_t admin);
+
+/*
+ * See if a table exists.
+ */
+HBASE_API int32_t hb_admin_table_exists(hb_admin_t admin,
+ char * name_space, size_t name_space_length,
+ char * table, size_t table_length, bool * exists);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif // __cplusplus
+
+#endif // SYNC_HBASE_ADMIN_H_
diff --git a/src/sync/hbase_connection.cc b/src/sync/hbase_connection.cc
new file mode 100644
index 0000000..df05376
--- /dev/null
+++ b/src/sync/hbase_connection.cc
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "sync/hbase_connection.h"
+
+#include "core/connection.h"
+#include "core/hbase_types.h"
+
+extern "C" {
+int32_t hb_connection_create(hb_connection_t * connection_ptr,
+ hb_connection_attr_t connection_attr) {
+ (*connection_ptr) = reinterpret_cast<hb_connection_t>(new Connection());
+ if ((*connection_ptr) == NULL)
+ return -1;
+ return 0;
+}
+int32_t hb_connection_destroy(hb_connection_t connection) {
+ free(connection);
+ return 0;
+}
+} // extern "C"
diff --git a/src/sync/hbase_connection.h b/src/sync/hbase_connection.h
new file mode 100644
index 0000000..af9284d
--- /dev/null
+++ b/src/sync/hbase_connection.h
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef SYNC_HBASE_CONNECTION_H_
+#define SYNC_HBASE_CONNECTION_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "core/hbase_macros.h"
+#include "core/hbase_types.h"
+#include "core/hbase_connection_attr.h"
+
+#include <stdlib.h>
+
+/**
+ * Create an hb_connection.
+ *
+ * if connection_attr is null everything will be left as default
+ */
+HBASE_API int32_t hb_connection_create(hb_connection_t * connection_ptr,
+ hb_connection_attr_t connection_attr);
+
+/**
+ * Destroy the connection and free all resources allocated at creation
+ * time.
+ */
+HBASE_API int32_t hb_connection_destroy(hb_connection_t connection);
+
+#ifdef __cplusplus
+} // extern "C"
+#endif // __cplusplus
+
+#endif // SYNC_HBASE_CONNECTION_H_
+
[hbase] 49/137: HBASE-17218 [C++] Use Google Style guide and
cpplint - ADDENDUM to add executable bit to the script
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 365b603c5ff6c89dca63e656856aa0e82067387d
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Thu Dec 15 16:19:09 2016 -0800
HBASE-17218 [C++] Use Google Style guide and cpplint - ADDENDUM to add executable bit to the script
---
bin/cpplint.sh | 0
1 file changed, 0 insertions(+), 0 deletions(-)
diff --git a/bin/cpplint.sh b/bin/cpplint.sh
old mode 100644
new mode 100755
[hbase] 85/137: HBASE-15602 Clean up using directives in cc files.
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 16beea20984a195baa252bf5ce635fc7ad9d1d81
Author: Scott Hunt <sc...@adobe.com>
AuthorDate: Tue May 30 16:26:04 2017 -0700
HBASE-15602 Clean up using directives in cc files.
Signed-off-by: Enis Soztutar <en...@apache.org>
---
connection/client-dispatcher.cc | 20 +++++----
connection/client-handler.cc | 21 ++++------
connection/connection-factory.cc | 7 ++--
connection/connection-factory.h | 6 +--
connection/connection-id.h | 17 ++++----
connection/connection-pool.cc | 22 ++++------
connection/connection-pool.h | 9 +---
connection/pipeline.cc | 15 ++++---
connection/request.cc | 3 +-
connection/rpc-client.cc | 3 +-
connection/rpc-client.h | 27 ++++--------
connection/rpc-connection.h | 3 +-
connection/sasl-handler.cc | 4 +-
core/async-batch-rpc-retrying-caller.cc | 14 -------
core/async-batch-rpc-retrying-caller.h | 40 +++++++++---------
core/async-rpc-retrying-caller-factory.h | 24 +++++------
core/async-rpc-retrying-caller.cc | 12 +++---
core/async-rpc-retrying-caller.h | 23 ++++-------
core/async-rpc-retrying-test.cc | 3 +-
core/client.cc | 2 +
core/client.h | 5 +--
core/connection-configuration.h | 50 +++++++++++-----------
core/filter.h | 12 +++---
core/get.h | 1 -
core/hbase-rpc-controller.h | 11 ++---
core/location-cache.cc | 71 ++++++++++++++------------------
core/meta-utils.cc | 9 ++--
core/multi-response.cc | 2 +
core/multi-response.h | 6 +--
core/raw-async-table.cc | 25 ++++++-----
core/raw-async-table.h | 26 +++++-------
core/region-request.h | 2 +-
core/region-result.cc | 1 -
core/region-result.h | 11 +++--
core/request-converter.cc | 2 +-
core/request-converter.h | 12 ++----
core/response-converter.h | 2 -
core/server-request.h | 3 --
core/table.h | 9 ++--
exceptions/exception.h | 3 +-
serde/rpc.cc | 20 ++++-----
serde/zk.cc | 11 +++--
utils/time-util.h | 38 +++++++++--------
utils/user-util.cc | 8 ++--
44 files changed, 269 insertions(+), 346 deletions(-)
diff --git a/connection/client-dispatcher.cc b/connection/client-dispatcher.cc
index 27201d2..35a1f7d 100644
--- a/connection/client-dispatcher.cc
+++ b/connection/client-dispatcher.cc
@@ -21,14 +21,13 @@
#include <utility>
-using namespace folly;
-using namespace hbase;
-using namespace wangle;
-using folly::exception_wrapper;
+using std::unique_ptr;
+
+namespace hbase {
ClientDispatcher::ClientDispatcher() : requests_(5000), current_call_id_(9) {}
-void ClientDispatcher::read(Context *ctx, std::unique_ptr<Response> in) {
+void ClientDispatcher::read(Context *ctx, unique_ptr<Response> in) {
auto call_id = in->call_id();
auto search = requests_.find(call_id);
@@ -44,10 +43,10 @@ void ClientDispatcher::read(Context *ctx, std::unique_ptr<Response> in) {
}
}
-Future<std::unique_ptr<Response>> ClientDispatcher::operator()(std::unique_ptr<Request> arg) {
+folly::Future<unique_ptr<Response>> ClientDispatcher::operator()(unique_ptr<Request> arg) {
auto call_id = current_call_id_++;
arg->set_call_id(call_id);
- requests_.insert(call_id, Promise<std::unique_ptr<Response>>{});
+ requests_.insert(call_id, folly::Promise<unique_ptr<Response>>{});
auto &p = requests_.find(call_id)->second;
auto f = p.getFuture();
p.setInterruptHandler([call_id, this](const folly::exception_wrapper &e) {
@@ -59,6 +58,9 @@ Future<std::unique_ptr<Response>> ClientDispatcher::operator()(std::unique_ptr<R
return f;
}
-Future<Unit> ClientDispatcher::close() { return ClientDispatcherBase::close(); }
+folly::Future<folly::Unit> ClientDispatcher::close() { return ClientDispatcherBase::close(); }
-Future<Unit> ClientDispatcher::close(Context *ctx) { return ClientDispatcherBase::close(ctx); }
+folly::Future<folly::Unit> ClientDispatcher::close(Context *ctx) {
+ return ClientDispatcherBase::close(ctx);
+}
+} // namespace hbase
diff --git a/connection/client-handler.cc b/connection/client-handler.cc
index e60382d..ac16197 100644
--- a/connection/client-handler.cc
+++ b/connection/client-handler.cc
@@ -30,28 +30,24 @@
#include "if/Client.pb.h"
#include "if/RPC.pb.h"
-using namespace hbase;
-using namespace folly;
-using namespace wangle;
-using hbase::pb::ResponseHeader;
-using hbase::pb::GetResponse;
using google::protobuf::Message;
+namespace hbase {
+
ClientHandler::ClientHandler(std::string user_name, std::shared_ptr<Codec> codec,
const std::string &server)
: user_name_(user_name),
serde_(codec),
server_(server),
once_flag_(std::make_unique<std::once_flag>()),
- resp_msgs_(
- make_unique<folly::AtomicHashMap<uint32_t, std::shared_ptr<google::protobuf::Message>>>(
- 5000)) {}
+ resp_msgs_(std::make_unique<folly::AtomicHashMap<uint32_t, std::shared_ptr<Message>>>(5000)) {
+}
-void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
+void ClientHandler::read(Context *ctx, std::unique_ptr<folly::IOBuf> buf) {
if (LIKELY(buf != nullptr)) {
buf->coalesce();
auto received = std::make_unique<Response>();
- ResponseHeader header;
+ pb::ResponseHeader header;
int used_bytes = serde_.ParseDelimited(buf.get(), &header);
VLOG(3) << "Read RPC ResponseHeader size=" << used_bytes << " call_id=" << header.call_id()
@@ -118,13 +114,13 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
VLOG(3) << "Exception RPC ResponseHeader, call_id=" << header.call_id()
<< " exception.what=" << remote_exception->what()
<< ", do_not_retry=" << remote_exception->do_not_retry();
- received->set_exception(::folly::exception_wrapper{*remote_exception});
+ received->set_exception(folly::exception_wrapper{*remote_exception});
}
ctx->fireRead(std::move(received));
}
}
-Future<Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
+folly::Future<folly::Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
// We need to send the header once.
// So use call_once to make sure that only one thread wins this.
std::call_once((*once_flag_), [ctx, this]() {
@@ -142,3 +138,4 @@ Future<Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
// Send the data down the pipeline.
return ctx->fireWrite(serde_.Request(r->call_id(), r->method(), r->req_msg().get()));
}
+} // namespace hbase
diff --git a/connection/connection-factory.cc b/connection/connection-factory.cc
index 9b06c84..d1bfbce 100644
--- a/connection/connection-factory.cc
+++ b/connection/connection-factory.cc
@@ -28,11 +28,11 @@
#include "connection/pipeline.h"
#include "connection/service.h"
-using namespace folly;
-using namespace hbase;
using std::chrono::milliseconds;
using std::chrono::nanoseconds;
+namespace hbase {
+
ConnectionFactory::ConnectionFactory(std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool,
std::shared_ptr<Codec> codec,
std::shared_ptr<Configuration> conf,
@@ -60,10 +60,11 @@ std::shared_ptr<HBaseService> ConnectionFactory::Connect(
// much nicer.
// TODO see about using shared promise for this.
auto pipeline = client
- ->connect(SocketAddress(hostname, port, true),
+ ->connect(folly::SocketAddress(hostname, port, true),
std::chrono::duration_cast<milliseconds>(connect_timeout_))
.get();
auto dispatcher = std::make_shared<ClientDispatcher>();
dispatcher->setPipeline(pipeline);
return dispatcher;
}
+} // namespace hbase
diff --git a/connection/connection-factory.h b/connection/connection-factory.h
index e1d7f6c..c96087d 100644
--- a/connection/connection-factory.h
+++ b/connection/connection-factory.h
@@ -30,8 +30,6 @@
#include "connection/service.h"
#include "security/user.h"
-using std::chrono::nanoseconds;
-
namespace hbase {
/**
@@ -46,7 +44,7 @@ class ConnectionFactory {
*/
ConnectionFactory(std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool,
std::shared_ptr<Codec> codec, std::shared_ptr<Configuration> conf,
- nanoseconds connect_timeout = nanoseconds(0));
+ std::chrono::nanoseconds connect_timeout = std::chrono::nanoseconds(0));
/** Default Destructor */
virtual ~ConnectionFactory() = default;
@@ -66,7 +64,7 @@ class ConnectionFactory {
const std::string &hostname, uint16_t port);
private:
- nanoseconds connect_timeout_;
+ std::chrono::nanoseconds connect_timeout_;
std::shared_ptr<Configuration> conf_;
std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool_;
std::shared_ptr<RpcPipelineFactory> pipeline_factory_;
diff --git a/connection/connection-id.h b/connection/connection-id.h
index 78b9780..7381103 100644
--- a/connection/connection-id.h
+++ b/connection/connection-id.h
@@ -25,31 +25,29 @@
#include <memory>
#include <utility>
-using hbase::pb::ServerName;
-using hbase::security::User;
-
namespace hbase {
+
class ConnectionId {
public:
ConnectionId(const std::string &host, uint16_t port)
- : ConnectionId(host, port, User::defaultUser(), "") {}
+ : ConnectionId(host, port, security::User::defaultUser(), "") {}
- ConnectionId(const std::string &host, uint16_t port, std::shared_ptr<User> user)
+ ConnectionId(const std::string &host, uint16_t port, std::shared_ptr<security::User> user)
: ConnectionId(host, port, user, "") {}
- ConnectionId(const std::string &host, uint16_t port, std::shared_ptr<User> user,
+ ConnectionId(const std::string &host, uint16_t port, std::shared_ptr<security::User> user,
const std::string &service_name)
: user_(user), service_name_(service_name), host_(host), port_(port) {}
virtual ~ConnectionId() = default;
- std::shared_ptr<User> user() const { return user_; }
+ std::shared_ptr<security::User> user() const { return user_; }
std::string service_name() const { return service_name_; }
std::string host() { return host_; }
uint16_t port() { return port_; }
private:
- std::shared_ptr<User> user_;
+ std::shared_ptr<security::User> user_;
std::string service_name_;
std::string host_;
uint16_t port_;
@@ -65,7 +63,8 @@ struct ConnectionIdEquals {
}
private:
- bool userEquals(const std::shared_ptr<User> &lhs, const std::shared_ptr<User> &rhs) const {
+ bool userEquals(const std::shared_ptr<security::User> &lhs,
+ const std::shared_ptr<security::User> &rhs) const {
return lhs == nullptr ? rhs == nullptr
: (rhs == nullptr ? false : lhs->user_name() == rhs->user_name());
}
diff --git a/connection/connection-pool.cc b/connection/connection-pool.cc
index d3ac3c2..e98759d 100644
--- a/connection/connection-pool.cc
+++ b/connection/connection-pool.cc
@@ -21,19 +21,14 @@
#include <folly/Conv.h>
#include <folly/Logging.h>
-#include <folly/SocketAddress.h>
#include <wangle/service/Service.h>
#include <memory>
#include <utility>
-using std::mutex;
-using std::unique_ptr;
-using std::shared_ptr;
-using hbase::ConnectionPool;
-using hbase::HBaseService;
-using folly::SharedMutexWritePriority;
-using folly::SocketAddress;
+using std::chrono::nanoseconds;
+
+namespace hbase {
ConnectionPool::ConnectionPool(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
std::shared_ptr<Codec> codec, std::shared_ptr<Configuration> conf,
@@ -62,7 +57,7 @@ std::shared_ptr<RpcConnection> ConnectionPool::GetConnection(
std::shared_ptr<RpcConnection> ConnectionPool::GetCachedConnection(
std::shared_ptr<ConnectionId> remote_id) {
- SharedMutexWritePriority::ReadHolder holder(map_mutex_);
+ folly::SharedMutexWritePriority::ReadHolder holder(map_mutex_);
auto found = connections_.find(remote_id);
if (found == connections_.end()) {
return nullptr;
@@ -74,7 +69,7 @@ std::shared_ptr<RpcConnection> ConnectionPool::GetNewConnection(
std::shared_ptr<ConnectionId> remote_id) {
// Grab the upgrade lock. While we are double checking other readers can
// continue on
- SharedMutexWritePriority::UpgradeHolder u_holder{map_mutex_};
+ folly::SharedMutexWritePriority::UpgradeHolder u_holder{map_mutex_};
// Now check if someone else created the connection before we got the lock
// This is safe since we hold the upgrade lock.
@@ -84,7 +79,7 @@ std::shared_ptr<RpcConnection> ConnectionPool::GetNewConnection(
return found->second;
} else {
// Yeah it looks a lot like there's no connection
- SharedMutexWritePriority::WriteHolder w_holder{std::move(u_holder)};
+ folly::SharedMutexWritePriority::WriteHolder w_holder{std::move(u_holder)};
// Make double sure there are not stale connections hanging around.
connections_.erase(remote_id);
@@ -102,7 +97,7 @@ std::shared_ptr<RpcConnection> ConnectionPool::GetNewConnection(
}
void ConnectionPool::Close(std::shared_ptr<ConnectionId> remote_id) {
- SharedMutexWritePriority::WriteHolder holder{map_mutex_};
+ folly::SharedMutexWritePriority::WriteHolder holder{map_mutex_};
DLOG(INFO) << "Closing RPC Connection to host:" << remote_id->host()
<< ", port:" << folly::to<std::string>(remote_id->port());
@@ -116,7 +111,7 @@ void ConnectionPool::Close(std::shared_ptr<ConnectionId> remote_id) {
}
void ConnectionPool::Close() {
- SharedMutexWritePriority::WriteHolder holder{map_mutex_};
+ folly::SharedMutexWritePriority::WriteHolder holder{map_mutex_};
for (auto &item : connections_) {
auto &con = item.second;
con->Close();
@@ -124,3 +119,4 @@ void ConnectionPool::Close() {
connections_.clear();
clients_.clear();
}
+} // namespace hbase
diff --git a/connection/connection-pool.h b/connection/connection-pool.h
index 0582d9b..c7c4246 100644
--- a/connection/connection-pool.h
+++ b/connection/connection-pool.h
@@ -31,13 +31,6 @@
#include "connection/service.h"
#include "if/HBase.pb.h"
-using hbase::ConnectionId;
-using hbase::ConnectionIdEquals;
-using hbase::ConnectionIdHash;
-using hbase::RpcConnection;
-
-using std::chrono::nanoseconds;
-
namespace hbase {
/**
@@ -51,7 +44,7 @@ class ConnectionPool {
/** Create connection pool wit default connection factory */
ConnectionPool(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor,
std::shared_ptr<Codec> codec, std::shared_ptr<Configuration> conf,
- nanoseconds connect_timeout = nanoseconds(0));
+ std::chrono::nanoseconds connect_timeout = std::chrono::nanoseconds(0));
/**
* Constructor that allows specifiying the connetion factory.
diff --git a/connection/pipeline.cc b/connection/pipeline.cc
index d27c849..2844752 100644
--- a/connection/pipeline.cc
+++ b/connection/pipeline.cc
@@ -27,26 +27,25 @@
#include "connection/client-handler.h"
#include "connection/sasl-handler.h"
-using namespace folly;
-using namespace hbase;
-using namespace wangle;
+namespace hbase {
RpcPipelineFactory::RpcPipelineFactory(std::shared_ptr<Codec> codec,
std::shared_ptr<Configuration> conf)
: user_util_(), codec_(codec), conf_(conf) {}
SerializePipeline::Ptr RpcPipelineFactory::newPipeline(
- std::shared_ptr<AsyncTransportWrapper> sock) {
- SocketAddress addr; // for logging
+ std::shared_ptr<folly::AsyncTransportWrapper> sock) {
+ folly::SocketAddress addr; // for logging
sock->getPeerAddress(&addr);
auto pipeline = SerializePipeline::create();
- pipeline->addBack(AsyncSocketHandler{sock});
- pipeline->addBack(EventBaseHandler{});
+ pipeline->addBack(wangle::AsyncSocketHandler{sock});
+ pipeline->addBack(wangle::EventBaseHandler{});
auto secure = security::User::IsSecurityEnabled(*conf_);
pipeline->addBack(SaslHandler{user_util_.user_name(secure), conf_});
- pipeline->addBack(LengthFieldBasedFrameDecoder{});
+ pipeline->addBack(wangle::LengthFieldBasedFrameDecoder{});
pipeline->addBack(ClientHandler{user_util_.user_name(secure), codec_, addr.describe()});
pipeline->finalize();
return pipeline;
}
+} // namespace hbase
diff --git a/connection/request.cc b/connection/request.cc
index 80883cc..8983726 100644
--- a/connection/request.cc
+++ b/connection/request.cc
@@ -21,7 +21,7 @@
#include "if/Client.pb.h"
-using namespace hbase;
+namespace hbase {
Request::Request(std::shared_ptr<google::protobuf::Message> req,
std::shared_ptr<google::protobuf::Message> resp, std::string method)
@@ -43,3 +43,4 @@ std::unique_ptr<Request> Request::multi() {
return std::make_unique<Request>(std::make_shared<hbase::pb::MultiRequest>(),
std::make_shared<hbase::pb::MultiResponse>(), "Multi");
}
+} // namespace hbase
diff --git a/connection/rpc-client.cc b/connection/rpc-client.cc
index 57df66d..fbbe3e7 100644
--- a/connection/rpc-client.cc
+++ b/connection/rpc-client.cc
@@ -25,7 +25,8 @@
#include <memory>
#include <string>
-using hbase::RpcClient;
+using hbase::security::User;
+using std::chrono::nanoseconds;
namespace hbase {
diff --git a/connection/rpc-client.h b/connection/rpc-client.h
index fbb773a..8b0abe7 100644
--- a/connection/rpc-client.h
+++ b/connection/rpc-client.h
@@ -29,43 +29,32 @@
#include <chrono>
#include <utility>
-using hbase::security::User;
-using hbase::pb::ServerName;
-using hbase::Request;
-using hbase::Response;
-using hbase::ConnectionId;
-using hbase::ConnectionPool;
-using hbase::RpcConnection;
-using hbase::security::User;
-
-using google::protobuf::Message;
-using std::chrono::nanoseconds;
-
namespace hbase {
class RpcClient {
public:
RpcClient(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor, std::shared_ptr<Codec> codec,
- std::shared_ptr<Configuration> conf, nanoseconds connect_timeout = nanoseconds(0));
+ std::shared_ptr<Configuration> conf,
+ std::chrono::nanoseconds connect_timeout = std::chrono::nanoseconds(0));
virtual ~RpcClient() { Close(); }
virtual std::unique_ptr<Response> SyncCall(const std::string &host, uint16_t port,
std::unique_ptr<Request> req,
- std::shared_ptr<User> ticket);
+ std::shared_ptr<security::User> ticket);
virtual std::unique_ptr<Response> SyncCall(const std::string &host, uint16_t port,
std::unique_ptr<Request> req,
- std::shared_ptr<User> ticket,
+ std::shared_ptr<security::User> ticket,
const std::string &service_name);
- virtual folly::Future<std::unique_ptr<Response>> AsyncCall(const std::string &host, uint16_t port,
- std::unique_ptr<Request> req,
- std::shared_ptr<User> ticket);
+ virtual folly::Future<std::unique_ptr<Response>> AsyncCall(
+ const std::string &host, uint16_t port, std::unique_ptr<Request> req,
+ std::shared_ptr<security::User> ticket);
virtual folly::Future<std::unique_ptr<Response>> AsyncCall(const std::string &host, uint16_t port,
std::unique_ptr<Request> req,
- std::shared_ptr<User> ticket,
+ std::shared_ptr<security::User> ticket,
const std::string &service_name);
virtual void Close();
diff --git a/connection/rpc-connection.h b/connection/rpc-connection.h
index c37b1e0..d9966a1 100644
--- a/connection/rpc-connection.h
+++ b/connection/rpc-connection.h
@@ -26,9 +26,8 @@
#include <memory>
#include <utility>
-using hbase::HBaseService;
-
namespace hbase {
+
class RpcConnection {
public:
RpcConnection(std::shared_ptr<ConnectionId> connection_id,
diff --git a/connection/sasl-handler.cc b/connection/sasl-handler.cc
index 02cddce..ea09595 100644
--- a/connection/sasl-handler.cc
+++ b/connection/sasl-handler.cc
@@ -134,7 +134,7 @@ folly::Future<folly::Unit> SaslHandler::WriteSaslOutput(Context *ctx, const char
return ctx->fireWrite(std::move(iob));
}
-void SaslHandler::FinishAuth(Context *ctx, folly::IOBufQueue* bufQueue) {
+void SaslHandler::FinishAuth(Context *ctx, folly::IOBufQueue *bufQueue) {
std::unique_ptr<folly::IOBuf> iob;
if (!bufQueue->empty()) {
iob = bufQueue->pop_front();
@@ -183,7 +183,7 @@ folly::Future<folly::Unit> SaslHandler::SaslInit(Context *ctx) {
return fut;
}
-void SaslHandler::ContinueSaslNegotiation(Context *ctx, folly::IOBufQueue* bufQueue) {
+void SaslHandler::ContinueSaslNegotiation(Context *ctx, folly::IOBufQueue *bufQueue) {
const char *out;
unsigned int outlen;
diff --git a/core/async-batch-rpc-retrying-caller.cc b/core/async-batch-rpc-retrying-caller.cc
index f3be637..05290f5 100644
--- a/core/async-batch-rpc-retrying-caller.cc
+++ b/core/async-batch-rpc-retrying-caller.cc
@@ -24,25 +24,11 @@
using folly::Future;
using folly::Promise;
using folly::Try;
-
-using folly::Future;
-using folly::Promise;
-using folly::Try;
-using hbase::Action;
-using hbase::LocationCache;
-using hbase::MultiResponse;
-using hbase::RegionLocation;
-using hbase::RegionRequest;
-using hbase::RequestConverter;
-using hbase::Result;
-using hbase::RpcClient;
-using hbase::ServerRequest;
using hbase::pb::ServerName;
using hbase::pb::TableName;
using hbase::security::User;
using std::chrono::nanoseconds;
using std::chrono::milliseconds;
-using wangle::CPUThreadPoolExecutor;
namespace hbase {
diff --git a/core/async-batch-rpc-retrying-caller.h b/core/async-batch-rpc-retrying-caller.h
index 6803a0e..29a0e6a 100644
--- a/core/async-batch-rpc-retrying-caller.h
+++ b/core/async-batch-rpc-retrying-caller.h
@@ -65,8 +65,8 @@
namespace hbase {
/* Equals function for ServerName */
struct ServerNameEquals {
- bool operator()(const std::shared_ptr<ServerName> &lhs,
- const std::shared_ptr<ServerName> &rhs) const {
+ bool operator()(const std::shared_ptr<pb::ServerName> &lhs,
+ const std::shared_ptr<pb::ServerName> &rhs) const {
return (lhs->start_code() == rhs->start_code() && lhs->host_name() == rhs->host_name() &&
lhs->port() == rhs->port());
}
@@ -74,7 +74,7 @@ struct ServerNameEquals {
struct ServerNameHash {
/** hash */
- std::size_t operator()(const std::shared_ptr<ServerName> &sn) const {
+ std::size_t operator()(const std::shared_ptr<pb::ServerName> &sn) const {
std::size_t h = 0;
boost::hash_combine(h, sn->start_code());
boost::hash_combine(h, sn->host_name());
@@ -86,16 +86,18 @@ struct ServerNameHash {
class AsyncBatchRpcRetryingCaller {
public:
using ActionsByServer =
- std::unordered_map<std::shared_ptr<ServerName>, std::shared_ptr<ServerRequest>,
+ std::unordered_map<std::shared_ptr<pb::ServerName>, std::shared_ptr<ServerRequest>,
ServerNameHash, ServerNameEquals>;
using ActionsByRegion = ServerRequest::ActionsByRegion;
AsyncBatchRpcRetryingCaller(std::shared_ptr<AsyncConnection> conn,
std::shared_ptr<folly::HHWheelTimer> retry_timer,
std::shared_ptr<pb::TableName> table_name,
- const std::vector<hbase::Get> &actions, nanoseconds pause_ns,
- int32_t max_attempts, nanoseconds operation_timeout_ns,
- nanoseconds rpc_timeout_ns, int32_t start_log_errors_count);
+ const std::vector<hbase::Get> &actions,
+ std::chrono::nanoseconds pause_ns, int32_t max_attempts,
+ std::chrono::nanoseconds operation_timeout_ns,
+ std::chrono::nanoseconds rpc_timeout_ns,
+ int32_t start_log_errors_count);
~AsyncBatchRpcRetryingCaller();
@@ -106,33 +108,33 @@ class AsyncBatchRpcRetryingCaller {
void LogException(int32_t tries, std::shared_ptr<RegionRequest> region_request,
std::shared_ptr<std::exception> &error,
- std::shared_ptr<ServerName> server_name);
+ std::shared_ptr<pb::ServerName> server_name);
void LogException(int32_t tries, std::vector<std::shared_ptr<RegionRequest>> ®ion_requests,
std::shared_ptr<std::exception> &error,
- std::shared_ptr<ServerName> server_name);
+ std::shared_ptr<pb::ServerName> server_name);
- const std::string GetExtraContextForError(std::shared_ptr<ServerName> server_name);
+ const std::string GetExtraContextForError(std::shared_ptr<pb::ServerName> server_name);
void AddError(const std::shared_ptr<Action> &action, std::shared_ptr<std::exception> error,
- std::shared_ptr<ServerName> server_name);
+ std::shared_ptr<pb::ServerName> server_name);
void AddError(const std::vector<std::shared_ptr<Action>> &actions,
- std::shared_ptr<std::exception> error, std::shared_ptr<ServerName> server_name);
+ std::shared_ptr<std::exception> error, std::shared_ptr<pb::ServerName> server_name);
void FailOne(const std::shared_ptr<Action> &action, int32_t tries,
std::shared_ptr<std::exception> error, int64_t current_time,
const std::string extras);
void FailAll(const std::vector<std::shared_ptr<Action>> &actions, int32_t tries,
- std::shared_ptr<std::exception> error, std::shared_ptr<ServerName> server_name);
+ std::shared_ptr<std::exception> error, std::shared_ptr<pb::ServerName> server_name);
void FailAll(const std::vector<std::shared_ptr<Action>> &actions, int32_t tries);
void AddAction2Error(uint64_t action_index, const ThrowableWithExtraContext &twec);
void OnError(const ActionsByRegion &actions_by_region, int32_t tries,
- std::shared_ptr<std::exception> exc, std::shared_ptr<ServerName> server_name);
+ std::shared_ptr<std::exception> exc, std::shared_ptr<pb::ServerName> server_name);
void TryResubmit(std::vector<std::shared_ptr<Action>> actions, int32_t tries);
@@ -147,12 +149,12 @@ class AsyncBatchRpcRetryingCaller {
void Send(ActionsByServer &actions_by_server, int32_t tries);
void OnComplete(const ActionsByRegion &actions_by_region, int32_t tries,
- const std::shared_ptr<ServerName> server_name,
+ const std::shared_ptr<pb::ServerName> server_name,
const std::unique_ptr<MultiResponse> multi_results);
void OnComplete(const std::shared_ptr<Action> &action,
const std::shared_ptr<RegionRequest> ®ion_request, int32_t tries,
- const std::shared_ptr<ServerName> &server_name,
+ const std::shared_ptr<pb::ServerName> &server_name,
const std::shared_ptr<RegionResult> ®ion_result,
std::vector<std::shared_ptr<Action>> &failed_actions);
@@ -161,10 +163,10 @@ class AsyncBatchRpcRetryingCaller {
std::shared_ptr<hbase::AsyncConnection> conn_;
std::shared_ptr<pb::TableName> table_name_;
std::vector<std::shared_ptr<Action>> actions_;
- nanoseconds pause_ns_;
+ std::chrono::nanoseconds pause_ns_;
int32_t max_attempts_ = 0;
- nanoseconds operation_timeout_ns_;
- nanoseconds rpc_timeout_ns_;
+ std::chrono::nanoseconds operation_timeout_ns_;
+ std::chrono::nanoseconds rpc_timeout_ns_;
int32_t start_log_errors_count_ = 0;
int64_t start_ns_ = TimeUtil::GetNowNanos();
diff --git a/core/async-rpc-retrying-caller-factory.h b/core/async-rpc-retrying-caller-factory.h
index f1ffdac..3c11f52 100644
--- a/core/async-rpc-retrying-caller-factory.h
+++ b/core/async-rpc-retrying-caller-factory.h
@@ -62,17 +62,17 @@ class SingleRequestCallerBuilder
return shared_this();
}
- SharedThisPtr rpc_timeout(nanoseconds rpc_timeout_nanos) {
+ SharedThisPtr rpc_timeout(std::chrono::nanoseconds rpc_timeout_nanos) {
rpc_timeout_nanos_ = rpc_timeout_nanos;
return shared_this();
}
- SharedThisPtr operation_timeout(nanoseconds operation_timeout_nanos) {
+ SharedThisPtr operation_timeout(std::chrono::nanoseconds operation_timeout_nanos) {
operation_timeout_nanos_ = operation_timeout_nanos;
return shared_this();
}
- SharedThisPtr pause(nanoseconds pause) {
+ SharedThisPtr pause(std::chrono::nanoseconds pause) {
pause_ = pause;
return shared_this();
}
@@ -119,9 +119,9 @@ class SingleRequestCallerBuilder
std::shared_ptr<AsyncConnection> conn_;
std::shared_ptr<folly::HHWheelTimer> retry_timer_;
std::shared_ptr<pb::TableName> table_name_;
- nanoseconds rpc_timeout_nanos_;
- nanoseconds operation_timeout_nanos_;
- nanoseconds pause_;
+ std::chrono::nanoseconds rpc_timeout_nanos_;
+ std::chrono::nanoseconds operation_timeout_nanos_;
+ std::chrono::nanoseconds pause_;
uint32_t max_retries_;
uint32_t start_log_errors_count_;
std::string row_;
@@ -149,17 +149,17 @@ class BatchCallerBuilder : public std::enable_shared_from_this<BatchCallerBuilde
return shared_this();
}
- SharedThisPtr operation_timeout(nanoseconds operation_timeout_nanos) {
+ SharedThisPtr operation_timeout(std::chrono::nanoseconds operation_timeout_nanos) {
operation_timeout_nanos_ = operation_timeout_nanos;
return shared_this();
}
- SharedThisPtr rpc_timeout(nanoseconds rpc_timeout_nanos) {
+ SharedThisPtr rpc_timeout(std::chrono::nanoseconds rpc_timeout_nanos) {
rpc_timeout_nanos_ = rpc_timeout_nanos;
return shared_this();
}
- SharedThisPtr pause(nanoseconds pause_ns) {
+ SharedThisPtr pause(std::chrono::nanoseconds pause_ns) {
pause_ns_ = pause_ns;
return shared_this();
}
@@ -192,10 +192,10 @@ class BatchCallerBuilder : public std::enable_shared_from_this<BatchCallerBuilde
std::shared_ptr<folly::HHWheelTimer> retry_timer_;
std::shared_ptr<hbase::pb::TableName> table_name_ = nullptr;
std::shared_ptr<std::vector<hbase::Get>> actions_ = nullptr;
- nanoseconds pause_ns_;
+ std::chrono::nanoseconds pause_ns_;
int32_t max_attempts_ = 0;
- nanoseconds operation_timeout_nanos_;
- nanoseconds rpc_timeout_nanos_;
+ std::chrono::nanoseconds operation_timeout_nanos_;
+ std::chrono::nanoseconds rpc_timeout_nanos_;
int32_t start_log_errors_count_ = 0;
};
class AsyncRpcRetryingCallerFactory {
diff --git a/core/async-rpc-retrying-caller.cc b/core/async-rpc-retrying-caller.cc
index f8b237b..a2c5f0e 100644
--- a/core/async-rpc-retrying-caller.cc
+++ b/core/async-rpc-retrying-caller.cc
@@ -44,9 +44,9 @@ template <typename RESP>
AsyncSingleRequestRpcRetryingCaller<RESP>::AsyncSingleRequestRpcRetryingCaller(
std::shared_ptr<AsyncConnection> conn, std::shared_ptr<folly::HHWheelTimer> retry_timer,
std::shared_ptr<hbase::pb::TableName> table_name, const std::string& row,
- RegionLocateType locate_type, Callable<RESP> callable, nanoseconds pause, uint32_t max_retries,
- nanoseconds operation_timeout_nanos, nanoseconds rpc_timeout_nanos,
- uint32_t start_log_errors_count)
+ RegionLocateType locate_type, Callable<RESP> callable, std::chrono::nanoseconds pause,
+ uint32_t max_retries, std::chrono::nanoseconds operation_timeout_nanos,
+ std::chrono::nanoseconds rpc_timeout_nanos, uint32_t start_log_errors_count)
: conn_(conn),
retry_timer_(retry_timer),
table_name_(table_name),
@@ -144,7 +144,7 @@ void AsyncSingleRequestRpcRetryingCaller<RESP>::OnError(
conn_->retry_executor()->add([&]() {
retry_timer_->scheduleTimeoutFn(
[this]() { conn_->cpu_executor()->add([&]() { LocateThenCall(); }); },
- milliseconds(TimeUtil::ToMillis(delay_ns)));
+ std::chrono::milliseconds(TimeUtil::ToMillis(delay_ns)));
});
}
@@ -212,8 +212,8 @@ void AsyncSingleRequestRpcRetryingCaller<RESP>::ResetController(
std::shared_ptr<HBaseRpcController> controller, const int64_t& timeout_ns) {
controller->Reset();
if (timeout_ns >= 0) {
- controller->set_call_timeout(
- milliseconds(std::min(static_cast<int64_t>(INT_MAX), TimeUtil::ToMillis(timeout_ns))));
+ controller->set_call_timeout(std::chrono::milliseconds(
+ std::min(static_cast<int64_t>(INT_MAX), TimeUtil::ToMillis(timeout_ns))));
}
}
diff --git a/core/async-rpc-retrying-caller.h b/core/async-rpc-retrying-caller.h
index c86ad0b5..fa3c288 100644
--- a/core/async-rpc-retrying-caller.h
+++ b/core/async-rpc-retrying-caller.h
@@ -37,9 +37,6 @@
#include "exceptions/exception.h"
#include "if/HBase.pb.h"
-using std::chrono::nanoseconds;
-using std::chrono::milliseconds;
-
namespace hbase {
template <typename T>
@@ -70,14 +67,12 @@ using Callable =
template <typename RESP>
class AsyncSingleRequestRpcRetryingCaller {
public:
- AsyncSingleRequestRpcRetryingCaller(std::shared_ptr<AsyncConnection> conn,
- std::shared_ptr<folly::HHWheelTimer> retry_timer,
- std::shared_ptr<hbase::pb::TableName> table_name,
- const std::string& row, RegionLocateType locate_type,
- Callable<RESP> callable, nanoseconds pause,
- uint32_t max_retries, nanoseconds operation_timeout_nanos,
- nanoseconds rpc_timeout_nanos,
- uint32_t start_log_errors_count);
+ AsyncSingleRequestRpcRetryingCaller(
+ std::shared_ptr<AsyncConnection> conn, std::shared_ptr<folly::HHWheelTimer> retry_timer,
+ std::shared_ptr<hbase::pb::TableName> table_name, const std::string& row,
+ RegionLocateType locate_type, Callable<RESP> callable, std::chrono::nanoseconds pause,
+ uint32_t max_retries, std::chrono::nanoseconds operation_timeout_nanos,
+ std::chrono::nanoseconds rpc_timeout_nanos, uint32_t start_log_errors_count);
virtual ~AsyncSingleRequestRpcRetryingCaller();
@@ -107,10 +102,10 @@ class AsyncSingleRequestRpcRetryingCaller {
std::string row_;
RegionLocateType locate_type_;
Callable<RESP> callable_;
- nanoseconds pause_;
+ std::chrono::nanoseconds pause_;
uint32_t max_retries_;
- nanoseconds operation_timeout_nanos_;
- nanoseconds rpc_timeout_nanos_;
+ std::chrono::nanoseconds operation_timeout_nanos_;
+ std::chrono::nanoseconds rpc_timeout_nanos_;
uint32_t start_log_errors_count_;
std::shared_ptr<folly::Promise<RESP>> promise_;
std::shared_ptr<HBaseRpcController> controller_;
diff --git a/core/async-rpc-retrying-test.cc b/core/async-rpc-retrying-test.cc
index 11750eb..0f83914 100644
--- a/core/async-rpc-retrying-test.cc
+++ b/core/async-rpc-retrying-test.cc
@@ -66,6 +66,7 @@ using hbase::RespConverter;
using hbase::Put;
using hbase::TimeUtil;
using hbase::Client;
+using hbase::security::User;
using ::testing::Return;
using ::testing::_;
@@ -350,7 +351,7 @@ void runTest(std::shared_ptr<AsyncRegionLocatorBase> region_locator, std::string
/* call with retry to get result */
auto async_caller =
- builder->table(std::make_shared<TableName>(tn))
+ builder->table(std::make_shared<hbase::pb::TableName>(tn))
->row(row)
->rpc_timeout(conn->connection_conf()->read_rpc_timeout())
->operation_timeout(conn->connection_conf()->operation_timeout())
diff --git a/core/client.cc b/core/client.cc
index 0053e19..e23aeae 100644
--- a/core/client.cc
+++ b/core/client.cc
@@ -25,6 +25,8 @@
#include <memory>
#include <utility>
+using hbase::pb::TableName;
+
namespace hbase {
Client::Client() {
diff --git a/core/client.h b/core/client.h
index 2719470..5563a15 100644
--- a/core/client.h
+++ b/core/client.h
@@ -29,9 +29,8 @@
#include "core/table.h"
#include "serde/table-name.h"
-using hbase::pb::TableName;
-
namespace hbase {
+
class Table;
/**
* Client.
@@ -54,7 +53,7 @@ class Client {
* @brief Retrieve a Table implementation for accessing a table.
* @param - table_name
*/
- std::unique_ptr<::hbase::Table> Table(const TableName& table_name);
+ std::unique_ptr<::hbase::Table> Table(const pb::TableName& table_name);
/**
* @brief Close the Client connection.
diff --git a/core/connection-configuration.h b/core/connection-configuration.h
index c6d1d60..adc8d5b 100644
--- a/core/connection-configuration.h
+++ b/core/connection-configuration.h
@@ -25,9 +25,6 @@
#include "core/configuration.h"
-using std::chrono::nanoseconds;
-using std::chrono::milliseconds;
-
namespace hbase {
/**
@@ -57,9 +54,10 @@ class ConnectionConfiguration {
}
// Used by tests
- ConnectionConfiguration(nanoseconds connect_timeout, nanoseconds operation_timeout,
- nanoseconds rpc_timeout, nanoseconds pause, uint32_t max_retries,
- uint32_t start_log_errors_count)
+ ConnectionConfiguration(std::chrono::nanoseconds connect_timeout,
+ std::chrono::nanoseconds operation_timeout,
+ std::chrono::nanoseconds rpc_timeout, std::chrono::nanoseconds pause,
+ uint32_t max_retries, uint32_t start_log_errors_count)
: connect_timeout_(connect_timeout),
operation_timeout_(operation_timeout),
meta_operation_timeout_(operation_timeout),
@@ -70,25 +68,25 @@ class ConnectionConfiguration {
max_retries_(max_retries),
start_log_errors_count_(start_log_errors_count) {}
- nanoseconds connect_timeout() const { return connect_timeout_; }
+ std::chrono::nanoseconds connect_timeout() const { return connect_timeout_; }
- nanoseconds meta_operation_timeout() const { return meta_operation_timeout_; }
+ std::chrono::nanoseconds meta_operation_timeout() const { return meta_operation_timeout_; }
// timeout for a whole operation such as get, put or delete. Notice that scan will not be effected
// by this value, see scanTimeoutNs.
- nanoseconds operation_timeout() const { return operation_timeout_; }
+ std::chrono::nanoseconds operation_timeout() const { return operation_timeout_; }
// timeout for each rpc request. Can be overridden by a more specific config, such as
// readRpcTimeout or writeRpcTimeout.
- nanoseconds rpc_timeout() const { return rpc_timeout_; }
+ std::chrono::nanoseconds rpc_timeout() const { return rpc_timeout_; }
// timeout for each read rpc request
- nanoseconds read_rpc_timeout() const { return read_rpc_timeout_; }
+ std::chrono::nanoseconds read_rpc_timeout() const { return read_rpc_timeout_; }
// timeout for each write rpc request
- nanoseconds write_rpc_timeout() const { return write_rpc_timeout_; }
+ std::chrono::nanoseconds write_rpc_timeout() const { return write_rpc_timeout_; }
- nanoseconds pause() const { return pause_; }
+ std::chrono::nanoseconds pause() const { return pause_; }
uint32_t max_retries() const { return max_retries_; }
@@ -97,7 +95,7 @@ class ConnectionConfiguration {
// The scan timeout is used as operation timeout for every
// operations in a scan, such as openScanner or next.
- nanoseconds scan_timeout() const { return scan_timeout_; }
+ std::chrono::nanoseconds scan_timeout() const { return scan_timeout_; }
uint32_t scanner_caching() const { return scanner_caching_; }
@@ -184,25 +182,25 @@ class ConnectionConfiguration {
*/
static constexpr const uint64_t kDefaultClientScannerMaxResultsSize = 2 * 1024 * 1024;
- nanoseconds connect_timeout_;
- nanoseconds meta_operation_timeout_;
- nanoseconds operation_timeout_;
- nanoseconds rpc_timeout_;
- nanoseconds read_rpc_timeout_;
- nanoseconds write_rpc_timeout_;
- nanoseconds pause_;
+ std::chrono::nanoseconds connect_timeout_;
+ std::chrono::nanoseconds meta_operation_timeout_;
+ std::chrono::nanoseconds operation_timeout_;
+ std::chrono::nanoseconds rpc_timeout_;
+ std::chrono::nanoseconds read_rpc_timeout_;
+ std::chrono::nanoseconds write_rpc_timeout_;
+ std::chrono::nanoseconds pause_;
uint32_t max_retries_;
uint32_t start_log_errors_count_;
- nanoseconds scan_timeout_;
+ std::chrono::nanoseconds scan_timeout_;
uint32_t scanner_caching_;
uint64_t scanner_max_result_size_;
- static nanoseconds ToNanos(const uint64_t& millis) {
- return std::chrono::duration_cast<nanoseconds>(milliseconds(millis));
+ static std::chrono::nanoseconds ToNanos(const uint64_t& millis) {
+ return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::milliseconds(millis));
}
- static uint64_t ToMillis(const nanoseconds& nanos) {
- return std::chrono::duration_cast<milliseconds>(nanos).count();
+ static uint64_t ToMillis(const std::chrono::nanoseconds& nanos) {
+ return std::chrono::duration_cast<std::chrono::milliseconds>(nanos).count();
}
};
diff --git a/core/filter.h b/core/filter.h
index 10accaa..1e870f9 100644
--- a/core/filter.h
+++ b/core/filter.h
@@ -29,8 +29,6 @@
#include "if/Filter.pb.h"
#include "if/HBase.pb.h"
-using google::protobuf::Message;
-
namespace hbase {
/**
@@ -59,13 +57,13 @@ namespace hbase {
*/
class Filter {
public:
- Filter(std::string java_class_name, std::unique_ptr<Message> data)
+ Filter(std::string java_class_name, std::unique_ptr<google::protobuf::Message> data)
: java_class_name_(java_class_name), data_(std::move(data)) {}
virtual ~Filter() {}
const std::string java_class_name() const { return java_class_name_; }
- const Message& data() const { return *data_; }
+ const google::protobuf::Message& data() const { return *data_; }
/**
* Serialize the filter data to the given buffer. Does protobuf encoding by default.
* Can be overriden if Filter does not use protobuf.
@@ -85,7 +83,7 @@ class Filter {
}
private:
- std::unique_ptr<Message> data_;
+ std::unique_ptr<google::protobuf::Message> data_;
std::string java_class_name_;
};
@@ -94,7 +92,7 @@ class Filter {
*/
class Comparator {
public:
- Comparator(std::string java_class_name, std::unique_ptr<Message> data)
+ Comparator(std::string java_class_name, std::unique_ptr<google::protobuf::Message> data)
: java_class_name_(java_class_name), data_(std::move(data)) {}
virtual ~Comparator() {}
@@ -119,7 +117,7 @@ class Comparator {
}
private:
- std::unique_ptr<Message> data_;
+ std::unique_ptr<google::protobuf::Message> data_;
std::string java_class_name_;
};
diff --git a/core/get.h b/core/get.h
index 098635d..c4cddfb 100644
--- a/core/get.h
+++ b/core/get.h
@@ -29,7 +29,6 @@
#include "core/time-range.h"
#include "if/Client.pb.h"
-using hbase::Row;
namespace hbase {
class Get : public Row, public Query {
diff --git a/core/hbase-rpc-controller.h b/core/hbase-rpc-controller.h
index 661c810..9bb89b9 100644
--- a/core/hbase-rpc-controller.h
+++ b/core/hbase-rpc-controller.h
@@ -22,19 +22,14 @@
#include <chrono>
#include <string>
-using google::protobuf::RpcController;
-using google::protobuf::Closure;
-
-using std::chrono::milliseconds;
-
namespace hbase {
-class HBaseRpcController : public RpcController {
+class HBaseRpcController : public google::protobuf::RpcController {
public:
HBaseRpcController() {}
virtual ~HBaseRpcController() = default;
- void set_call_timeout(const milliseconds& call_timeout) {
+ void set_call_timeout(const std::chrono::milliseconds& call_timeout) {
// TODO:
}
@@ -50,7 +45,7 @@ class HBaseRpcController : public RpcController {
bool IsCanceled() const override { return false; }
- void NotifyOnCancel(Closure* callback) override {}
+ void NotifyOnCancel(google::protobuf::Closure* callback) override {}
};
} /* namespace hbase */
diff --git a/core/location-cache.cc b/core/location-cache.cc
index e0afcfb..d368314 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -35,22 +35,11 @@
#include <utility>
-using namespace std;
-using namespace folly;
-using hbase::RpcConnection;
-
-using wangle::ServiceFilter;
-using hbase::Request;
-using hbase::Response;
-using hbase::LocationCache;
-using hbase::RegionLocation;
-using hbase::HBaseService;
-using hbase::ConnectionPool;
-using hbase::pb::ScanResponse;
-using hbase::pb::TableName;
-using hbase::pb::ServerName;
using hbase::pb::MetaRegionServer;
-using hbase::pb::RegionInfo;
+using hbase::pb::ServerName;
+using hbase::pb::TableName;
+
+namespace hbase {
LocationCache::LocationCache(std::shared_ptr<hbase::Configuration> conf,
std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
@@ -74,8 +63,8 @@ LocationCache::~LocationCache() {
LOG(INFO) << "Closed connection to ZooKeeper.";
}
-Future<ServerName> LocationCache::LocateMeta() {
- lock_guard<mutex> g(meta_lock_);
+folly::Future<ServerName> LocationCache::LocateMeta() {
+ std::lock_guard<std::mutex> g(meta_lock_);
if (meta_promise_ == nullptr) {
this->RefreshMetaLocation();
}
@@ -84,19 +73,19 @@ Future<ServerName> LocationCache::LocateMeta() {
void LocationCache::InvalidateMeta() {
if (meta_promise_ != nullptr) {
- lock_guard<mutex> g(meta_lock_);
+ std::lock_guard<std::mutex> g(meta_lock_);
meta_promise_ = nullptr;
}
}
/// MUST hold the meta_lock_
void LocationCache::RefreshMetaLocation() {
- meta_promise_ = make_unique<SharedPromise<ServerName>>();
+ meta_promise_ = std::make_unique<folly::SharedPromise<ServerName>>();
cpu_executor_->add([&] { meta_promise_->setWith([&] { return this->ReadMetaLocation(); }); });
}
ServerName LocationCache::ReadMetaLocation() {
- auto buf = IOBuf::create(4096);
+ auto buf = folly::IOBuf::create(4096);
ZkDeserializer derser;
// This needs to be int rather than size_t as that's what ZK expects.
@@ -107,7 +96,7 @@ ServerName LocationCache::ReadMetaLocation() {
reinterpret_cast<char *>(buf->writableData()), &len, nullptr);
if (zk_result != ZOK || len < 9) {
LOG(ERROR) << "Error getting meta location.";
- throw runtime_error("Error getting meta location. Quorum: " + zk_quorum_);
+ throw std::runtime_error("Error getting meta location. Quorum: " + zk_quorum_);
}
buf->append(len);
@@ -118,8 +107,8 @@ ServerName LocationCache::ReadMetaLocation() {
return mrs.server();
}
-Future<std::shared_ptr<RegionLocation>> LocationCache::LocateFromMeta(const TableName &tn,
- const string &row) {
+folly::Future<std::shared_ptr<RegionLocation>> LocationCache::LocateFromMeta(
+ const TableName &tn, const std::string &row) {
return this->LocateMeta()
.via(cpu_executor_.get())
.then([this](ServerName sn) {
@@ -150,17 +139,16 @@ Future<std::shared_ptr<RegionLocation>> LocationCache::LocateFromMeta(const Tabl
// assertion errors
return rl;
})
- .then([tn, this](shared_ptr<RegionLocation> rl) {
+ .then([tn, this](std::shared_ptr<RegionLocation> rl) {
// now add fetched location to the cache.
this->CacheLocation(tn, rl);
return rl;
});
}
-Future<shared_ptr<RegionLocation>> LocationCache::LocateRegion(const hbase::pb::TableName &tn,
- const std::string &row,
- const RegionLocateType locate_type,
- const int64_t locate_ns) {
+folly::Future<std::shared_ptr<RegionLocation>> LocationCache::LocateRegion(
+ const TableName &tn, const std::string &row, const RegionLocateType locate_type,
+ const int64_t locate_ns) {
// TODO: implement region locate type and timeout
auto cached_loc = this->GetCachedLocation(tn, row);
if (cached_loc != nullptr) {
@@ -171,8 +159,8 @@ Future<shared_ptr<RegionLocation>> LocationCache::LocateRegion(const hbase::pb::
}
// must hold shared lock on locations_lock_
-shared_ptr<RegionLocation> LocationCache::GetCachedLocation(const hbase::pb::TableName &tn,
- const std::string &row) {
+std::shared_ptr<RegionLocation> LocationCache::GetCachedLocation(const hbase::pb::TableName &tn,
+ const std::string &row) {
auto t_locs = this->GetTableLocations(tn);
std::shared_lock<folly::SharedMutexWritePriority> lock(locations_lock_);
@@ -213,7 +201,7 @@ shared_ptr<RegionLocation> LocationCache::GetCachedLocation(const hbase::pb::Tab
// must hold unique lock on locations_lock_
void LocationCache::CacheLocation(const hbase::pb::TableName &tn,
- const shared_ptr<RegionLocation> loc) {
+ const std::shared_ptr<RegionLocation> loc) {
auto t_locs = this->GetTableLocations(tn);
std::unique_lock<folly::SharedMutexWritePriority> lock(locations_lock_);
@@ -228,7 +216,7 @@ bool LocationCache::IsLocationCached(const hbase::pb::TableName &tn, const std::
// shared lock needed for cases when this table has been requested before;
// in the rare case it hasn't, unique lock will be grabbed to add it to cache
-shared_ptr<hbase::PerTableLocationMap> LocationCache::GetTableLocations(
+std::shared_ptr<hbase::PerTableLocationMap> LocationCache::GetTableLocations(
const hbase::pb::TableName &tn) {
auto found_locs = this->GetCachedTableLocations(tn);
if (found_locs == nullptr) {
@@ -237,9 +225,9 @@ shared_ptr<hbase::PerTableLocationMap> LocationCache::GetTableLocations(
return found_locs;
}
-shared_ptr<hbase::PerTableLocationMap> LocationCache::GetCachedTableLocations(
+std::shared_ptr<hbase::PerTableLocationMap> LocationCache::GetCachedTableLocations(
const hbase::pb::TableName &tn) {
- SharedMutexWritePriority::ReadHolder r_holder{locations_lock_};
+ folly::SharedMutexWritePriority::ReadHolder r_holder{locations_lock_};
auto table_locs = cached_locations_.find(tn);
if (table_locs != cached_locations_.end()) {
@@ -249,38 +237,38 @@ shared_ptr<hbase::PerTableLocationMap> LocationCache::GetCachedTableLocations(
}
}
-shared_ptr<hbase::PerTableLocationMap> LocationCache::GetNewTableLocations(
+std::shared_ptr<hbase::PerTableLocationMap> LocationCache::GetNewTableLocations(
const hbase::pb::TableName &tn) {
// double-check locking under upgradable lock
- SharedMutexWritePriority::UpgradeHolder u_holder{locations_lock_};
+ folly::SharedMutexWritePriority::UpgradeHolder u_holder{locations_lock_};
auto table_locs = cached_locations_.find(tn);
if (table_locs != cached_locations_.end()) {
return table_locs->second;
}
- SharedMutexWritePriority::WriteHolder w_holder{std::move(u_holder)};
+ folly::SharedMutexWritePriority::WriteHolder w_holder{std::move(u_holder)};
- auto t_locs_p = make_shared<map<std::string, shared_ptr<RegionLocation>>>();
+ auto t_locs_p = std::make_shared<std::map<std::string, std::shared_ptr<RegionLocation>>>();
cached_locations_.insert(std::make_pair(tn, t_locs_p));
return t_locs_p;
}
// must hold unique lock on locations_lock_
void LocationCache::ClearCache() {
- unique_lock<SharedMutexWritePriority> lock(locations_lock_);
+ std::unique_lock<folly::SharedMutexWritePriority> lock(locations_lock_);
cached_locations_.clear();
}
// must hold unique lock on locations_lock_
void LocationCache::ClearCachedLocations(const hbase::pb::TableName &tn) {
- unique_lock<SharedMutexWritePriority> lock(locations_lock_);
+ std::unique_lock<folly::SharedMutexWritePriority> lock(locations_lock_);
cached_locations_.erase(tn);
}
// must hold unique lock on locations_lock_
void LocationCache::ClearCachedLocation(const hbase::pb::TableName &tn, const std::string &row) {
auto table_locs = this->GetTableLocations(tn);
- unique_lock<folly::SharedMutexWritePriority> lock(locations_lock_);
+ std::unique_lock<folly::SharedMutexWritePriority> lock(locations_lock_);
table_locs->erase(row);
}
@@ -289,3 +277,4 @@ void LocationCache::UpdateCachedLocation(const RegionLocation &loc,
// TODO: just clears the location for now. We can inspect RegionMovedExceptions, etc later.
ClearCachedLocation(loc.region_info().table_name(), loc.region_info().start_key());
}
+} // namespace hbase
diff --git a/core/meta-utils.cc b/core/meta-utils.cc
index 119520f..0577bfc 100644
--- a/core/meta-utils.cc
+++ b/core/meta-utils.cc
@@ -31,14 +31,12 @@
#include "serde/table-name.h"
using hbase::pb::TableName;
-using hbase::MetaUtil;
-using hbase::Request;
-using hbase::Response;
-using hbase::RegionLocation;
using hbase::pb::RegionInfo;
+using hbase::pb::RegionSpecifier_RegionSpecifierType;
using hbase::pb::ScanRequest;
using hbase::pb::ServerName;
-using hbase::pb::RegionSpecifier_RegionSpecifierType;
+
+namespace hbase {
static const std::string META_REGION = "1588230740";
static const std::string CATALOG_FAMILY = "info";
@@ -113,3 +111,4 @@ std::shared_ptr<RegionLocation> MetaUtil::CreateLocation(const Response &resp) {
auto server_name = folly::to<ServerName>(*server_str);
return std::make_shared<RegionLocation>(row, std::move(region_info), server_name, nullptr);
}
+} // namespace hbase
diff --git a/core/multi-response.cc b/core/multi-response.cc
index 562f3b6..f620c98 100644
--- a/core/multi-response.cc
+++ b/core/multi-response.cc
@@ -20,6 +20,8 @@
#include "core/multi-response.h"
#include "core/region-result.h"
+using hbase::pb::RegionLoadStats;
+
namespace hbase {
MultiResponse::MultiResponse() {}
diff --git a/core/multi-response.h b/core/multi-response.h
index cebd2b7..96883fd 100644
--- a/core/multi-response.h
+++ b/core/multi-response.h
@@ -28,10 +28,6 @@
#include "core/result.h"
#include "if/Client.pb.h"
-using hbase::RegionResult;
-using hbase::Result;
-using hbase::pb::RegionLoadStats;
-
namespace hbase {
class MultiResponse {
@@ -62,7 +58,7 @@ class MultiResponse {
const std::map<std::string, std::shared_ptr<std::exception>>& RegionExceptions() const;
- void AddStatistic(const std::string& region_name, std::shared_ptr<RegionLoadStats> stat);
+ void AddStatistic(const std::string& region_name, std::shared_ptr<pb::RegionLoadStats> stat);
const std::map<std::string, std::shared_ptr<RegionResult>>& RegionResults() const;
diff --git a/core/raw-async-table.cc b/core/raw-async-table.cc
index 9e0d4a3..1a66a04 100644
--- a/core/raw-async-table.cc
+++ b/core/raw-async-table.cc
@@ -22,11 +22,13 @@
#include "core/request-converter.h"
#include "core/response-converter.h"
+using hbase::security::User;
+
namespace hbase {
template <typename RESP>
std::shared_ptr<SingleRequestCallerBuilder<RESP>> RawAsyncTable::CreateCallerBuilder(
- std::string row, nanoseconds rpc_timeout) {
+ std::string row, std::chrono::nanoseconds rpc_timeout) {
return connection_->caller_factory()
->Single<RESP>()
->table(table_name_)
@@ -54,7 +56,7 @@ folly::Future<RESP> RawAsyncTable::Call(
[resp_converter](const std::unique_ptr<PRESP>& presp) { return resp_converter(*presp); });
}
-Future<std::shared_ptr<Result>> RawAsyncTable::Get(const hbase::Get& get) {
+folly::Future<std::shared_ptr<Result>> RawAsyncTable::Get(const hbase::Get& get) {
auto caller =
CreateCallerBuilder<std::shared_ptr<Result>>(get.row(), connection_conf_->read_rpc_timeout())
->action([=, &get](std::shared_ptr<hbase::HBaseRpcController> controller,
@@ -76,27 +78,28 @@ Future<std::shared_ptr<Result>> RawAsyncTable::Get(const hbase::Get& get) {
return caller->Call().then([caller](const auto r) { return r; });
}
-Future<Unit> RawAsyncTable::Put(const hbase::Put& put) {
+folly::Future<folly::Unit> RawAsyncTable::Put(const hbase::Put& put) {
auto caller =
- CreateCallerBuilder<Unit>(put.row(), connection_conf_->write_rpc_timeout())
- ->action([=, &put](std::shared_ptr<hbase::HBaseRpcController> controller,
- std::shared_ptr<hbase::RegionLocation> loc,
- std::shared_ptr<hbase::RpcClient> rpc_client) -> folly::Future<Unit> {
- return Call<hbase::Put, hbase::Request, hbase::Response, Unit>(
+ CreateCallerBuilder<folly::Unit>(put.row(), connection_conf_->write_rpc_timeout())
+ ->action([=, &put](
+ std::shared_ptr<hbase::HBaseRpcController> controller,
+ std::shared_ptr<hbase::RegionLocation> loc,
+ std::shared_ptr<hbase::RpcClient> rpc_client) -> folly::Future<folly::Unit> {
+ return Call<hbase::Put, hbase::Request, hbase::Response, folly::Unit>(
rpc_client, controller, loc, put, &hbase::RequestConverter::ToMutateRequest,
- [](const Response& r) -> Unit { return folly::unit; });
+ [](const Response& r) -> folly::Unit { return folly::unit; });
})
->Build();
return caller->Call().then([caller](const auto r) { return r; });
}
-Future<std::vector<Try<std::shared_ptr<Result>>>> RawAsyncTable::Get(
+folly::Future<std::vector<folly::Try<std::shared_ptr<Result>>>> RawAsyncTable::Get(
const std::vector<hbase::Get>& gets) {
return this->Batch(gets);
}
-Future<std::vector<Try<std::shared_ptr<Result>>>> RawAsyncTable::Batch(
+folly::Future<std::vector<folly::Try<std::shared_ptr<Result>>>> RawAsyncTable::Batch(
const std::vector<hbase::Get>& gets) {
auto caller = connection_->caller_factory()
->Batch()
diff --git a/core/raw-async-table.h b/core/raw-async-table.h
index e26d46e..9d9ed91 100644
--- a/core/raw-async-table.h
+++ b/core/raw-async-table.h
@@ -33,13 +33,6 @@
#include "core/put.h"
#include "core/result.h"
-using folly::Future;
-using folly::Try;
-using folly::Unit;
-using hbase::pb::TableName;
-using std::chrono::nanoseconds;
-using std::chrono::milliseconds;
-
namespace hbase {
/**
@@ -48,26 +41,29 @@ namespace hbase {
*/
class RawAsyncTable {
public:
- RawAsyncTable(std::shared_ptr<TableName> table_name, std::shared_ptr<AsyncConnection> connection)
+ RawAsyncTable(std::shared_ptr<pb::TableName> table_name,
+ std::shared_ptr<AsyncConnection> connection)
: connection_(connection),
connection_conf_(connection->connection_conf()),
table_name_(table_name),
rpc_client_(connection->rpc_client()) {}
virtual ~RawAsyncTable() = default;
- Future<std::shared_ptr<Result>> Get(const hbase::Get& get);
+ folly::Future<std::shared_ptr<Result>> Get(const hbase::Get& get);
- Future<Unit> Put(const hbase::Put& put);
+ folly::Future<folly::Unit> Put(const hbase::Put& put);
void Close() {}
- Future<std::vector<Try<std::shared_ptr<Result>>>> Get(const std::vector<hbase::Get>& gets);
- Future<std::vector<Try<std::shared_ptr<Result>>>> Batch(const std::vector<hbase::Get>& gets);
+ folly::Future<std::vector<folly::Try<std::shared_ptr<Result>>>> Get(
+ const std::vector<hbase::Get>& gets);
+ folly::Future<std::vector<folly::Try<std::shared_ptr<Result>>>> Batch(
+ const std::vector<hbase::Get>& gets);
private:
/* Data */
std::shared_ptr<AsyncConnection> connection_;
std::shared_ptr<ConnectionConfiguration> connection_conf_;
- std::shared_ptr<TableName> table_name_;
+ std::shared_ptr<pb::TableName> table_name_;
std::shared_ptr<RpcClient> rpc_client_;
/* Methods */
@@ -79,7 +75,7 @@ class RawAsyncTable {
const RespConverter<RESP, PRESP> resp_converter);
template <typename RESP>
- std::shared_ptr<SingleRequestCallerBuilder<RESP>> CreateCallerBuilder(std::string row,
- nanoseconds rpc_timeout);
+ std::shared_ptr<SingleRequestCallerBuilder<RESP>> CreateCallerBuilder(
+ std::string row, std::chrono::nanoseconds rpc_timeout);
};
} // namespace hbase
diff --git a/core/region-request.h b/core/region-request.h
index 7ce7c96..aded3a9 100644
--- a/core/region-request.h
+++ b/core/region-request.h
@@ -18,13 +18,13 @@
*/
#pragma once
+
#include <memory>
#include <queue>
#include <vector>
#include "core/action.h"
#include "core/region-location.h"
-using hbase::Action;
namespace hbase {
class RegionRequest {
diff --git a/core/region-result.cc b/core/region-result.cc
index d9ab942..05ab274 100644
--- a/core/region-result.cc
+++ b/core/region-result.cc
@@ -21,7 +21,6 @@
#include <glog/logging.h>
#include <stdexcept>
-using hbase::Result;
using hbase::pb::RegionLoadStats;
namespace hbase {
diff --git a/core/region-result.h b/core/region-result.h
index 9b7ca03..cfd9e5a 100644
--- a/core/region-result.h
+++ b/core/region-result.h
@@ -26,30 +26,29 @@
#include "core/result.h"
#include "if/Client.pb.h"
-using hbase::Result;
-using hbase::pb::RegionLoadStats;
-
namespace hbase {
+
using ResultOrExceptionTuple =
std::tuple<std::shared_ptr<hbase::Result>, std::shared_ptr<std::exception>>;
+
class RegionResult {
public:
RegionResult();
void AddResultOrException(int32_t index, std::shared_ptr<hbase::Result> result,
std::shared_ptr<std::exception> exc);
- void set_stat(std::shared_ptr<RegionLoadStats> stat);
+ void set_stat(std::shared_ptr<pb::RegionLoadStats> stat);
int ResultOrExceptionSize() const;
std::shared_ptr<ResultOrExceptionTuple> ResultOrException(int32_t index) const;
- const std::shared_ptr<RegionLoadStats>& stat() const;
+ const std::shared_ptr<pb::RegionLoadStats>& stat() const;
~RegionResult();
private:
std::map<int, ResultOrExceptionTuple> result_or_excption_;
- std::shared_ptr<RegionLoadStats> stat_;
+ std::shared_ptr<pb::RegionLoadStats> stat_;
};
} /* namespace hbase */
diff --git a/core/request-converter.cc b/core/request-converter.cc
index c90e1ab..eb293f5 100644
--- a/core/request-converter.cc
+++ b/core/request-converter.cc
@@ -24,8 +24,8 @@
#include <utility>
#include "if/Client.pb.h"
-using hbase::Request;
using hbase::pb::GetRequest;
+using hbase::pb::MutationProto;
using hbase::pb::RegionAction;
using hbase::pb::RegionSpecifier;
using hbase::pb::RegionSpecifier_RegionSpecifierType;
diff --git a/core/request-converter.h b/core/request-converter.h
index 6861604..b604d18 100644
--- a/core/request-converter.h
+++ b/core/request-converter.h
@@ -34,11 +34,6 @@
#include "if/Client.pb.h"
#include "if/HBase.pb.h"
-using hbase::pb::MutationProto;
-using hbase::pb::RegionAction;
-using hbase::pb::RegionSpecifier;
-using hbase::pb::ServerName;
-using hbase::ServerRequest;
using MutationType = hbase::pb::MutationProto_MutationType;
using DeleteType = hbase::pb::MutationProto_DeleteType;
@@ -73,8 +68,9 @@ class RequestConverter {
static std::unique_ptr<Request> ToMutateRequest(const Put &put, const std::string ®ion_name);
- static std::unique_ptr<MutationProto> ToMutation(const MutationType type,
- const Mutation &mutation, const int64_t nonce);
+ static std::unique_ptr<pb::MutationProto> ToMutation(const MutationType type,
+ const Mutation &mutation,
+ const int64_t nonce);
private:
// Constructor not required. We have all static methods to create PB requests.
@@ -86,7 +82,7 @@ class RequestConverter {
* @param region_specifier - RegionSpecifier to be filled and passed in PB
* Request.
*/
- static void SetRegion(const std::string ®ion_name, RegionSpecifier *region_specifier);
+ static void SetRegion(const std::string ®ion_name, pb::RegionSpecifier *region_specifier);
static std::unique_ptr<hbase::pb::Get> ToGet(const Get &get);
static DeleteType ToDeleteType(const CellType type);
static bool IsDelete(const CellType type);
diff --git a/core/response-converter.h b/core/response-converter.h
index a5095fd..443527d 100644
--- a/core/response-converter.h
+++ b/core/response-converter.h
@@ -28,8 +28,6 @@
#include "if/Client.pb.h"
#include "serde/cell-scanner.h"
-using hbase::Request;
-using hbase::Response;
namespace hbase {
/**
diff --git a/core/server-request.h b/core/server-request.h
index 827b2e7..7f31c2b 100644
--- a/core/server-request.h
+++ b/core/server-request.h
@@ -27,9 +27,6 @@
#include "core/region-location.h"
#include "core/region-request.h"
-using hbase::Action;
-using hbase::RegionRequest;
-
namespace hbase {
class ServerRequest {
diff --git a/core/table.h b/core/table.h
index 142baae..0ee8312 100644
--- a/core/table.h
+++ b/core/table.h
@@ -35,9 +35,8 @@
#include "core/result.h"
#include "serde/table-name.h"
-using hbase::pb::TableName;
-
namespace hbase {
+
class Client;
class Table {
@@ -45,7 +44,7 @@ class Table {
/**
* Constructors
*/
- Table(const TableName &table_name, std::shared_ptr<AsyncConnection> async_connection);
+ Table(const pb::TableName &table_name, std::shared_ptr<AsyncConnection> async_connection);
~Table();
/**
@@ -75,12 +74,12 @@ class Table {
std::shared_ptr<RegionLocation> GetRegionLocation(const std::string &row);
private:
- std::shared_ptr<TableName> table_name_;
+ std::shared_ptr<pb::TableName> table_name_;
std::shared_ptr<AsyncConnection> async_connection_;
std::shared_ptr<hbase::Configuration> conf_;
std::unique_ptr<RawAsyncTable> async_table_;
private:
- milliseconds operation_timeout() const;
+ std::chrono::milliseconds operation_timeout() const;
};
} /* namespace hbase */
diff --git a/exceptions/exception.h b/exceptions/exception.h
index 2943d57..f25fbea 100644
--- a/exceptions/exception.h
+++ b/exceptions/exception.h
@@ -80,7 +80,8 @@ public:
std::shared_ptr<std::vector<ThrowableWithExtraContext>> exceptions) :
IOException(
GetMessage(num_retries, exceptions),
- exceptions->empty() ? folly::exception_wrapper{} : (*exceptions)[exceptions->size() - 1].cause()){
+ exceptions->empty() ? folly::exception_wrapper{}
+ : (*exceptions)[exceptions->size() - 1].cause()){
}
virtual ~RetriesExhaustedException() = default;
diff --git a/serde/rpc.cc b/serde/rpc.cc
index 968cd5b..957a317 100644
--- a/serde/rpc.cc
+++ b/serde/rpc.cc
@@ -32,19 +32,16 @@
#include "if/RPC.pb.h"
#include "utils/version.h"
-using namespace hbase;
-
using folly::IOBuf;
using folly::io::RWPrivateCursor;
using google::protobuf::Message;
-using google::protobuf::Message;
using google::protobuf::io::ArrayInputStream;
using google::protobuf::io::ArrayOutputStream;
using google::protobuf::io::CodedInputStream;
using google::protobuf::io::CodedOutputStream;
using google::protobuf::io::ZeroCopyOutputStream;
-using std::string;
-using std::unique_ptr;
+
+namespace hbase {
static const std::string PREAMBLE = "HBas";
static const std::string INTERFACE = "ClientService";
@@ -104,7 +101,7 @@ std::unique_ptr<IOBuf> RpcSerde::Preamble(bool secure) {
return magic;
}
-unique_ptr<IOBuf> RpcSerde::Header(const string &user) {
+std::unique_ptr<IOBuf> RpcSerde::Header(const std::string &user) {
pb::ConnectionHeader h;
// TODO(eclark): Make this not a total lie.
@@ -150,8 +147,8 @@ std::unique_ptr<pb::VersionInfo> RpcSerde::CreateVersionInfo() {
return version_info;
}
-unique_ptr<IOBuf> RpcSerde::Request(const uint32_t call_id, const string &method,
- const Message *msg) {
+std::unique_ptr<IOBuf> RpcSerde::Request(const uint32_t call_id, const std::string &method,
+ const Message *msg) {
pb::RequestHeader rq;
rq.set_method_name(method);
rq.set_call_id(call_id);
@@ -173,7 +170,7 @@ std::unique_ptr<CellScanner> RpcSerde::CreateCellScanner(std::unique_ptr<folly::
return codec_->CreateDecoder(std::move(buf), offset, length);
}
-unique_ptr<IOBuf> RpcSerde::PrependLength(unique_ptr<IOBuf> msg) {
+std::unique_ptr<IOBuf> RpcSerde::PrependLength(std::unique_ptr<IOBuf> msg) {
// Java ints are 4 long. So create a buffer that large
auto len_buf = IOBuf::create(4);
// Then make those bytes visible.
@@ -191,7 +188,7 @@ unique_ptr<IOBuf> RpcSerde::PrependLength(unique_ptr<IOBuf> msg) {
return len_buf;
}
-unique_ptr<IOBuf> RpcSerde::SerializeDelimited(const Message &msg) {
+std::unique_ptr<IOBuf> RpcSerde::SerializeDelimited(const Message &msg) {
// Get the buffer size needed for just the message.
int msg_size = msg.ByteSize();
int buf_size = CodedOutputStream::VarintSize32(msg_size) + msg_size;
@@ -218,7 +215,8 @@ unique_ptr<IOBuf> RpcSerde::SerializeDelimited(const Message &msg) {
return buf;
}
// TODO(eclark): Make this 1 copy.
-unique_ptr<IOBuf> RpcSerde::SerializeMessage(const Message &msg) {
+std::unique_ptr<IOBuf> RpcSerde::SerializeMessage(const Message &msg) {
auto buf = IOBuf::copyBuffer(msg.SerializeAsString());
return buf;
}
+} // namespace hbase
diff --git a/serde/zk.cc b/serde/zk.cc
index bf68400..a71eb87 100644
--- a/serde/zk.cc
+++ b/serde/zk.cc
@@ -25,15 +25,13 @@
#include <string>
-using hbase::ZkDeserializer;
using std::runtime_error;
-using folly::IOBuf;
-using folly::io::Cursor;
-using google::protobuf::Message;
+
+namespace hbase {
static const std::string MAGIC_STRING = "PBUF";
-bool ZkDeserializer::Parse(IOBuf *buf, Message *out) {
+bool ZkDeserializer::Parse(folly::IOBuf *buf, google::protobuf::Message *out) {
// The format is like this
// 1 byte of magic number. 255
// 4 bytes of id length.
@@ -41,7 +39,7 @@ bool ZkDeserializer::Parse(IOBuf *buf, Message *out) {
// 4 bytes of a magic string PBUF
// Then the protobuf serialized without a varint header.
- Cursor c{buf};
+ folly::io::Cursor c{buf};
// There should be a magic number for recoverable zk
uint8_t magic_num = c.read<uint8_t>();
@@ -76,3 +74,4 @@ bool ZkDeserializer::Parse(IOBuf *buf, Message *out) {
return true;
}
+} // namespace hbase
diff --git a/utils/time-util.h b/utils/time-util.h
index 183260b..165e9f1 100644
--- a/utils/time-util.h
+++ b/utils/time-util.h
@@ -21,49 +21,51 @@
#include <chrono>
#include <string>
-using std::chrono::nanoseconds;
-using std::chrono::milliseconds;
-using std::chrono::seconds;
namespace hbase {
+
class TimeUtil {
public:
static inline int64_t ToMillis(const int64_t& nanos) {
- return std::chrono::duration_cast<milliseconds>(nanoseconds(nanos)).count();
+ return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::nanoseconds(nanos))
+ .count();
}
- static inline milliseconds ToMillis(const nanoseconds& nanos) {
- return std::chrono::duration_cast<milliseconds>(nanoseconds(nanos));
+ static inline std::chrono::milliseconds ToMillis(const std::chrono::nanoseconds& nanos) {
+ return std::chrono::duration_cast<std::chrono::milliseconds>(std::chrono::nanoseconds(nanos));
}
- static inline nanoseconds ToNanos(const milliseconds& millis) {
- return std::chrono::duration_cast<nanoseconds>(millis);
+ static inline std::chrono::nanoseconds ToNanos(const std::chrono::milliseconds& millis) {
+ return std::chrono::duration_cast<std::chrono::nanoseconds>(millis);
}
- static inline nanoseconds MillisToNanos(const int64_t& millis) {
- return std::chrono::duration_cast<nanoseconds>(milliseconds(millis));
+ static inline std::chrono::nanoseconds MillisToNanos(const int64_t& millis) {
+ return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::milliseconds(millis));
}
- static inline nanoseconds SecondsToNanos(const int64_t& secs) {
- return std::chrono::duration_cast<nanoseconds>(seconds(secs));
+ static inline std::chrono::nanoseconds SecondsToNanos(const int64_t& secs) {
+ return std::chrono::duration_cast<std::chrono::nanoseconds>(std::chrono::seconds(secs));
}
- static inline std::string ToMillisStr(const nanoseconds& nanos) {
- return std::to_string(std::chrono::duration_cast<milliseconds>(nanos).count());
+ static inline std::string ToMillisStr(const std::chrono::nanoseconds& nanos) {
+ return std::to_string(std::chrono::duration_cast<std::chrono::milliseconds>(nanos).count());
}
static inline int64_t GetNowNanos() {
auto duration = std::chrono::high_resolution_clock::now().time_since_epoch();
- return std::chrono::duration_cast<nanoseconds>(duration).count();
+ return std::chrono::duration_cast<std::chrono::nanoseconds>(duration).count();
}
static inline int64_t ElapsedMillis(const int64_t& start_ns) {
- return std::chrono::duration_cast<milliseconds>(nanoseconds(GetNowNanos() - start_ns)).count();
+ return std::chrono::duration_cast<std::chrono::milliseconds>(
+ std::chrono::nanoseconds(GetNowNanos() - start_ns))
+ .count();
}
static inline std::string ElapsedMillisStr(const int64_t& start_ns) {
- return std::to_string(
- std::chrono::duration_cast<milliseconds>(nanoseconds(GetNowNanos() - start_ns)).count());
+ return std::to_string(std::chrono::duration_cast<std::chrono::milliseconds>(
+ std::chrono::nanoseconds(GetNowNanos() - start_ns))
+ .count());
}
};
} /* namespace hbase */
diff --git a/utils/user-util.cc b/utils/user-util.cc
index 71f0012..092d54c 100644
--- a/utils/user-util.cc
+++ b/utils/user-util.cc
@@ -25,12 +25,11 @@
#include <sys/types.h>
#include <unistd.h>
-using namespace hbase;
-using namespace std;
+namespace hbase {
UserUtil::UserUtil() : once_flag_{} {}
-string UserUtil::user_name(bool secure) {
+std::string UserUtil::user_name(bool secure) {
std::call_once(once_flag_, [this, secure]() { compute_user_name(secure); });
return user_name_;
}
@@ -44,7 +43,7 @@ void UserUtil::compute_user_name(bool secure) {
// make sure that we got something.
if (passwd && passwd->pw_name) {
- user_name_ = string{passwd->pw_name};
+ user_name_ = std::string{passwd->pw_name};
}
if (!secure) return;
krb5_context ctx;
@@ -75,3 +74,4 @@ void UserUtil::compute_user_name(bool secure) {
krb5_free_principal(ctx, princ);
krb5_free_context(ctx);
}
+} // namespace hbase
[hbase] 122/137: HBASE-18578 [C++] Add pause for RPC test
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 2509f5298cf856484ccf83bdd7212b41f16ec880
Author: Xiaobing Zhou <xz...@hortonworks.com>
AuthorDate: Tue Aug 22 12:01:21 2017 -0700
HBASE-18578 [C++] Add pause for RPC test
Signed-off-by: Enis Soztutar <en...@apache.org>
---
connection/rpc-test-server.cc | 9 ++++++-
connection/rpc-test.cc | 61 ++++++++++++++++++++++++++++++++++++-------
2 files changed, 59 insertions(+), 11 deletions(-)
diff --git a/connection/rpc-test-server.cc b/connection/rpc-test-server.cc
index 6132fbb..707bca7 100644
--- a/connection/rpc-test-server.cc
+++ b/connection/rpc-test-server.cc
@@ -88,7 +88,14 @@ Future<std::unique_ptr<Response>> RpcTestService::operator()(std::unique_ptr<Req
response->set_exception(folly::make_exception_wrapper<RpcTestException>("server error!"));
} else if (method_name == "pause") {
- // TODO:
+ auto pb_resp_msg = std::make_shared<EmptyResponseProto>();
+ /* sleeping */
+ auto pb_req_msg = std::static_pointer_cast<PauseRequestProto>(request->req_msg());
+ std::this_thread::sleep_for(std::chrono::milliseconds(pb_req_msg->ms()));
+ response->set_resp_msg(pb_resp_msg);
+ VLOG(1) << "RPC server:"
+ << " pause called, " << pb_req_msg->ms() << " ms";
+
} else if (method_name == "addr") {
// TODO:
} else if (method_name == "socketNotOpen") {
diff --git a/connection/rpc-test.cc b/connection/rpc-test.cc
index 4688950..d541397 100644
--- a/connection/rpc-test.cc
+++ b/connection/rpc-test.cc
@@ -30,6 +30,7 @@
#include <glog/logging.h>
#include <gtest/gtest.h>
#include <boost/thread.hpp>
+#include <chrono>
#include "connection/rpc-client.h"
#include "exceptions/exception.h"
@@ -41,11 +42,14 @@
using namespace wangle;
using namespace folly;
using namespace hbase;
+using namespace std::chrono;
DEFINE_int32(port, 0, "test server port");
DEFINE_string(result_format, "RPC {} returned: {}.", "output format of RPC result");
-DEFINE_string(fail_format, "Shouldn't get here, exception is expected for RPC {}.",
- "output format of enforcing fail");
+DEFINE_string(fail_ex_format, "Shouldn't get here, exception is expected for RPC {}.",
+ "output format of enforcing fail with exception");
+DEFINE_string(fail_no_ex_format, "Shouldn't get here, exception is not expected for RPC {}.",
+ "output format of enforcing fail without exception");
typedef ServerBootstrap<RpcTestServerSerializePipeline> ServerTestBootstrap;
typedef std::shared_ptr<ServerTestBootstrap> ServerPtr;
@@ -110,8 +114,8 @@ TEST_F(RpcTest, Ping) {
VLOG(1) << folly::sformat(FLAGS_result_format, method, "");
})
.onError([&](const folly::exception_wrapper& ew) {
- FAIL() << folly::sformat(FLAGS_fail_format, method);
- });
+ FAIL() << folly::sformat(FLAGS_fail_no_ex_format, method);
+ }).get();
server->stop();
server->join();
@@ -144,8 +148,8 @@ TEST_F(RpcTest, Echo) {
EXPECT_EQ(greetings, pb_resp->message());
})
.onError([&](const folly::exception_wrapper& ew) {
- FAIL() << folly::sformat(FLAGS_fail_format, method);
- });
+ FAIL() << folly::sformat(FLAGS_fail_no_ex_format, method);
+ }).get();
server->stop();
server->join();
@@ -168,7 +172,7 @@ TEST_F(RpcTest, Error) {
->AsyncCall(server_addr->getAddressStr(), server_addr->getPort(), std::move(request),
hbase::security::User::defaultUser())
.then([&](std::unique_ptr<Response> response) {
- FAIL() << folly::sformat(FLAGS_fail_format, method);
+ FAIL() << folly::sformat(FLAGS_fail_ex_format, method);
})
.onError([&](const folly::exception_wrapper& ew) {
VLOG(1) << folly::sformat(FLAGS_result_format, method, ew.what());
@@ -184,7 +188,7 @@ TEST_F(RpcTest, Error) {
EXPECT_EQ(kRpcTestException, e.exception_class_name());
EXPECT_EQ(kRpcTestException + ": server error!", e.stack_trace());
}));
- });
+ }).get();
server->stop();
server->join();
@@ -208,7 +212,7 @@ TEST_F(RpcTest, SocketNotOpen) {
->AsyncCall(server_addr->getAddressStr(), server_addr->getPort(), std::move(request),
hbase::security::User::defaultUser())
.then([&](std::unique_ptr<Response> response) {
- FAIL() << folly::sformat(FLAGS_fail_format, method);
+ FAIL() << folly::sformat(FLAGS_fail_ex_format, method);
})
.onError([&](const folly::exception_wrapper& ew) {
VLOG(1) << folly::sformat(FLAGS_result_format, method, ew.what());
@@ -231,5 +235,42 @@ TEST_F(RpcTest, SocketNotOpen) {
EXPECT_EQ(111 /*ECONNREFUSED*/, ase.getErrno());
});
}));
- });
+ }).get();
+}
+
+/**
+ * test pause
+ */
+TEST_F(RpcTest, Pause) {
+ int ms = 500;
+
+ auto conf = CreateConf();
+ auto server = CreateRpcServer();
+ auto server_addr = GetRpcServerAddress(server);
+ auto client =
+ CreateRpcClient(conf, std::chrono::duration_cast<nanoseconds>(milliseconds(2 * ms)));
+
+ auto method = "pause";
+ auto request = std::make_unique<Request>(std::make_shared<PauseRequestProto>(),
+ std::make_shared<EmptyResponseProto>(), method);
+ auto pb_msg = std::static_pointer_cast<PauseRequestProto>(request->req_msg());
+
+ pb_msg->set_ms(ms);
+
+ /* sending out request */
+ client
+ ->AsyncCall(server_addr->getAddressStr(), server_addr->getPort(), std::move(request),
+ hbase::security::User::defaultUser())
+ .then([&](std::unique_ptr<Response> response) {
+ auto pb_resp = std::static_pointer_cast<EmptyResponseProto>(response->resp_msg());
+ EXPECT_TRUE(pb_resp != nullptr);
+ VLOG(1) << folly::sformat(FLAGS_result_format, method, "");
+ })
+ .onError([&](const folly::exception_wrapper& ew) {
+ VLOG(1) << folly::sformat(FLAGS_result_format, method, ew.what());
+ FAIL() << folly::sformat(FLAGS_fail_no_ex_format, method);
+ }).get();
+
+ server->stop();
+ server->join();
}
[hbase] 21/137: HBASE-15731 Add on a connection pool
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit a28f85bd6af9d19125c935f28d017c857642270a
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Thu Apr 28 19:53:42 2016 -0700
HBASE-15731 Add on a connection pool
Summary:
Add on a connection pool protected by read write mutex.
Add on a service filter that will remove a connection from a connection pool when closed
Test Plan: Need to add on tests.
Differential Revision: https://reviews.facebook.net/D57411
---
connection/BUCK | 5 ++
connection/connection-factory.cc | 4 +-
connection/connection-factory.h | 5 +-
connection/connection-pool-test.cc | 77 +++++++++++++++++++++++++++
connection/connection-pool.cc | 89 +++++++++++++++++++++++++++++++
connection/connection-pool.h | 59 +++++++++++++++++++++
connection/service.h | 4 +-
core/BUCK | 4 --
core/client.h | 3 --
core/get-request.cc | 19 -------
core/get-request.h | 35 -------------
core/get-result.cc | 19 -------
core/get-result.h | 32 -----------
core/location-cache.cc | 1 -
core/simple-client.cc | 105 +++++++++++++++++++++----------------
15 files changed, 297 insertions(+), 164 deletions(-)
diff --git a/connection/BUCK b/connection/BUCK
index d393885..96f2136 100644
--- a/connection/BUCK
+++ b/connection/BUCK
@@ -22,6 +22,7 @@ cxx_library(name="connection",
"client-dispatcher.h",
"client-handler.h",
"connection-factory.h",
+ "connection-pool.h",
"pipeline.h",
"request.h",
"response.h",
@@ -31,6 +32,7 @@ cxx_library(name="connection",
"client-dispatcher.cc",
"client-handler.cc",
"connection-factory.cc",
+ "connection-pool.cc",
"pipeline.cc",
"request.cc",
],
@@ -42,3 +44,6 @@ cxx_library(name="connection",
"//third-party:wangle",
],
visibility=['//core/...', ], )
+cxx_test(name="connection-pool-test",
+ srcs=["connection-pool-test.cc", ],
+ deps=[":connection", ], )
diff --git a/connection/connection-factory.cc b/connection/connection-factory.cc
index 7073f9d..b546269 100644
--- a/connection/connection-factory.cc
+++ b/connection/connection-factory.cc
@@ -45,8 +45,8 @@ ConnectionFactory::ConnectionFactory() {
bootstrap_.pipelineFactory(std::make_shared<RpcPipelineFactory>());
}
-std::shared_ptr<Service<std::unique_ptr<Request>, Response>>
-ConnectionFactory::make_connection(std::string host, int port) {
+std::shared_ptr<HBaseService>
+ConnectionFactory::make_connection(const std::string &host, int port) {
// Connect to a given server
// Then when connected create a ClientDispactcher.
auto pipeline = bootstrap_.connect(SocketAddress(host, port, true)).get();
diff --git a/connection/connection-factory.h b/connection/connection-factory.h
index 8d1d2f0..5a45316 100644
--- a/connection/connection-factory.h
+++ b/connection/connection-factory.h
@@ -31,8 +31,9 @@ namespace hbase {
class ConnectionFactory {
public:
ConnectionFactory();
- std::shared_ptr<wangle::Service<std::unique_ptr<Request>, Response>>
- make_connection(std::string host, int port);
+
+ virtual std::shared_ptr<HBaseService> make_connection(const std::string &host,
+ int port);
private:
wangle::ClientBootstrap<SerializePipeline> bootstrap_;
diff --git a/connection/connection-pool-test.cc b/connection/connection-pool-test.cc
new file mode 100644
index 0000000..975bc5e
--- /dev/null
+++ b/connection/connection-pool-test.cc
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "connection/connection-pool.h"
+
+#include <folly/Logging.h>
+#include <gmock/gmock.h>
+
+#include "connection/connection-factory.h"
+#include "if/HBase.pb.h"
+
+using namespace hbase;
+
+using hbase::pb::ServerName;
+using ::testing::Return;
+using ::testing::_;
+
+class MockConnectionFactory : public ConnectionFactory {
+public:
+ MOCK_METHOD2(make_connection,
+ std::shared_ptr<HBaseService>(const std::string &hostname,
+ int port));
+};
+
+class MockServiceBase : public HBaseService {
+public:
+ folly::Future<Response> operator()(std::unique_ptr<Request> req) override {
+ return do_operation(req.get());
+ }
+ virtual folly::Future<Response> do_operation(Request *req) {
+ return folly::makeFuture<Response>(Response{});
+ }
+};
+
+class MockService : public MockServiceBase {
+public:
+ MOCK_METHOD1(do_operation, folly::Future<Response>(Request *));
+};
+
+TEST(TestConnectionPool, TestOnlyCreateOnce) {
+ std::string hostname{"hostname"};
+ auto mock_service = std::make_shared<MockService>();
+ uint32_t port{999};
+
+ LOG(ERROR) << "About to make a MockConnectionFactory";
+ auto mock_cf = std::make_shared<MockConnectionFactory>();
+ EXPECT_CALL((*mock_cf), make_connection(_, _))
+ .Times(1)
+ .WillRepeatedly(Return(mock_service));
+ ConnectionPool cp{mock_cf};
+
+ LOG(ERROR) << "Created ConnectionPool";
+
+ ServerName sn;
+ sn.set_host_name(hostname);
+ sn.set_port(port);
+
+ auto result = cp.get(sn);
+ ASSERT_TRUE(result != nullptr);
+ result = cp.get(sn);
+}
diff --git a/connection/connection-pool.cc b/connection/connection-pool.cc
new file mode 100644
index 0000000..a967df2
--- /dev/null
+++ b/connection/connection-pool.cc
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "connection/connection-pool.h"
+
+#include <wangle/service/Service.h>
+
+using std::mutex;
+using std::unique_ptr;
+using std::shared_ptr;
+using hbase::pb::ServerName;
+using wangle::ServiceFilter;
+using folly::SharedMutexWritePriority;
+
+namespace hbase {
+
+class RemoveServiceFilter
+ : public ServiceFilter<unique_ptr<Request>, Response> {
+
+public:
+ RemoveServiceFilter(std::shared_ptr<HBaseService> service, ServerName sn,
+ ConnectionPool *cp)
+ : ServiceFilter<unique_ptr<Request>, Response>(service), sn_(sn),
+ cp_(cp) {}
+
+ folly::Future<folly::Unit> close() override {
+ if (!released.exchange(true)) {
+ return this->service_->close().then(
+ [this]() { this->cp_->close(this->sn_); });
+ } else {
+ return folly::makeFuture();
+ }
+ }
+
+ virtual bool isAvailable() override { return service_->isAvailable(); }
+
+ folly::Future<Response> operator()(unique_ptr<Request> req) override {
+ return (*this->service_)(std::move(req));
+ }
+
+private:
+ std::atomic<bool> released{false};
+ hbase::pb::ServerName sn_;
+ ConnectionPool *cp_;
+};
+
+ConnectionPool::ConnectionPool() : cf_(std::make_shared<ConnectionFactory>()) {}
+ConnectionPool::ConnectionPool(std::shared_ptr<ConnectionFactory> cf)
+ : cf_(cf) {}
+
+std::shared_ptr<HBaseService> ConnectionPool::get(const ServerName &sn) {
+ SharedMutexWritePriority::UpgradeHolder holder(map_mutex_);
+ auto found = connections_.find(sn);
+ if (found == connections_.end() || found->second == nullptr) {
+ SharedMutexWritePriority::WriteHolder holder(std::move(holder));
+ auto new_con = cf_->make_connection(sn.host_name(), sn.port());
+ auto wrapped = std::make_shared<RemoveServiceFilter>(new_con, sn, this);
+ connections_[sn] = wrapped;
+ return new_con;
+ }
+ return found->second;
+}
+void ConnectionPool::close(ServerName sn) {
+ SharedMutexWritePriority::WriteHolder holder(map_mutex_);
+
+ auto found = connections_.find(sn);
+ if (found == connections_.end() || found->second == nullptr) {
+ return;
+ }
+ auto service = found->second;
+ connections_.erase(found);
+}
+}
diff --git a/connection/connection-pool.h b/connection/connection-pool.h
new file mode 100644
index 0000000..394cd71
--- /dev/null
+++ b/connection/connection-pool.h
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <folly/SharedMutex.h>
+#include <mutex>
+#include <unordered_map>
+
+#include "connection/connection-factory.h"
+#include "connection/service.h"
+#include "if/HBase.pb.h"
+
+namespace hbase {
+struct MyServerNameEquals {
+ bool operator()(const hbase::pb::ServerName &lhs,
+ const hbase::pb::ServerName &rhs) const {
+ return lhs.host_name() == rhs.host_name() && lhs.port() == rhs.port();
+ }
+};
+struct MyServerNameHash {
+ std::size_t operator()(hbase::pb::ServerName const &s) const {
+ std::size_t h1 = std::hash<std::string>()(s.host_name());
+ std::size_t h2 = std::hash<uint32_t>()(s.port());
+ return h1 ^ (h2 << 1);
+ }
+};
+
+class ConnectionPool {
+public:
+ ConnectionPool();
+ explicit ConnectionPool(std::shared_ptr<ConnectionFactory> cf);
+ std::shared_ptr<HBaseService> get(const hbase::pb::ServerName &sn);
+ void close(hbase::pb::ServerName sn);
+
+private:
+ std::shared_ptr<ConnectionFactory> cf_;
+ std::unordered_map<hbase::pb::ServerName, std::shared_ptr<HBaseService>,
+ MyServerNameHash, MyServerNameEquals>
+ connections_;
+ folly::SharedMutexWritePriority map_mutex_;
+};
+
+} // namespace hbase
diff --git a/connection/service.h b/connection/service.h
index feb14ec..79f087d 100644
--- a/connection/service.h
+++ b/connection/service.h
@@ -18,9 +18,11 @@
*/
#pragma once
+#include <wangle/service/Service.h>
+
#include "connection/request.h"
#include "connection/response.h"
namespace hbase {
-using HBaseService = wangle::Service<Request, Response>;
+using HBaseService = wangle::Service<std::unique_ptr<Request>, Response>;
} // namespace hbase
diff --git a/core/BUCK b/core/BUCK
index 9db6fda..e555ba4 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -21,8 +21,6 @@ cxx_library(
exported_headers=[
"client.h",
"connection.h",
- "get-request.h",
- "get-result.h",
"hbase_macros.h",
"location-cache.h",
"table-name.h",
@@ -32,8 +30,6 @@ cxx_library(
],
srcs=[
"client.cc",
- "get-request.cc",
- "get-result.cc",
"location-cache.cc",
"meta-utils.cc",
"table-name.cc",
diff --git a/core/client.h b/core/client.h
index b583285..4bed751 100644
--- a/core/client.h
+++ b/core/client.h
@@ -24,8 +24,6 @@
#include <string>
-#include "core/get-request.h"
-#include "core/get-result.h"
#include "core/location-cache.h"
#include "if/Cell.pb.h"
@@ -33,7 +31,6 @@ namespace hbase {
class Client {
public:
explicit Client(std::string quorum_spec);
- folly::Future<GetResult> get(const GetRequest &get_request);
private:
LocationCache location_cache_;
diff --git a/core/get-request.cc b/core/get-request.cc
deleted file mode 100644
index e927ccc..0000000
--- a/core/get-request.cc
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#include "core/get-request.h"
diff --git a/core/get-request.h b/core/get-request.h
deleted file mode 100644
index bb755c5..0000000
--- a/core/get-request.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#pragma once
-
-#include <string>
-
-#include "core/table-name.h"
-
-namespace hbase {
-
-class GetRequest {
-public:
- GetRequest(TableName table_name, std::string key);
-
-private:
- TableName table_name_;
- std::string key_;
-};
-} // namespace hbase
diff --git a/core/get-result.cc b/core/get-result.cc
deleted file mode 100644
index 7eea483..0000000
--- a/core/get-result.cc
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#include "core/get-result.h"
diff --git a/core/get-result.h b/core/get-result.h
deleted file mode 100644
index a49ad98..0000000
--- a/core/get-result.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#pragma once
-
-#include <string>
-
-namespace hbase {
-
-class GetResult {
-public:
- explicit GetResult(std::string key);
-
-private:
- std::string key_;
-};
-} // namespace hbase
diff --git a/core/location-cache.cc b/core/location-cache.cc
index 5925f4a..c81deba 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -77,7 +77,6 @@ ServerName LocationCache::ReadMetaLocation() {
int zk_result =
zoo_get(this->zk_, META_ZNODE_NAME, 0,
reinterpret_cast<char *>(buf->writableData()), &len, nullptr);
- LOG(ERROR) << "len = " << len;
if (zk_result != ZOK || len < 9) {
LOG(ERROR) << "Error getting meta location.";
throw runtime_error("Error getting meta location");
diff --git a/core/simple-client.cc b/core/simple-client.cc
index 2cb6200..11dcd68 100644
--- a/core/simple-client.cc
+++ b/core/simple-client.cc
@@ -25,7 +25,7 @@
#include <chrono>
#include <iostream>
-#include "connection/connection-factory.h"
+#include "connection/connection-pool.h"
#include "core/client.h"
#include "if/Client.pb.h"
#include "if/ZooKeeper.pb.h"
@@ -33,13 +33,20 @@
using namespace folly;
using namespace std;
using namespace std::chrono;
-using namespace hbase;
-using namespace hbase::pb;
-using namespace google::protobuf;
+using hbase::Response;
+using hbase::Request;
+using hbase::HBaseService;
+using hbase::LocationCache;
+using hbase::ConnectionPool;
+using hbase::pb::ServerName;
+using hbase::pb::RegionSpecifier_RegionSpecifierType;
+using hbase::pb::GetRequest;
+using hbase::pb::GetResponse;
// TODO(eclark): remove the need for this.
DEFINE_string(region, "1588230740", "What region to send a get to");
DEFINE_string(row, "test", "What row to get");
+DEFINE_string(zookeeper, "localhost:2181", "What zk quorum to talk to");
int main(int argc, char *argv[]) {
google::SetUsageMessage(
@@ -48,46 +55,52 @@ int main(int argc, char *argv[]) {
google::InitGoogleLogging(argv[0]);
// Create a connection factory
- ConnectionFactory cf;
-
- LocationCache cache{"localhost:2181", wangle::getCPUExecutor()};
-
- auto result = cache.LocateMeta().get();
-
- // Create a connection to the local host
- auto conn = cf.make_connection(result.host_name(), result.port());
-
- // Send the request
- auto r = Request::get();
-
- // This is a get request so make that
- auto req_msg = static_pointer_cast<hbase::pb::GetRequest>(r->req_msg());
-
- // Set what region
- req_msg->mutable_region()->set_value(FLAGS_region);
- // It's always this.
- req_msg->mutable_region()->set_type(
- RegionSpecifier_RegionSpecifierType::
- RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
-
- // What row.
- req_msg->mutable_get()->set_row(FLAGS_row);
-
- // Send it.
- auto resp = (*conn)(std::move(r)).get(milliseconds(5000));
-
- auto get_resp = std::static_pointer_cast<GetResponse>(resp.response());
- cout << "GetResponse has_result = " << get_resp->has_result() << '\n';
- if (get_resp->has_result()) {
- auto &r = get_resp->result();
- cout << "Result cell_size = " << r.cell_size() << endl;
- for (auto &cell : r.cell()) {
- cout << "\trow = " << cell.row() << " family = " << cell.family()
- << " qualifier = " << cell.qualifier()
- << " timestamp = " << cell.timestamp() << " value = " << cell.value()
- << endl;
- }
- }
-
- return 0;
+ ConnectionPool cp;
+ auto cpu_ex = wangle::getCPUExecutor();
+ LocationCache cache{FLAGS_zookeeper, cpu_ex};
+ auto result =
+ cache.LocateMeta()
+ .then([&cp = cp](ServerName sn) { return cp.get(sn); })
+ .then([](shared_ptr<HBaseService> con) {
+ // Send the request
+ auto r = Request::get();
+ // This is a get request so make that
+ auto req_msg = static_pointer_cast<GetRequest>(r->req_msg());
+ // Set what region
+ req_msg->mutable_region()->set_value(FLAGS_region);
+ // It's always this.
+ req_msg->mutable_region()->set_type(
+ RegionSpecifier_RegionSpecifierType::
+ RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
+
+ // What row.
+ req_msg->mutable_get()->set_row(FLAGS_row);
+
+ return (*con)(std::move(r));
+ })
+ .then([](Response resp) {
+ return static_pointer_cast<GetResponse>(resp.response());
+ })
+ .via(cpu_ex.get())
+ .then([](shared_ptr<GetResponse> get_resp) {
+ cout << "GetResponse has_result = " << get_resp->has_result()
+ << '\n';
+ if (get_resp->has_result()) {
+ auto &r = get_resp->result();
+ cout << "Result cell_size = " << r.cell_size() << endl;
+ for (auto &cell : r.cell()) {
+ cout << "\trow = " << cell.row()
+ << " family = " << cell.family()
+ << " qualifier = " << cell.qualifier()
+ << " timestamp = " << cell.timestamp()
+ << " value = " << cell.value() << endl;
+ }
+ return 0;
+ }
+
+ return 1;
+ })
+ .get(milliseconds(5000));
+
+ return result;
}
[hbase] 88/137: HBASE-15903 Delete Object - addnedum makes
PutGetDelete robust
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 8a2c2459ae28cd1522858a2730604a3e499fd027
Author: tedyu <yu...@gmail.com>
AuthorDate: Wed May 31 17:18:10 2017 -0700
HBASE-15903 Delete Object - addnedum makes PutGetDelete robust
---
core/client-test.cc | 1 +
1 file changed, 1 insertion(+)
diff --git a/core/client-test.cc b/core/client-test.cc
index b58ca0a..d0791b8 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -139,6 +139,7 @@ TEST_F(ClientTest, PutGetDelete) {
// Put two values for column "extra"
table->Put(Put{row}.AddColumn("d", "extra", "1st val extra"));
table->Put(Put{row}.AddColumn("d", "extra", valExtra));
+ usleep(1000);
table->Put(Put{row}.AddColumn("d", "ext", valExt));
// Perform the Get
[hbase] 14/137: HBASE-15690 Add utility to get current username
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit a7b33dd70bff6738b748d3e64e83768e4f9db1e9
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Thu Apr 21 21:17:11 2016 -0700
HBASE-15690 Add utility to get current username
Summary:
Add a class to fine the username of the current process.
It will only call out once and is multithread safe
Test Plan: Unit Test
Differential Revision: https://reviews.facebook.net/D57081
---
utils/BUCK | 40 +++++++++++++++++++++++++++++++++++
utils/user-util-test.cc | 34 ++++++++++++++++++++++++++++++
utils/user-util.cc | 55 +++++++++++++++++++++++++++++++++++++++++++++++++
utils/user-util.h | 37 +++++++++++++++++++++++++++++++++
4 files changed, 166 insertions(+)
diff --git a/utils/BUCK b/utils/BUCK
new file mode 100644
index 0000000..2b65b12
--- /dev/null
+++ b/utils/BUCK
@@ -0,0 +1,40 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cxx_library(name="utils",
+ exported_headers=[
+ "user-util.h",
+ ],
+ srcs=[
+ "user-util.cc",
+ ],
+ deps=[
+ '//third-party:folly',
+ ],
+ visibility=[
+ 'PUBLIC',
+ ],
+ tests=[
+ ":user-util-test"
+ ],)
+cxx_test(name="user-util-test",
+ srcs=[
+ "user-util-test.cc",
+ ],
+ deps=[
+ ":utils",
+ ],)
diff --git a/utils/user-util-test.cc b/utils/user-util-test.cc
new file mode 100644
index 0000000..2a7434f
--- /dev/null
+++ b/utils/user-util-test.cc
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <gtest/gtest.h>
+#include <string>
+#include <folly/Logging.h>
+
+#include "utils/user-util.h"
+
+using namespace std;
+using namespace hbase;
+
+TEST(TestUserUtil, TestGetSomething) {
+ UserUtil u_util;
+ string name = u_util.user_name();
+
+ ASSERT_GT(name.length(), 0);
+}
diff --git a/utils/user-util.cc b/utils/user-util.cc
new file mode 100644
index 0000000..c4427e3
--- /dev/null
+++ b/utils/user-util.cc
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "utils/user-util.h"
+
+#include <pwd.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <folly/Logging.h>
+
+using namespace hbase;
+using namespace std;
+
+string UserUtil::user_name() {
+ if (!init_) {
+ compute_user_name();
+ }
+ return user_name_;
+}
+
+void UserUtil::compute_user_name() {
+ lock_guard<mutex> lock(m_);
+
+ if (init_) {
+ return;
+ }
+
+ // According to the man page of getpwuid
+ // this should never be free'd
+ //
+ // So yeah a raw pointer with no ownership....
+ struct passwd *passwd = getpwuid(getuid());
+
+ // make sure that we got something.
+ if (passwd && passwd->pw_name) {
+ user_name_ = string{passwd->pw_name};
+ init_ = true;
+ }
+}
diff --git a/utils/user-util.h b/utils/user-util.h
new file mode 100644
index 0000000..4f74948
--- /dev/null
+++ b/utils/user-util.h
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <memory>
+#include <atomic>
+#include <mutex>
+
+namespace hbase {
+class UserUtil {
+public:
+ std::string user_name();
+
+private:
+ void compute_user_name();
+ std::atomic<bool> init_{false};
+ std::string user_name_;
+ std::mutex m_;
+};
+} // namespace hbase
[hbase] 81/137: HBASE-17726 [C++] Move implementation from header
to cc for request retry
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit eb8fe1851e0830fe3a11acbcecfad331d3a90c6c
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Tue Apr 11 06:14:41 2017 -0700
HBASE-17726 [C++] Move implementation from header to cc for request retry
---
core/BUCK | 7 +-
core/async-connection.h | 10 +-
core/async-region-locator.h | 1 +
core/async-rpc-retrying-caller.cc | 197 +++++++++++++++++++++++++++++++++++++-
core/async-rpc-retrying-caller.h | 168 ++------------------------------
core/async-rpc-retrying-test.cc | 30 ++++--
core/get-test.cc | 2 +-
core/mutation.cc | 4 +-
core/mutation.h | 2 +-
core/region-request.h | 4 +-
core/request-converter.h | 4 +-
core/response-converter.cc | 42 ++++----
12 files changed, 265 insertions(+), 206 deletions(-)
diff --git a/core/BUCK b/core/BUCK
index 30c3390..412ee3b 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -21,6 +21,8 @@ cxx_library(
exported_headers=[
"async-connection.h",
"async-region-locator.h",
+ "async-rpc-retrying-caller-factory.h",
+ "async-rpc-retrying-caller.h",
"client.h",
"cell.h",
"hbase-macros.h",
@@ -42,8 +44,6 @@ cxx_library(
"response-converter.h",
"table.h",
"raw-async-table.h",
- "async-rpc-retrying-caller-factory.h",
- "async-rpc-retrying-caller.h",
"hbase-rpc-controller.h",
"time-range.h",
"zk-util.h",
@@ -56,8 +56,11 @@ cxx_library(
],
srcs=[
"async-connection.cc",
+ "async-rpc-retrying-caller-factory.cc",
+ "async-rpc-retrying-caller.cc",
"cell.cc",
"client.cc",
+ "hbase-rpc-controller.cc",
"keyvalue-codec.cc",
"location-cache.cc",
"meta-utils.cc",
diff --git a/core/async-connection.h b/core/async-connection.h
index 6a61124..ff11577 100644
--- a/core/async-connection.h
+++ b/core/async-connection.h
@@ -26,12 +26,12 @@
#include <memory>
#include <string>
+#include <utility>
#include "connection/rpc-client.h"
#include "core/async-region-locator.h"
#include "core/configuration.h"
#include "core/connection-configuration.h"
-#include "core/connection-configuration.h"
#include "core/hbase-configuration-loader.h"
#include "core/hbase-rpc-controller.h"
#include "core/keyvalue-codec.h"
@@ -45,8 +45,8 @@ class AsyncRpcRetryingCallerFactory;
class AsyncConnection {
public:
- AsyncConnection(){};
- virtual ~AsyncConnection(){};
+ AsyncConnection() {}
+ virtual ~AsyncConnection() {}
virtual std::shared_ptr<Configuration> conf() = 0;
virtual std::shared_ptr<ConnectionConfiguration> connection_conf() = 0;
virtual std::shared_ptr<AsyncRpcRetryingCallerFactory> caller_factory() = 0;
@@ -82,7 +82,7 @@ class AsyncConnectionImpl : public AsyncConnection,
return std::make_shared<HBaseRpcController>();
}
- virtual void Close() override;
+ void Close() override;
protected:
AsyncConnectionImpl() {}
@@ -105,7 +105,7 @@ class AsyncConnectionImpl : public AsyncConnection,
bool is_closed_ = false;
private:
- AsyncConnectionImpl(std::shared_ptr<Configuration> conf) : conf_(conf) {}
+ explicit AsyncConnectionImpl(std::shared_ptr<Configuration> conf) : conf_(conf) {}
void Init();
};
} // namespace hbase
diff --git a/core/async-region-locator.h b/core/async-region-locator.h
index b0019e0..c606dcb 100644
--- a/core/async-region-locator.h
+++ b/core/async-region-locator.h
@@ -20,6 +20,7 @@
#pragma once
#include <folly/futures/Future.h>
+#include <memory>
#include <string>
#include "core/region-location.h"
diff --git a/core/async-rpc-retrying-caller.cc b/core/async-rpc-retrying-caller.cc
index 743b6bb..965a44b 100644
--- a/core/async-rpc-retrying-caller.cc
+++ b/core/async-rpc-retrying-caller.cc
@@ -19,4 +19,199 @@
#include "core/async-rpc-retrying-caller.h"
-namespace hbase {} /* namespace hbase */
+#include <folly/Format.h>
+#include <folly/Logging.h>
+#include <folly/futures/Unit.h>
+
+#include "connection/rpc-client.h"
+#include "core/async-connection.h"
+#include "core/hbase-rpc-controller.h"
+#include "core/region-location.h"
+#include "core/result.h"
+#include "exceptions/exception.h"
+#include "if/HBase.pb.h"
+#include "utils/connection-util.h"
+#include "utils/sys-util.h"
+#include "utils/time-util.h"
+
+namespace hbase {
+
+template <typename RESP>
+AsyncSingleRequestRpcRetryingCaller<RESP>::AsyncSingleRequestRpcRetryingCaller(
+ std::shared_ptr<AsyncConnection> conn, std::shared_ptr<hbase::pb::TableName> table_name,
+ const std::string& row, RegionLocateType locate_type, Callable<RESP> callable,
+ nanoseconds pause, uint32_t max_retries, nanoseconds operation_timeout_nanos,
+ nanoseconds rpc_timeout_nanos, uint32_t start_log_errors_count)
+ : conn_(conn),
+ table_name_(table_name),
+ row_(row),
+ locate_type_(locate_type),
+ callable_(callable),
+ pause_(pause),
+ max_retries_(max_retries),
+ operation_timeout_nanos_(operation_timeout_nanos),
+ rpc_timeout_nanos_(rpc_timeout_nanos),
+ start_log_errors_count_(start_log_errors_count),
+ promise_(std::make_shared<folly::Promise<RESP>>()),
+ tries_(1) {
+ controller_ = conn_->CreateRpcController();
+ start_ns_ = TimeUtil::GetNowNanos();
+ max_attempts_ = ConnectionUtils::Retries2Attempts(max_retries);
+ exceptions_ = std::make_shared<std::vector<ThrowableWithExtraContext>>();
+ retry_timer_ = folly::HHWheelTimer::newTimer(&event_base_);
+}
+
+template <typename RESP>
+AsyncSingleRequestRpcRetryingCaller<RESP>::~AsyncSingleRequestRpcRetryingCaller() {}
+
+template <typename RESP>
+folly::Future<RESP> AsyncSingleRequestRpcRetryingCaller<RESP>::Call() {
+ auto f = promise_->getFuture();
+ LocateThenCall();
+ return f;
+}
+
+template <typename RESP>
+void AsyncSingleRequestRpcRetryingCaller<RESP>::LocateThenCall() {
+ int64_t locate_timeout_ns;
+ if (operation_timeout_nanos_.count() > 0) {
+ locate_timeout_ns = RemainingTimeNs();
+ if (locate_timeout_ns <= 0) {
+ CompleteExceptionally();
+ return;
+ }
+ } else {
+ locate_timeout_ns = -1L;
+ }
+
+ conn_->region_locator()
+ ->LocateRegion(*table_name_, row_, locate_type_, locate_timeout_ns)
+ .then([this](std::shared_ptr<RegionLocation> loc) { Call(*loc); })
+ .onError([this](const std::exception& e) {
+ OnError(e,
+ [this]() -> std::string {
+ return "Locate '" + row_ + "' in " + table_name_->namespace_() + "::" +
+ table_name_->qualifier() + " failed, tries = " + std::to_string(tries_) +
+ ", maxAttempts = " + std::to_string(max_attempts_) + ", timeout = " +
+ TimeUtil::ToMillisStr(operation_timeout_nanos_) + " ms, time elapsed = " +
+ TimeUtil::ElapsedMillisStr(this->start_ns_) + " ms";
+ },
+ [](const std::exception& error) {});
+ });
+}
+
+template <typename RESP>
+void AsyncSingleRequestRpcRetryingCaller<RESP>::OnError(
+ const std::exception& error, Supplier<std::string> err_msg,
+ Consumer<std::exception> update_cached_location) {
+ ThrowableWithExtraContext twec(std::make_shared<std::exception>(error), TimeUtil::GetNowNanos());
+ exceptions_->push_back(twec);
+ if (SysUtil::InstanceOf<DoNotRetryIOException, std::exception>(error) || tries_ >= max_retries_) {
+ CompleteExceptionally();
+ return;
+ }
+
+ int64_t delay_ns;
+ if (operation_timeout_nanos_.count() > 0) {
+ int64_t max_delay_ns = RemainingTimeNs() - ConnectionUtils::kSleepDeltaNs;
+ if (max_delay_ns <= 0) {
+ CompleteExceptionally();
+ return;
+ }
+ delay_ns = std::min(max_delay_ns, ConnectionUtils::GetPauseTime(pause_.count(), tries_ - 1));
+ } else {
+ delay_ns = ConnectionUtils::GetPauseTime(pause_.count(), tries_ - 1);
+ }
+ update_cached_location(error);
+ tries_++;
+ retry_timer_->scheduleTimeoutFn([this]() { LocateThenCall(); },
+ milliseconds(TimeUtil::ToMillis(delay_ns)));
+}
+
+template <typename RESP>
+void AsyncSingleRequestRpcRetryingCaller<RESP>::Call(const RegionLocation& loc) {
+ int64_t call_timeout_ns;
+ if (operation_timeout_nanos_.count() > 0) {
+ call_timeout_ns = this->RemainingTimeNs();
+ if (call_timeout_ns <= 0) {
+ this->CompleteExceptionally();
+ return;
+ }
+ call_timeout_ns = std::min(call_timeout_ns, rpc_timeout_nanos_.count());
+ } else {
+ call_timeout_ns = rpc_timeout_nanos_.count();
+ }
+
+ std::shared_ptr<RpcClient> rpc_client;
+ try {
+ // TODO: There is no connection attempt happening here, no need to try-catch.
+ rpc_client = conn_->rpc_client();
+ } catch (const IOException& e) {
+ OnError(e,
+ [&, this]() -> std::string {
+ return "Get async rpc_client to " +
+ folly::sformat("{0}:{1}", loc.server_name().host_name(),
+ loc.server_name().port()) +
+ " for '" + row_ + "' in " + loc.DebugString() + " of " +
+ table_name_->namespace_() + "::" + table_name_->qualifier() +
+ " failed, tries = " + std::to_string(tries_) + ", maxAttempts = " +
+ std::to_string(max_attempts_) + ", timeout = " +
+ TimeUtil::ToMillisStr(this->operation_timeout_nanos_) +
+ " ms, time elapsed = " + TimeUtil::ElapsedMillisStr(this->start_ns_) + " ms";
+ },
+ [&, this](const std::exception& error) {
+ conn_->region_locator()->UpdateCachedLocation(loc, error);
+ });
+ return;
+ }
+
+ ResetController(controller_, call_timeout_ns);
+
+ callable_(controller_, std::make_shared<RegionLocation>(loc), rpc_client)
+ .then([this](const RESP& resp) { this->promise_->setValue(std::move(resp)); })
+ .onError([&, this](const std::exception& e) {
+ OnError(e,
+ [&, this]() -> std::string {
+ return "Call to " + folly::sformat("{0}:{1}", loc.server_name().host_name(),
+ loc.server_name().port()) +
+ " for '" + row_ + "' in " + loc.DebugString() + " of " +
+ table_name_->namespace_() + "::" + table_name_->qualifier() +
+ " failed, tries = " + std::to_string(tries_) + ", maxAttempts = " +
+ std::to_string(max_attempts_) + ", timeout = " +
+ TimeUtil::ToMillisStr(this->operation_timeout_nanos_) +
+ " ms, time elapsed = " + TimeUtil::ElapsedMillisStr(this->start_ns_) +
+ " ms";
+ },
+ [&, this](const std::exception& error) {
+ conn_->region_locator()->UpdateCachedLocation(loc, error);
+ });
+ return;
+ });
+}
+
+template <typename RESP>
+void AsyncSingleRequestRpcRetryingCaller<RESP>::CompleteExceptionally() {
+ this->promise_->setException(RetriesExhaustedException(tries_ - 1, exceptions_));
+}
+
+template <typename RESP>
+int64_t AsyncSingleRequestRpcRetryingCaller<RESP>::RemainingTimeNs() {
+ return operation_timeout_nanos_.count() - (TimeUtil::GetNowNanos() - start_ns_);
+}
+
+template <typename RESP>
+void AsyncSingleRequestRpcRetryingCaller<RESP>::ResetController(
+ std::shared_ptr<HBaseRpcController> controller, const int64_t& timeout_ns) {
+ controller->Reset();
+ if (timeout_ns >= 0) {
+ controller->set_call_timeout(
+ milliseconds(std::min(static_cast<int64_t>(INT_MAX), TimeUtil::ToMillis(timeout_ns))));
+ }
+}
+
+// explicit instantiations for the linker. Otherwise, you have to #include the .cc file for the
+// templetized
+// class definitions.
+template class AsyncSingleRequestRpcRetryingCaller<std::shared_ptr<hbase::Result>>;
+template class AsyncSingleRequestRpcRetryingCaller<folly::Unit>;
+} /* namespace hbase */
diff --git a/core/async-rpc-retrying-caller.h b/core/async-rpc-retrying-caller.h
index 6503301..6006388 100644
--- a/core/async-rpc-retrying-caller.h
+++ b/core/async-rpc-retrying-caller.h
@@ -18,11 +18,10 @@
*/
#pragma once
-#include <folly/Format.h>
-#include <folly/Logging.h>
#include <folly/futures/Future.h>
#include <folly/io/async/EventBase.h>
#include <folly/io/async/HHWheelTimer.h>
+
#include <algorithm>
#include <chrono>
#include <functional>
@@ -31,15 +30,11 @@
#include <type_traits>
#include <utility>
#include <vector>
-#include "connection/rpc-client.h"
#include "core/async-connection.h"
#include "core/hbase-rpc-controller.h"
#include "core/region-location.h"
#include "exceptions/exception.h"
#include "if/HBase.pb.h"
-#include "utils/connection-util.h"
-#include "utils/sys-util.h"
-#include "utils/time-util.h"
using std::chrono::nanoseconds;
using std::chrono::milliseconds;
@@ -80,168 +75,26 @@ class AsyncSingleRequestRpcRetryingCaller {
Callable<RESP> callable, nanoseconds pause,
uint32_t max_retries, nanoseconds operation_timeout_nanos,
nanoseconds rpc_timeout_nanos,
- uint32_t start_log_errors_count)
- : conn_(conn),
- table_name_(table_name),
- row_(row),
- locate_type_(locate_type),
- callable_(callable),
- pause_(pause),
- max_retries_(max_retries),
- operation_timeout_nanos_(operation_timeout_nanos),
- rpc_timeout_nanos_(rpc_timeout_nanos),
- start_log_errors_count_(start_log_errors_count),
- promise_(std::make_shared<folly::Promise<RESP>>()),
- tries_(1) {
- controller_ = conn_->CreateRpcController();
- start_ns_ = TimeUtil::GetNowNanos();
- max_attempts_ = ConnectionUtils::Retries2Attempts(max_retries);
- exceptions_ = std::make_shared<std::vector<ThrowableWithExtraContext>>();
- retry_timer_ = folly::HHWheelTimer::newTimer(&event_base_);
- }
+ uint32_t start_log_errors_count);
- virtual ~AsyncSingleRequestRpcRetryingCaller() {}
+ virtual ~AsyncSingleRequestRpcRetryingCaller();
- folly::Future<RESP> Call() {
- auto f = promise_->getFuture();
- LocateThenCall();
- return f;
- }
+ folly::Future<RESP> Call();
private:
- void LocateThenCall() {
- int64_t locate_timeout_ns;
- if (operation_timeout_nanos_.count() > 0) {
- locate_timeout_ns = RemainingTimeNs();
- if (locate_timeout_ns <= 0) {
- CompleteExceptionally();
- return;
- }
- } else {
- locate_timeout_ns = -1L;
- }
-
- conn_->region_locator()
- ->LocateRegion(*table_name_, row_, locate_type_, locate_timeout_ns)
- .then([this](std::shared_ptr<RegionLocation> loc) { Call(*loc); })
- .onError([this](const std::exception& e) {
- OnError(e,
- [this]() -> std::string {
- return "Locate '" + row_ + "' in " + table_name_->namespace_() + "::" +
- table_name_->qualifier() + " failed, tries = " + std::to_string(tries_) +
- ", maxAttempts = " + std::to_string(max_attempts_) + ", timeout = " +
- TimeUtil::ToMillisStr(operation_timeout_nanos_) +
- " ms, time elapsed = " + TimeUtil::ElapsedMillisStr(this->start_ns_) +
- " ms";
- },
- [](const std::exception& error) {});
- });
- }
+ void LocateThenCall();
void OnError(const std::exception& error, Supplier<std::string> err_msg,
- Consumer<std::exception> update_cached_location) {
- ThrowableWithExtraContext twec(std::make_shared<std::exception>(error),
- TimeUtil::GetNowNanos());
- exceptions_->push_back(twec);
- if (SysUtil::InstanceOf<DoNotRetryIOException, std::exception>(error) ||
- tries_ >= max_retries_) {
- CompleteExceptionally();
- return;
- }
-
- int64_t delay_ns;
- if (operation_timeout_nanos_.count() > 0) {
- int64_t max_delay_ns = RemainingTimeNs() - ConnectionUtils::kSleepDeltaNs;
- if (max_delay_ns <= 0) {
- CompleteExceptionally();
- return;
- }
- delay_ns = std::min(max_delay_ns, ConnectionUtils::GetPauseTime(pause_.count(), tries_ - 1));
- } else {
- delay_ns = ConnectionUtils::GetPauseTime(pause_.count(), tries_ - 1);
- }
- update_cached_location(error);
- tries_++;
- retry_timer_->scheduleTimeoutFn([this]() { LocateThenCall(); },
- milliseconds(TimeUtil::ToMillis(delay_ns)));
- }
-
- void Call(const RegionLocation& loc) {
- int64_t call_timeout_ns;
- if (operation_timeout_nanos_.count() > 0) {
- call_timeout_ns = this->RemainingTimeNs();
- if (call_timeout_ns <= 0) {
- this->CompleteExceptionally();
- return;
- }
- call_timeout_ns = std::min(call_timeout_ns, rpc_timeout_nanos_.count());
- } else {
- call_timeout_ns = rpc_timeout_nanos_.count();
- }
+ Consumer<std::exception> update_cached_location);
- std::shared_ptr<RpcClient> rpc_client;
- try {
- // TODO: There is no connection attempt happening here, no need to try-catch.
- rpc_client = conn_->rpc_client();
- } catch (const IOException& e) {
- OnError(e,
- [&, this]() -> std::string {
- return "Get async rpc_client to " +
- folly::sformat("{0}:{1}", loc.server_name().host_name(),
- loc.server_name().port()) +
- " for '" + row_ + "' in " + loc.DebugString() + " of " +
- table_name_->namespace_() + "::" + table_name_->qualifier() +
- " failed, tries = " + std::to_string(tries_) + ", maxAttempts = " +
- std::to_string(max_attempts_) + ", timeout = " +
- TimeUtil::ToMillisStr(this->operation_timeout_nanos_) +
- " ms, time elapsed = " + TimeUtil::ElapsedMillisStr(this->start_ns_) + " ms";
- },
- [&, this](const std::exception& error) {
- conn_->region_locator()->UpdateCachedLocation(loc, error);
- });
- return;
- }
+ void Call(const RegionLocation& loc);
- ResetController(controller_, call_timeout_ns);
+ void CompleteExceptionally();
- callable_(controller_, std::make_shared<RegionLocation>(loc), rpc_client)
- .then([this](const RESP& resp) { this->promise_->setValue(std::move(resp)); })
- .onError([&, this](const std::exception& e) {
- OnError(e,
- [&, this]() -> std::string {
- return "Call to " + folly::sformat("{0}:{1}", loc.server_name().host_name(),
- loc.server_name().port()) +
- " for '" + row_ + "' in " + loc.DebugString() + " of " +
- table_name_->namespace_() + "::" + table_name_->qualifier() +
- " failed, tries = " + std::to_string(tries_) + ", maxAttempts = " +
- std::to_string(max_attempts_) + ", timeout = " +
- TimeUtil::ToMillisStr(this->operation_timeout_nanos_) +
- " ms, time elapsed = " + TimeUtil::ElapsedMillisStr(this->start_ns_) +
- " ms";
- },
- [&, this](const std::exception& error) {
- conn_->region_locator()->UpdateCachedLocation(loc, error);
- });
- return;
- });
- }
-
- void CompleteExceptionally() {
- this->promise_->setException(RetriesExhaustedException(tries_ - 1, exceptions_));
- }
-
- int64_t RemainingTimeNs() {
- return operation_timeout_nanos_.count() - (TimeUtil::GetNowNanos() - start_ns_);
- }
+ int64_t RemainingTimeNs();
static void ResetController(std::shared_ptr<HBaseRpcController> controller,
- const int64_t& timeout_ns) {
- controller->Reset();
- if (timeout_ns >= 0) {
- controller->set_call_timeout(
- milliseconds(std::min(static_cast<int64_t>(INT_MAX), TimeUtil::ToMillis(timeout_ns))));
- }
- }
+ const int64_t& timeout_ns);
private:
folly::HHWheelTimer::UniquePtr retry_timer_;
@@ -263,5 +116,4 @@ class AsyncSingleRequestRpcRetryingCaller {
uint32_t max_attempts_;
folly::EventBase event_base_;
};
-
} /* namespace hbase */
diff --git a/core/async-rpc-retrying-test.cc b/core/async-rpc-retrying-test.cc
index 3ed6866..4956972 100644
--- a/core/async-rpc-retrying-test.cc
+++ b/core/async-rpc-retrying-test.cc
@@ -47,11 +47,23 @@
#include "test-util/test-util.h"
#include "utils/time-util.h"
-using namespace google::protobuf;
-using namespace hbase;
-using namespace hbase::pb;
-using namespace std::placeholders;
-using namespace testing;
+using hbase::AsyncRpcRetryingCallerFactory;
+using hbase::AsyncConnection;
+using hbase::AsyncRegionLocator;
+using hbase::ConnectionConfiguration;
+using hbase::Configuration;
+using hbase::HBaseRpcController;
+using hbase::RegionLocation;
+using hbase::RegionLocateType;
+using hbase::RpcClient;
+using hbase::RequestConverter;
+using hbase::ResponseConverter;
+using hbase::ReqConverter;
+using hbase::RespConverter;
+using hbase::Put;
+using hbase::TimeUtil;
+using hbase::Client;
+
using ::testing::Return;
using ::testing::_;
using std::chrono::nanoseconds;
@@ -62,10 +74,10 @@ class MockAsyncRegionLocator : public AsyncRegionLocator {
: region_location_(region_location) {}
~MockAsyncRegionLocator() = default;
- folly::Future<std::shared_ptr<RegionLocation>> LocateRegion(const hbase::pb::TableName&,
- const std::string&,
- const RegionLocateType,
- const int64_t) override {
+ folly::Future<std::shared_ptr<hbase::RegionLocation>> LocateRegion(const hbase::pb::TableName&,
+ const std::string&,
+ const RegionLocateType,
+ const int64_t) override {
folly::Promise<std::shared_ptr<RegionLocation>> promise;
promise.setValue(region_location_);
return promise.getFuture();
diff --git a/core/get-test.cc b/core/get-test.cc
index 6127e23..4a44a26 100644
--- a/core/get-test.cc
+++ b/core/get-test.cc
@@ -17,8 +17,8 @@
*
*/
-#include "core/cell.h"
#include "core/get.h"
+#include "core/cell.h"
#include <glog/logging.h>
#include <gtest/gtest.h>
diff --git a/core/mutation.cc b/core/mutation.cc
index ab33105..7182202 100644
--- a/core/mutation.cc
+++ b/core/mutation.cc
@@ -26,8 +26,8 @@
namespace hbase {
-Mutation::Mutation(const std::string &row) : Row(row) { }
-Mutation::Mutation(const std::string &row, int64_t timestamp) : Row(row), timestamp_(timestamp) { }
+Mutation::Mutation(const std::string &row) : Row(row) {}
+Mutation::Mutation(const std::string &row, int64_t timestamp) : Row(row), timestamp_(timestamp) {}
Mutation::Mutation(const Mutation &mutation) {
row_ = mutation.row_;
diff --git a/core/mutation.h b/core/mutation.h
index 5e0381b..496891e 100644
--- a/core/mutation.h
+++ b/core/mutation.h
@@ -31,7 +31,7 @@
namespace hbase {
-class Mutation: public Row {
+class Mutation : public Row {
public:
/**
* Constructors
diff --git a/core/region-request.h b/core/region-request.h
index 6f29d44..7ce7c96 100644
--- a/core/region-request.h
+++ b/core/region-request.h
@@ -34,9 +34,7 @@ class RegionRequest {
explicit RegionRequest(const std::shared_ptr<hbase::RegionLocation> ®ion_loc)
: region_loc_(region_loc) {}
~RegionRequest() {}
- void AddAction(std::shared_ptr<Action> action) {
- actions_.push_back(action);
- }
+ void AddAction(std::shared_ptr<Action> action) { actions_.push_back(action); }
std::shared_ptr<hbase::RegionLocation> region_location() const { return region_loc_; }
const ActionList &actions() const { return actions_; }
diff --git a/core/request-converter.h b/core/request-converter.h
index ff6b290..6861604 100644
--- a/core/request-converter.h
+++ b/core/request-converter.h
@@ -26,11 +26,11 @@
#include "core/action.h"
#include "core/cell.h"
#include "core/get.h"
+#include "core/mutation.h"
+#include "core/put.h"
#include "core/region-request.h"
#include "core/scan.h"
#include "core/server-request.h"
-#include "core/mutation.h"
-#include "core/put.h"
#include "if/Client.pb.h"
#include "if/HBase.pb.h"
diff --git a/core/response-converter.cc b/core/response-converter.cc
index 7729257..b29a819 100644
--- a/core/response-converter.cc
+++ b/core/response-converter.cc
@@ -61,11 +61,11 @@ std::shared_ptr<Result> ResponseConverter::ToResult(
int cells_read = 0;
while (cells_read != result.associated_cell_count()) {
if (cell_scanner->Advance()) {
- vcells.push_back(cell_scanner->Current());
+ vcells.push_back(cell_scanner->Current());
cells_read += 1;
} else {
- LOG(ERROR)<< "CellScanner::Advance() returned false unexpectedly. Cells Read:- "
- << cells_read << "; Expected Cell Count:- " << result.associated_cell_count();
+ LOG(ERROR) << "CellScanner::Advance() returned false unexpectedly. Cells Read:- "
+ << cells_read << "; Expected Cell Count:- " << result.associated_cell_count();
std::runtime_error("CellScanner::Advance() returned false unexpectedly");
}
}
@@ -111,16 +111,15 @@ std::vector<std::shared_ptr<Result>> ResponseConverter::FromScanResponse(const R
std::unique_ptr<hbase::MultiResponse> ResponseConverter::GetResults(std::shared_ptr<Request> req,
const Response& resp) {
- auto multi_req = std::static_pointer_cast < hbase::pb::MultiRequest > (req->req_msg());
- auto multi_resp = std::static_pointer_cast < hbase::pb::MultiResponse > (resp.resp_msg());
+ auto multi_req = std::static_pointer_cast<hbase::pb::MultiRequest>(req->req_msg());
+ auto multi_resp = std::static_pointer_cast<hbase::pb::MultiResponse>(resp.resp_msg());
VLOG(3) << "GetResults:" << multi_resp->ShortDebugString();
int req_region_action_count = multi_req->regionaction_size();
int res_region_action_count = multi_resp->regionactionresult_size();
if (req_region_action_count != res_region_action_count) {
- throw std::runtime_error(
- "Request mutation count=" + std::to_string(req_region_action_count)
- + " does not match response mutation result count="
- + std::to_string(res_region_action_count));
+ throw std::runtime_error("Request mutation count=" + std::to_string(req_region_action_count) +
+ " does not match response mutation result count=" +
+ std::to_string(res_region_action_count));
}
auto multi_response = std::make_unique<hbase::MultiResponse>();
for (int32_t num = 0; num < res_region_action_count; num++) {
@@ -134,7 +133,7 @@ std::unique_ptr<hbase::MultiResponse> ResponseConverter::GetResults(std::shared_
auto region_name = rs.value();
if (action_result.has_exception()) {
if (action_result.exception().has_value()) {
- auto exc = std::make_shared < hbase::IOException > (action_result.exception().value());
+ auto exc = std::make_shared<hbase::IOException>(action_result.exception().value());
VLOG(8) << "Store Region Exception:- " << exc->what();
multi_response->AddRegionException(region_name, exc);
}
@@ -142,19 +141,18 @@ std::unique_ptr<hbase::MultiResponse> ResponseConverter::GetResults(std::shared_
}
if (actions.action_size() != action_result.resultorexception_size()) {
- throw std::runtime_error(
- "actions.action_size=" + std::to_string(actions.action_size())
- + ", action_result.resultorexception_size="
- + std::to_string(action_result.resultorexception_size()) + " for region "
- + actions.region().value());
+ throw std::runtime_error("actions.action_size=" + std::to_string(actions.action_size()) +
+ ", action_result.resultorexception_size=" +
+ std::to_string(action_result.resultorexception_size()) +
+ " for region " + actions.region().value());
}
for (hbase::pb::ResultOrException roe : action_result.resultorexception()) {
- std::shared_ptr < Result > result;
- std::shared_ptr < std::exception > exc;
+ std::shared_ptr<Result> result;
+ std::shared_ptr<std::exception> exc;
if (roe.has_exception()) {
if (roe.exception().has_value()) {
- exc = std::make_shared < hbase::IOException > (roe.exception().value());
+ exc = std::make_shared<hbase::IOException>(roe.exception().value());
VLOG(8) << "Store ResultOrException:- " << exc->what();
}
} else if (roe.has_result()) {
@@ -165,9 +163,9 @@ std::unique_ptr<hbase::MultiResponse> ResponseConverter::GetResults(std::shared_
// Sometimes, the response is just "it was processed". Generally, this occurs for things
// like mutateRows where either we get back 'processed' (or not) and optionally some
// statistics about the regions we touched.
- std::vector < std::shared_ptr < Cell >> empty_cells;
- result = std::make_shared < Result
- > (empty_cells, multi_resp->processed() ? true : false, false, false);
+ std::vector<std::shared_ptr<Cell>> empty_cells;
+ result = std::make_shared<Result>(empty_cells, multi_resp->processed() ? true : false,
+ false, false);
}
multi_response->AddRegionResult(region_name, roe.index(), std::move(result), exc);
}
@@ -177,7 +175,7 @@ std::unique_ptr<hbase::MultiResponse> ResponseConverter::GetResults(std::shared_
hbase::pb::MultiRegionLoadStats stats = multi_resp->regionstatistics();
for (int i = 0; i < stats.region_size(); i++) {
multi_response->AddStatistic(stats.region(i).value(),
- std::make_shared < RegionLoadStats > (stats.stat(i)));
+ std::make_shared<RegionLoadStats>(stats.stat(i)));
}
}
return multi_response;
[hbase] 137/137: HBASE-22024 Clean up BUCK related things for C++
native client
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 10f12831ca4c6ae3017717d0aeeaae771ebd27d6
Author: xujunhong <xu...@xiaomi.com>
AuthorDate: Mon Mar 11 19:11:41 2019 -0700
HBASE-22024 Clean up BUCK related things for C++ native client
Signed-off-by: Guanghao Zhang <zg...@apache.org>
---
.buckconfig | 2 --
.gitignore | 5 -----
BUILDING.md | 8 +++++---
bin/copy-protobuf.sh | 2 +-
bin/start-docker.sh | 11 -----------
5 files changed, 6 insertions(+), 22 deletions(-)
diff --git a/.buckconfig b/.buckconfig
deleted file mode 100644
index 402ef27..0000000
--- a/.buckconfig
+++ /dev/null
@@ -1,2 +0,0 @@
-[cxx]
- gtest_dep = //third-party:google-test
diff --git a/.gitignore b/.gitignore
index b83bb81..202b096 100644
--- a/.gitignore
+++ b/.gitignore
@@ -14,11 +14,6 @@
#python
*.pyc
-# Buck
-buck-out
-.buckd
-
-
*.swp
# Thirdparty dirs
diff --git a/BUILDING.md b/BUILDING.md
index 53eb6d9..035af4b 100644
--- a/BUILDING.md
+++ b/BUILDING.md
@@ -19,7 +19,7 @@ under the License.
# Building HBase native client
-The HBase native client build using buck and produces a linux library.
+The HBase native client build using cmake and produces a linux library.
# Dependencies
@@ -35,7 +35,9 @@ slow. If things get too slow using [dinghy](https://github.com/codekitchen/dingh
can help speed things up by using nfs.
If possible pairing virtual box with dinghy will result in the fastest,
-most stable docker environment. However none of it is needed.
+most stable docker environment.
+
+However none of them is a must.
# Building using docker
@@ -45,7 +47,7 @@ hbase instance from the jar's created.
Then go into the hbase-native-client directory and run `./bin/start-docker.sh`
that will build the docker development environment and when complete will
-drop you into a shell on a linux vm with all the tools needed installed.
+drop you into a shell on a linux vm with all the dependencies needed installed.
# CMake
diff --git a/bin/copy-protobuf.sh b/bin/copy-protobuf.sh
index c928461..2e2a241 100755
--- a/bin/copy-protobuf.sh
+++ b/bin/copy-protobuf.sh
@@ -21,4 +21,4 @@ IFS=$'\n\t'
BIN_DIR=$(dirname "$0")
PB_SOURCE_DIR="${BIN_DIR}/../../hbase-protocol-shaded/src/main/protobuf/"
PB_DEST_DIR="${BIN_DIR}/../src/hbase/if/"
-rsync -r --exclude BUCK ${PB_SOURCE_DIR} ${PB_DEST_DIR}
+rsync -r "${PB_SOURCE_DIR}" "${PB_DEST_DIR}"
diff --git a/bin/start-docker.sh b/bin/start-docker.sh
index 391238d..f6899a0 100755
--- a/bin/start-docker.sh
+++ b/bin/start-docker.sh
@@ -26,23 +26,12 @@ eval "$(docker-machine env dinghy)"
BIN_DIR=$(pushd `dirname "$0"` 2>&1 > /dev/null && pwd && popd 2>&1 > /dev/null)
BASE_DIR=$(pushd "${BIN_DIR}/../" 2>&1 > /dev/null && pwd && popd 2>&1 > /dev/null)
-# TODO: this is wrong. These should be called from BUCK, but I could not find a
-# way to execute scripts in BUCK that are in under bin/.
${BIN_DIR}/copy-protobuf.sh
${BIN_DIR}/copy-version.sh
# Go into the base dir. This just makes things cleaner.
pushd ${BASE_DIR}
-# Make sure that there is a thrid-party dir.
-mkdir third-party || true
-
-# Get gtest
-# TODO(eclark): Remove this ( see HBASE-15427 )
-if [[ ! -d third-party/googletest ]]; then
- git clone https://github.com/google/googletest.git third-party/googletest
-fi
-
# We don't want to have to re-download all the jars in docker if we can help it
if [[ ! -d ~/.m2 ]]; then
echo "~/.m2 directory doesn't exist. Check Apache Maven is installed."
[hbase] 19/137: HBASE-15687 Allow decoding more than GetResponse
from the server
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 58e710c1b7eeed36bb99add99528bca7422b9f3c
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Wed Apr 27 15:27:09 2016 -0700
HBASE-15687 Allow decoding more than GetResponse from the server
Summary:
We'll need more than get's for the client to be usable. So now the
Request class contains the protobufs needed to encode and decode
rpc's.
I also added some helper methods to create initial requests.
Test Plan: It compiles and still gets data from HBase meta
Differential Revision: https://reviews.facebook.net/D57327
---
connection/BUCK | 1 +
connection/client-dispatcher.cc | 6 +++---
connection/client-dispatcher.h | 6 +++---
connection/client-handler.cc | 30 +++++++++++++++++++++------
connection/client-handler.h | 18 ++++++++++++----
connection/connection-factory.cc | 6 +++---
connection/connection-factory.h | 2 +-
connection/pipeline.h | 3 ++-
connection/request.cc | 45 ++++++++++++++++++++++++++++++++++++++++
connection/request.h | 25 ++++++++++++++++++----
core/simple-client.cc | 13 ++++++------
11 files changed, 123 insertions(+), 32 deletions(-)
diff --git a/connection/BUCK b/connection/BUCK
index 5067708..d393885 100644
--- a/connection/BUCK
+++ b/connection/BUCK
@@ -32,6 +32,7 @@ cxx_library(name="connection",
"client-handler.cc",
"connection-factory.cc",
"pipeline.cc",
+ "request.cc",
],
deps=[
"//if:if",
diff --git a/connection/client-dispatcher.cc b/connection/client-dispatcher.cc
index 25cff7d..eea0a17 100644
--- a/connection/client-dispatcher.cc
+++ b/connection/client-dispatcher.cc
@@ -35,16 +35,16 @@ void ClientDispatcher::read(Context *ctx, Response in) {
p.setValue(in);
}
-Future<Response> ClientDispatcher::operator()(Request arg) {
+Future<Response> ClientDispatcher::operator()(std::unique_ptr<Request> arg) {
auto call_id = ++current_call_id_;
- arg.set_call_id(call_id);
+ arg->set_call_id(call_id);
auto &p = requests_[call_id];
auto f = p.getFuture();
p.setInterruptHandler([call_id, this](const folly::exception_wrapper &e) {
this->requests_.erase(call_id);
});
- this->pipeline_->write(arg);
+ this->pipeline_->write(std::move(arg));
return f;
}
diff --git a/connection/client-dispatcher.h b/connection/client-dispatcher.h
index 89c7119..877e877 100644
--- a/connection/client-dispatcher.h
+++ b/connection/client-dispatcher.h
@@ -27,11 +27,11 @@
namespace hbase {
class ClientDispatcher
- : public wangle::ClientDispatcherBase<SerializePipeline, Request,
- Response> {
+ : public wangle::ClientDispatcherBase<SerializePipeline,
+ std::unique_ptr<Request>, Response> {
public:
void read(Context *ctx, Response in) override;
- folly::Future<Response> operator()(Request arg) override;
+ folly::Future<Response> operator()(std::unique_ptr<Request> arg) override;
folly::Future<folly::Unit> close(Context *ctx) override;
folly::Future<folly::Unit> close() override;
diff --git a/connection/client-handler.cc b/connection/client-handler.cc
index 205993a7..abcf5c1 100644
--- a/connection/client-handler.cc
+++ b/connection/client-handler.cc
@@ -47,14 +47,30 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
LOG(INFO) << "Read ResponseHeader size=" << used_bytes
<< " call_id=" << header.call_id()
<< " has_exception=" << header.has_exception();
+
+ // Get the response protobuf from the map
+ auto search = resp_msgs_.find(header.call_id());
+ // It's an error if it's not there.
+ CHECK(search != resp_msgs_.end());
+ auto resp_msg = search->second;
+ CHECK(resp_msg != nullptr);
+
+ // Make sure we don't leak the protobuf
+ resp_msgs_.erase(search);
+
+ // set the call_id.
+ // This will be used to by the dispatcher to match up
+ // the promise with the response.
received.set_call_id(header.call_id());
+ // If there was an exception then there's no
+ // data left on the wire.
if (header.has_exception() == false) {
buf->trimStart(used_bytes);
- // For now assume that everything was a get.
- // We'll need to set this up later.
- received.set_response(std::make_shared<GetResponse>());
- used_bytes = deser_.parse_delimited(buf.get(), received.response().get());
+ used_bytes = deser_.parse_delimited(buf.get(), resp_msg.get());
+ // Make sure that bytes were parsed.
+ CHECK(used_bytes == buf->length());
+ received.set_response(resp_msg);
}
ctx->fireRead(std::move(received));
}
@@ -62,7 +78,7 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
// TODO(eclark): Figure out how to handle the
// network errors that are going to come.
-Future<Unit> ClientHandler::write(Context *ctx, Request r) {
+Future<Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
// Keep track of if we have sent the header.
if (UNLIKELY(need_send_header_)) {
need_send_header_ = false;
@@ -78,5 +94,7 @@ Future<Unit> ClientHandler::write(Context *ctx, Request r) {
ctx->fireWrite(std::move(pre));
}
- return ctx->fireWrite(ser_.request(r.call_id(), r.method(), r.msg()));
+ resp_msgs_[r->call_id()] = r->resp_msg();
+ return ctx->fireWrite(
+ ser_.request(r->call_id(), r->method(), r->req_msg().get()));
}
diff --git a/connection/client-handler.h b/connection/client-handler.h
index dbaf5a0..41bb883 100644
--- a/connection/client-handler.h
+++ b/connection/client-handler.h
@@ -30,20 +30,30 @@ namespace hbase {
class Request;
class Response;
}
+namespace google {
+namespace protobuf {
+class Message;
+}
+}
namespace hbase {
-class ClientHandler
- : public wangle::Handler<std::unique_ptr<folly::IOBuf>, Response, Request,
- std::unique_ptr<folly::IOBuf>> {
+class ClientHandler : public wangle::Handler<std::unique_ptr<folly::IOBuf>,
+ Response, std::unique_ptr<Request>,
+ std::unique_ptr<folly::IOBuf>> {
public:
ClientHandler(std::string user_name);
void read(Context *ctx, std::unique_ptr<folly::IOBuf> msg) override;
- folly::Future<folly::Unit> write(Context *ctx, Request r) override;
+ folly::Future<folly::Unit> write(Context *ctx,
+ std::unique_ptr<Request> r) override;
private:
bool need_send_header_ = true;
std::string user_name_;
ClientSerializer ser_;
ClientDeserializer deser_;
+
+ // in flight requests
+ std::unordered_map<uint32_t, std::shared_ptr<google::protobuf::Message>>
+ resp_msgs_;
};
} // namespace hbase
diff --git a/connection/connection-factory.cc b/connection/connection-factory.cc
index 5d1b0da..7073f9d 100644
--- a/connection/connection-factory.cc
+++ b/connection/connection-factory.cc
@@ -45,14 +45,14 @@ ConnectionFactory::ConnectionFactory() {
bootstrap_.pipelineFactory(std::make_shared<RpcPipelineFactory>());
}
-std::shared_ptr<Service<Request, Response>>
+std::shared_ptr<Service<std::unique_ptr<Request>, Response>>
ConnectionFactory::make_connection(std::string host, int port) {
// Connect to a given server
// Then when connected create a ClientDispactcher.
auto pipeline = bootstrap_.connect(SocketAddress(host, port, true)).get();
auto dispatcher = std::make_shared<ClientDispatcher>();
dispatcher->setPipeline(pipeline);
- auto service =
- std::make_shared<CloseOnReleaseFilter<Request, Response>>(dispatcher);
+ auto service = std::make_shared<
+ CloseOnReleaseFilter<std::unique_ptr<Request>, Response>>(dispatcher);
return service;
}
diff --git a/connection/connection-factory.h b/connection/connection-factory.h
index 73ac032..8d1d2f0 100644
--- a/connection/connection-factory.h
+++ b/connection/connection-factory.h
@@ -31,7 +31,7 @@ namespace hbase {
class ConnectionFactory {
public:
ConnectionFactory();
- std::shared_ptr<wangle::Service<Request, Response>>
+ std::shared_ptr<wangle::Service<std::unique_ptr<Request>, Response>>
make_connection(std::string host, int port);
private:
diff --git a/connection/pipeline.h b/connection/pipeline.h
index 8114fab..6c4f4ff 100644
--- a/connection/pipeline.h
+++ b/connection/pipeline.h
@@ -26,7 +26,8 @@
#include "utils/user-util.h"
namespace hbase {
-using SerializePipeline = wangle::Pipeline<folly::IOBufQueue &, Request>;
+using SerializePipeline =
+ wangle::Pipeline<folly::IOBufQueue &, std::unique_ptr<Request>>;
class RpcPipelineFactory : public wangle::PipelineFactory<SerializePipeline> {
public:
diff --git a/connection/request.cc b/connection/request.cc
new file mode 100644
index 0000000..50ea029
--- /dev/null
+++ b/connection/request.cc
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "connection/request.h"
+
+#include "if/Client.pb.h"
+
+using namespace hbase;
+
+Request::Request(std::shared_ptr<google::protobuf::Message> req,
+ std::shared_ptr<google::protobuf::Message> resp,
+ std::string method)
+ : req_msg_(req), resp_msg_(resp), method_(method), call_id_(0) {}
+
+std::unique_ptr<Request> Request::get() {
+ return std::make_unique<Request>(std::make_shared<hbase::pb::GetRequest>(),
+ std::make_shared<hbase::pb::GetResponse>(),
+ "Get");
+}
+std::unique_ptr<Request> Request::mutate() {
+ return std::make_unique<Request>(std::make_shared<hbase::pb::MutateRequest>(),
+ std::make_shared<hbase::pb::MutateResponse>(),
+ "Mutate");
+}
+std::unique_ptr<Request> Request::scan() {
+ return std::make_unique<Request>(std::make_shared<hbase::pb::ScanRequest>(),
+ std::make_shared<hbase::pb::ScanResponse>(),
+ "Scan");
+}
diff --git a/connection/request.h b/connection/request.h
index e9e3e88..743c469 100644
--- a/connection/request.h
+++ b/connection/request.h
@@ -21,22 +21,39 @@
#include <google/protobuf/message.h>
#include <cstdint>
+#include <memory>
#include <string>
namespace hbase {
class Request {
public:
- Request() : call_id_(0) {}
+ static std::unique_ptr<Request> get();
+ static std::unique_ptr<Request> mutate();
+ static std::unique_ptr<Request> scan();
+
+ Request(std::shared_ptr<google::protobuf::Message> req,
+ std::shared_ptr<google::protobuf::Message> resp, std::string method);
+
uint32_t call_id() { return call_id_; }
void set_call_id(uint32_t call_id) { call_id_ = call_id; }
- google::protobuf::Message *msg() { return msg_.get(); }
- void set_msg(std::shared_ptr<google::protobuf::Message> msg) { msg_ = msg; }
+
+ std::shared_ptr<google::protobuf::Message> req_msg() { return req_msg_; }
+ std::shared_ptr<google::protobuf::Message> resp_msg() { return resp_msg_; }
+
+ void set_req_msg(std::shared_ptr<google::protobuf::Message> msg) {
+ req_msg_ = msg;
+ }
+ void set_resp_msg(std::shared_ptr<google::protobuf::Message> msg) {
+ resp_msg_ = msg;
+ }
+
std::string method() { return method_; }
void set_method(std::string method) { method_ = method; }
private:
uint32_t call_id_;
- std::shared_ptr<google::protobuf::Message> msg_ = nullptr;
+ std::shared_ptr<google::protobuf::Message> req_msg_ = nullptr;
+ std::shared_ptr<google::protobuf::Message> resp_msg_ = nullptr;
std::string method_ = "Get";
};
} // namespace hbase
diff --git a/core/simple-client.cc b/core/simple-client.cc
index 8b2fae5..2cb6200 100644
--- a/core/simple-client.cc
+++ b/core/simple-client.cc
@@ -58,24 +58,23 @@ int main(int argc, char *argv[]) {
auto conn = cf.make_connection(result.host_name(), result.port());
// Send the request
- Request r;
+ auto r = Request::get();
// This is a get request so make that
- auto msg = make_shared<hbase::pb::GetRequest>();
+ auto req_msg = static_pointer_cast<hbase::pb::GetRequest>(r->req_msg());
// Set what region
- msg->mutable_region()->set_value(FLAGS_region);
+ req_msg->mutable_region()->set_value(FLAGS_region);
// It's always this.
- msg->mutable_region()->set_type(
+ req_msg->mutable_region()->set_type(
RegionSpecifier_RegionSpecifierType::
RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
// What row.
- msg->mutable_get()->set_row(FLAGS_row);
+ req_msg->mutable_get()->set_row(FLAGS_row);
// Send it.
- r.set_msg(msg);
- auto resp = (*conn)(r).get(milliseconds(5000));
+ auto resp = (*conn)(std::move(r)).get(milliseconds(5000));
auto get_resp = std::static_pointer_cast<GetResponse>(resp.response());
cout << "GetResponse has_result = " << get_resp->has_result() << '\n';
[hbase] 130/137: HBASE-18802 [C++] Remove hbase-macros.h
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 4810f57895f86c245355174210242c1c4518b372
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Thu Sep 14 14:07:34 2017 -0700
HBASE-18802 [C++] Remove hbase-macros.h
---
core/hbase-macros.h | 56 -----------------------------------------------------
1 file changed, 56 deletions(-)
diff --git a/core/hbase-macros.h b/core/hbase-macros.h
deleted file mode 100644
index d70c247..0000000
--- a/core/hbase-macros.h
+++ /dev/null
@@ -1,56 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#pragma once
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/**
- * The following code block define API as the tag for exported
- * functions. The library should be compiled with symbols visibility
- * set to hidden by default and only the exported functions should be
- * tagged as HBASE_API.
- *
- * When building the library on Windows, compile with compiler flag
- * "-D_LIBHBASE_IMPLEMENTATION_", whereas when linking application with
- * this library, this compiler flag should not be used.
- */
-#if defined _WIN32 || defined __CYGWIN__
-#ifdef _LIBHBASE_IMPLEMENTATION_
-#define API __declspec(dllexport)
-#else
-#ifdef _LIBHBASE_TEST_
-#define HBASE_API
-#else
-#define HBASE_API __declspec(dllimport)
-#endif
-#endif
-#else
-#if __GNUC__ >= 4
-#define HBASE_API __attribute__((visibility("default")))
-#else
-#define HBASE_API
-#endif
-#endif
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
[hbase] 60/137: HBASE-15705 Add on meta cache (Mikhail Antonov)
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 9a4a6cb61f5a363ca8c7fbda5b128edba111b433
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Thu Jan 19 15:01:11 2017 -0800
HBASE-15705 Add on meta cache (Mikhail Antonov)
---
connection/connection-pool.cc | 20 +++---
connection/rpc-client.cc | 5 +-
connection/rpc-client.h | 6 +-
core/client.cc | 17 +++--
core/client.h | 9 ++-
core/location-cache-test.cc | 82 +++++++++++++++++++++++-
core/location-cache.cc | 142 ++++++++++++++++++++++++++++++++++++++++--
core/location-cache.h | 97 ++++++++++++++++++++++++++++-
core/region-location.h | 4 ++
core/simple-client.cc | 2 +-
10 files changed, 349 insertions(+), 35 deletions(-)
diff --git a/connection/connection-pool.cc b/connection/connection-pool.cc
index 07518c5..15dd64e 100644
--- a/connection/connection-pool.cc
+++ b/connection/connection-pool.cc
@@ -41,15 +41,7 @@ ConnectionPool::ConnectionPool(std::shared_ptr<wangle::IOThreadPoolExecutor> io_
ConnectionPool::ConnectionPool(std::shared_ptr<ConnectionFactory> cf)
: cf_(cf), clients_(), connections_(), map_mutex_() {}
-ConnectionPool::~ConnectionPool() {
- SharedMutexWritePriority::WriteHolder holder(map_mutex_);
- for (auto &item : connections_) {
- auto &con = item.second;
- con->Close();
- }
- connections_.clear();
- clients_.clear();
-}
+ConnectionPool::~ConnectionPool() { Close(); }
std::shared_ptr<RpcConnection> ConnectionPool::GetConnection(
std::shared_ptr<ConnectionId> remote_id) {
@@ -116,4 +108,12 @@ void ConnectionPool::Close(std::shared_ptr<ConnectionId> remote_id) {
connections_.erase(found);
}
-void ConnectionPool::Close() {}
+void ConnectionPool::Close() {
+ SharedMutexWritePriority::WriteHolder holder{map_mutex_};
+ for (auto &item : connections_) {
+ auto &con = item.second;
+ con->Close();
+ }
+ connections_.clear();
+ clients_.clear();
+}
diff --git a/connection/rpc-client.cc b/connection/rpc-client.cc
index 9cfefb8..7621193 100644
--- a/connection/rpc-client.cc
+++ b/connection/rpc-client.cc
@@ -39,9 +39,8 @@ class RpcChannelImplementation : public AbstractRpcChannel {
};
} // namespace hbase
-RpcClient::RpcClient() {
- io_executor_ = std::make_shared<wangle::IOThreadPoolExecutor>(sysconf(_SC_NPROCESSORS_ONLN));
-
+RpcClient::RpcClient(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor)
+ : io_executor_(io_executor) {
cp_ = std::make_shared<ConnectionPool>(io_executor_);
}
diff --git a/connection/rpc-client.h b/connection/rpc-client.h
index 407d588..aeb9b56 100644
--- a/connection/rpc-client.h
+++ b/connection/rpc-client.h
@@ -26,6 +26,8 @@
#include <google/protobuf/service.h>
+#include <utility>
+
using hbase::security::User;
using hbase::pb::ServerName;
using hbase::Request;
@@ -49,7 +51,7 @@ class RpcClient : public std::enable_shared_from_this<RpcClient> {
friend class RpcChannelImplementation;
public:
- RpcClient();
+ RpcClient(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor);
virtual ~RpcClient() { Close(); }
@@ -77,6 +79,8 @@ class RpcClient : public std::enable_shared_from_this<RpcClient> {
std::shared_ptr<User> ticket,
int rpc_timeout);
+ std::shared_ptr<ConnectionPool> connection_pool() const { return cp_; }
+
private:
void CallMethod(const MethodDescriptor *method, RpcController *controller, const Message *req_msg,
Message *resp_msg, Closure *done, const std::string &host, uint16_t port,
diff --git a/core/client.cc b/core/client.cc
index 6eb3d8f..dd568ce 100644
--- a/core/client.cc
+++ b/core/client.cc
@@ -33,15 +33,22 @@ Client::Client() {
"hbase-site.xml is absent in the search path or problems in XML parsing";
throw std::runtime_error("Configuration object not present.");
}
- conf_ = std::make_shared<hbase::Configuration>(conf.value());
- auto zk_quorum = conf_->Get(kHBaseZookeeperQuorum_, kDefHBaseZookeeperQuorum_);
- location_cache_ = std::make_shared<hbase::LocationCache>(zk_quorum, cpu_executor_, io_executor_);
+ init(conf.value());
}
-Client::Client(const hbase::Configuration &conf) {
+Client::Client(const hbase::Configuration &conf) { init(conf); }
+
+void Client::init(const hbase::Configuration &conf) {
conf_ = std::make_shared<hbase::Configuration>(conf);
auto zk_quorum = conf_->Get(kHBaseZookeeperQuorum_, kDefHBaseZookeeperQuorum_);
- location_cache_ = std::make_shared<hbase::LocationCache>(zk_quorum, cpu_executor_, io_executor_);
+
+ cpu_executor_ =
+ std::make_shared<wangle::CPUThreadPoolExecutor>(4); // TODO: read num threads from conf
+ io_executor_ = std::make_shared<wangle::IOThreadPoolExecutor>(sysconf(_SC_NPROCESSORS_ONLN));
+
+ rpc_client_ = std::make_shared<hbase::RpcClient>(io_executor_);
+ location_cache_ = std::make_shared<hbase::LocationCache>(zk_quorum, cpu_executor_,
+ rpc_client_->connection_pool());
}
// We can't have the threads continue running after everything is done
diff --git a/core/client.h b/core/client.h
index da71624..730981d 100644
--- a/core/client.h
+++ b/core/client.h
@@ -67,14 +67,13 @@ class Client {
void Close();
private:
+ void init(const hbase::Configuration &conf);
const std::string kHBaseZookeeperQuorum_ = "hbase.zookeeper.quorum";
const std::string kDefHBaseZookeeperQuorum_ = "localhost:2181";
- std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_ =
- std::make_shared<wangle::CPUThreadPoolExecutor>(4);
- std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_ =
- std::make_shared<wangle::IOThreadPoolExecutor>(sysconf(_SC_NPROCESSORS_ONLN));
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_;
std::shared_ptr<hbase::LocationCache> location_cache_;
- std::shared_ptr<hbase::RpcClient> rpc_client_ = std::make_shared<hbase::RpcClient>();
+ std::shared_ptr<hbase::RpcClient> rpc_client_;
std::shared_ptr<hbase::Configuration> conf_;
bool is_closed_ = false;
};
diff --git a/core/location-cache-test.cc b/core/location-cache-test.cc
index 53db6fc..42e7bb3 100644
--- a/core/location-cache-test.cc
+++ b/core/location-cache-test.cc
@@ -34,19 +34,24 @@ TEST(LocationCacheTest, TestGetMetaNodeContents) {
auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
- LocationCache cache{"localhost:2181", cpu, io};
+ auto cp = std::make_shared<ConnectionPool>(io);
+ LocationCache cache{"localhost:2181", cpu, cp};
auto f = cache.LocateMeta();
auto result = f.get();
ASSERT_FALSE(f.hasException());
ASSERT_TRUE(result.has_port());
ASSERT_TRUE(result.has_host_name());
+ cpu->stop();
+ io->stop();
+ cp->Close();
}
TEST(LocationCacheTest, TestGetRegionLocation) {
TestUtil test_util{};
auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
- LocationCache cache{"localhost:2181", cpu, io};
+ auto cp = std::make_shared<ConnectionPool>(io);
+ LocationCache cache{"localhost:2181", cpu, cp};
// If there is no table this should throw an exception
auto tn = folly::to<hbase::pb::TableName>("t");
@@ -57,4 +62,77 @@ TEST(LocationCacheTest, TestGetRegionLocation) {
ASSERT_TRUE(loc != nullptr);
cpu->stop();
io->stop();
+ cp->Close();
+}
+
+TEST(LocationCacheTest, TestCaching) {
+ TestUtil test_util{};
+ auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
+ auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
+ auto cp = std::make_shared<ConnectionPool>(io);
+ LocationCache cache{"localhost:2181", cpu, cp};
+
+ auto tn_1 = folly::to<hbase::pb::TableName>("t1");
+ auto tn_2 = folly::to<hbase::pb::TableName>("t2");
+ auto tn_3 = folly::to<hbase::pb::TableName>("t3");
+ auto row_a = "a";
+
+ // test location pulled from meta gets cached
+ ASSERT_ANY_THROW(cache.LocateRegion(tn_1, row_a).get(milliseconds(1000)));
+ ASSERT_ANY_THROW(cache.LocateFromMeta(tn_1, row_a).get(milliseconds(1000)));
+
+ test_util.RunShellCmd("create 't1', 'd'");
+
+ ASSERT_FALSE(cache.IsLocationCached(tn_1, row_a));
+ auto loc = cache.LocateRegion(tn_1, row_a).get(milliseconds(1000));
+ ASSERT_TRUE(cache.IsLocationCached(tn_1, row_a));
+ ASSERT_EQ(loc, cache.GetCachedLocation(tn_1, row_a));
+
+ // test with two regions
+ test_util.RunShellCmd("create 't2', 'd', SPLITS => ['b']");
+
+ ASSERT_FALSE(cache.IsLocationCached(tn_2, "a"));
+ loc = cache.LocateRegion(tn_2, "a").get(milliseconds(1000));
+ ASSERT_TRUE(cache.IsLocationCached(tn_2, "a"));
+ ASSERT_EQ(loc, cache.GetCachedLocation(tn_2, "a"));
+
+ ASSERT_FALSE(cache.IsLocationCached(tn_2, "b"));
+ loc = cache.LocateRegion(tn_2, "b").get(milliseconds(1000));
+ ASSERT_TRUE(cache.IsLocationCached(tn_2, "b"));
+ ASSERT_EQ(loc, cache.GetCachedLocation(tn_2, "b"));
+ ASSERT_TRUE(cache.IsLocationCached(tn_2, "ba"));
+ ASSERT_EQ(loc, cache.GetCachedLocation(tn_2, "ba"));
+
+ // test with three regions
+ test_util.RunShellCmd("create 't3', 'd', SPLITS => ['b', 'c']");
+
+ ASSERT_FALSE(cache.IsLocationCached(tn_3, "c"));
+ ASSERT_FALSE(cache.IsLocationCached(tn_3, "ca"));
+ loc = cache.LocateRegion(tn_3, "ca").get(milliseconds(1000));
+ ASSERT_TRUE(cache.IsLocationCached(tn_3, "c"));
+ ASSERT_EQ(loc, cache.GetCachedLocation(tn_3, "c"));
+ ASSERT_TRUE(cache.IsLocationCached(tn_3, "ca"));
+ ASSERT_EQ(loc, cache.GetCachedLocation(tn_3, "ca"));
+
+ ASSERT_FALSE(cache.IsLocationCached(tn_3, "b"));
+ loc = cache.LocateRegion(tn_3, "b").get(milliseconds(1000));
+ ASSERT_TRUE(cache.IsLocationCached(tn_3, "b"));
+ ASSERT_EQ(loc, cache.GetCachedLocation(tn_3, "b"));
+ ASSERT_TRUE(cache.IsLocationCached(tn_3, "ba"));
+ ASSERT_EQ(loc, cache.GetCachedLocation(tn_3, "ba"));
+
+ // clear second region
+ cache.ClearCachedLocation(tn_3, "b");
+ ASSERT_FALSE(cache.IsLocationCached(tn_3, "b"));
+
+ ASSERT_FALSE(cache.IsLocationCached(tn_3, "a"));
+ loc = cache.LocateRegion(tn_3, "a").get(milliseconds(1000));
+ ASSERT_TRUE(cache.IsLocationCached(tn_3, "a"));
+ ASSERT_EQ(loc, cache.GetCachedLocation(tn_3, "a"));
+ ASSERT_TRUE(cache.IsLocationCached(tn_3, "abc"));
+ ASSERT_EQ(loc, cache.GetCachedLocation(tn_3, "abc"));
+
+ cpu->stop();
+ io->stop();
+ cp->Close();
}
diff --git a/core/location-cache.cc b/core/location-cache.cc
index 6c2a790..66f3eb7 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -55,14 +55,16 @@ static const char META_ZNODE_NAME[] = "/hbase/meta-region-server";
LocationCache::LocationCache(std::string quorum_spec,
std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
- std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor)
+ std::shared_ptr<ConnectionPool> cp)
: quorum_spec_(quorum_spec),
cpu_executor_(cpu_executor),
meta_promise_(nullptr),
meta_lock_(),
- cp_(io_executor),
+ cp_(cp),
meta_util_(),
- zk_(nullptr) {
+ zk_(nullptr),
+ cached_locations_(),
+ locations_lock_() {
zk_ = zookeeper_init(quorum_spec.c_str(), nullptr, 1000, 0, 0, 0);
}
@@ -121,7 +123,7 @@ Future<std::shared_ptr<RegionLocation>> LocationCache::LocateFromMeta(const Tabl
.via(cpu_executor_.get())
.then([this](ServerName sn) {
auto remote_id = std::make_shared<ConnectionId>(sn.host_name(), sn.port());
- return this->cp_.GetConnection(remote_id);
+ return this->cp_->GetConnection(remote_id);
})
.then([tn, row, this](std::shared_ptr<RpcConnection> rpc_connection) {
return (*rpc_connection->get_service())(std::move(meta_util_.MetaRequest(tn, row)));
@@ -143,11 +145,27 @@ Future<std::shared_ptr<RegionLocation>> LocationCache::LocateFromMeta(const Tabl
auto remote_id =
std::make_shared<ConnectionId>(rl->server_name().host_name(), rl->server_name().port());
// Now fill out the connection.
- rl->set_service(cp_.GetConnection(remote_id)->get_service());
+ // rl->set_service(cp_->GetConnection(remote_id)->get_service()); TODO: this causes wangle
+ // assertion errors
+ return rl;
+ })
+ .then([tn, this](shared_ptr<RegionLocation> rl) {
+ // now add fetched location to the cache.
+ this->CacheLocation(tn, rl);
return rl;
});
}
+Future<shared_ptr<RegionLocation>> LocationCache::LocateRegion(const hbase::pb::TableName &tn,
+ const std::string &row) {
+ auto cached_loc = this->GetCachedLocation(tn, row);
+ if (cached_loc != nullptr) {
+ return cached_loc;
+ } else {
+ return this->LocateFromMeta(tn, row);
+ }
+}
+
std::shared_ptr<RegionLocation> LocationCache::CreateLocation(const Response &resp) {
auto resp_msg = static_pointer_cast<ScanResponse>(resp.resp_msg());
auto &results = resp_msg->results().Get(0);
@@ -163,3 +181,117 @@ std::shared_ptr<RegionLocation> LocationCache::CreateLocation(const Response &re
auto server_name = folly::to<ServerName>(cell_one);
return std::make_shared<RegionLocation>(row, std::move(region_info), server_name, nullptr);
}
+
+// must hold shared lock on locations_lock_
+shared_ptr<RegionLocation> LocationCache::GetCachedLocation(const hbase::pb::TableName &tn,
+ const std::string &row) {
+ auto t_locs = this->GetTableLocations(tn);
+ std::shared_lock<folly::SharedMutexWritePriority> lock(locations_lock_);
+
+ if (VLOG_IS_ON(2)) {
+ for (const auto &p : *t_locs) {
+ VLOG(2) << "t_locs[" << p.first << "] = " << p.second->DebugString();
+ }
+ }
+
+ // looking for the "floor" key as a start key
+ auto possible_region = t_locs->upper_bound(row);
+
+ if (t_locs->empty()) {
+ VLOG(2) << "Could not find region in cache, table map is empty";
+ return nullptr;
+ }
+
+ if (possible_region == t_locs->begin()) {
+ VLOG(2) << "Could not find region in cache, all keys are greater, row:" << row
+ << " ,possible_region:" << possible_region->second->DebugString();
+ return nullptr;
+ }
+ --possible_region;
+
+ VLOG(2) << "Found possible region in cache for row:" << row
+ << " ,possible_region:" << possible_region->second->DebugString();
+
+ // found possible start key, now need to check end key
+ if (possible_region->second->region_info().end_key() == "" ||
+ possible_region->second->region_info().end_key() > row) {
+ VLOG(1) << "Found region in cache for row:" << row
+ << " ,region:" << possible_region->second->DebugString();
+ return possible_region->second;
+ } else {
+ return nullptr;
+ }
+}
+
+// must hold unique lock on locations_lock_
+void LocationCache::CacheLocation(const hbase::pb::TableName &tn,
+ const shared_ptr<RegionLocation> loc) {
+ auto t_locs = this->GetTableLocations(tn);
+ std::unique_lock<folly::SharedMutexWritePriority> lock(locations_lock_);
+
+ (*t_locs)[loc->region_info().start_key()] = loc;
+ VLOG(1) << "Cached location for region:" << loc->DebugString();
+}
+
+// must hold shared lock on locations_lock_
+bool LocationCache::IsLocationCached(const hbase::pb::TableName &tn, const std::string &row) {
+ return (this->GetCachedLocation(tn, row) != nullptr);
+}
+
+// shared lock needed for cases when this table has been requested before;
+// in the rare case it hasn't, unique lock will be grabbed to add it to cache
+shared_ptr<hbase::PerTableLocationMap> LocationCache::GetTableLocations(
+ const hbase::pb::TableName &tn) {
+ auto found_locs = this->GetCachedTableLocations(tn);
+ if (found_locs == nullptr) {
+ found_locs = this->GetNewTableLocations(tn);
+ }
+ return found_locs;
+}
+
+shared_ptr<hbase::PerTableLocationMap> LocationCache::GetCachedTableLocations(
+ const hbase::pb::TableName &tn) {
+ SharedMutexWritePriority::ReadHolder r_holder{locations_lock_};
+
+ auto table_locs = cached_locations_.find(tn);
+ if (table_locs != cached_locations_.end()) {
+ return table_locs->second;
+ } else {
+ return nullptr;
+ }
+}
+
+shared_ptr<hbase::PerTableLocationMap> LocationCache::GetNewTableLocations(
+ const hbase::pb::TableName &tn) {
+ // double-check locking under upgradable lock
+ SharedMutexWritePriority::UpgradeHolder u_holder{locations_lock_};
+
+ auto table_locs = cached_locations_.find(tn);
+ if (table_locs != cached_locations_.end()) {
+ return table_locs->second;
+ }
+ SharedMutexWritePriority::WriteHolder w_holder{std::move(u_holder)};
+
+ auto t_locs_p = make_shared<map<std::string, shared_ptr<RegionLocation>>>();
+ cached_locations_.insert(std::make_pair(tn, t_locs_p));
+ return t_locs_p;
+}
+
+// must hold unique lock on locations_lock_
+void LocationCache::ClearCache() {
+ unique_lock<SharedMutexWritePriority> lock(locations_lock_);
+ cached_locations_.clear();
+}
+
+// must hold unique lock on locations_lock_
+void LocationCache::ClearCachedLocations(const hbase::pb::TableName &tn) {
+ unique_lock<SharedMutexWritePriority> lock(locations_lock_);
+ cached_locations_.erase(tn);
+}
+
+// must hold unique lock on locations_lock_
+void LocationCache::ClearCachedLocation(const hbase::pb::TableName &tn, const std::string &row) {
+ auto table_locs = this->GetTableLocations(tn);
+ unique_lock<folly::SharedMutexWritePriority> lock(locations_lock_);
+ table_locs->erase(row);
+}
diff --git a/core/location-cache.h b/core/location-cache.h
index b290a1f..22a8ad5 100644
--- a/core/location-cache.h
+++ b/core/location-cache.h
@@ -19,6 +19,7 @@
#pragma once
#include <folly/Executor.h>
+#include <folly/SharedMutex.h>
#include <folly/futures/Future.h>
#include <folly/futures/SharedPromise.h>
#include <wangle/concurrent/CPUThreadPoolExecutor.h>
@@ -27,6 +28,7 @@
#include <memory>
#include <mutex>
+#include <shared_mutex>
#include <string>
#include "connection/connection-pool.h"
@@ -40,8 +42,34 @@ class Request;
class Response;
namespace pb {
class ServerName;
+class TableName;
}
+/** Equals function for TableName (uses namespace and table name) */
+struct TableNameEquals {
+ /** equals */
+ bool operator()(const hbase::pb::TableName &lht, const hbase::pb::TableName &rht) const {
+ return lht.namespace_() == rht.namespace_() && lht.qualifier() == rht.qualifier();
+ }
+};
+
+/** Hash for TableName. */
+struct TableNameHash {
+ /** hash */
+ std::size_t operator()(hbase::pb::TableName const &t) const {
+ std::size_t h = 0;
+ boost::hash_combine(h, t.namespace_());
+ boost::hash_combine(h, t.qualifier());
+ return h;
+ }
+};
+
+// typedefs for location cache
+typedef std::map<std::string, std::shared_ptr<RegionLocation>> PerTableLocationMap;
+typedef std::unordered_map<hbase::pb::TableName, std::shared_ptr<PerTableLocationMap>,
+ TableNameHash, TableNameEquals>
+ RegionLocationMap;
+
/**
* Class that can look up and cache locations.
*/
@@ -56,7 +84,7 @@ class LocationCache {
*/
LocationCache(std::string quorum_spec,
std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
- std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor);
+ std::shared_ptr<ConnectionPool> cp);
/**
* Destructor.
* This will clean up the zookeeper connections.
@@ -71,7 +99,8 @@ class LocationCache {
folly::Future<hbase::pb::ServerName> LocateMeta();
/**
- * Go read meta and find out where a region is located.
+ * Go read meta and find out where a region is located. Most users should
+ * never call this method directly and should use LocateRegion() instead.
*
* @param tn Table name of the table to look up. This object must live until
* after the future is returned
@@ -83,14 +112,72 @@ class LocationCache {
const std::string &row);
/**
+ * The only method clients should use for meta lookups. If corresponding
+ * location is cached, it's returned from the cache, otherwise lookup
+ * in meta table is done, location is cached and then returned.
+ * It's expected that tiny fraction of invocations incurs meta scan.
+ * This method is to look up non-meta regions; use LocateMeta() to get the
+ * location of hbase:meta region.
+ *
+ * @param tn Table name of the table to look up. This object must live until
+ * after the future is returned
+ *
+ * @param row of the table to look up. This object must live until after the
+ * future is returned
+ */
+ folly::Future<std::shared_ptr<RegionLocation>> LocateRegion(const hbase::pb::TableName &tn,
+ const std::string &row);
+
+ /**
* Remove the cached location of meta.
*/
void InvalidateMeta();
+ /**
+ * Return cached region location corresponding to this row,
+ * nullptr if this location isn't cached.
+ */
+ std::shared_ptr<RegionLocation> GetCachedLocation(const hbase::pb::TableName &tn,
+ const std::string &row);
+
+ /**
+ * Add non-meta region location in the cache (location of meta itself
+ * is cached separately).
+ */
+ void CacheLocation(const hbase::pb::TableName &tn, const std::shared_ptr<RegionLocation> loc);
+
+ /**
+ * Check if location corresponding to this row key is cached.
+ */
+ bool IsLocationCached(const hbase::pb::TableName &tn, const std::string &row);
+
+ /**
+ * Return cached location for all region of this table.
+ */
+ std::shared_ptr<PerTableLocationMap> GetTableLocations(const hbase::pb::TableName &tn);
+
+ /**
+ * Completely clear location cache.
+ */
+ void ClearCache();
+
+ /**
+ * Clear all cached locations for one table.
+ */
+ void ClearCachedLocations(const hbase::pb::TableName &tn);
+
+ /**
+ * Clear cached region location.
+ */
+ void ClearCachedLocation(const hbase::pb::TableName &tn, const std::string &row);
+
private:
void RefreshMetaLocation();
hbase::pb::ServerName ReadMetaLocation();
std::shared_ptr<RegionLocation> CreateLocation(const Response &resp);
+ std::shared_ptr<hbase::PerTableLocationMap> GetCachedTableLocations(
+ const hbase::pb::TableName &tn);
+ std::shared_ptr<hbase::PerTableLocationMap> GetNewTableLocations(const hbase::pb::TableName &tn);
/* data */
std::string quorum_spec_;
@@ -98,7 +185,11 @@ class LocationCache {
std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> meta_promise_;
std::mutex meta_lock_;
MetaUtil meta_util_;
- ConnectionPool cp_;
+ std::shared_ptr<ConnectionPool> cp_;
+
+ // cached region locations
+ RegionLocationMap cached_locations_;
+ folly::SharedMutexWritePriority locations_lock_;
// TODO: migrate this to a smart pointer with a deleter.
zhandle_t *zk_;
diff --git a/core/region-location.h b/core/region-location.h
index e7b76d3..b0411cb 100644
--- a/core/region-location.h
+++ b/core/region-location.h
@@ -79,6 +79,10 @@ class RegionLocation {
*/
void set_server_name(hbase::pb::ServerName sn) { sn_ = sn; }
+ const std::string DebugString() {
+ return "region_info:" + ri_.ShortDebugString() + ", server_name:" + sn_.ShortDebugString();
+ }
+
private:
std::string region_name_;
hbase::pb::RegionInfo ri_;
diff --git a/core/simple-client.cc b/core/simple-client.cc
index 90e7cd4..dac0d10 100644
--- a/core/simple-client.cc
+++ b/core/simple-client.cc
@@ -94,7 +94,7 @@ int main(int argc, char *argv[]) {
auto row = FLAGS_row;
auto tn = folly::to<TableName>(FLAGS_table);
- auto loc = cache.LocateFromMeta(tn, row).get(milliseconds(5000));
+ auto loc = cache.LocateRegion(tn, row).get(milliseconds(5000));
auto connection = loc->service();
auto num_puts = FLAGS_columns;
[hbase] 24/137: HBASE-15750 Add on meta deserialization
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 1fa8212c3a16ef672c824c3130f4008b88b7847d
Author: Elliott Clark <ec...@apache.org>
AuthorDate: Tue May 3 12:17:07 2016 -0700
HBASE-15750 Add on meta deserialization
Summary: Add on meta region info deserialization
Test Plan:
Unit tests.
Simple client connects.
Differential Revision: https://reviews.facebook.net/D57555
---
connection/client-dispatcher.cc | 1 +
connection/client-handler.cc | 13 ++--
connection/client-handler.h | 6 +-
connection/connection-pool.cc | 36 +----------
connection/connection-pool.h | 8 +--
core/BUCK | 4 --
core/location-cache-test.cc | 1 +
core/location-cache.cc | 73 +++++++++++++++++++---
core/location-cache.h | 7 +--
core/meta-utils.cc | 9 +--
core/meta-utils.h | 7 ++-
core/region-location.h | 5 +-
core/simple-client.cc | 6 +-
serde/BUCK | 56 ++++++++---------
serde/client-deserializer-test.cc | 25 ++++----
serde/client-deserializer.cc | 68 --------------------
serde/client-serializer-test.cc | 26 ++++----
.../region-info-deserializer-test.cc | 58 +++++++++--------
core/table-name.h => serde/region-info.h | 31 ++++-----
serde/{client-serializer.cc => rpc.cc} | 70 ++++++++++++++++-----
serde/{client-serializer.h => rpc.h} | 31 ++++-----
.../{client-deserializer.h => server-name-test.cc} | 24 +++----
serde/server-name.h | 21 +++++++
{core => serde}/table-name-test.cc | 2 +-
{core => serde}/table-name.h | 22 ++++---
serde/zk-deserializer-test.cc | 8 +--
serde/{zk-deserializer.cc => zk.cc} | 4 +-
serde/{zk-deserializer.h => zk.h} | 2 +-
28 files changed, 316 insertions(+), 308 deletions(-)
diff --git a/connection/client-dispatcher.cc b/connection/client-dispatcher.cc
index 817adc1..6e2dc54 100644
--- a/connection/client-dispatcher.cc
+++ b/connection/client-dispatcher.cc
@@ -44,6 +44,7 @@ Future<Response> ClientDispatcher::operator()(std::unique_ptr<Request> arg) {
auto &p = requests_[call_id];
auto f = p.getFuture();
p.setInterruptHandler([call_id, this](const folly::exception_wrapper &e) {
+ LOG(ERROR) << "e = " << call_id;
this->requests_.erase(call_id);
});
this->pipeline_->write(std::move(arg));
diff --git a/connection/client-handler.cc b/connection/client-handler.cc
index 3180f4e..496e4f2 100644
--- a/connection/client-handler.cc
+++ b/connection/client-handler.cc
@@ -37,8 +37,7 @@ using hbase::pb::GetResponse;
using google::protobuf::Message;
ClientHandler::ClientHandler(std::string user_name)
- : user_name_(user_name), need_send_header_(true), ser_(), deser_(),
- resp_msgs_() {}
+ : user_name_(user_name), need_send_header_(true), serde_(), resp_msgs_() {}
void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
if (LIKELY(buf != nullptr)) {
@@ -46,7 +45,7 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
Response received;
ResponseHeader header;
- int used_bytes = deser_.parse_delimited(buf.get(), &header);
+ int used_bytes = serde_.ParseDelimited(buf.get(), &header);
LOG(INFO) << "Read ResponseHeader size=" << used_bytes
<< " call_id=" << header.call_id()
<< " has_exception=" << header.has_exception();
@@ -70,7 +69,7 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
// data left on the wire.
if (header.has_exception() == false) {
buf->trimStart(used_bytes);
- used_bytes = deser_.parse_delimited(buf.get(), resp_msg.get());
+ used_bytes = serde_.ParseDelimited(buf.get(), resp_msg.get());
// Make sure that bytes were parsed.
CHECK(used_bytes == buf->length());
received.set_response(resp_msg);
@@ -91,13 +90,13 @@ Future<Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
// and one for the request.
//
// That doesn't seem like too bad, but who knows.
- auto pre = ser_.preamble();
- auto header = ser_.header(user_name_);
+ auto pre = serde_.Preamble();
+ auto header = serde_.Header(user_name_);
pre->appendChain(std::move(header));
ctx->fireWrite(std::move(pre));
}
resp_msgs_[r->call_id()] = r->resp_msg();
return ctx->fireWrite(
- ser_.request(r->call_id(), r->method(), r->req_msg().get()));
+ serde_.Request(r->call_id(), r->method(), r->req_msg().get()));
}
diff --git a/connection/client-handler.h b/connection/client-handler.h
index 68513de..ce99c9e 100644
--- a/connection/client-handler.h
+++ b/connection/client-handler.h
@@ -22,8 +22,7 @@
#include <string>
-#include "serde/client-deserializer.h"
-#include "serde/client-serializer.h"
+#include "serde/rpc.h"
// Forward decs.
namespace hbase {
@@ -49,8 +48,7 @@ public:
private:
bool need_send_header_;
std::string user_name_;
- ClientSerializer ser_;
- ClientDeserializer deser_;
+ RpcSerde serde_;
// in flight requests
std::unordered_map<uint32_t, std::shared_ptr<google::protobuf::Message>>
diff --git a/connection/connection-pool.cc b/connection/connection-pool.cc
index 72c1306..eafe60a 100644
--- a/connection/connection-pool.cc
+++ b/connection/connection-pool.cc
@@ -25,41 +25,10 @@ using std::mutex;
using std::unique_ptr;
using std::shared_ptr;
using hbase::pb::ServerName;
-using wangle::ServiceFilter;
using folly::SharedMutexWritePriority;
namespace hbase {
-class RemoveServiceFilter
- : public ServiceFilter<unique_ptr<Request>, Response> {
-
-public:
- RemoveServiceFilter(std::shared_ptr<HBaseService> service, ServerName sn,
- ConnectionPool &cp)
- : ServiceFilter<unique_ptr<Request>, Response>(service), sn_(sn),
- cp_(cp) {}
-
- folly::Future<folly::Unit> close() override {
- if (!released.exchange(true)) {
- return this->service_->close().then(
- [this]() { this->cp_.close(this->sn_); });
- } else {
- return folly::makeFuture();
- }
- }
-
- virtual bool isAvailable() override { return service_->isAvailable(); }
-
- folly::Future<Response> operator()(unique_ptr<Request> req) override {
- return (*this->service_)(std::move(req));
- }
-
-private:
- std::atomic<bool> released{false};
- hbase::pb::ServerName sn_;
- ConnectionPool &cp_;
-};
-
ConnectionPool::ConnectionPool()
: cf_(std::make_shared<ConnectionFactory>()), connections_(), map_mutex_() {
}
@@ -72,13 +41,12 @@ std::shared_ptr<HBaseService> ConnectionPool::get(const ServerName &sn) {
if (found == connections_.end() || found->second == nullptr) {
SharedMutexWritePriority::WriteHolder holder(std::move(holder));
auto new_con = cf_->make_connection(sn.host_name(), sn.port());
- auto wrapped = std::make_shared<RemoveServiceFilter>(new_con, sn, *this);
- connections_[sn] = wrapped;
+ connections_[sn] = new_con;
return new_con;
}
return found->second;
}
-void ConnectionPool::close(ServerName sn) {
+void ConnectionPool::close(const ServerName &sn) {
SharedMutexWritePriority::WriteHolder holder(map_mutex_);
auto found = connections_.find(sn);
diff --git a/connection/connection-pool.h b/connection/connection-pool.h
index 394cd71..b8330e3 100644
--- a/connection/connection-pool.h
+++ b/connection/connection-pool.h
@@ -27,13 +27,13 @@
#include "if/HBase.pb.h"
namespace hbase {
-struct MyServerNameEquals {
+struct ServerNameEquals {
bool operator()(const hbase::pb::ServerName &lhs,
const hbase::pb::ServerName &rhs) const {
return lhs.host_name() == rhs.host_name() && lhs.port() == rhs.port();
}
};
-struct MyServerNameHash {
+struct ServerNameHash {
std::size_t operator()(hbase::pb::ServerName const &s) const {
std::size_t h1 = std::hash<std::string>()(s.host_name());
std::size_t h2 = std::hash<uint32_t>()(s.port());
@@ -46,12 +46,12 @@ public:
ConnectionPool();
explicit ConnectionPool(std::shared_ptr<ConnectionFactory> cf);
std::shared_ptr<HBaseService> get(const hbase::pb::ServerName &sn);
- void close(hbase::pb::ServerName sn);
+ void close(const hbase::pb::ServerName &sn);
private:
std::shared_ptr<ConnectionFactory> cf_;
std::unordered_map<hbase::pb::ServerName, std::shared_ptr<HBaseService>,
- MyServerNameHash, MyServerNameEquals>
+ ServerNameHash, ServerNameEquals>
connections_;
folly::SharedMutexWritePriority map_mutex_;
};
diff --git a/core/BUCK b/core/BUCK
index 447248b..ef8c2f8 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -24,7 +24,6 @@ cxx_library(
"hbase_macros.h",
"region-location.h",
"location-cache.h",
- "table-name.h",
# TODO: move this out of exported
# Once meta lookup works
"meta-utils.h",
@@ -53,9 +52,6 @@ cxx_test(name="location-cache-test",
],
deps=[":core", ],
run_test_separately=True, )
-cxx_test(name="table-name-test",
- srcs=["table-name-test.cc", ],
- deps=[":core", ], )
cxx_binary(name="simple-client",
srcs=["simple-client.cc", ],
deps=[":core", "//connection:connection"], )
diff --git a/core/location-cache-test.cc b/core/location-cache-test.cc
index f3166fb..172799d 100644
--- a/core/location-cache-test.cc
+++ b/core/location-cache-test.cc
@@ -30,4 +30,5 @@ TEST(LocationCacheTest, TestGetMetaNodeContents) {
auto result = f.get();
ASSERT_FALSE(f.hasException());
ASSERT_TRUE(result.has_port());
+ ASSERT_TRUE(result.has_host_name());
}
diff --git a/core/location-cache.cc b/core/location-cache.cc
index 539051a..2667f11 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -20,19 +20,25 @@
#include <folly/Logging.h>
#include <folly/io/IOBuf.h>
+#include <wangle/concurrent/GlobalExecutor.h>
#include "connection/response.h"
#include "if/Client.pb.h"
#include "if/ZooKeeper.pb.h"
-#include "serde/zk-deserializer.h"
+#include "serde/server-name.h"
+#include "serde/region-info.h"
+#include "serde/zk.h"
using namespace std;
using namespace folly;
+using wangle::ServiceFilter;
+using hbase::Request;
using hbase::Response;
using hbase::LocationCache;
using hbase::RegionLocation;
using hbase::HBaseService;
+using hbase::ConnectionPool;
using hbase::pb::ScanResponse;
using hbase::pb::TableName;
using hbase::pb::ServerName;
@@ -45,7 +51,7 @@ static const char META_ZNODE_NAME[] = "/hbase/meta-region-server";
LocationCache::LocationCache(string quorum_spec,
shared_ptr<folly::Executor> executor)
: quorum_spec_(quorum_spec), executor_(executor), meta_promise_(nullptr),
- meta_lock_(), cp_(), meta_util_() {
+ meta_lock_(), cp_(), meta_util_(), zk_(nullptr) {
zk_ = zookeeper_init(quorum_spec.c_str(), nullptr, 1000, 0, 0, 0);
}
@@ -95,28 +101,77 @@ ServerName LocationCache::ReadMetaLocation() {
buf->append(len);
MetaRegionServer mrs;
- if (derser.parse(buf.get(), &mrs) == false) {
+ if (derser.Parse(buf.get(), &mrs) == false) {
LOG(ERROR) << "Unable to decode";
}
return mrs.server();
}
-Future<RegionLocation> LocationCache::locateFromMeta(const TableName &tn,
- const string &row) {
+Future<std::shared_ptr<RegionLocation>>
+LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
+ auto exc = wangle::getIOExecutor();
return this->LocateMeta()
.then([&](ServerName sn) { return this->cp_.get(sn); })
+ .via(exc.get()) // Need to handle all rpc's on the IOExecutor.
.then([&](std::shared_ptr<HBaseService> service) {
- return (*service)(std::move(meta_util_.make_meta_request(tn, row)));
+ return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
})
.then([&](Response resp) {
// take the protobuf response and make it into
// a region location.
- return this->parse_response(std::move(resp));
+ return this->CreateLocation(std::move(resp));
});
}
-RegionLocation LocationCache::parse_response(const Response &resp) {
+class RemoveServiceFilter
+ : public ServiceFilter<std::unique_ptr<Request>, Response> {
+
+public:
+ RemoveServiceFilter(std::shared_ptr<HBaseService> service, ServerName sn,
+ ConnectionPool &cp)
+ : ServiceFilter<unique_ptr<Request>, Response>(service), sn_(sn),
+ cp_(cp) {}
+
+ folly::Future<folly::Unit> close() override {
+ if (!released.exchange(true)) {
+ return this->service_->close().then([this]() {
+ // TODO(eclark): remove the service from the meta cache.
+ this->cp_.close(this->sn_);
+ });
+ } else {
+ return folly::makeFuture();
+ }
+ }
+
+ virtual bool isAvailable() override {
+ return !released && service_->isAvailable();
+ }
+
+ folly::Future<Response> operator()(unique_ptr<Request> req) override {
+ // TODO(eclark): add in an on error handler that will
+ // remove the region location from the cache if needed.
+ // Also close the connection if this is likely to be an error
+ // that needs to get a new connection.
+ return (*this->service_)(std::move(req));
+ }
+
+private:
+ std::atomic<bool> released{false};
+ hbase::pb::ServerName sn_;
+ ConnectionPool &cp_;
+};
+
+std::shared_ptr<RegionLocation>
+LocationCache::CreateLocation(const Response &resp){
auto resp_msg = static_pointer_cast<ScanResponse>(resp.response());
+ auto &results = resp_msg->results().Get(0);
+ auto &cells = results.cell();
LOG(ERROR) << "resp_msg = " << resp_msg->DebugString();
- return RegionLocation{RegionInfo{}, ServerName{}, nullptr};
+ auto ri = folly::to<RegionInfo>(cells.Get(0).value());
+ auto sn = folly::to<ServerName>(cells.Get(1).value());
+
+ LOG(ERROR) << "RegionInfo = " << ri.DebugString();
+ LOG(ERROR) << "ServerName = " << sn.DebugString();
+ auto wrapped = make_shared<RemoveServiceFilter>(cp_.get(sn), sn, this->cp_);
+ return std::make_shared<RegionLocation>(std::move(ri), std::move(sn), wrapped);
}
diff --git a/core/location-cache.h b/core/location-cache.h
index cfd6838..99b5e5e 100644
--- a/core/location-cache.h
+++ b/core/location-cache.h
@@ -29,8 +29,8 @@
#include "connection/connection-pool.h"
#include "core/meta-utils.h"
-#include "core/table-name.h"
#include "core/region-location.h"
+#include "serde/table-name.h"
namespace hbase {
@@ -48,14 +48,14 @@ public:
// Meta Related Methods.
// These are only public until testing is complete
folly::Future<hbase::pb::ServerName> LocateMeta();
- folly::Future<RegionLocation> locateFromMeta(const hbase::pb::TableName &tn,
+ folly::Future<std::shared_ptr<RegionLocation>> LocateFromMeta(const hbase::pb::TableName &tn,
const std::string &row);
- RegionLocation parse_response(const Response &resp);
void InvalidateMeta();
private:
void RefreshMetaLocation();
hbase::pb::ServerName ReadMetaLocation();
+ std::shared_ptr<RegionLocation> CreateLocation(const Response &resp);
std::string quorum_spec_;
std::shared_ptr<folly::Executor> executor_;
@@ -64,7 +64,6 @@ private:
ConnectionPool cp_;
MetaUtil meta_util_;
-
// TODO: migrate this to a smart pointer with a deleter.
zhandle_t *zk_;
};
diff --git a/core/meta-utils.cc b/core/meta-utils.cc
index d2fdd88..1325d83 100644
--- a/core/meta-utils.cc
+++ b/core/meta-utils.cc
@@ -23,25 +23,26 @@
#include "connection/request.h"
#include "connection/response.h"
-#include "core/table-name.h"
#include "if/Client.pb.h"
+#include "serde/table-name.h"
using hbase::pb::TableName;
using hbase::MetaUtil;
using hbase::Request;
using hbase::Response;
using hbase::pb::ScanRequest;
+using hbase::pb::ServerName;
using hbase::pb::RegionSpecifier_RegionSpecifierType;
static const std::string META_REGION = "1588230740";
-std::string MetaUtil::region_lookup_rowkey(const TableName &tn,
+std::string MetaUtil::RegionLookupRowkey(const TableName &tn,
const std::string &row) const {
return folly::to<std::string>(tn, ",", row, ",", "999999999999999999");
}
std::unique_ptr<Request>
-MetaUtil::make_meta_request(const TableName tn, const std::string &row) const {
+MetaUtil::MetaRequest(const TableName tn, const std::string &row) const {
auto request = Request::scan();
auto msg = std::static_pointer_cast<ScanRequest>(request->req_msg());
@@ -76,6 +77,6 @@ MetaUtil::make_meta_request(const TableName tn, const std::string &row) const {
info_col->add_qualifier("server");
info_col->add_qualifier("regioninfo");
- scan->set_start_row(region_lookup_rowkey(tn, row));
+ scan->set_start_row(RegionLookupRowkey(tn, row));
return request;
}
diff --git a/core/meta-utils.h b/core/meta-utils.h
index e007d02..5a659f3 100644
--- a/core/meta-utils.h
+++ b/core/meta-utils.h
@@ -21,15 +21,16 @@
#include <string>
#include "connection/Request.h"
-#include "core/table-name.h"
+#include "if/HBase.pb.h"
+#include "serde/table-name.h"
namespace hbase {
class MetaUtil {
public:
- std::string region_lookup_rowkey(const hbase::pb::TableName &tn,
+ std::string RegionLookupRowkey(const hbase::pb::TableName &tn,
const std::string &row) const;
- std::unique_ptr<Request> make_meta_request(const hbase::pb::TableName tn,
+ std::unique_ptr<Request> MetaRequest(const hbase::pb::TableName tn,
const std::string &row) const;
};
} // namespace hbase
diff --git a/core/region-location.h b/core/region-location.h
index a46b8e2..7922c95 100644
--- a/core/region-location.h
+++ b/core/region-location.h
@@ -23,7 +23,6 @@
#include "connection/service.h"
#include "if/HBase.pb.h"
-
namespace hbase {
class RegionLocation {
@@ -32,8 +31,8 @@ public:
std::shared_ptr<HBaseService> service)
: ri_(ri), sn_(sn), service_(service) {}
- const hbase::pb::RegionInfo& region_info() { return ri_; }
- const hbase::pb::ServerName& server_name() { return sn_; }
+ const hbase::pb::RegionInfo ®ion_info() { return ri_; }
+ const hbase::pb::ServerName &server_name() { return sn_; }
std::shared_ptr<HBaseService> service() { return service_; }
private:
diff --git a/core/simple-client.cc b/core/simple-client.cc
index ab614e4..00e3369 100644
--- a/core/simple-client.cc
+++ b/core/simple-client.cc
@@ -27,9 +27,9 @@
#include "connection/connection-pool.h"
#include "core/client.h"
-#include "core/table-name.h"
#include "if/Client.pb.h"
#include "if/ZooKeeper.pb.h"
+#include "serde/table-name.h"
using namespace folly;
using namespace std;
@@ -39,7 +39,7 @@ using hbase::Request;
using hbase::HBaseService;
using hbase::LocationCache;
using hbase::ConnectionPool;
-using hbase::TableNameUtil;
+using hbase::pb::TableName;
using hbase::pb::ServerName;
using hbase::pb::RegionSpecifier_RegionSpecifierType;
using hbase::pb::GetRequest;
@@ -61,7 +61,7 @@ int main(int argc, char *argv[]) {
auto cpu_ex = wangle::getCPUExecutor();
LocationCache cache{FLAGS_zookeeper, cpu_ex};
auto result =
- cache.locateFromMeta(TableNameUtil::create(FLAGS_table), FLAGS_row)
+ cache.LocateFromMeta(folly::to<TableName>(FLAGS_table), FLAGS_row)
.get(milliseconds(5000));
return 0;
diff --git a/serde/BUCK b/serde/BUCK
index 539a221..db15026 100644
--- a/serde/BUCK
+++ b/serde/BUCK
@@ -17,49 +17,47 @@
cxx_library(name="serde",
exported_headers=[
- "client-serializer.h",
- "client-deserializer.h",
- "zk-deserializer.h",
+ "region-info.h",
+ "rpc.h",
+ "server-name.h",
+ "table-name.h",
+ "zk.h",
],
srcs=[
- "client-serializer.cc",
- "client-deserializer.cc",
- "zk-deserializer.cc",
+ "rpc.cc",
+ "zk.cc",
],
deps=[
"//if:if",
"//third-party:folly",
],
tests=[
- ":client-serializer-test",
":client-deserializer-test",
+ ":client-serializer-test",
+ ":server-name-test",
+ ":table-name-test",
+ ":zk-deserializer-test",
+ ":region-info-deserializer-test",
],
compiler_flags=['-Weffc++'],
visibility=[
'PUBLIC',
], )
-
+cxx_test(name="table-name-test",
+ srcs=["table-name-test.cc", ],
+ deps=[":serde", ], )
+cxx_test(name="server-name-test",
+ srcs=["server-name-test.cc", ],
+ deps=[":serde", ], )
cxx_test(name="client-serializer-test",
- srcs=[
- "client-serializer-test.cc",
- ],
- deps=[
- ":serde",
- "//if:if",
- ], )
+ srcs=["client-serializer-test.cc", ],
+ deps=[":serde", ], )
cxx_test(name="client-deserializer-test",
- srcs=[
- "client-deserializer-test.cc",
- ],
- deps=[
- ":serde",
- "//if:if",
- ], )
+ srcs=["client-deserializer-test.cc", ],
+ deps=[":serde", ], )
cxx_test(name="zk-deserializer-test",
- srcs=[
- "zk-deserializer-test.cc",
- ],
- deps=[
- ":serde",
- "//if:if",
- ], )
+ srcs=["zk-deserializer-test.cc", ],
+ deps=[":serde", ], )
+cxx_test(name="region-info-deserializer-test",
+ srcs=["region-info-deserializer-test.cc", ],
+ deps=[":serde", ], )
diff --git a/serde/client-deserializer-test.cc b/serde/client-deserializer-test.cc
index 9fef093..8c571b1 100644
--- a/serde/client-deserializer-test.cc
+++ b/serde/client-deserializer-test.cc
@@ -16,13 +16,12 @@
* limitations under the License.
*
*/
+#include "serde/rpc.h"
#include <folly/io/IOBuf.h>
#include <gtest/gtest.h>
#include "if/Client.pb.h"
-#include "serde/client-deserializer.h"
-#include "serde/client-serializer.h"
using namespace hbase;
using folly::IOBuf;
@@ -30,23 +29,23 @@ using hbase::pb::GetRequest;
using hbase::pb::RegionSpecifier;
using hbase::pb::RegionSpecifier_RegionSpecifierType;
-TEST(TestClientDeserializer, TestReturnFalseOnNullPtr) {
- ClientDeserializer deser;
- ASSERT_LT(deser.parse_delimited(nullptr, nullptr), 0);
+TEST(TestRpcSerde, TestReturnFalseOnNullPtr) {
+ RpcSerde deser;
+ ASSERT_LT(deser.ParseDelimited(nullptr, nullptr), 0);
}
-TEST(TestClientDeserializer, TestReturnFalseOnBadInput) {
- ClientDeserializer deser;
+TEST(TestRpcSerde, TestReturnFalseOnBadInput) {
+ RpcSerde deser;
auto buf = IOBuf::copyBuffer("test");
GetRequest gr;
- ASSERT_LT(deser.parse_delimited(buf.get(), &gr), 0);
+ ASSERT_LT(deser.ParseDelimited(buf.get(), &gr), 0);
}
-TEST(TestClientDeserializer, TestGoodGetRequestFullRoundTrip) {
+TEST(TestRpcSerde, TestGoodGetRequestFullRoundTrip) {
GetRequest in;
- ClientSerializer ser;
- ClientDeserializer deser;
+ RpcSerde ser;
+ RpcSerde deser;
// fill up the GetRequest.
in.mutable_region()->set_value("test_region_id");
@@ -56,11 +55,11 @@ TEST(TestClientDeserializer, TestGoodGetRequestFullRoundTrip) {
in.mutable_get()->set_row("test_row");
// Create the buffer
- auto buf = ser.serialize_delimited(in);
+ auto buf = ser.SerializeDelimited(in);
GetRequest out;
- int used_bytes = deser.parse_delimited(buf.get(), &out);
+ int used_bytes = deser.ParseDelimited(buf.get(), &out);
ASSERT_GT(used_bytes, 0);
ASSERT_EQ(used_bytes, buf->length());
diff --git a/serde/client-deserializer.cc b/serde/client-deserializer.cc
deleted file mode 100644
index acca7ea..0000000
--- a/serde/client-deserializer.cc
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "serde/client-deserializer.h"
-
-#include <folly/Logging.h>
-#include <google/protobuf/io/coded_stream.h>
-#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
-#include <google/protobuf/message.h>
-
-using namespace hbase;
-
-using folly::IOBuf;
-using google::protobuf::Message;
-using google::protobuf::io::ArrayInputStream;
-using google::protobuf::io::CodedInputStream;
-
-int ClientDeserializer::parse_delimited(const IOBuf *buf, Message *msg) {
- if (buf == nullptr || msg == nullptr) {
- return -2;
- }
-
- DCHECK(!buf->isChained());
-
- ArrayInputStream ais{buf->data(), static_cast<int>(buf->length())};
- CodedInputStream coded_stream{&ais};
-
- uint32_t msg_size;
-
- // Try and read the varint.
- if (coded_stream.ReadVarint32(&msg_size) == false) {
- FB_LOG_EVERY_MS(ERROR, 1000) << "Unable to read a var uint32_t";
- return -3;
- }
-
- coded_stream.PushLimit(msg_size);
- // Parse the message.
- if (msg->MergeFromCodedStream(&coded_stream) == false) {
- FB_LOG_EVERY_MS(ERROR, 1000)
- << "Unable to read a protobuf message from data.";
- return -4;
- }
-
- // Make sure all the data was consumed.
- if (coded_stream.ConsumedEntireMessage() == false) {
- FB_LOG_EVERY_MS(ERROR, 1000)
- << "Orphaned data left after reading protobuf message";
- return -5;
- }
-
- return coded_stream.CurrentPosition();
-}
diff --git a/serde/client-serializer-test.cc b/serde/client-serializer-test.cc
index 9bf38af..2bd17fb 100644
--- a/serde/client-serializer-test.cc
+++ b/serde/client-serializer-test.cc
@@ -24,16 +24,16 @@
#include "if/HBase.pb.h"
#include "if/RPC.pb.h"
-#include "serde/client-serializer.h"
+#include "serde/rpc.h"
using namespace hbase;
using namespace hbase::pb;
using namespace folly;
using namespace folly::io;
-TEST(ClientSerializerTest, PreambleIncludesHBas) {
- ClientSerializer ser;
- auto buf = ser.preamble();
+TEST(RpcSerdeTest, PreambleIncludesHBas) {
+ RpcSerde ser;
+ auto buf = ser.Preamble();
const char *p = reinterpret_cast<const char *>(buf->data());
// Take the first for chars and make sure they are the
// magic string
@@ -42,16 +42,16 @@ TEST(ClientSerializerTest, PreambleIncludesHBas) {
EXPECT_EQ(6, buf->computeChainDataLength());
}
-TEST(ClientSerializerTest, PreambleIncludesVersion) {
- ClientSerializer ser;
- auto buf = ser.preamble();
+TEST(RpcSerdeTest, PreambleIncludesVersion) {
+ RpcSerde ser;
+ auto buf = ser.Preamble();
EXPECT_EQ(0, static_cast<const uint8_t *>(buf->data())[4]);
EXPECT_EQ(80, static_cast<const uint8_t *>(buf->data())[5]);
}
-TEST(ClientSerializerTest, TestHeaderLengthPrefixed) {
- ClientSerializer ser;
- auto header = ser.header("elliott");
+TEST(RpcSerdeTest, TestHeaderLengthPrefixed) {
+ RpcSerde ser;
+ auto header = ser.Header("elliott");
// The header should be prefixed by 4 bytes of length.
EXPECT_EQ(4, header->length());
@@ -64,9 +64,9 @@ TEST(ClientSerializerTest, TestHeaderLengthPrefixed) {
EXPECT_EQ(prefixed_len, header->next()->length());
}
-TEST(ClientSerializerTest, TestHeaderDecode) {
- ClientSerializer ser;
- auto buf = ser.header("elliott");
+TEST(RpcSerdeTest, TestHeaderDecode) {
+ RpcSerde ser;
+ auto buf = ser.Header("elliott");
auto header_buf = buf->next();
ConnectionHeader h;
diff --git a/core/table-name.h b/serde/region-info-deserializer-test.cc
similarity index 53%
copy from core/table-name.h
copy to serde/region-info-deserializer-test.cc
index 1612667..ce8dedf 100644
--- a/core/table-name.h
+++ b/serde/region-info-deserializer-test.cc
@@ -16,35 +16,39 @@
* limitations under the License.
*
*/
-#pragma once
-#include <memory>
+#include "serde/region-info.h"
+
+#include <gtest/gtest.h>
+
#include <string>
#include "if/HBase.pb.h"
-#include <folly/Conv.h>
-
-namespace hbase {
-namespace pb {
-
-// Provide folly::to<std::string>(TableName);
-template <class String> void toAppend(const TableName &in, String *result) {
- if (!in.has_namespace_() || in.namespace_() == "default") {
- folly::toAppend(in.qualifier(), result);
- } else {
- folly::toAppend(in.namespace_(), ':', in.qualifier(), result);
- }
-}
+#include "serde/table-name.h"
+
+using std::string;
+using hbase::pb::RegionInfo;
+using hbase::pb::TableName;
+
+TEST(TestRegionInfoDesializer, TestDeserialize) {
+ string ns{"test_ns"};
+ string tn{"table_name"};
+ string start_row{"AAAAAA"};
+ string stop_row{"BBBBBBBBBBBB"};
+ uint64_t region_id = 2345678;
-} // namespace pb
-
-class TableNameUtil {
-public:
- static ::hbase::pb::TableName create(std::string table_name) {
- ::hbase::pb::TableName tn;
- tn.set_namespace_("default");
- tn.set_qualifier(table_name);
- return tn;
- }
-};
-} // namespace hbase
+ RegionInfo ri_out;
+ ri_out.set_region_id(region_id);
+ ri_out.mutable_table_name()->set_namespace_(ns);
+ ri_out.mutable_table_name()->set_qualifier(tn);
+ ri_out.set_start_key(start_row);
+ ri_out.set_end_key(stop_row);
+
+
+ string header{"PBUF"};
+ string ser = header + ri_out.SerializeAsString();
+
+ auto out = folly::to<RegionInfo>(ser);
+
+ EXPECT_EQ(region_id, out.region_id());
+}
diff --git a/core/table-name.h b/serde/region-info.h
similarity index 63%
copy from core/table-name.h
copy to serde/region-info.h
index 1612667..6af351c 100644
--- a/core/table-name.h
+++ b/serde/region-info.h
@@ -16,35 +16,26 @@
* limitations under the License.
*
*/
-#pragma once
-#include <memory>
-#include <string>
+#pragma once
#include "if/HBase.pb.h"
+
#include <folly/Conv.h>
+#include <boost/algorithm/string/predicate.hpp>
namespace hbase {
namespace pb {
+template <class String> void parseTo(String in, RegionInfo& out) {
+ // TODO(eclark): there has to be something better.
+ std::string s = folly::to<std::string>(in);
-// Provide folly::to<std::string>(TableName);
-template <class String> void toAppend(const TableName &in, String *result) {
- if (!in.has_namespace_() || in.namespace_() == "default") {
- folly::toAppend(in.qualifier(), result);
- } else {
- folly::toAppend(in.namespace_(), ':', in.qualifier(), result);
+ if (!boost::starts_with(s, "PBUF") ) {
+ throw std::runtime_error("Region Info field doesn't contain preamble");
+ }
+ if (!out.ParseFromArray(s.data() + 4, s.size() - 4)) {
+ throw std::runtime_error("Bad protobuf for RegionInfo");
}
}
-
} // namespace pb
-
-class TableNameUtil {
-public:
- static ::hbase::pb::TableName create(std::string table_name) {
- ::hbase::pb::TableName tn;
- tn.set_namespace_("default");
- tn.set_qualifier(table_name);
- return tn;
- }
-};
} // namespace hbase
diff --git a/serde/client-serializer.cc b/serde/rpc.cc
similarity index 67%
rename from serde/client-serializer.cc
rename to serde/rpc.cc
index 09b81c8..4c3c999 100644
--- a/serde/client-serializer.cc
+++ b/serde/rpc.cc
@@ -16,12 +16,15 @@
* limitations under the License.
*
*/
-#include "serde/client-serializer.h"
+#include "serde/rpc.h"
+
+#include <folly/Logging.h>
#include <folly/Logging.h>
#include <folly/io/Cursor.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
+#include <google/protobuf/message.h>
#include "if/HBase.pb.h"
#include "if/RPC.pb.h"
@@ -31,7 +34,10 @@ using namespace hbase;
using folly::IOBuf;
using folly::io::RWPrivateCursor;
using google::protobuf::Message;
+using google::protobuf::Message;
+using google::protobuf::io::ArrayInputStream;
using google::protobuf::io::ArrayOutputStream;
+using google::protobuf::io::CodedInputStream;
using google::protobuf::io::CodedOutputStream;
using google::protobuf::io::ZeroCopyOutputStream;
using std::string;
@@ -42,9 +48,46 @@ static const std::string INTERFACE = "ClientService";
static const uint8_t RPC_VERSION = 0;
static const uint8_t DEFAULT_AUTH_TYPE = 80;
-ClientSerializer::ClientSerializer() : auth_type_(DEFAULT_AUTH_TYPE) {}
+int RpcSerde::ParseDelimited(const IOBuf *buf, Message *msg) {
+ if (buf == nullptr || msg == nullptr) {
+ return -2;
+ }
+
+ DCHECK(!buf->isChained());
+
+ ArrayInputStream ais{buf->data(), static_cast<int>(buf->length())};
+ CodedInputStream coded_stream{&ais};
+
+ uint32_t msg_size;
+
+ // Try and read the varint.
+ if (coded_stream.ReadVarint32(&msg_size) == false) {
+ FB_LOG_EVERY_MS(ERROR, 1000) << "Unable to read a var uint32_t";
+ return -3;
+ }
+
+ coded_stream.PushLimit(msg_size);
+ // Parse the message.
+ if (msg->MergeFromCodedStream(&coded_stream) == false) {
+ FB_LOG_EVERY_MS(ERROR, 1000)
+ << "Unable to read a protobuf message from data.";
+ return -4;
+ }
+
+ // Make sure all the data was consumed.
+ if (coded_stream.ConsumedEntireMessage() == false) {
+ FB_LOG_EVERY_MS(ERROR, 1000)
+ << "Orphaned data left after reading protobuf message";
+ return -5;
+ }
+
+ return coded_stream.CurrentPosition();
+}
+
+RpcSerde::RpcSerde() : auth_type_(DEFAULT_AUTH_TYPE) {}
+RpcSerde::~RpcSerde() {}
-unique_ptr<IOBuf> ClientSerializer::preamble() {
+unique_ptr<IOBuf> RpcSerde::Preamble() {
auto magic = IOBuf::copyBuffer(PREAMBLE, 0, 2);
magic->append(2);
RWPrivateCursor c(magic.get());
@@ -56,7 +99,7 @@ unique_ptr<IOBuf> ClientSerializer::preamble() {
return magic;
}
-unique_ptr<IOBuf> ClientSerializer::header(const string &user) {
+unique_ptr<IOBuf> RpcSerde::Header(const string &user) {
pb::ConnectionHeader h;
// TODO(eclark): Make this not a total lie.
@@ -68,26 +111,25 @@ unique_ptr<IOBuf> ClientSerializer::header(const string &user) {
// It worked for a while with the java client; until it
// didn't.
h.set_service_name(INTERFACE);
- return prepend_length(serialize_message(h));
+ return PrependLength(SerializeMessage(h));
}
-unique_ptr<IOBuf> ClientSerializer::request(const uint32_t call_id,
- const string &method,
- const Message *msg) {
+unique_ptr<IOBuf> RpcSerde::Request(const uint32_t call_id,
+ const string &method, const Message *msg) {
pb::RequestHeader rq;
rq.set_method_name(method);
rq.set_call_id(call_id);
rq.set_request_param(msg != nullptr);
- auto ser_header = serialize_delimited(rq);
+ auto ser_header = SerializeDelimited(rq);
if (msg != nullptr) {
- auto ser_req = serialize_delimited(*msg);
+ auto ser_req = SerializeDelimited(*msg);
ser_header->appendChain(std::move(ser_req));
}
- return prepend_length(std::move(ser_header));
+ return PrependLength(std::move(ser_header));
}
-unique_ptr<IOBuf> ClientSerializer::prepend_length(unique_ptr<IOBuf> msg) {
+unique_ptr<IOBuf> RpcSerde::PrependLength(unique_ptr<IOBuf> msg) {
// Java ints are 4 long. So create a buffer that large
auto len_buf = IOBuf::create(4);
// Then make those bytes visible.
@@ -105,7 +147,7 @@ unique_ptr<IOBuf> ClientSerializer::prepend_length(unique_ptr<IOBuf> msg) {
return len_buf;
}
-unique_ptr<IOBuf> ClientSerializer::serialize_delimited(const Message &msg) {
+unique_ptr<IOBuf> RpcSerde::SerializeDelimited(const Message &msg) {
// Get the buffer size needed for just the message.
int msg_size = msg.ByteSize();
int buf_size = CodedOutputStream::VarintSize32(msg_size) + msg_size;
@@ -133,7 +175,7 @@ unique_ptr<IOBuf> ClientSerializer::serialize_delimited(const Message &msg) {
return buf;
}
// TODO(eclark): Make this 1 copy.
-unique_ptr<IOBuf> ClientSerializer::serialize_message(const Message &msg) {
+unique_ptr<IOBuf> RpcSerde::SerializeMessage(const Message &msg) {
auto buf = IOBuf::copyBuffer(msg.SerializeAsString());
return buf;
}
diff --git a/serde/client-serializer.h b/serde/rpc.h
similarity index 68%
rename from serde/client-serializer.h
rename to serde/rpc.h
index 9c819fe..cefb583 100644
--- a/serde/client-serializer.h
+++ b/serde/rpc.h
@@ -18,38 +18,41 @@
*/
#pragma once
-#include <cstdint>
-#include <folly/io/IOBuf.h>
+#include <memory>
#include <string>
// Forward
+namespace folly {
+class IOBuf;
+}
namespace google {
namespace protobuf {
class Message;
}
}
-namespace hbase {
-class Request;
-}
namespace hbase {
-class ClientSerializer {
+class RpcSerde {
public:
- ClientSerializer();
- std::unique_ptr<folly::IOBuf> preamble();
- std::unique_ptr<folly::IOBuf> header(const std::string &user);
- std::unique_ptr<folly::IOBuf> request(const uint32_t call_id,
+ RpcSerde();
+ virtual ~RpcSerde();
+ int ParseDelimited(const folly::IOBuf *buf, google::protobuf::Message *msg);
+ std::unique_ptr<folly::IOBuf> Preamble();
+ std::unique_ptr<folly::IOBuf> Header(const std::string &user);
+ std::unique_ptr<folly::IOBuf> Request(const uint32_t call_id,
const std::string &method,
const google::protobuf::Message *msg);
std::unique_ptr<folly::IOBuf>
- serialize_delimited(const google::protobuf::Message &msg);
+ SerializeDelimited(const google::protobuf::Message &msg);
std::unique_ptr<folly::IOBuf>
- serialize_message(const google::protobuf::Message &msg);
+ SerializeMessage(const google::protobuf::Message &msg);
std::unique_ptr<folly::IOBuf>
- prepend_length(std::unique_ptr<folly::IOBuf> msg);
+ PrependLength(std::unique_ptr<folly::IOBuf> msg);
+private:
+ /* data */
uint8_t auth_type_;
};
-} // namespace hbase
+}
diff --git a/serde/client-deserializer.h b/serde/server-name-test.cc
similarity index 75%
rename from serde/client-deserializer.h
rename to serde/server-name-test.cc
index b9664b0..35dcbc1 100644
--- a/serde/client-deserializer.h
+++ b/serde/server-name-test.cc
@@ -16,21 +16,17 @@
* limitations under the License.
*
*/
-#pragma once
-#include <folly/io/IOBuf.h>
+#include "serde/server-name.h"
-// Forward
-namespace google {
-namespace protobuf {
-class Message;
-}
-}
+#include <gtest/gtest.h>
+#include <string>
-namespace hbase {
-class ClientDeserializer {
-public:
- int parse_delimited(const folly::IOBuf *buf, google::protobuf::Message *msg);
-};
+using hbase::pb::ServerName;
-} // namespace hbase
+TEST(TestServerName, TestMakeServerName) {
+ auto sn = folly::to<ServerName>("test:123");
+
+ ASSERT_EQ("test", sn.host_name());
+ ASSERT_EQ(123, sn.port());
+}
diff --git a/serde/server-name.h b/serde/server-name.h
new file mode 100644
index 0000000..bdba087
--- /dev/null
+++ b/serde/server-name.h
@@ -0,0 +1,21 @@
+#pragma once
+
+#include "if/HBase.pb.h"
+#include <folly/Conv.h>
+#include <folly/String.h>
+
+namespace hbase {
+namespace pb {
+
+template <class String> void parseTo(String in, ServerName &out) {
+ // TODO see about getting rsplit into folly.
+ std::string s = folly::to<std::string>(in);
+
+ auto delim = s.rfind(":");
+ DCHECK(delim != std::string::npos);
+ out.set_host_name(s.substr(0, delim));
+ // Now keep everything after the : (delim + 1) to the end.
+ out.set_port(folly::to<int>(s.substr(delim + 1)));
+}
+}
+}
diff --git a/core/table-name-test.cc b/serde/table-name-test.cc
similarity index 98%
rename from core/table-name-test.cc
rename to serde/table-name-test.cc
index 7bad3f1..877d522 100644
--- a/core/table-name-test.cc
+++ b/serde/table-name-test.cc
@@ -22,7 +22,7 @@
#include <string>
-#include "core/table-name.h"
+#include "serde/table-name.h"
using namespace hbase;
using hbase::pb::TableName;
diff --git a/core/table-name.h b/serde/table-name.h
similarity index 80%
rename from core/table-name.h
rename to serde/table-name.h
index 1612667..c81e166 100644
--- a/core/table-name.h
+++ b/serde/table-name.h
@@ -23,6 +23,7 @@
#include "if/HBase.pb.h"
#include <folly/Conv.h>
+#include <folly/String.h>
namespace hbase {
namespace pb {
@@ -36,15 +37,18 @@ template <class String> void toAppend(const TableName &in, String *result) {
}
}
-} // namespace pb
+template <class String> void parseTo(String in, TableName &out) {
+ std::vector<std::string> v;
+ folly::split(":", in, v);
-class TableNameUtil {
-public:
- static ::hbase::pb::TableName create(std::string table_name) {
- ::hbase::pb::TableName tn;
- tn.set_namespace_("default");
- tn.set_qualifier(table_name);
- return tn;
+ if (v.size() == 1) {
+ out.set_namespace_("default");
+ out.set_qualifier(v[0]);
+ } else {
+ out.set_namespace_(v[0]);
+ out.set_qualifier(v[1]);
}
-};
+}
+
+} // namespace pb
} // namespace hbase
diff --git a/serde/zk-deserializer-test.cc b/serde/zk-deserializer-test.cc
index 92d85a0..f07eecf 100644
--- a/serde/zk-deserializer-test.cc
+++ b/serde/zk-deserializer-test.cc
@@ -17,7 +17,7 @@
*
*/
-#include "serde/zk-deserializer.h"
+#include "serde/zk.h"
#include <folly/Logging.h>
#include <folly/io/Cursor.h>
@@ -41,7 +41,7 @@ TEST(TestZkDesializer, TestThrowNoMagicNum) {
buf->append(100);
RWPrivateCursor c{buf.get()};
c.write<uint8_t>(99);
- ASSERT_THROW(deser.parse(buf.get(), &mrs), runtime_error);
+ ASSERT_THROW(deser.Parse(buf.get(), &mrs), runtime_error);
}
// Test if the protobuf is in a format that we can't decode
@@ -78,7 +78,7 @@ TEST(TestZkDesializer, TestBadProtoThrow) {
// Create the protobuf
MetaRegionServer out;
- ASSERT_THROW(deser.parse(buf.get(), &out), runtime_error);
+ ASSERT_THROW(deser.Parse(buf.get(), &out), runtime_error);
}
// Test to make sure the whole thing works.
@@ -118,6 +118,6 @@ TEST(TestZkDesializer, TestNoThrow) {
// Create the protobuf
MetaRegionServer out;
- ASSERT_TRUE(deser.parse(buf.get(), &out));
+ ASSERT_TRUE(deser.Parse(buf.get(), &out));
ASSERT_EQ(mrs.server().host_name(), out.server().host_name());
}
diff --git a/serde/zk-deserializer.cc b/serde/zk.cc
similarity index 96%
rename from serde/zk-deserializer.cc
rename to serde/zk.cc
index 33cf809..59871a5 100644
--- a/serde/zk-deserializer.cc
+++ b/serde/zk.cc
@@ -17,7 +17,7 @@
*
*/
-#include "serde/zk-deserializer.h"
+#include "serde/zk.h"
#include <folly/io/Cursor.h>
#include <folly/io/IOBuf.h>
@@ -31,7 +31,7 @@ using google::protobuf::Message;
static const std::string MAGIC_STRING = "PBUF";
-bool ZkDeserializer::parse(IOBuf *buf, Message *out) {
+bool ZkDeserializer::Parse(IOBuf *buf, Message *out) {
// The format is like this
// 1 byte of magic number. 255
diff --git a/serde/zk-deserializer.h b/serde/zk.h
similarity index 93%
rename from serde/zk-deserializer.h
rename to serde/zk.h
index aa91661..b672bf4 100644
--- a/serde/zk-deserializer.h
+++ b/serde/zk.h
@@ -30,6 +30,6 @@ class IOBuf;
namespace hbase {
class ZkDeserializer {
public:
- bool parse(folly::IOBuf *buf, google::protobuf::Message *out);
+ bool Parse(folly::IOBuf *buf, google::protobuf::Message *out);
};
} // namespace hbase
[hbase] 52/137: HBASE-15905 Makefile build env incorrectly links in
tests (Priyadharshini karthikeyan and Sudeep Sunthankar)
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 5c967300e6cc3218003c38eee5d427cb2d6c8028
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Thu Dec 29 17:49:20 2016 -0800
HBASE-15905 Makefile build env incorrectly links in tests (Priyadharshini karthikeyan and Sudeep Sunthankar)
---
Makefile | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/Makefile b/Makefile
index 96c275e..99e38ef 100644
--- a/Makefile
+++ b/Makefile
@@ -35,13 +35,16 @@ LDFLAGS = -lprotobuf -lzookeeper_mt -lsasl2 -lfolly -lwangle
LINKFLAG = -shared
#define list of source files and object files
-SRC = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc))
+ALLSRC = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc))
+EXCLUDE_SRC = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*-test.cc)) core/simple-client.cc
+SRC = $(filter-out $(EXCLUDE_SRC), $(ALLSRC))
PROTOSRC = $(patsubst %.proto, $(addprefix build/,%.pb.cc),$(wildcard if/*.proto))
DEPS = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.h))
PROTODEPS = $(patsubst %.proto, $(addprefix build/,%.pb.h),$(wildcard if/*.proto))
DEBUG_OBJ = $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(SRC))
DEBUG_OBJ += $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(PROTOSRC))
RELEASE_OBJ = $(patsubst %.cc,$(RELEASE_PATH)/%.o,$(SRC))
+RELEASE_OBJ += $(patsubst %.cc,$(RELEASE_PATH)/%.o,$(PROTOSRC))
INCLUDES = $(addprefix -I,$(INCLUDE_DIR))
LIB_DIR = /usr/local
@@ -54,6 +57,7 @@ ARC_DEBUG=$(DEBUG_PATH)/libHbaseClient_d.a
vpath %.cc $(SRC_DIR)
+build: checkdirs protos $(LIB_DEBUG) $(LIB_RELEASE) $(ARC_DEBUG) $(ARC_RELEASE)
$(LIB_DEBUG): $(DEBUG_BUILD_DIR)
define make-goal-dbg
$1/%.o: %.cc $(DEPS) $(PROTODEPS) $(PROTOSRC)
@@ -68,7 +72,7 @@ endef
.PHONY: all clean install
-build: checkdirs protos $(LIB_DEBUG) $(LIB_RELEASE) $(ARC_DEBUG) $(ARC_RELEASE)
+
checkdirs: $(DEBUG_BUILD_DIR) $(RELEASE_BUILD_DIR) $(PROTO_SRC_DIR)
@@ -130,6 +134,6 @@ help:
@echo " install : will copy the libs to $(LIB_LIBDIR). super user priviliege would be required."
@echo " check : will test everything."
@echo " protos : will build the corresponding sources for protobufs present in if/ directory."
+ @echo " lint : will ensure that code conforms to Google coding style."
all: build doc check
-
[hbase] 71/137: HBASE-17741 [C++] rename some files to use (dash)
instead of _
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit e71fff0ea157b937cd7add4c70aab761e8b9eb19
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Mon Mar 6 19:27:20 2017 -0800
HBASE-17741 [C++] rename some files to use (dash) instead of _
---
core/BUCK | 30 +++++++++++-----------
core/async-rpc-retrying-test.cc | 4 +--
core/client-test.cc | 2 +-
core/client.h | 2 +-
core/get.h | 2 +-
...ion_loader.cc => hbase-configuration-loader.cc} | 2 +-
...ation_loader.h => hbase-configuration-loader.h} | 0
...uration-test.cc => hbase-configuration-test.cc} | 2 +-
core/{hbase_macros.h => hbase-macros.h} | 0
core/meta-utils.cc | 2 +-
...converter-test.cc => request-converter-test.cc} | 2 +-
.../{request_converter.cc => request-converter.cc} | 2 +-
core/{request_converter.h => request-converter.h} | 0
...response_converter.cc => response-converter.cc} | 2 +-
.../{response_converter.h => response-converter.h} | 0
core/scan.h | 2 +-
core/table.cc | 4 +--
core/{time_range-test.cc => time-range-test.cc} | 2 +-
core/{time_range.cc => time-range.cc} | 2 +-
core/{time_range.h => time-range.h} | 0
test-util/mini-cluster.cc | 10 ++------
test-util/mini-cluster.h | 10 ++------
22 files changed, 35 insertions(+), 47 deletions(-)
diff --git a/core/BUCK b/core/BUCK
index 95fb8b5..e40236f 100644
--- a/core/BUCK
+++ b/core/BUCK
@@ -21,7 +21,7 @@ cxx_library(
exported_headers=[
"client.h",
"cell.h",
- "hbase_macros.h",
+ "hbase-macros.h",
"filter.h",
"query.h",
"keyvalue-codec.h",
@@ -32,13 +32,13 @@ cxx_library(
# Once meta lookup works
"meta-utils.h",
"get.h",
- "time_range.h",
+ "time-range.h",
"configuration.h",
- "hbase_configuration_loader.h",
+ "hbase-configuration-loader.h",
"scan.h",
"result.h",
- "request_converter.h",
- "response_converter.h",
+ "request-converter.h",
+ "response-converter.h",
"table.h",
"async-rpc-retrying-caller-factory.h",
"async-rpc-retrying-caller.h",
@@ -51,13 +51,13 @@ cxx_library(
"location-cache.cc",
"meta-utils.cc",
"get.cc",
- "time_range.cc",
+ "time-range.cc",
"configuration.cc",
- "hbase_configuration_loader.cc",
+ "hbase-configuration-loader.cc",
"scan.cc",
"result.cc",
- "request_converter.cc",
- "response_converter.cc",
+ "request-converter.cc",
+ "response-converter.cc",
"table.cc",
],
deps=[
@@ -120,8 +120,8 @@ cxx_test(
],
run_test_separately=True,)
cxx_test(
- name="time_range-test",
- srcs=["time_range-test.cc",],
+ name="time-range-test",
+ srcs=["time-range-test.cc",],
deps=[":core",],
run_test_separately=True,)
cxx_test(
@@ -130,8 +130,8 @@ cxx_test(
deps=[":core",],
run_test_separately=True,)
cxx_test(
- name="hbase_configuration-test",
- srcs=["hbase_configuration-test.cc",],
+ name="hbase-configuration-test",
+ srcs=["hbase-configuration-test.cc",],
deps=[":core",],
run_test_separately=True,)
cxx_test(
@@ -145,8 +145,8 @@ cxx_test(
deps=[":core",],
run_test_separately=True,)
cxx_test(
- name="request_converter-test",
- srcs=["request_converter-test.cc",],
+ name="request-converter-test",
+ srcs=["request-converter-test.cc",],
deps=[
":core",
"//connection:connection",
diff --git a/core/async-rpc-retrying-test.cc b/core/async-rpc-retrying-test.cc
index d2435b9..f2a79a1 100644
--- a/core/async-rpc-retrying-test.cc
+++ b/core/async-rpc-retrying-test.cc
@@ -36,8 +36,8 @@
#include "core/hbase-rpc-controller.h"
#include "core/keyvalue-codec.h"
#include "core/region-location.h"
-#include "core/request_converter.h"
-#include "core/response_converter.h"
+#include "core/request-converter.h"
+#include "core/response-converter.h"
#include "core/result.h"
#include "exceptions/exception.h"
#include "if/Client.pb.h"
diff --git a/core/client-test.cc b/core/client-test.cc
index afca847..6f74e9c 100644
--- a/core/client-test.cc
+++ b/core/client-test.cc
@@ -21,7 +21,7 @@
#include <gtest/gtest.h>
#include "core/configuration.h"
#include "core/get.h"
-#include "core/hbase_configuration_loader.h"
+#include "core/hbase-configuration-loader.h"
#include "core/result.h"
#include "core/table.h"
#include "serde/table-name.h"
diff --git a/core/client.h b/core/client.h
index e73ab70..6e3c889 100644
--- a/core/client.h
+++ b/core/client.h
@@ -30,7 +30,7 @@
#include "connection/rpc-client.h"
#include "core/configuration.h"
#include "core/connection-configuration.h"
-#include "core/hbase_configuration_loader.h"
+#include "core/hbase-configuration-loader.h"
#include "core/keyvalue-codec.h"
#include "core/location-cache.h"
#include "core/table.h"
diff --git a/core/get.h b/core/get.h
index 92d1fee..5492f21 100644
--- a/core/get.h
+++ b/core/get.h
@@ -25,7 +25,7 @@
#include <string>
#include <vector>
#include "core/query.h"
-#include "core/time_range.h"
+#include "core/time-range.h"
#include "if/Client.pb.h"
namespace hbase {
diff --git a/core/hbase_configuration_loader.cc b/core/hbase-configuration-loader.cc
similarity index 99%
rename from core/hbase_configuration_loader.cc
rename to core/hbase-configuration-loader.cc
index 38a617b..4d6a3e9 100644
--- a/core/hbase_configuration_loader.cc
+++ b/core/hbase-configuration-loader.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/hbase_configuration_loader.h"
+#include "core/hbase-configuration-loader.h"
#include <glog/logging.h>
#include <boost/foreach.hpp>
diff --git a/core/hbase_configuration_loader.h b/core/hbase-configuration-loader.h
similarity index 100%
rename from core/hbase_configuration_loader.h
rename to core/hbase-configuration-loader.h
diff --git a/core/hbase_configuration-test.cc b/core/hbase-configuration-test.cc
similarity index 99%
rename from core/hbase_configuration-test.cc
rename to core/hbase-configuration-test.cc
index 07a6358..ebf0817 100644
--- a/core/hbase_configuration-test.cc
+++ b/core/hbase-configuration-test.cc
@@ -24,7 +24,7 @@
#include <gtest/gtest.h>
#include <boost/filesystem.hpp>
#include "core/configuration.h"
-#include "core/hbase_configuration_loader.h"
+#include "core/hbase-configuration-loader.h"
using namespace hbase;
diff --git a/core/hbase_macros.h b/core/hbase-macros.h
similarity index 100%
rename from core/hbase_macros.h
rename to core/hbase-macros.h
diff --git a/core/meta-utils.cc b/core/meta-utils.cc
index bd26607..24a0360 100644
--- a/core/meta-utils.cc
+++ b/core/meta-utils.cc
@@ -24,7 +24,7 @@
#include "connection/request.h"
#include "connection/response.h"
-#include "core/response_converter.h"
+#include "core/response-converter.h"
#include "if/Client.pb.h"
#include "serde/region-info.h"
#include "serde/server-name.h"
diff --git a/core/request_converter-test.cc b/core/request-converter-test.cc
similarity index 99%
rename from core/request_converter-test.cc
rename to core/request-converter-test.cc
index 84d5e8b..c01f16c 100644
--- a/core/request_converter-test.cc
+++ b/core/request-converter-test.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/request_converter.h"
+#include "core/request-converter.h"
#include <gtest/gtest.h>
#include <limits>
diff --git a/core/request_converter.cc b/core/request-converter.cc
similarity index 99%
rename from core/request_converter.cc
rename to core/request-converter.cc
index 149202e..227e04a 100644
--- a/core/request_converter.cc
+++ b/core/request-converter.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/request_converter.h"
+#include "core/request-converter.h"
#include <utility>
#include "if/Client.pb.h"
diff --git a/core/request_converter.h b/core/request-converter.h
similarity index 100%
rename from core/request_converter.h
rename to core/request-converter.h
diff --git a/core/response_converter.cc b/core/response-converter.cc
similarity index 99%
rename from core/response_converter.cc
rename to core/response-converter.cc
index 2497306..7bb5e5d 100644
--- a/core/response_converter.cc
+++ b/core/response-converter.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/response_converter.h"
+#include "core/response-converter.h"
#include <string>
#include <vector>
diff --git a/core/response_converter.h b/core/response-converter.h
similarity index 100%
rename from core/response_converter.h
rename to core/response-converter.h
diff --git a/core/scan.h b/core/scan.h
index 7e8c7bd..cedbe0a 100644
--- a/core/scan.h
+++ b/core/scan.h
@@ -25,7 +25,7 @@
#include <string>
#include <vector>
#include "core/get.h"
-#include "core/time_range.h"
+#include "core/time-range.h"
#include "if/Client.pb.h"
namespace hbase {
diff --git a/core/table.cc b/core/table.cc
index ba4dc29..2ce8fcd 100644
--- a/core/table.cc
+++ b/core/table.cc
@@ -25,8 +25,8 @@
#include <utility>
#include <vector>
-#include "core/request_converter.h"
-#include "core/response_converter.h"
+#include "core/request-converter.h"
+#include "core/response-converter.h"
#include "if/Client.pb.h"
#include "security/user.h"
#include "serde/server-name.h"
diff --git a/core/time_range-test.cc b/core/time-range-test.cc
similarity index 98%
rename from core/time_range-test.cc
rename to core/time-range-test.cc
index 49cb359..136ab54 100644
--- a/core/time_range-test.cc
+++ b/core/time-range-test.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/time_range.h"
+#include "core/time-range.h"
#include <glog/logging.h>
#include <gtest/gtest.h>
diff --git a/core/time_range.cc b/core/time-range.cc
similarity index 98%
rename from core/time_range.cc
rename to core/time-range.cc
index 04b70ec..e6b789b 100644
--- a/core/time_range.cc
+++ b/core/time-range.cc
@@ -17,7 +17,7 @@
*
*/
-#include "core/time_range.h"
+#include "core/time-range.h"
#include <limits>
#include <stdexcept>
#include <string>
diff --git a/core/time_range.h b/core/time-range.h
similarity index 100%
rename from core/time_range.h
rename to core/time-range.h
diff --git a/test-util/mini-cluster.cc b/test-util/mini-cluster.cc
index b40b689..88a091c 100644
--- a/test-util/mini-cluster.cc
+++ b/test-util/mini-cluster.cc
@@ -19,17 +19,11 @@
#include "test-util/mini-cluster.h"
#include <glog/logging.h>
-#include <gtest/gtest.h>
+#include <fcntl.h>
#include <boost/filesystem/operations.hpp>
#include <boost/filesystem/fstream.hpp>
#include <fstream>
-#include "core/client.h"
-#include "core/configuration.h"
-#include "core/get.h"
-#include "core/hbase_configuration_loader.h"
-#include "core/result.h"
-#include "core/table.h"
-#include "serde/table-name.h"
+
using hbase::MiniCluster;
diff --git a/test-util/mini-cluster.h b/test-util/mini-cluster.h
index 9b8ca92..54842e1 100644
--- a/test-util/mini-cluster.h
+++ b/test-util/mini-cluster.h
@@ -18,17 +18,11 @@
*/
#pragma once
-#include <gtest/gtest.h>
#include <string>
-#include "core/client.h"
-#include "core/configuration.h"
-#include "core/get.h"
-#include "core/hbase_configuration_loader.h"
-#include "core/result.h"
-#include "core/table.h"
#include "jni.h"
-#include "serde/table-name.h"
+
namespace hbase {
+
class MiniCluster {
public:
jobject StartCluster(int numRegionServers, std::string conf_path);
[hbase] 36/137: HBASE-15851 Makefile update for build env
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 54beb115bb0d2f0df6272ab13995e9cf0948e376
Author: sudeeps <su...@sudeepsunthankar.net>
AuthorDate: Sun May 22 09:38:47 2016 +1000
HBASE-15851 Makefile update for build env
1) Makefile to compile protobuf sources which are extracted in build
2) Added -O2 and -D_GLIBCXX_USE_CXX11_ABI=0 compilation flags
3) Header files added in Makefile
---
Makefile | 132 +++++++++++++++++++++++++++++++++++++++++++-----------
Makefile.protos | 53 ++++++++++++++++++++++
core/meta-utils.h | 2 +-
3 files changed, 159 insertions(+), 28 deletions(-)
diff --git a/Makefile b/Makefile
index 826233f..7e68b6a 100644
--- a/Makefile
+++ b/Makefile
@@ -1,37 +1,115 @@
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-build:
- $(shell buck build core/... )
+#use "gcc" to compile source files
+CC:=g++
+LD:=g++
+
+DEBUG_PATH = build/debug
+RELEASE_PATH = build/release
+PROTO_SRC_DIR = build/if
+MODULES = connection core serde test-util utils
+SRC_DIR = $(MODULES)
+DEBUG_BUILD_DIR = $(addprefix $(DEBUG_PATH)/,$(MODULES))
+RELEASE_BUILD_DIR = $(addprefix $(RELEASE_PATH)/,$(MODULES))
+INCLUDE_DIR = . build/
-check:
- $(shell buck test --all --no-results-cache )
+#flags to pass to the CPP compiler & linker
+CPPFLAGS_DEBUG = -D_GLIBCXX_USE_CXX11_ABI=0 -g -Wall -std=c++14 -pedantic -fPIC
+CPPFLAGS_RELEASE = -D_GLIBCXX_USE_CXX11_ABI=0 -DNDEBUG -O2 -Wall -std=c++14 -pedantic -fPIC
+LDFLAGS = -lprotobuf -lzookeeper_mt -lsasl2 -lfolly -lwangle
+LINKFLAG = -shared
-doc:
- $(shell doxygen hbase.doxygen > /dev/null )
+#define list of source files and object files
+SRC = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc))
+PROTOSRC = $(patsubst %.proto, $(addprefix build/,%.pb.cc),$(wildcard if/*.proto))
+DEPS = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.h))
+PROTODEPS = $(patsubst %.proto, $(addprefix build/,%.pb.h),$(wildcard if/*.proto))
+DEBUG_OBJ = $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(SRC))
+DEBUG_OBJ += $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(PROTOSRC))
+RELEASE_OBJ = $(patsubst %.cc,$(RELEASE_PATH)/%.o,$(SRC))
+INCLUDES = $(addprefix -I,$(INCLUDE_DIR))
+
+LIB_DIR = /usr/local
+LIB_LIBDIR = $(LIB_DIR)/lib
+LIB_INCDIR = $(LIB_DIR)/include
+LIB_RELEASE=$(RELEASE_PATH)/libHbaseClient.so
+ARC_RELEASE=$(RELEASE_PATH)/libHbaseClient.a
+LIB_DEBUG=$(DEBUG_PATH)/libHbaseClient_d.so
+ARC_DEBUG=$(DEBUG_PATH)/libHbaseClient_d.a
+
+vpath %.cc $(SRC_DIR)
+
+$(LIB_DEBUG): $(DEBUG_BUILD_DIR)
+define make-goal-dbg
+$1/%.o: %.cc $(DEPS) $(PROTODEPS) $(PROTOSRC)
+ $(CC) -c $$< -o $$@ $(CPPFLAGS_DEBUG) $(INCLUDES)
+endef
+
+$(LIB_RELEASE): $(RELEASE_BUILD_DIR)
+define make-goal-rel
+$1/%.o: %.cc $(DEPS) $(PROTODEPS) $(PROTOSRC)
+ $(CC) -c $$< -o $$@ $(CPPFLAGS_RELEASE) $(INCLUDES)
+endef
+
+.PHONY: all clean install
+
+build: checkdirs protos $(LIB_DEBUG) $(LIB_RELEASE) $(ARC_DEBUG) $(ARC_RELEASE)
+
+checkdirs: $(DEBUG_BUILD_DIR) $(RELEASE_BUILD_DIR) $(PROTO_SRC_DIR)
+
+protos: createprotosrc
+ @make all -f Makefile.protos
+
+createprotosrc: $(PROTO_SRC_DIR)
+ @protoc --proto_path=if --cpp_out=$(PROTO_SRC_DIR) if/*.proto
+
+install:
+ cp $(LIB_RELEASE) $(LIB_LIBDIR)/libHbaseClient.so
+ cp $(ARC_RELEASE) $(LIB_LIBDIR)/libHbaseClient.a
+ cp $(LIB_DEBUG) $(LIB_LIBDIR)/libHbaseClient_d.so
+ cp $(ARC_DEBUG) $(LIB_LIBDIR)/libHbaseClient_d.a
+
+$(PROTO_SRC_DIR):
+ @mkdir -p $@
+
+$(DEBUG_BUILD_DIR):
+ @mkdir -p $@
+$(RELEASE_BUILD_DIR):
+ @mkdir -p $@
+
+$(ARC_DEBUG): $(DEBUG_OBJ)
+ ar rcs $@ $^
+
+$(ARC_RELEASE): $(RELEASE_OBJ)
+ ar rcs $@ $^
+
+$(LIB_RELEASE): $(RELEASE_OBJ)
+ $(LD) $(LINKFLAG) -o $@ $(LDFLAGS) $(RELEASE_OBJ)
+
+$(LIB_DEBUG): $(DEBUG_OBJ)
+ $(LD) $(LINKFLAG) -o $@ $(LDFLAGS) $(DEBUG_OBJ)
+
+#to clean re-compilable files
clean:
- $(shell rm -rf docs )
+ @rm -rf $(DEBUG_BUILD_DIR) $(RELEASE_BUILD_DIR) $(LIB_RELEASE) $(LIB_DEBUG) $(ARC_RELEASE) $(ARC_DEBUG) docs buck-out build
+
+$(foreach bdir,$(DEBUG_BUILD_DIR), $(eval $(call make-goal-dbg,$(bdir))))
+
+$(foreach bdir,$(RELEASE_BUILD_DIR),$(eval $(call make-goal-rel,$(bdir))))
+
+check:
+ $(shell buck test --all --no-results-cache)
+
+doc:
+ $(shell doxygen hbase.doxygen > /dev/null)
help:
@echo "Available targets:"
@echo ""
- @echo " build : will build everything."
- @echo " clean : will remove the docs folder"
- @echo " check : will test everything."
+ @echo " build : will build everything."
+ @echo " clean : will remove the docs folder, object files and local libraries"
+ @echo " install : will copy the libs to $(LIB_LIBDIR). super user priviliege would be required."
+ @echo " check : will test everything."
+ @echo " protos : will build the corresponding sources for protobufs present in if/ directory."
all: build doc check
+
diff --git a/Makefile.protos b/Makefile.protos
new file mode 100644
index 0000000..4f1727a
--- /dev/null
+++ b/Makefile.protos
@@ -0,0 +1,53 @@
+#use "gcc" to compile source files
+CC:=g++
+LD:=g++
+
+DEBUG_PATH = build/debug
+RELEASE_PATH = build/release
+MODULES = build/if
+SRC_DIR = $(MODULES)
+DEBUG_BUILD_DIR = $(addprefix $(DEBUG_PATH)/,$(MODULES))
+RELEASE_BUILD_DIR = $(addprefix $(RELEASE_PATH)/,$(MODULES))
+INCLUDE_DIR = .
+
+#flags to pass to the CPP compiler & linker
+CPPFLAGS_DEBUG = -D_GLIBCXX_USE_CXX11_ABI=0 -g -Wall -std=c++14 -pedantic -fPIC
+CPPFLAGS_RELEASE = -D_GLIBCXX_USE_CXX11_ABI=0 -DNDEBUG -O2 -Wall -std=c++14 -pedantic -fPIC
+
+#define list of source files and object files
+SRC = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc))
+DEPS = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.h))
+DEBUG_OBJ = $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(SRC))
+RELEASE_OBJ = $(patsubst %.cc,$(RELEASE_PATH)/%.o,$(SRC))
+INCLUDES = $(addprefix -I,$(INCLUDE_DIR))
+
+vpath %.cc $(SRC_DIR)
+
+$(DEBUG_OBJ): $(DEBUG_BUILD_DIR)
+define make-goal-dbg
+$1/%.o: %.cc $(DEPS)
+ $(CC) -c $$< -o $$@ $(CPPFLAGS_DEBUG) $(INCLUDES)
+endef
+
+$(RELEASE_OBJ): $(RELEASE_BUILD_DIR)
+define make-goal-rel
+$1/%.o: %.cc $(DEPS)
+ $(CC) -c $$< -o $$@ $(CPPFLAGS_RELEASE) $(INCLUDES)
+endef
+
+$(DEBUG_BUILD_DIR):
+ @mkdir -p $@
+
+$(RELEASE_BUILD_DIR):
+ @mkdir -p $@
+
+.PHONY: all clean
+
+all: $(DEBUG_OBJ) $(RELEASE_OBJ)
+
+clean:
+ @rm -rf $(DEBUG_OBJ) $(RELEASE_OBJ)
+
+$(foreach bdir,$(DEBUG_BUILD_DIR), $(eval $(call make-goal-dbg,$(bdir))))
+
+$(foreach bdir,$(RELEASE_BUILD_DIR),$(eval $(call make-goal-rel,$(bdir))))
diff --git a/core/meta-utils.h b/core/meta-utils.h
index f6cc84f..294e17d 100644
--- a/core/meta-utils.h
+++ b/core/meta-utils.h
@@ -20,7 +20,7 @@
#include <string>
-#include "connection/Request.h"
+#include "connection/request.h"
#include "if/HBase.pb.h"
#include "serde/table-name.h"
[hbase] 112/137: HBASE-18459 [C++] Fix Segfault in location-cache
Posted by zg...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
zghao pushed a commit to branch hbase-native-client
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit f799780c47b0f426dd319418cf08bb2eabc856f4
Author: Enis Soztutar <en...@apache.org>
AuthorDate: Tue Aug 1 16:02:20 2017 -0700
HBASE-18459 [C++] Fix Segfault in location-cache
---
core/configuration.cc | 2 +-
core/location-cache.cc | 18 ++++++++++++------
core/location-cache.h | 4 ++--
3 files changed, 15 insertions(+), 9 deletions(-)
diff --git a/core/configuration.cc b/core/configuration.cc
index f4fc46d..1fd2851 100644
--- a/core/configuration.cc
+++ b/core/configuration.cc
@@ -24,8 +24,8 @@
#include <utility>
#include <glog/logging.h>
-#include <boost/lexical_cast.hpp>
#include <boost/format.hpp>
+#include <boost/lexical_cast.hpp>
namespace hbase {
diff --git a/core/location-cache.cc b/core/location-cache.cc
index 5f68420..ed5f5dc 100644
--- a/core/location-cache.cc
+++ b/core/location-cache.cc
@@ -84,17 +84,20 @@ folly::Future<ServerName> LocationCache::LocateMeta() {
}
return meta_promise_->getFuture().onError([&](const folly::exception_wrapper &ew) {
auto promise = InvalidateMeta();
- promise->setException(ew);
+ if (promise) {
+ promise->setException(ew);
+ }
+ throw ew;
return ServerName{};
});
}
-std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> LocationCache::InvalidateMeta() {
+std::shared_ptr<folly::SharedPromise<hbase::pb::ServerName>> LocationCache::InvalidateMeta() {
VLOG(2) << "Invalidating meta location";
std::lock_guard<std::recursive_mutex> g(meta_lock_);
if (meta_promise_ != nullptr) {
// return the unique_ptr back to the caller.
- std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> ret = nullptr;
+ std::shared_ptr<folly::SharedPromise<hbase::pb::ServerName>> ret = nullptr;
std::swap(ret, meta_promise_);
return ret;
} else {
@@ -102,10 +105,13 @@ std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> LocationCache::Inva
}
}
-/// MUST hold the meta_lock_
void LocationCache::RefreshMetaLocation() {
- meta_promise_ = std::make_unique<folly::SharedPromise<ServerName>>();
- cpu_executor_->add([&] { meta_promise_->setWith([&] { return this->ReadMetaLocation(); }); });
+ meta_promise_ = std::make_shared<folly::SharedPromise<ServerName>>();
+ auto p = meta_promise_;
+ cpu_executor_->add([this, p] {
+ std::lock_guard<std::recursive_mutex> g(meta_lock_);
+ p->setWith([&] { return this->ReadMetaLocation(); });
+ });
}
// Note: this is a blocking call to zookeeper
diff --git a/core/location-cache.h b/core/location-cache.h
index a374fb6..932bef7 100644
--- a/core/location-cache.h
+++ b/core/location-cache.h
@@ -137,7 +137,7 @@ class LocationCache : public AsyncRegionLocator {
/**
* Remove the cached location of meta.
*/
- std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> InvalidateMeta();
+ std::shared_ptr<folly::SharedPromise<hbase::pb::ServerName>> InvalidateMeta();
/**
* Return cached region location corresponding to this row,
@@ -201,7 +201,7 @@ class LocationCache : public AsyncRegionLocator {
std::shared_ptr<hbase::Configuration> conf_;
std::string zk_quorum_;
std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
- std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> meta_promise_;
+ std::shared_ptr<folly::SharedPromise<hbase::pb::ServerName>> meta_promise_;
std::recursive_mutex meta_lock_;
MetaUtil meta_util_;
std::shared_ptr<ConnectionPool> cp_;