You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ec...@apache.org on 2016/07/11 23:47:54 UTC
[01/50] [abbrv] hbase git commit: HBASE-16171 Fix the potential
problems in TestHCM.testConnectionCloseAllowsInterrupt (Colin Ma) [Forced
Update!]
Repository: hbase
Updated Branches:
refs/heads/HBASE-14850 114817253 -> 0eb150f9d (forced update)
HBASE-16171 Fix the potential problems in TestHCM.testConnectionCloseAllowsInterrupt (Colin Ma)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5e0d97e0
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5e0d97e0
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5e0d97e0
Branch: refs/heads/HBASE-14850
Commit: 5e0d97e0e95434b03ba886279180a031ed997dfd
Parents: a33097e
Author: stack <st...@apache.org>
Authored: Fri Jul 8 10:17:55 2016 -0700
Committer: stack <st...@apache.org>
Committed: Fri Jul 8 10:18:36 2016 -0700
----------------------------------------------------------------------
.../src/test/java/org/apache/hadoop/hbase/client/TestHCM.java | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/5e0d97e0/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
index 4723fa8..1b20b76 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestHCM.java
@@ -464,6 +464,10 @@ public class TestHCM {
c2.setInt(HConstants.HBASE_CLIENT_PAUSE, 1); // don't wait between retries.
c2.setInt(RpcClient.FAILED_SERVER_EXPIRY_KEY, 0); // Server do not really expire
c2.setBoolean(RpcClient.SPECIFIC_WRITE_THREAD, allowsInterrupt);
+ // to avoid the client to be stuck when do the Get
+ c2.setInt(HConstants.HBASE_CLIENT_META_OPERATION_TIMEOUT, 10000);
+ c2.setInt(HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, 10000);
+ c2.setInt(HConstants.HBASE_RPC_TIMEOUT_KEY, 5000);
Connection connection = ConnectionFactory.createConnection(c2);
final Table table = connection.getTable(tableName);
@@ -488,6 +492,9 @@ public class TestHCM {
done++;
if (done % 100 == 0)
LOG.info("done=" + done);
+ // without the sleep, will cause the exception for too many files in
+ // org.apache.hadoop.hdfs.server.datanode.DataXceiver
+ Thread.sleep(100);
}
} catch (Throwable t) {
failed.set(t);
[45/50] [abbrv] hbase git commit: HBASE-15945 Patch for Cell
Posted by ec...@apache.org.
HBASE-15945 Patch for Cell
This patch consists of Cell implementation without additional interfaces.
Signed-off-by: Elliott Clark <ec...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fa3ab420
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fa3ab420
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fa3ab420
Branch: refs/heads/HBASE-14850
Commit: fa3ab420d19bba853d96576176257b5c12f1a48f
Parents: 2b6341d
Author: Sudeep Sunthankar <su...@gmail.com>
Authored: Mon Jul 4 21:02:25 2016 +1000
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/core/cell-test.cc | 175 +++++++++++++++++++++++++++++
hbase-native-client/core/cell.cc | 77 +++++++++++++
hbase-native-client/core/cell.h | 60 ++++++++++
3 files changed, 312 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa3ab420/hbase-native-client/core/cell-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/cell-test.cc b/hbase-native-client/core/cell-test.cc
new file mode 100644
index 0000000..cbe50eb
--- /dev/null
+++ b/hbase-native-client/core/cell-test.cc
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/cell.h"
+
+#include <memory>
+#include <gtest/gtest.h>
+#include <glog/logging.h>
+
+using namespace hbase;
+TEST(CellTest, CellFailureTest) {
+ CellType cell_type = CellType::PUT;
+ std::string row = "row";
+ std::string family = "family";
+ std::string column = "column";
+ std::string value = "value";
+ long timestamp = std::numeric_limits<long>::max();
+ std::string tags = "";
+ std::unique_ptr<Cell> cell(
+ new Cell(row, family, column, timestamp, value, cell_type));
+ if (cell.get()) {
+ EXPECT_NE("row-value", cell.get()->Row());
+ EXPECT_NE("family-value", cell.get()->Family());
+ EXPECT_NE("column-value", cell.get()->Qualifier());
+ EXPECT_NE("value-value", cell.get()->Value());
+ EXPECT_NE(8975431260, cell.get()->Timestamp());
+ EXPECT_NE(CellType::MAXIMUM, cell.get()->Type());
+ }
+}
+
+TEST(CellTest, CellSuceessTest) {
+ std::string row = "row-value";
+ std::string family = "family-value";
+ std::string column = "column-value";
+ std::string value = "value-value";
+ long timestamp = std::numeric_limits<long>::max();
+ CellType cell_type = CellType::PUT;
+ const std::unique_ptr<Cell> cell(
+ new Cell(row, family, column, timestamp, value, cell_type));
+ if (cell.get()) {
+ EXPECT_EQ(row, cell.get()->Row());
+ EXPECT_EQ(family, cell.get()->Family());
+ EXPECT_EQ(column, cell.get()->Qualifier());
+ EXPECT_EQ(value, cell.get()->Value());
+ EXPECT_EQ(timestamp, cell.get()->Timestamp());
+ EXPECT_EQ(cell_type, cell.get()->Type());
+ }
+}
+
+TEST(CellTest, MultipleCellsTest) {
+ std::vector<const Cell *> cells;
+ for (int i = 0; i < 5; i++) {
+ std::string row = "row-value";
+ std::string family = "family-value";
+ std::string column = "column-value";
+ std::string value = "value-value";
+ long timestamp = std::numeric_limits<long>::max();
+ row += std::to_string(i);
+ value += std::to_string(i);
+ CellType cell_type = CellType::PUT;
+ const Cell *cell = new Cell(row, family, column, timestamp, value,
+ cell_type);
+ cells.push_back(cell);
+ }
+ int i = 0;
+ for (const auto cell : cells) {
+ std::string row = "row-value";
+ std::string value = "value-value";
+ row += std::to_string(i);
+ value += std::to_string(i);
+ EXPECT_EQ(row, cell->Row());
+ EXPECT_EQ("family-value", cell->Family());
+ EXPECT_EQ("column-value", cell->Qualifier());
+ EXPECT_EQ(value, cell->Value());
+ EXPECT_EQ(std::numeric_limits<long>::max(), cell->Timestamp());
+ EXPECT_EQ(CellType::PUT, cell->Type());
+ i += 1;
+ }
+ for (const auto cell : cells) {
+ delete cell;
+ }
+ cells.clear();
+}
+
+TEST(CellTest, CellRowTest) {
+ std::string row = "only-row";
+ std::string family = "";
+ std::string column = "";
+ std::string value = "";
+ long timestamp = std::numeric_limits<long>::max();
+ CellType cell_type = CellType::PUT;
+ std::unique_ptr<Cell> cell(
+ new Cell(row, family, column, timestamp, value, cell_type));
+ if (cell.get()) {
+ EXPECT_EQ(row, cell.get()->Row());
+ EXPECT_EQ(family, cell.get()->Family());
+ EXPECT_EQ(column, cell.get()->Qualifier());
+ EXPECT_EQ(value, cell.get()->Value());
+ EXPECT_EQ(timestamp, cell.get()->Timestamp());
+ EXPECT_EQ(cell_type, cell.get()->Type());
+ }
+}
+
+TEST(CellTest, CellRowFamilyTest) {
+ std::string row = "only-row";
+ std::string family = "only-family";
+ std::string column = "";
+ std::string value = "";
+ long timestamp = std::numeric_limits<long>::max();
+ CellType cell_type = CellType::PUT;
+ const std::unique_ptr<Cell> cell(
+ new Cell(row, family, column, timestamp, value, cell_type));
+ if (cell.get()) {
+ EXPECT_EQ(row, cell.get()->Row());
+ EXPECT_EQ(family, cell.get()->Family());
+ EXPECT_EQ(column, cell.get()->Qualifier());
+ EXPECT_EQ(value, cell.get()->Value());
+ EXPECT_EQ(timestamp, cell.get()->Timestamp());
+ EXPECT_EQ(cell_type, cell.get()->Type());
+ }
+}
+
+TEST(CellTest, CellRowFamilyValueTest) {
+ std::string row = "only-row";
+ std::string family = "only-family";
+ std::string column = "";
+ std::string value = "only-value";
+ long timestamp = std::numeric_limits<long>::max();
+ CellType cell_type = CellType::PUT;
+ const std::unique_ptr<Cell> cell(
+ new Cell(row, family, column, timestamp, value, cell_type));
+ if (cell.get()) {
+ EXPECT_EQ(row, cell.get()->Row());
+ EXPECT_EQ(family, cell.get()->Family());
+ EXPECT_EQ(column, cell.get()->Qualifier());
+ EXPECT_EQ(value, cell.get()->Value());
+ EXPECT_EQ(timestamp, cell.get()->Timestamp());
+ EXPECT_EQ(cell_type, cell.get()->Type());
+ }
+}
+
+TEST(CellTest, CellRowFamilyColumnValueTest) {
+ std::string row = "only-row";
+ std::string family = "only-family";
+ std::string column = "only-column";
+ std::string value = "only-value";
+ long timestamp = std::numeric_limits<long>::max();
+ CellType cell_type = CellType::PUT;
+ std::unique_ptr<Cell> cell(
+ new Cell(row, family, column, timestamp, value, cell_type));
+ if (cell.get()) {
+ EXPECT_EQ(row, cell.get()->Row());
+ EXPECT_EQ(family, cell.get()->Family());
+ EXPECT_EQ(column, cell.get()->Qualifier());
+ EXPECT_EQ(value, cell.get()->Value());
+ EXPECT_EQ(timestamp, cell.get()->Timestamp());
+ EXPECT_EQ(cell_type, cell.get()->Type());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa3ab420/hbase-native-client/core/cell.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/cell.cc b/hbase-native-client/core/cell.cc
new file mode 100644
index 0000000..f214479
--- /dev/null
+++ b/hbase-native-client/core/cell.cc
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/cell.h"
+#include <stdexcept>
+
+namespace hbase {
+
+Cell::Cell(const std::string &row, const std::string &family,
+ const std::string &qualifier, const long ×tamp,
+ const std::string &value, const hbase::CellType &cell_type)
+ : row_(row),
+ family_(family),
+ qualifier_(qualifier),
+ timestamp_(timestamp),
+ cell_type_(cell_type),
+ value_(value),
+ sequence_id_(0) {
+
+ if (0 == row.size())
+ throw std::runtime_error("Row size should be greater than 0");
+
+ if (0 == family.size())
+ throw std::runtime_error("Column family size should be greater than 0");
+
+ if (0 >= timestamp)
+ throw std::runtime_error("Timestamp should be greater than 0");
+}
+
+Cell::~Cell() {
+}
+
+const std::string &Cell::Row() const {
+ return row_;
+}
+
+const std::string &Cell::Family() const {
+ return family_;
+}
+
+const std::string &Cell::Qualifier() const {
+ return qualifier_;
+}
+
+unsigned long Cell::Timestamp() const {
+ return timestamp_;
+}
+
+const std::string &Cell::Value() const {
+ return value_;
+}
+
+hbase::CellType Cell::Type() const {
+ return cell_type_;
+}
+
+long Cell::SequenceId() const {
+ return sequence_id_;
+}
+
+} /* namespace hbase */
http://git-wip-us.apache.org/repos/asf/hbase/blob/fa3ab420/hbase-native-client/core/cell.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/cell.h b/hbase-native-client/core/cell.h
new file mode 100644
index 0000000..16ed280
--- /dev/null
+++ b/hbase-native-client/core/cell.h
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <string>
+
+namespace hbase {
+
+enum CellType {
+ MINIMUM = 0,
+ PUT = 4,
+ DELETE = 8,
+ DELETEFAMILYVERSION = 10,
+ DELETE_COLUMN = 12,
+ DELETE_FAMILY = 14,
+ MAXIMUM = 255
+};
+
+class Cell {
+ public:
+ Cell(const std::string &row, const std::string &family,
+ const std::string &qualifier, const long ×tamp,
+ const std::string &value, const hbase::CellType &cell_type);
+ virtual ~Cell();
+ const std::string &Row() const;
+ const std::string &Family() const;
+ const std::string &Qualifier() const;
+ unsigned long Timestamp() const;
+ const std::string &Value() const;
+ CellType Type() const;
+ long SequenceId() const;
+
+ private:
+ std::string row_;
+ std::string family_;
+ std::string qualifier_;
+ unsigned long timestamp_;
+ hbase::CellType cell_type_;
+ std::string value_;
+ long sequence_id_;
+};
+
+} /* namespace hbase */
[33/50] [abbrv] hbase git commit: HBASE-15687 Allow decoding more
than GetResponse from the server
Posted by ec...@apache.org.
HBASE-15687 Allow decoding more than GetResponse from the server
Summary:
We'll need more than get's for the client to be usable. So now the
Request class contains the protobufs needed to encode and decode
rpc's.
I also added some helper methods to create initial requests.
Test Plan: It compiles and still gets data from HBase meta
Differential Revision: https://reviews.facebook.net/D57327
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/14a33d7d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/14a33d7d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/14a33d7d
Branch: refs/heads/HBASE-14850
Commit: 14a33d7dbc7c9118f83be24925d1ff5c603cc352
Parents: a1069eb
Author: Elliott Clark <ec...@apache.org>
Authored: Wed Apr 27 15:27:09 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/connection/BUCK | 1 +
.../connection/client-dispatcher.cc | 6 +--
.../connection/client-dispatcher.h | 6 +--
.../connection/client-handler.cc | 30 ++++++++++---
hbase-native-client/connection/client-handler.h | 18 ++++++--
.../connection/connection-factory.cc | 6 +--
.../connection/connection-factory.h | 2 +-
hbase-native-client/connection/pipeline.h | 3 +-
hbase-native-client/connection/request.cc | 45 ++++++++++++++++++++
hbase-native-client/connection/request.h | 25 +++++++++--
hbase-native-client/core/simple-client.cc | 13 +++---
11 files changed, 123 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/14a33d7d/hbase-native-client/connection/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/BUCK b/hbase-native-client/connection/BUCK
index 5067708..d393885 100644
--- a/hbase-native-client/connection/BUCK
+++ b/hbase-native-client/connection/BUCK
@@ -32,6 +32,7 @@ cxx_library(name="connection",
"client-handler.cc",
"connection-factory.cc",
"pipeline.cc",
+ "request.cc",
],
deps=[
"//if:if",
http://git-wip-us.apache.org/repos/asf/hbase/blob/14a33d7d/hbase-native-client/connection/client-dispatcher.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-dispatcher.cc b/hbase-native-client/connection/client-dispatcher.cc
index 25cff7d..eea0a17 100644
--- a/hbase-native-client/connection/client-dispatcher.cc
+++ b/hbase-native-client/connection/client-dispatcher.cc
@@ -35,16 +35,16 @@ void ClientDispatcher::read(Context *ctx, Response in) {
p.setValue(in);
}
-Future<Response> ClientDispatcher::operator()(Request arg) {
+Future<Response> ClientDispatcher::operator()(std::unique_ptr<Request> arg) {
auto call_id = ++current_call_id_;
- arg.set_call_id(call_id);
+ arg->set_call_id(call_id);
auto &p = requests_[call_id];
auto f = p.getFuture();
p.setInterruptHandler([call_id, this](const folly::exception_wrapper &e) {
this->requests_.erase(call_id);
});
- this->pipeline_->write(arg);
+ this->pipeline_->write(std::move(arg));
return f;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/14a33d7d/hbase-native-client/connection/client-dispatcher.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-dispatcher.h b/hbase-native-client/connection/client-dispatcher.h
index 89c7119..877e877 100644
--- a/hbase-native-client/connection/client-dispatcher.h
+++ b/hbase-native-client/connection/client-dispatcher.h
@@ -27,11 +27,11 @@
namespace hbase {
class ClientDispatcher
- : public wangle::ClientDispatcherBase<SerializePipeline, Request,
- Response> {
+ : public wangle::ClientDispatcherBase<SerializePipeline,
+ std::unique_ptr<Request>, Response> {
public:
void read(Context *ctx, Response in) override;
- folly::Future<Response> operator()(Request arg) override;
+ folly::Future<Response> operator()(std::unique_ptr<Request> arg) override;
folly::Future<folly::Unit> close(Context *ctx) override;
folly::Future<folly::Unit> close() override;
http://git-wip-us.apache.org/repos/asf/hbase/blob/14a33d7d/hbase-native-client/connection/client-handler.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-handler.cc b/hbase-native-client/connection/client-handler.cc
index 205993a7..abcf5c1 100644
--- a/hbase-native-client/connection/client-handler.cc
+++ b/hbase-native-client/connection/client-handler.cc
@@ -47,14 +47,30 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
LOG(INFO) << "Read ResponseHeader size=" << used_bytes
<< " call_id=" << header.call_id()
<< " has_exception=" << header.has_exception();
+
+ // Get the response protobuf from the map
+ auto search = resp_msgs_.find(header.call_id());
+ // It's an error if it's not there.
+ CHECK(search != resp_msgs_.end());
+ auto resp_msg = search->second;
+ CHECK(resp_msg != nullptr);
+
+ // Make sure we don't leak the protobuf
+ resp_msgs_.erase(search);
+
+ // set the call_id.
+ // This will be used to by the dispatcher to match up
+ // the promise with the response.
received.set_call_id(header.call_id());
+ // If there was an exception then there's no
+ // data left on the wire.
if (header.has_exception() == false) {
buf->trimStart(used_bytes);
- // For now assume that everything was a get.
- // We'll need to set this up later.
- received.set_response(std::make_shared<GetResponse>());
- used_bytes = deser_.parse_delimited(buf.get(), received.response().get());
+ used_bytes = deser_.parse_delimited(buf.get(), resp_msg.get());
+ // Make sure that bytes were parsed.
+ CHECK(used_bytes == buf->length());
+ received.set_response(resp_msg);
}
ctx->fireRead(std::move(received));
}
@@ -62,7 +78,7 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
// TODO(eclark): Figure out how to handle the
// network errors that are going to come.
-Future<Unit> ClientHandler::write(Context *ctx, Request r) {
+Future<Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
// Keep track of if we have sent the header.
if (UNLIKELY(need_send_header_)) {
need_send_header_ = false;
@@ -78,5 +94,7 @@ Future<Unit> ClientHandler::write(Context *ctx, Request r) {
ctx->fireWrite(std::move(pre));
}
- return ctx->fireWrite(ser_.request(r.call_id(), r.method(), r.msg()));
+ resp_msgs_[r->call_id()] = r->resp_msg();
+ return ctx->fireWrite(
+ ser_.request(r->call_id(), r->method(), r->req_msg().get()));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/14a33d7d/hbase-native-client/connection/client-handler.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-handler.h b/hbase-native-client/connection/client-handler.h
index dbaf5a0..41bb883 100644
--- a/hbase-native-client/connection/client-handler.h
+++ b/hbase-native-client/connection/client-handler.h
@@ -30,20 +30,30 @@ namespace hbase {
class Request;
class Response;
}
+namespace google {
+namespace protobuf {
+class Message;
+}
+}
namespace hbase {
-class ClientHandler
- : public wangle::Handler<std::unique_ptr<folly::IOBuf>, Response, Request,
- std::unique_ptr<folly::IOBuf>> {
+class ClientHandler : public wangle::Handler<std::unique_ptr<folly::IOBuf>,
+ Response, std::unique_ptr<Request>,
+ std::unique_ptr<folly::IOBuf>> {
public:
ClientHandler(std::string user_name);
void read(Context *ctx, std::unique_ptr<folly::IOBuf> msg) override;
- folly::Future<folly::Unit> write(Context *ctx, Request r) override;
+ folly::Future<folly::Unit> write(Context *ctx,
+ std::unique_ptr<Request> r) override;
private:
bool need_send_header_ = true;
std::string user_name_;
ClientSerializer ser_;
ClientDeserializer deser_;
+
+ // in flight requests
+ std::unordered_map<uint32_t, std::shared_ptr<google::protobuf::Message>>
+ resp_msgs_;
};
} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/14a33d7d/hbase-native-client/connection/connection-factory.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-factory.cc b/hbase-native-client/connection/connection-factory.cc
index 5d1b0da..7073f9d 100644
--- a/hbase-native-client/connection/connection-factory.cc
+++ b/hbase-native-client/connection/connection-factory.cc
@@ -45,14 +45,14 @@ ConnectionFactory::ConnectionFactory() {
bootstrap_.pipelineFactory(std::make_shared<RpcPipelineFactory>());
}
-std::shared_ptr<Service<Request, Response>>
+std::shared_ptr<Service<std::unique_ptr<Request>, Response>>
ConnectionFactory::make_connection(std::string host, int port) {
// Connect to a given server
// Then when connected create a ClientDispactcher.
auto pipeline = bootstrap_.connect(SocketAddress(host, port, true)).get();
auto dispatcher = std::make_shared<ClientDispatcher>();
dispatcher->setPipeline(pipeline);
- auto service =
- std::make_shared<CloseOnReleaseFilter<Request, Response>>(dispatcher);
+ auto service = std::make_shared<
+ CloseOnReleaseFilter<std::unique_ptr<Request>, Response>>(dispatcher);
return service;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/14a33d7d/hbase-native-client/connection/connection-factory.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-factory.h b/hbase-native-client/connection/connection-factory.h
index 73ac032..8d1d2f0 100644
--- a/hbase-native-client/connection/connection-factory.h
+++ b/hbase-native-client/connection/connection-factory.h
@@ -31,7 +31,7 @@ namespace hbase {
class ConnectionFactory {
public:
ConnectionFactory();
- std::shared_ptr<wangle::Service<Request, Response>>
+ std::shared_ptr<wangle::Service<std::unique_ptr<Request>, Response>>
make_connection(std::string host, int port);
private:
http://git-wip-us.apache.org/repos/asf/hbase/blob/14a33d7d/hbase-native-client/connection/pipeline.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/pipeline.h b/hbase-native-client/connection/pipeline.h
index 8114fab..6c4f4ff 100644
--- a/hbase-native-client/connection/pipeline.h
+++ b/hbase-native-client/connection/pipeline.h
@@ -26,7 +26,8 @@
#include "utils/user-util.h"
namespace hbase {
-using SerializePipeline = wangle::Pipeline<folly::IOBufQueue &, Request>;
+using SerializePipeline =
+ wangle::Pipeline<folly::IOBufQueue &, std::unique_ptr<Request>>;
class RpcPipelineFactory : public wangle::PipelineFactory<SerializePipeline> {
public:
http://git-wip-us.apache.org/repos/asf/hbase/blob/14a33d7d/hbase-native-client/connection/request.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/request.cc b/hbase-native-client/connection/request.cc
new file mode 100644
index 0000000..50ea029
--- /dev/null
+++ b/hbase-native-client/connection/request.cc
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "connection/request.h"
+
+#include "if/Client.pb.h"
+
+using namespace hbase;
+
+Request::Request(std::shared_ptr<google::protobuf::Message> req,
+ std::shared_ptr<google::protobuf::Message> resp,
+ std::string method)
+ : req_msg_(req), resp_msg_(resp), method_(method), call_id_(0) {}
+
+std::unique_ptr<Request> Request::get() {
+ return std::make_unique<Request>(std::make_shared<hbase::pb::GetRequest>(),
+ std::make_shared<hbase::pb::GetResponse>(),
+ "Get");
+}
+std::unique_ptr<Request> Request::mutate() {
+ return std::make_unique<Request>(std::make_shared<hbase::pb::MutateRequest>(),
+ std::make_shared<hbase::pb::MutateResponse>(),
+ "Mutate");
+}
+std::unique_ptr<Request> Request::scan() {
+ return std::make_unique<Request>(std::make_shared<hbase::pb::ScanRequest>(),
+ std::make_shared<hbase::pb::ScanResponse>(),
+ "Scan");
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/14a33d7d/hbase-native-client/connection/request.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/request.h b/hbase-native-client/connection/request.h
index e9e3e88..743c469 100644
--- a/hbase-native-client/connection/request.h
+++ b/hbase-native-client/connection/request.h
@@ -21,22 +21,39 @@
#include <google/protobuf/message.h>
#include <cstdint>
+#include <memory>
#include <string>
namespace hbase {
class Request {
public:
- Request() : call_id_(0) {}
+ static std::unique_ptr<Request> get();
+ static std::unique_ptr<Request> mutate();
+ static std::unique_ptr<Request> scan();
+
+ Request(std::shared_ptr<google::protobuf::Message> req,
+ std::shared_ptr<google::protobuf::Message> resp, std::string method);
+
uint32_t call_id() { return call_id_; }
void set_call_id(uint32_t call_id) { call_id_ = call_id; }
- google::protobuf::Message *msg() { return msg_.get(); }
- void set_msg(std::shared_ptr<google::protobuf::Message> msg) { msg_ = msg; }
+
+ std::shared_ptr<google::protobuf::Message> req_msg() { return req_msg_; }
+ std::shared_ptr<google::protobuf::Message> resp_msg() { return resp_msg_; }
+
+ void set_req_msg(std::shared_ptr<google::protobuf::Message> msg) {
+ req_msg_ = msg;
+ }
+ void set_resp_msg(std::shared_ptr<google::protobuf::Message> msg) {
+ resp_msg_ = msg;
+ }
+
std::string method() { return method_; }
void set_method(std::string method) { method_ = method; }
private:
uint32_t call_id_;
- std::shared_ptr<google::protobuf::Message> msg_ = nullptr;
+ std::shared_ptr<google::protobuf::Message> req_msg_ = nullptr;
+ std::shared_ptr<google::protobuf::Message> resp_msg_ = nullptr;
std::string method_ = "Get";
};
} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/14a33d7d/hbase-native-client/core/simple-client.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/simple-client.cc b/hbase-native-client/core/simple-client.cc
index 8b2fae5..2cb6200 100644
--- a/hbase-native-client/core/simple-client.cc
+++ b/hbase-native-client/core/simple-client.cc
@@ -58,24 +58,23 @@ int main(int argc, char *argv[]) {
auto conn = cf.make_connection(result.host_name(), result.port());
// Send the request
- Request r;
+ auto r = Request::get();
// This is a get request so make that
- auto msg = make_shared<hbase::pb::GetRequest>();
+ auto req_msg = static_pointer_cast<hbase::pb::GetRequest>(r->req_msg());
// Set what region
- msg->mutable_region()->set_value(FLAGS_region);
+ req_msg->mutable_region()->set_value(FLAGS_region);
// It's always this.
- msg->mutable_region()->set_type(
+ req_msg->mutable_region()->set_type(
RegionSpecifier_RegionSpecifierType::
RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
// What row.
- msg->mutable_get()->set_row(FLAGS_row);
+ req_msg->mutable_get()->set_row(FLAGS_row);
// Send it.
- r.set_msg(msg);
- auto resp = (*conn)(r).get(milliseconds(5000));
+ auto resp = (*conn)(std::move(r)).get(milliseconds(5000));
auto get_resp = std::static_pointer_cast<GetResponse>(resp.response());
cout << "GetResponse has_result = " << get_resp->has_result() << '\n';
[23/50] [abbrv] hbase git commit: HBASE-15649 Ignore buck-out when
building docker image
Posted by ec...@apache.org.
HBASE-15649 Ignore buck-out when building docker image
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/098028c5
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/098028c5
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/098028c5
Branch: refs/heads/HBASE-14850
Commit: 098028c5c61ae7d318dde73fece2c0758f28d113
Parents: bfabc71
Author: Elliott Clark <ec...@apache.org>
Authored: Wed Apr 13 17:02:38 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/.dockerignore | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/098028c5/hbase-native-client/.dockerignore
----------------------------------------------------------------------
diff --git a/hbase-native-client/.dockerignore b/hbase-native-client/.dockerignore
new file mode 100644
index 0000000..2cfffb7
--- /dev/null
+++ b/hbase-native-client/.dockerignore
@@ -0,0 +1,2 @@
+buck-out
+.buckd
[13/50] [abbrv] hbase git commit: HBASE-15604 Add a good readme on
the build.
Posted by ec...@apache.org.
HBASE-15604 Add a good readme on the build.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bfabc715
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bfabc715
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bfabc715
Branch: refs/heads/HBASE-14850
Commit: bfabc715c30350ec1ee1953826ab844ceb60d750
Parents: 45cf565
Author: Elliott Clark <ec...@apache.org>
Authored: Fri Apr 8 13:44:45 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/BUILDING.md | 63 ++++++++++++++++++++++++++++++++++++
1 file changed, 63 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/bfabc715/hbase-native-client/BUILDING.md
----------------------------------------------------------------------
diff --git a/hbase-native-client/BUILDING.md b/hbase-native-client/BUILDING.md
new file mode 100644
index 0000000..20ef2a0
--- /dev/null
+++ b/hbase-native-client/BUILDING.md
@@ -0,0 +1,63 @@
+<!---
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+#Building HBase native client
+
+The HBase native client build using buck and produces a linux library.
+
+
+# Dependencies
+
+The easiest way to build hbase-native-client is to
+use [Docker](https://www.docker.com/). This will mean that any platform
+with docker can be used to build the hbase-native-client. It will also
+mean that you're building with the same versions of all dependencies that
+the client is tested with.
+
+On OSX the current boot2docker solution will work however it's pretty
+slow. If things get too slow using [dinghy](https://github.com/codekitchen/dinghy)
+can help speed things up by using nfs.
+
+If possible pairing virtual box with dinghy will result in the fastest,
+most stable docker environment. However none of it is needed.
+
+# Building using docker
+
+To start with make sure that you have built the java project using
+`mvn package -DskipTests`. That will allow all tests to spin up a standalone
+hbase instance from the jar's created.
+
+Then go into the hbase-native-client directory and run `./bin/start-docker.sh`
+that will build the docker development environment and when complete will
+drop you into a shell on a linux vm with all the tools needed installed.
+
+# Buck
+
+From then on we can use [buck](https://buckbuild.com/) to build everything.
+For example:
+```
+buck build //core:core
+buck test --all
+buck build //core:simple-client
+```
+
+That will build the library, then build and test everything, then build
+the simple-client binary. Buck will find all modules used, and compile
+them in parallel, caching the results. Output from buck is in the buck-out
+foulder. Generated binaries are in buck-out/gen logs are in buck-out/logs
[48/50] [abbrv] hbase git commit: HBASE-15770 Stop using wangle's
global executor
Posted by ec...@apache.org.
HBASE-15770 Stop using wangle's global executor
Summary: Connection pool and connection factory now get thread pools through their constructor. This means that the client has the whole control over the threads.
Test Plan: simple-client still writes.
Differential Revision: https://reviews.facebook.net/D57801
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/d3a6efd7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/d3a6efd7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/d3a6efd7
Branch: refs/heads/HBASE-14850
Commit: d3a6efd717241188a2541efdb4658fccd3c319f1
Parents: 297a4df
Author: Elliott Clark <ec...@apache.org>
Authored: Fri May 6 14:32:16 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
.../connection/client-dispatcher.h | 3 ++-
.../connection/client-handler.cc | 2 +-
.../connection/connection-factory.cc | 8 +++---
.../connection/connection-factory.h | 2 +-
.../connection/connection-pool-test.cc | 1 +
.../connection/connection-pool.cc | 7 ++---
.../connection/connection-pool.h | 2 +-
hbase-native-client/core/client.cc | 23 +++++++++++-----
hbase-native-client/core/client.h | 9 +++++--
hbase-native-client/core/location-cache-test.cc | 5 ++--
hbase-native-client/core/location-cache.cc | 28 +++++++++++---------
hbase-native-client/core/location-cache.h | 8 ++++--
hbase-native-client/core/simple-client.cc | 8 +++---
hbase-native-client/serde/server-name-test.cc | 1 -
14 files changed, 63 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/d3a6efd7/hbase-native-client/connection/client-dispatcher.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-dispatcher.h b/hbase-native-client/connection/client-dispatcher.h
index 4bfb35d..2497cc7 100644
--- a/hbase-native-client/connection/client-dispatcher.h
+++ b/hbase-native-client/connection/client-dispatcher.h
@@ -31,7 +31,8 @@
namespace hbase {
/**
- * Dispatcher that assigns a call_id and then routes the response back to the future.
+ * Dispatcher that assigns a call_id and then routes the response back to the
+ * future.
*/
class ClientDispatcher
: public wangle::ClientDispatcherBase<SerializePipeline,
http://git-wip-us.apache.org/repos/asf/hbase/blob/d3a6efd7/hbase-native-client/connection/client-handler.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-handler.cc b/hbase-native-client/connection/client-handler.cc
index cae03c7..2e3fcd3 100644
--- a/hbase-native-client/connection/client-handler.cc
+++ b/hbase-native-client/connection/client-handler.cc
@@ -37,7 +37,7 @@ using hbase::pb::GetResponse;
using google::protobuf::Message;
ClientHandler::ClientHandler(std::string user_name)
- : user_name_(user_name), serde_(), once_flag_(),
+ : user_name_(user_name), serde_(), once_flag_(std::make_unique<std::once_flag>()),
resp_msgs_(
make_unique<folly::AtomicHashMap<
uint32_t, std::shared_ptr<google::protobuf::Message>>>(5000)) {}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d3a6efd7/hbase-native-client/connection/connection-factory.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-factory.cc b/hbase-native-client/connection/connection-factory.cc
index 635d12d..beec6d5 100644
--- a/hbase-native-client/connection/connection-factory.cc
+++ b/hbase-native-client/connection/connection-factory.cc
@@ -19,8 +19,6 @@
#include "connection/connection-factory.h"
-#include <wangle/concurrent/GlobalExecutor.h>
-
#include "connection/client-dispatcher.h"
#include "connection/pipeline.h"
#include "connection/service.h"
@@ -28,9 +26,9 @@
using namespace folly;
using namespace hbase;
-ConnectionFactory::ConnectionFactory()
- : io_pool_(std::static_pointer_cast<wangle::IOThreadPoolExecutor>(
- wangle::getIOExecutor())),
+ConnectionFactory::ConnectionFactory(
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool)
+ : io_pool_(io_pool),
pipeline_factory_(std::make_shared<RpcPipelineFactory>()) {}
std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>
http://git-wip-us.apache.org/repos/asf/hbase/blob/d3a6efd7/hbase-native-client/connection/connection-factory.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-factory.h b/hbase-native-client/connection/connection-factory.h
index 8b6d8d8..fb5d9fe 100644
--- a/hbase-native-client/connection/connection-factory.h
+++ b/hbase-native-client/connection/connection-factory.h
@@ -39,7 +39,7 @@ public:
* Constructor.
* There should only be one ConnectionFactory per client.
*/
- ConnectionFactory();
+ ConnectionFactory(std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool);
/** Default Desctructor */
virtual ~ConnectionFactory() = default;
http://git-wip-us.apache.org/repos/asf/hbase/blob/d3a6efd7/hbase-native-client/connection/connection-pool-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-pool-test.cc b/hbase-native-client/connection/connection-pool-test.cc
index b1a0ba0..c0c346f 100644
--- a/hbase-native-client/connection/connection-pool-test.cc
+++ b/hbase-native-client/connection/connection-pool-test.cc
@@ -34,6 +34,7 @@ using ::testing::_;
class MockConnectionFactory : public ConnectionFactory {
public:
+ MockConnectionFactory() : ConnectionFactory(nullptr) {}
MOCK_METHOD0(MakeBootstrap,
std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>());
MOCK_METHOD3(Connect,
http://git-wip-us.apache.org/repos/asf/hbase/blob/d3a6efd7/hbase-native-client/connection/connection-pool.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-pool.cc b/hbase-native-client/connection/connection-pool.cc
index 75f343e..90e2056 100644
--- a/hbase-native-client/connection/connection-pool.cc
+++ b/hbase-native-client/connection/connection-pool.cc
@@ -31,9 +31,10 @@ using hbase::HBaseService;
using folly::SharedMutexWritePriority;
using folly::SocketAddress;
-ConnectionPool::ConnectionPool()
- : cf_(std::make_shared<ConnectionFactory>()), clients_(), connections_(),
- map_mutex_() {}
+ConnectionPool::ConnectionPool(
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor)
+ : cf_(std::make_shared<ConnectionFactory>(io_executor)), clients_(),
+ connections_(), map_mutex_() {}
ConnectionPool::ConnectionPool(std::shared_ptr<ConnectionFactory> cf)
: cf_(cf), clients_(), connections_(), map_mutex_() {}
http://git-wip-us.apache.org/repos/asf/hbase/blob/d3a6efd7/hbase-native-client/connection/connection-pool.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-pool.h b/hbase-native-client/connection/connection-pool.h
index 5edd407..60f00de 100644
--- a/hbase-native-client/connection/connection-pool.h
+++ b/hbase-native-client/connection/connection-pool.h
@@ -61,7 +61,7 @@ struct ServerNameHash {
class ConnectionPool {
public:
/** Create connection pool wit default connection factory */
- ConnectionPool();
+ ConnectionPool(std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor);
/**
* Desctructor.
http://git-wip-us.apache.org/repos/asf/hbase/blob/d3a6efd7/hbase-native-client/core/client.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client.cc b/hbase-native-client/core/client.cc
index 4b9f844..1e80998 100644
--- a/hbase-native-client/core/client.cc
+++ b/hbase-native-client/core/client.cc
@@ -19,18 +19,27 @@
#include "core/client.h"
-#include <folly/Logging.h>
-#include <folly/Random.h>
-#include <gflags/gflags.h>
#include <glog/logging.h>
-#include <wangle/concurrent/GlobalExecutor.h>
#include <string>
-#include "if/ZooKeeper.pb.h"
-
using namespace folly;
using namespace std;
using namespace hbase::pb;
-namespace hbase {}
+namespace hbase {
+
+Client::Client(std::string zk_quorum)
+ : cpu_executor_(std::make_shared<wangle::CPUThreadPoolExecutor>(4)),
+ io_executor_(std::make_shared<wangle::IOThreadPoolExecutor>(
+ sysconf(_SC_NPROCESSORS_ONLN))),
+ location_cache_(zk_quorum, cpu_executor_, io_executor_) {}
+
+// We can't have the threads continue running after everything is done
+// that leads to an error.
+Client::~Client() {
+ cpu_executor_->stop();
+ io_executor_->stop();
+}
+
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/d3a6efd7/hbase-native-client/core/client.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client.h b/hbase-native-client/core/client.h
index 4a6d23b..4db82c4 100644
--- a/hbase-native-client/core/client.h
+++ b/hbase-native-client/core/client.h
@@ -21,6 +21,8 @@
#include <folly/futures/Future.h>
#include <folly/io/IOBuf.h>
+#include <wangle/concurrent/CPUThreadPoolExecutor.h>
+#include <wangle/concurrent/IOThreadPoolExecutor.h>
#include <string>
@@ -33,19 +35,22 @@ namespace hbase {
* Client.
*
* This is the class that provides access to an HBase cluster.
- * It is thread safe and does connection pooling. Current recommendations are to have only one Client per cluster around.
+ * It is thread safe and does connection pooling. Current recommendations are to
+ * have only one Client per cluster around.
*/
class Client {
public:
-
/**
* Create a new client.
* @param quorum_spec Where to connect to get Zookeeper bootstrap information.
*/
explicit Client(std::string quorum_spec);
+ ~Client();
private:
LocationCache location_cache_;
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_;
};
} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/d3a6efd7/hbase-native-client/core/location-cache-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache-test.cc b/hbase-native-client/core/location-cache-test.cc
index 172799d..8bc6383 100644
--- a/hbase-native-client/core/location-cache-test.cc
+++ b/hbase-native-client/core/location-cache-test.cc
@@ -18,14 +18,15 @@
*/
#include <folly/Memory.h>
#include <gtest/gtest.h>
-#include <wangle/concurrent/GlobalExecutor.h>
#include "location-cache.h"
using namespace hbase;
TEST(LocationCacheTest, TestGetMetaNodeContents) {
// TODO(elliott): need to make a test utility for this.
- LocationCache cache{"localhost:2181", wangle::getCPUExecutor()};
+ auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
+ auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
+ LocationCache cache{"localhost:2181", cpu, io};
auto f = cache.LocateMeta();
auto result = f.get();
ASSERT_FALSE(f.hasException());
http://git-wip-us.apache.org/repos/asf/hbase/blob/d3a6efd7/hbase-native-client/core/location-cache.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
index 583d305..6c018f9 100644
--- a/hbase-native-client/core/location-cache.cc
+++ b/hbase-native-client/core/location-cache.cc
@@ -20,7 +20,8 @@
#include <folly/Logging.h>
#include <folly/io/IOBuf.h>
-#include <wangle/concurrent/GlobalExecutor.h>
+#include <wangle/concurrent/CPUThreadPoolExecutor.h>
+#include <wangle/concurrent/IOThreadPoolExecutor.h>
#include "connection/response.h"
#include "if/Client.pb.h"
@@ -48,10 +49,13 @@ using hbase::pb::RegionInfo;
// TODO(eclark): make this configurable on client creation
static const char META_ZNODE_NAME[] = "/hbase/meta-region-server";
-LocationCache::LocationCache(string quorum_spec,
- shared_ptr<folly::Executor> executor)
- : quorum_spec_(quorum_spec), executor_(executor), meta_promise_(nullptr),
- meta_lock_(), cp_(), meta_util_(), zk_(nullptr) {
+LocationCache::LocationCache(
+ std::string quorum_spec,
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor)
+ : quorum_spec_(quorum_spec), cpu_executor_(cpu_executor),
+ meta_promise_(nullptr), meta_lock_(), cp_(io_executor), meta_util_(),
+ zk_(nullptr) {
zk_ = zookeeper_init(quorum_spec.c_str(), nullptr, 1000, 0, 0, 0);
}
@@ -79,7 +83,7 @@ void LocationCache::InvalidateMeta() {
/// MUST hold the meta_lock_
void LocationCache::RefreshMetaLocation() {
meta_promise_ = make_unique<SharedPromise<ServerName>>();
- executor_->add([&] {
+ cpu_executor_->add([&] {
meta_promise_->setWith([&] { return this->ReadMetaLocation(); });
});
}
@@ -109,10 +113,9 @@ ServerName LocationCache::ReadMetaLocation() {
Future<std::shared_ptr<RegionLocation>>
LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
- auto exec = wangle::getCPUExecutor();
return this->LocateMeta()
- .via(exec.get())
- .then([ exec = exec, this ](ServerName sn) { return this->cp_.get(sn); })
+ .via(cpu_executor_.get())
+ .then([this](ServerName sn) { return this->cp_.get(sn); })
.then([&](std::shared_ptr<HBaseService> service) {
return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
})
@@ -121,7 +124,7 @@ LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
// a region location.
return this->CreateLocation(std::move(resp));
})
- .then([ exec = exec, this ](std::shared_ptr<RegionLocation> rl) {
+ .then([this](std::shared_ptr<RegionLocation> rl) {
// Now fill out the connection.
rl->set_service(cp_.get(rl->server_name()));
return rl;
@@ -129,14 +132,14 @@ LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
}
/**
- * Filter to remove a service from the location cache and the connection cache on errors
+ * Filter to remove a service from the location cache and the connection cache
+ * on errors
* or on cloase.
*/
class RemoveServiceFilter
: public ServiceFilter<std::unique_ptr<Request>, Response> {
public:
-
/** Create a new filter. */
RemoveServiceFilter(std::shared_ptr<HBaseService> service, ServerName sn,
ConnectionPool &cp)
@@ -157,7 +160,6 @@ public:
}
}
-
/** Has this been closed */
virtual bool isAvailable() override {
return !released && service_->isAvailable();
http://git-wip-us.apache.org/repos/asf/hbase/blob/d3a6efd7/hbase-native-client/core/location-cache.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h
index 88bec18..e077750 100644
--- a/hbase-native-client/core/location-cache.h
+++ b/hbase-native-client/core/location-cache.h
@@ -21,6 +21,8 @@
#include <folly/Executor.h>
#include <folly/futures/Future.h>
#include <folly/futures/SharedPromise.h>
+#include <wangle/concurrent/CPUThreadPoolExecutor.h>
+#include <wangle/concurrent/IOThreadPoolExecutor.h>
#include <zookeeper/zookeeper.h>
#include <memory>
@@ -51,7 +53,8 @@ public:
* @param executor The cpu executor to run on.
*/
LocationCache(std::string quorum_spec,
- std::shared_ptr<folly::Executor> executor);
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_exector,
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor);
/**
* Destructor.
* This will clean up the zookeeper connections.
@@ -81,8 +84,9 @@ private:
hbase::pb::ServerName ReadMetaLocation();
std::shared_ptr<RegionLocation> CreateLocation(const Response &resp);
+ /* data */
std::string quorum_spec_;
- std::shared_ptr<folly::Executor> executor_;
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> meta_promise_;
std::mutex meta_lock_;
MetaUtil meta_util_;
http://git-wip-us.apache.org/repos/asf/hbase/blob/d3a6efd7/hbase-native-client/core/simple-client.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/simple-client.cc b/hbase-native-client/core/simple-client.cc
index 39c82c3..f3f6c42 100644
--- a/hbase-native-client/core/simple-client.cc
+++ b/hbase-native-client/core/simple-client.cc
@@ -22,7 +22,7 @@
#include <folly/futures/Future.h>
#include <gflags/gflags.h>
#include <wangle/concurrent/CPUThreadPoolExecutor.h>
-#include <wangle/concurrent/GlobalExecutor.h>
+#include <wangle/concurrent/IOThreadPoolExecutor.h>
#include <atomic>
#include <chrono>
@@ -89,12 +89,10 @@ int main(int argc, char *argv[]) {
// Set up thread pools.
auto cpu_pool =
std::make_shared<wangle::CPUThreadPoolExecutor>(FLAGS_threads);
- wangle::setCPUExecutor(cpu_pool);
auto io_pool = std::make_shared<wangle::IOThreadPoolExecutor>(5);
- wangle::setIOExecutor(io_pool);
// Create the cache.
- LocationCache cache{FLAGS_zookeeper, cpu_pool};
+ LocationCache cache{FLAGS_zookeeper, cpu_pool, io_pool};
auto row = FLAGS_row;
auto tn = folly::to<TableName>(FLAGS_table);
@@ -105,7 +103,7 @@ int main(int argc, char *argv[]) {
auto num_puts = FLAGS_columns;
auto results = std::vector<Future<Response>>{};
- uint64_t col{0};
+ auto col = uint64_t{0};
for (; col < num_puts; col++) {
results.push_back(folly::makeFuture(col)
.via(cpu_pool.get())
http://git-wip-us.apache.org/repos/asf/hbase/blob/d3a6efd7/hbase-native-client/serde/server-name-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/server-name-test.cc b/hbase-native-client/serde/server-name-test.cc
index 2281fa2..73a68d6 100644
--- a/hbase-native-client/serde/server-name-test.cc
+++ b/hbase-native-client/serde/server-name-test.cc
@@ -46,5 +46,4 @@ TEST(TestServerName, TestIPV6) {
ASSERT_EQ("[::::1]", sn.host_name());
ASSERT_EQ(123, sn.port());
-
}
[12/50] [abbrv] hbase git commit: HBASE-15766 Show working puts
Posted by ec...@apache.org.
HBASE-15766 Show working puts
Summary:
Add on showing how a set of puts to a single connection will work.
This still needs retries and looking up what region each request is going to
Test Plan:
./buck-out/gen/core/simple-client -columns 100
../bin/hbase shell
count 't'
100 row(s) in 0.2470 seconds
Differential Revision: https://reviews.facebook.net/D57603
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/79b5085d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/79b5085d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/79b5085d
Branch: refs/heads/HBASE-14850
Commit: 79b5085db5ca6283714ec5be2f5439d4d5d5eb72
Parents: f034294
Author: Elliott Clark <ec...@apache.org>
Authored: Wed May 4 01:54:21 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
.../connection/client-dispatcher.cc | 9 +--
.../connection/client-dispatcher.h | 11 ++-
.../connection/client-handler.cc | 13 ++--
hbase-native-client/connection/client-handler.h | 4 +-
.../connection/connection-factory.cc | 46 ++++++------
.../connection/connection-factory.h | 11 ++-
.../connection/connection-pool-test.cc | 54 +++++++++++---
.../connection/connection-pool.cc | 33 +++++++--
.../connection/connection-pool.h | 10 ++-
hbase-native-client/core/client.cc | 6 +-
hbase-native-client/core/location-cache.cc | 25 +++----
hbase-native-client/core/location-cache.h | 6 +-
hbase-native-client/core/meta-utils.cc | 6 +-
hbase-native-client/core/meta-utils.h | 4 +-
hbase-native-client/core/region-location.h | 8 ++-
hbase-native-client/core/simple-client.cc | 76 +++++++++++++++++---
.../serde/region-info-deserializer-test.cc | 1 -
hbase-native-client/serde/region-info.h | 6 +-
18 files changed, 231 insertions(+), 98 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/79b5085d/hbase-native-client/connection/client-dispatcher.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-dispatcher.cc b/hbase-native-client/connection/client-dispatcher.cc
index 6e2dc54..655d765 100644
--- a/hbase-native-client/connection/client-dispatcher.cc
+++ b/hbase-native-client/connection/client-dispatcher.cc
@@ -22,10 +22,11 @@ using namespace folly;
using namespace hbase;
using namespace wangle;
-ClientDispatcher::ClientDispatcher() : requests_(), current_call_id_(9) {}
+ClientDispatcher::ClientDispatcher() : requests_(5000), current_call_id_(9) {}
void ClientDispatcher::read(Context *ctx, Response in) {
auto call_id = in.call_id();
+
auto search = requests_.find(call_id);
CHECK(search != requests_.end());
auto p = std::move(search->second);
@@ -38,10 +39,10 @@ void ClientDispatcher::read(Context *ctx, Response in) {
}
Future<Response> ClientDispatcher::operator()(std::unique_ptr<Request> arg) {
- auto call_id = ++current_call_id_;
-
+ auto call_id = current_call_id_++;
arg->set_call_id(call_id);
- auto &p = requests_[call_id];
+ requests_.insert(call_id, Promise<Response>{});
+ auto &p = requests_.find(call_id)->second;
auto f = p.getFuture();
p.setInterruptHandler([call_id, this](const folly::exception_wrapper &e) {
LOG(ERROR) << "e = " << call_id;
http://git-wip-us.apache.org/repos/asf/hbase/blob/79b5085d/hbase-native-client/connection/client-dispatcher.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-dispatcher.h b/hbase-native-client/connection/client-dispatcher.h
index 826fc6a..4435a1b 100644
--- a/hbase-native-client/connection/client-dispatcher.h
+++ b/hbase-native-client/connection/client-dispatcher.h
@@ -19,8 +19,12 @@
#pragma once
+#include <folly/AtomicHashMap.h>
+#include <folly/Logging.h>
#include <wangle/service/ClientDispatcher.h>
+#include <atomic>
+
#include "connection/pipeline.h"
#include "connection/request.h"
#include "connection/response.h"
@@ -31,13 +35,16 @@ class ClientDispatcher
std::unique_ptr<Request>, Response> {
public:
ClientDispatcher();
+ ~ClientDispatcher() {
+ LOG(ERROR) << "Killing ClientDispatcher call_id = " << current_call_id_;
+ }
void read(Context *ctx, Response in) override;
folly::Future<Response> operator()(std::unique_ptr<Request> arg) override;
folly::Future<folly::Unit> close(Context *ctx) override;
folly::Future<folly::Unit> close() override;
private:
- std::unordered_map<uint32_t, folly::Promise<Response>> requests_;
+ folly::AtomicHashMap<uint32_t, folly::Promise<Response>> requests_;
// Start at some number way above what could
// be there for un-initialized call id counters.
//
@@ -46,6 +53,6 @@ private:
//
// uint32_t has a max of 4Billion so 10 more or less is
// not a big deal.
- uint32_t current_call_id_;
+ std::atomic<uint32_t> current_call_id_;
};
} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/79b5085d/hbase-native-client/connection/client-handler.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-handler.cc b/hbase-native-client/connection/client-handler.cc
index 496e4f2..b92ad89 100644
--- a/hbase-native-client/connection/client-handler.cc
+++ b/hbase-native-client/connection/client-handler.cc
@@ -37,7 +37,10 @@ using hbase::pb::GetResponse;
using google::protobuf::Message;
ClientHandler::ClientHandler(std::string user_name)
- : user_name_(user_name), need_send_header_(true), serde_(), resp_msgs_() {}
+ : user_name_(user_name), need_send_header_(true), serde_(),
+ resp_msgs_(
+ make_unique<folly::AtomicHashMap<
+ uint32_t, std::shared_ptr<google::protobuf::Message>>>(5000)) {}
void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
if (LIKELY(buf != nullptr)) {
@@ -51,14 +54,14 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
<< " has_exception=" << header.has_exception();
// Get the response protobuf from the map
- auto search = resp_msgs_.find(header.call_id());
+ auto search = resp_msgs_->find(header.call_id());
// It's an error if it's not there.
- CHECK(search != resp_msgs_.end());
+ CHECK(search != resp_msgs_->end());
auto resp_msg = search->second;
CHECK(resp_msg != nullptr);
// Make sure we don't leak the protobuf
- resp_msgs_.erase(search);
+ resp_msgs_->erase(header.call_id());
// set the call_id.
// This will be used to by the dispatcher to match up
@@ -96,7 +99,7 @@ Future<Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
ctx->fireWrite(std::move(pre));
}
- resp_msgs_[r->call_id()] = r->resp_msg();
+ resp_msgs_->insert(r->call_id(), r->resp_msg());
return ctx->fireWrite(
serde_.Request(r->call_id(), r->method(), r->req_msg().get()));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/79b5085d/hbase-native-client/connection/client-handler.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-handler.h b/hbase-native-client/connection/client-handler.h
index ce99c9e..be5143c 100644
--- a/hbase-native-client/connection/client-handler.h
+++ b/hbase-native-client/connection/client-handler.h
@@ -18,6 +18,7 @@
*/
#pragma once
+#include <folly/AtomicHashMap.h>
#include <wangle/channel/Handler.h>
#include <string>
@@ -51,7 +52,8 @@ private:
RpcSerde serde_;
// in flight requests
- std::unordered_map<uint32_t, std::shared_ptr<google::protobuf::Message>>
+ std::unique_ptr<folly::AtomicHashMap<
+ uint32_t, std::shared_ptr<google::protobuf::Message>>>
resp_msgs_;
};
} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/79b5085d/hbase-native-client/connection/connection-factory.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-factory.cc b/hbase-native-client/connection/connection-factory.cc
index 9102d60..635d12d 100644
--- a/hbase-native-client/connection/connection-factory.cc
+++ b/hbase-native-client/connection/connection-factory.cc
@@ -19,40 +19,36 @@
#include "connection/connection-factory.h"
-#include <folly/futures/Future.h>
-#include <wangle/bootstrap/ClientBootstrap.h>
-#include <wangle/channel/AsyncSocketHandler.h>
-#include <wangle/channel/EventBaseHandler.h>
-#include <wangle/channel/OutputBufferingHandler.h>
-#include <wangle/service/ClientDispatcher.h>
-#include <wangle/service/CloseOnReleaseFilter.h>
-#include <wangle/service/ExpiringFilter.h>
-
-#include <string>
+#include <wangle/concurrent/GlobalExecutor.h>
#include "connection/client-dispatcher.h"
#include "connection/pipeline.h"
-#include "connection/request.h"
-#include "connection/response.h"
#include "connection/service.h"
using namespace folly;
using namespace hbase;
-using namespace wangle;
-ConnectionFactory::ConnectionFactory() : bootstrap_() {
- bootstrap_.group(std::make_shared<wangle::IOThreadPoolExecutor>(1));
- bootstrap_.pipelineFactory(std::make_shared<RpcPipelineFactory>());
-}
+ConnectionFactory::ConnectionFactory()
+ : io_pool_(std::static_pointer_cast<wangle::IOThreadPoolExecutor>(
+ wangle::getIOExecutor())),
+ pipeline_factory_(std::make_shared<RpcPipelineFactory>()) {}
-std::shared_ptr<HBaseService>
-ConnectionFactory::make_connection(const std::string &host, int port) {
- // Connect to a given server
- // Then when connected create a ClientDispactcher.
- auto pipeline = bootstrap_.connect(SocketAddress(host, port, true)).get();
+std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>
+ConnectionFactory::MakeBootstrap() {
+ auto client = std::make_shared<wangle::ClientBootstrap<SerializePipeline>>();
+ client->group(io_pool_);
+ client->pipelineFactory(pipeline_factory_);
+
+ return client;
+}
+std::shared_ptr<HBaseService> ConnectionFactory::Connect(
+ std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> client,
+ const std::string &hostname, int port) {
+ // Yes this will block however it makes dealing with connection pool soooooo
+ // much nicer.
+ // TODO see about using shared promise for this.
+ auto pipeline = client->connect(SocketAddress(hostname, port, true)).get();
auto dispatcher = std::make_shared<ClientDispatcher>();
dispatcher->setPipeline(pipeline);
- auto service = std::make_shared<
- CloseOnReleaseFilter<std::unique_ptr<Request>, Response>>(dispatcher);
- return service;
+ return dispatcher;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/79b5085d/hbase-native-client/connection/connection-factory.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-factory.h b/hbase-native-client/connection/connection-factory.h
index fc4e161..2284a7c 100644
--- a/hbase-native-client/connection/connection-factory.h
+++ b/hbase-native-client/connection/connection-factory.h
@@ -33,10 +33,15 @@ public:
ConnectionFactory();
virtual ~ConnectionFactory() = default;
- virtual std::shared_ptr<HBaseService> make_connection(const std::string &host,
- int port);
+ virtual std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>
+ MakeBootstrap();
+
+ virtual std::shared_ptr<HBaseService>
+ Connect(std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> client,
+ const std::string &hostname, int port);
private:
- wangle::ClientBootstrap<SerializePipeline> bootstrap_;
+ std::shared_ptr<wangle::IOThreadPoolExecutor> io_pool_;
+ std::shared_ptr<RpcPipelineFactory> pipeline_factory_;
};
} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/79b5085d/hbase-native-client/connection/connection-pool-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-pool-test.cc b/hbase-native-client/connection/connection-pool-test.cc
index 975bc5e..b1a0ba0 100644
--- a/hbase-native-client/connection/connection-pool-test.cc
+++ b/hbase-native-client/connection/connection-pool-test.cc
@@ -24,6 +24,7 @@
#include "connection/connection-factory.h"
#include "if/HBase.pb.h"
+#include "serde/server-name.h"
using namespace hbase;
@@ -33,11 +34,16 @@ using ::testing::_;
class MockConnectionFactory : public ConnectionFactory {
public:
- MOCK_METHOD2(make_connection,
- std::shared_ptr<HBaseService>(const std::string &hostname,
- int port));
+ MOCK_METHOD0(MakeBootstrap,
+ std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>());
+ MOCK_METHOD3(Connect,
+ std::shared_ptr<HBaseService>(
+ std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>,
+ const std::string &hostname, int port));
};
+class MockBootstrap : public wangle::ClientBootstrap<SerializePipeline> {};
+
class MockServiceBase : public HBaseService {
public:
folly::Future<Response> operator()(std::unique_ptr<Request> req) override {
@@ -54,19 +60,20 @@ public:
};
TEST(TestConnectionPool, TestOnlyCreateOnce) {
- std::string hostname{"hostname"};
+ auto hostname = std::string{"hostname"};
+ auto mock_boot = std::make_shared<MockBootstrap>();
auto mock_service = std::make_shared<MockService>();
+ auto mock_cf = std::make_shared<MockConnectionFactory>();
uint32_t port{999};
- LOG(ERROR) << "About to make a MockConnectionFactory";
- auto mock_cf = std::make_shared<MockConnectionFactory>();
- EXPECT_CALL((*mock_cf), make_connection(_, _))
+ EXPECT_CALL((*mock_cf), Connect(_, _, _))
.Times(1)
.WillRepeatedly(Return(mock_service));
+ EXPECT_CALL((*mock_cf), MakeBootstrap())
+ .Times(1)
+ .WillRepeatedly(Return(mock_boot));
ConnectionPool cp{mock_cf};
- LOG(ERROR) << "Created ConnectionPool";
-
ServerName sn;
sn.set_host_name(hostname);
sn.set_port(port);
@@ -75,3 +82,32 @@ TEST(TestConnectionPool, TestOnlyCreateOnce) {
ASSERT_TRUE(result != nullptr);
result = cp.get(sn);
}
+
+TEST(TestConnectionPool, TestOnlyCreateMultipleDispose) {
+ std::string hostname_one{"hostname"};
+ std::string hostname_two{"hostname_two"};
+ uint32_t port{999};
+
+ auto mock_boot = std::make_shared<MockBootstrap>();
+ auto mock_service = std::make_shared<MockService>();
+ auto mock_cf = std::make_shared<MockConnectionFactory>();
+
+ EXPECT_CALL((*mock_cf), Connect(_, _, _))
+ .Times(2)
+ .WillRepeatedly(Return(mock_service));
+ EXPECT_CALL((*mock_cf), MakeBootstrap())
+ .Times(2)
+ .WillRepeatedly(Return(mock_boot));
+ ConnectionPool cp{mock_cf};
+
+ {
+ auto result_one = cp.get(folly::to<ServerName>(
+ hostname_one + ":" + folly::to<std::string>(port)));
+ auto result_two = cp.get(folly::to<ServerName>(
+ hostname_two + ":" + folly::to<std::string>(port)));
+ }
+ auto result_one = cp.get(
+ folly::to<ServerName>(hostname_one + ":" + folly::to<std::string>(port)));
+ auto result_two = cp.get(
+ folly::to<ServerName>(hostname_two + ":" + folly::to<std::string>(port)));
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/79b5085d/hbase-native-client/connection/connection-pool.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-pool.cc b/hbase-native-client/connection/connection-pool.cc
index eafe60a..6ed5ad9 100644
--- a/hbase-native-client/connection/connection-pool.cc
+++ b/hbase-native-client/connection/connection-pool.cc
@@ -19,6 +19,7 @@
#include "connection/connection-pool.h"
+#include <folly/SocketAddress.h>
#include <wangle/service/Service.h>
using std::mutex;
@@ -26,28 +27,46 @@ using std::unique_ptr;
using std::shared_ptr;
using hbase::pb::ServerName;
using folly::SharedMutexWritePriority;
+using folly::SocketAddress;
namespace hbase {
ConnectionPool::ConnectionPool()
- : cf_(std::make_shared<ConnectionFactory>()), connections_(), map_mutex_() {
-}
+ : cf_(std::make_shared<ConnectionFactory>()), clients_(), connections_(),
+ map_mutex_() {}
ConnectionPool::ConnectionPool(std::shared_ptr<ConnectionFactory> cf)
- : cf_(cf), connections_(), map_mutex_() {}
+ : cf_(cf), clients_(), connections_(), map_mutex_() {}
+
+ConnectionPool::~ConnectionPool() {
+ SharedMutexWritePriority::WriteHolder holder(map_mutex_);
+ for (auto &item : connections_) {
+ auto &con = item.second;
+ con->close();
+ }
+ connections_.clear();
+ clients_.clear();
+}
std::shared_ptr<HBaseService> ConnectionPool::get(const ServerName &sn) {
+ // Create a read lock.
SharedMutexWritePriority::UpgradeHolder holder(map_mutex_);
+
auto found = connections_.find(sn);
if (found == connections_.end() || found->second == nullptr) {
+ // Move the upgradable lock into the write lock if the connection
+ // hasn't been found.
SharedMutexWritePriority::WriteHolder holder(std::move(holder));
- auto new_con = cf_->make_connection(sn.host_name(), sn.port());
- connections_[sn] = new_con;
- return new_con;
+ auto client = cf_->MakeBootstrap();
+ auto dispatcher = cf_->Connect(client, sn.host_name(), sn.port());
+ clients_.insert(std::make_pair(sn, client));
+ connections_.insert(std::make_pair(sn, dispatcher));
+ return dispatcher;
}
return found->second;
}
+
void ConnectionPool::close(const ServerName &sn) {
- SharedMutexWritePriority::WriteHolder holder(map_mutex_);
+ SharedMutexWritePriority::WriteHolder holder{map_mutex_};
auto found = connections_.find(sn);
if (found == connections_.end() || found->second == nullptr) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/79b5085d/hbase-native-client/connection/connection-pool.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-pool.h b/hbase-native-client/connection/connection-pool.h
index b8330e3..907afdb 100644
--- a/hbase-native-client/connection/connection-pool.h
+++ b/hbase-native-client/connection/connection-pool.h
@@ -37,23 +37,29 @@ struct ServerNameHash {
std::size_t operator()(hbase::pb::ServerName const &s) const {
std::size_t h1 = std::hash<std::string>()(s.host_name());
std::size_t h2 = std::hash<uint32_t>()(s.port());
- return h1 ^ (h2 << 1);
+ return h1 ^ (h2 << 2);
}
};
class ConnectionPool {
public:
ConnectionPool();
+ ~ConnectionPool();
explicit ConnectionPool(std::shared_ptr<ConnectionFactory> cf);
std::shared_ptr<HBaseService> get(const hbase::pb::ServerName &sn);
void close(const hbase::pb::ServerName &sn);
private:
- std::shared_ptr<ConnectionFactory> cf_;
std::unordered_map<hbase::pb::ServerName, std::shared_ptr<HBaseService>,
ServerNameHash, ServerNameEquals>
connections_;
+ std::unordered_map<
+ hbase::pb::ServerName,
+ std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>,
+ ServerNameHash, ServerNameEquals>
+ clients_;
folly::SharedMutexWritePriority map_mutex_;
+ std::shared_ptr<ConnectionFactory> cf_;
};
} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/79b5085d/hbase-native-client/core/client.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client.cc b/hbase-native-client/core/client.cc
index 266c239..4b9f844 100644
--- a/hbase-native-client/core/client.cc
+++ b/hbase-native-client/core/client.cc
@@ -33,8 +33,4 @@ using namespace folly;
using namespace std;
using namespace hbase::pb;
-namespace hbase {
-
-Client::Client(string quorum_spec)
- : location_cache_(quorum_spec, wangle::getCPUExecutor()) {}
-}
+namespace hbase {}
http://git-wip-us.apache.org/repos/asf/hbase/blob/79b5085d/hbase-native-client/core/location-cache.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
index 2667f11..e2a6251 100644
--- a/hbase-native-client/core/location-cache.cc
+++ b/hbase-native-client/core/location-cache.cc
@@ -25,8 +25,8 @@
#include "connection/response.h"
#include "if/Client.pb.h"
#include "if/ZooKeeper.pb.h"
-#include "serde/server-name.h"
#include "serde/region-info.h"
+#include "serde/server-name.h"
#include "serde/zk.h"
using namespace std;
@@ -109,17 +109,22 @@ ServerName LocationCache::ReadMetaLocation() {
Future<std::shared_ptr<RegionLocation>>
LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
- auto exc = wangle::getIOExecutor();
+ auto exec = wangle::getCPUExecutor();
return this->LocateMeta()
- .then([&](ServerName sn) { return this->cp_.get(sn); })
- .via(exc.get()) // Need to handle all rpc's on the IOExecutor.
+ .via(exec.get())
+ .then([ exec = exec, this ](ServerName sn) { return this->cp_.get(sn); })
.then([&](std::shared_ptr<HBaseService> service) {
return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
})
- .then([&](Response resp) {
+ .then([this](Response resp) {
// take the protobuf response and make it into
// a region location.
return this->CreateLocation(std::move(resp));
+ })
+ .then([ exec = exec, this ](std::shared_ptr<RegionLocation> rl) {
+ // Now fill out the connection.
+ rl->set_service(cp_.get(rl->server_name()));
+ return rl;
});
}
@@ -162,16 +167,12 @@ private:
};
std::shared_ptr<RegionLocation>
-LocationCache::CreateLocation(const Response &resp){
+LocationCache::CreateLocation(const Response &resp) {
auto resp_msg = static_pointer_cast<ScanResponse>(resp.response());
auto &results = resp_msg->results().Get(0);
auto &cells = results.cell();
- LOG(ERROR) << "resp_msg = " << resp_msg->DebugString();
auto ri = folly::to<RegionInfo>(cells.Get(0).value());
auto sn = folly::to<ServerName>(cells.Get(1).value());
-
- LOG(ERROR) << "RegionInfo = " << ri.DebugString();
- LOG(ERROR) << "ServerName = " << sn.DebugString();
- auto wrapped = make_shared<RemoveServiceFilter>(cp_.get(sn), sn, this->cp_);
- return std::make_shared<RegionLocation>(std::move(ri), std::move(sn), wrapped);
+ return std::make_shared<RegionLocation>(cells.Get(0).row(), std::move(ri), sn,
+ nullptr);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/79b5085d/hbase-native-client/core/location-cache.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h
index 99b5e5e..7f76428 100644
--- a/hbase-native-client/core/location-cache.h
+++ b/hbase-native-client/core/location-cache.h
@@ -48,9 +48,10 @@ public:
// Meta Related Methods.
// These are only public until testing is complete
folly::Future<hbase::pb::ServerName> LocateMeta();
- folly::Future<std::shared_ptr<RegionLocation>> LocateFromMeta(const hbase::pb::TableName &tn,
- const std::string &row);
+ folly::Future<std::shared_ptr<RegionLocation>>
+ LocateFromMeta(const hbase::pb::TableName &tn, const std::string &row);
void InvalidateMeta();
+ ConnectionPool cp_;
private:
void RefreshMetaLocation();
@@ -61,7 +62,6 @@ private:
std::shared_ptr<folly::Executor> executor_;
std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> meta_promise_;
std::mutex meta_lock_;
- ConnectionPool cp_;
MetaUtil meta_util_;
// TODO: migrate this to a smart pointer with a deleter.
http://git-wip-us.apache.org/repos/asf/hbase/blob/79b5085d/hbase-native-client/core/meta-utils.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/meta-utils.cc b/hbase-native-client/core/meta-utils.cc
index 1325d83..23d2041 100644
--- a/hbase-native-client/core/meta-utils.cc
+++ b/hbase-native-client/core/meta-utils.cc
@@ -37,12 +37,12 @@ using hbase::pb::RegionSpecifier_RegionSpecifierType;
static const std::string META_REGION = "1588230740";
std::string MetaUtil::RegionLookupRowkey(const TableName &tn,
- const std::string &row) const {
+ const std::string &row) const {
return folly::to<std::string>(tn, ",", row, ",", "999999999999999999");
}
-std::unique_ptr<Request>
-MetaUtil::MetaRequest(const TableName tn, const std::string &row) const {
+std::unique_ptr<Request> MetaUtil::MetaRequest(const TableName tn,
+ const std::string &row) const {
auto request = Request::scan();
auto msg = std::static_pointer_cast<ScanRequest>(request->req_msg());
http://git-wip-us.apache.org/repos/asf/hbase/blob/79b5085d/hbase-native-client/core/meta-utils.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/meta-utils.h b/hbase-native-client/core/meta-utils.h
index 5a659f3..dfef065 100644
--- a/hbase-native-client/core/meta-utils.h
+++ b/hbase-native-client/core/meta-utils.h
@@ -29,8 +29,8 @@ namespace hbase {
class MetaUtil {
public:
std::string RegionLookupRowkey(const hbase::pb::TableName &tn,
- const std::string &row) const;
+ const std::string &row) const;
std::unique_ptr<Request> MetaRequest(const hbase::pb::TableName tn,
- const std::string &row) const;
+ const std::string &row) const;
};
} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/79b5085d/hbase-native-client/core/region-location.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/region-location.h b/hbase-native-client/core/region-location.h
index 7922c95..7887526 100644
--- a/hbase-native-client/core/region-location.h
+++ b/hbase-native-client/core/region-location.h
@@ -27,15 +27,19 @@ namespace hbase {
class RegionLocation {
public:
- RegionLocation(hbase::pb::RegionInfo ri, hbase::pb::ServerName sn,
+ RegionLocation(std::string region_name, hbase::pb::RegionInfo ri,
+ hbase::pb::ServerName sn,
std::shared_ptr<HBaseService> service)
- : ri_(ri), sn_(sn), service_(service) {}
+ : region_name_(region_name), ri_(ri), sn_(sn), service_(service) {}
const hbase::pb::RegionInfo ®ion_info() { return ri_; }
const hbase::pb::ServerName &server_name() { return sn_; }
+ const std::string ®ion_name() { return region_name_; }
std::shared_ptr<HBaseService> service() { return service_; }
+ void set_service(std::shared_ptr<HBaseService> s) { service_ = s; }
private:
+ std::string region_name_;
hbase::pb::RegionInfo ri_;
hbase::pb::ServerName sn_;
std::shared_ptr<HBaseService> service_;
http://git-wip-us.apache.org/repos/asf/hbase/blob/79b5085d/hbase-native-client/core/simple-client.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/simple-client.cc b/hbase-native-client/core/simple-client.cc
index 00e3369..39c82c3 100644
--- a/hbase-native-client/core/simple-client.cc
+++ b/hbase-native-client/core/simple-client.cc
@@ -19,16 +19,21 @@
#include <folly/Logging.h>
#include <folly/Random.h>
+#include <folly/futures/Future.h>
#include <gflags/gflags.h>
+#include <wangle/concurrent/CPUThreadPoolExecutor.h>
#include <wangle/concurrent/GlobalExecutor.h>
+#include <atomic>
#include <chrono>
#include <iostream>
+#include <thread>
#include "connection/connection-pool.h"
#include "core/client.h"
#include "if/Client.pb.h"
#include "if/ZooKeeper.pb.h"
+#include "serde/server-name.h"
#include "serde/table-name.h"
using namespace folly;
@@ -39,16 +44,41 @@ using hbase::Request;
using hbase::HBaseService;
using hbase::LocationCache;
using hbase::ConnectionPool;
+using hbase::ConnectionFactory;
using hbase::pb::TableName;
using hbase::pb::ServerName;
using hbase::pb::RegionSpecifier_RegionSpecifierType;
-using hbase::pb::GetRequest;
-using hbase::pb::GetResponse;
+using hbase::pb::MutateRequest;
+using hbase::pb::MutationProto_MutationType;
// TODO(eclark): remove the need for this.
DEFINE_string(table, "t", "What region to send a get");
DEFINE_string(row, "test", "What row to get");
DEFINE_string(zookeeper, "localhost:2181", "What zk quorum to talk to");
+DEFINE_uint64(columns, 10000, "How many columns to write");
+DEFINE_int32(threads, 6, "How many cpu threads");
+
+std::unique_ptr<Request> MakeRequest(uint64_t col, std::string region_name) {
+ auto req = Request::mutate();
+ auto msg = std::static_pointer_cast<MutateRequest>(req->req_msg());
+ auto region = msg->mutable_region();
+ auto suf = folly::to<std::string>(col);
+
+ region->set_value(region_name);
+ region->set_type(RegionSpecifier_RegionSpecifierType::
+ RegionSpecifier_RegionSpecifierType_REGION_NAME);
+ auto mutation = msg->mutable_mutation();
+ mutation->set_row(FLAGS_row + suf);
+ mutation->set_mutate_type(
+ MutationProto_MutationType::MutationProto_MutationType_PUT);
+ auto column = mutation->add_column_value();
+ column->set_family("d");
+ auto qual = column->add_qualifier_value();
+ qual->set_qualifier(suf);
+ qual->set_value(".");
+
+ return std::move(req);
+}
int main(int argc, char *argv[]) {
google::SetUsageMessage(
@@ -56,13 +86,41 @@ int main(int argc, char *argv[]) {
google::ParseCommandLineFlags(&argc, &argv, true);
google::InitGoogleLogging(argv[0]);
- // Create a connection factory
- ConnectionPool cp;
- auto cpu_ex = wangle::getCPUExecutor();
- LocationCache cache{FLAGS_zookeeper, cpu_ex};
- auto result =
- cache.LocateFromMeta(folly::to<TableName>(FLAGS_table), FLAGS_row)
- .get(milliseconds(5000));
+ // Set up thread pools.
+ auto cpu_pool =
+ std::make_shared<wangle::CPUThreadPoolExecutor>(FLAGS_threads);
+ wangle::setCPUExecutor(cpu_pool);
+ auto io_pool = std::make_shared<wangle::IOThreadPoolExecutor>(5);
+ wangle::setIOExecutor(io_pool);
+
+ // Create the cache.
+ LocationCache cache{FLAGS_zookeeper, cpu_pool};
+
+ auto row = FLAGS_row;
+ auto tn = folly::to<TableName>(FLAGS_table);
+
+ auto loc = cache.LocateFromMeta(tn, row).get(milliseconds(5000));
+ auto connection = loc->service();
+
+ auto num_puts = FLAGS_columns;
+
+ auto results = std::vector<Future<Response>>{};
+ uint64_t col{0};
+ for (; col < num_puts; col++) {
+ results.push_back(folly::makeFuture(col)
+ .via(cpu_pool.get())
+ .then([loc](uint64_t col) {
+ return MakeRequest(col, loc->region_name());
+ })
+ .then([connection](std::unique_ptr<Request> req) {
+ return (*connection)(std::move(req));
+ }));
+ }
+ auto allf = folly::collect(results).get();
+
+ LOG(ERROR) << "Successfully sent " << allf.size() << " requests.";
+
+ io_pool->stop();
return 0;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/79b5085d/hbase-native-client/serde/region-info-deserializer-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/region-info-deserializer-test.cc b/hbase-native-client/serde/region-info-deserializer-test.cc
index ce8dedf..5cb8482 100644
--- a/hbase-native-client/serde/region-info-deserializer-test.cc
+++ b/hbase-native-client/serde/region-info-deserializer-test.cc
@@ -44,7 +44,6 @@ TEST(TestRegionInfoDesializer, TestDeserialize) {
ri_out.set_start_key(start_row);
ri_out.set_end_key(stop_row);
-
string header{"PBUF"};
string ser = header + ri_out.SerializeAsString();
http://git-wip-us.apache.org/repos/asf/hbase/blob/79b5085d/hbase-native-client/serde/region-info.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/region-info.h b/hbase-native-client/serde/region-info.h
index 6af351c..e2ecfc9 100644
--- a/hbase-native-client/serde/region-info.h
+++ b/hbase-native-client/serde/region-info.h
@@ -21,16 +21,16 @@
#include "if/HBase.pb.h"
-#include <folly/Conv.h>
#include <boost/algorithm/string/predicate.hpp>
+#include <folly/Conv.h>
namespace hbase {
namespace pb {
-template <class String> void parseTo(String in, RegionInfo& out) {
+template <class String> void parseTo(String in, RegionInfo &out) {
// TODO(eclark): there has to be something better.
std::string s = folly::to<std::string>(in);
- if (!boost::starts_with(s, "PBUF") ) {
+ if (!boost::starts_with(s, "PBUF")) {
throw std::runtime_error("Region Info field doesn't contain preamble");
}
if (!out.ParseFromArray(s.data() + 4, s.size() - 4)) {
[26/50] [abbrv] hbase git commit: HBASE-14855 Connect to regionserver
Posted by ec...@apache.org.
HBASE-14855 Connect to regionserver
Summary:
* Client dispatcher to put request and response together with call id. Though right now this is a pretty empty version of that.
* pipeline. The Serialization pipeline.
* client-serilization-handler.h This does the work of sending the preamable, the connection header, and length prepending data. This will obviously need to be split up into h and cc files
Test Plan: Use simple-client to connect to a standalone server
Differential Revision: https://reviews.facebook.net/D56385
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b23e8630
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b23e8630
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b23e8630
Branch: refs/heads/HBASE-14850
Commit: b23e8630cfe9148c858c4860e64e1693db0d8bd5
Parents: bbeaacb
Author: Elliott Clark <ec...@apache.org>
Authored: Mon Apr 4 13:52:05 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/core/BUCK | 30 +++++-
hbase-native-client/core/client-dispatcher.cc | 54 ++++++++++
hbase-native-client/core/client-dispatcher.h | 42 ++++++++
.../core/client-serialize-handler.cc | 104 ++++++++++++++++++
.../core/client-serialize-handler.h | 44 ++++++++
hbase-native-client/core/client.cc | 16 ++-
hbase-native-client/core/client.h | 18 +++-
hbase-native-client/core/connection-factory.cc | 57 ++++++++++
hbase-native-client/core/connection-factory.h | 41 ++++++++
hbase-native-client/core/get-request.cc | 19 ++++
hbase-native-client/core/get-request.h | 35 +++++++
hbase-native-client/core/get-result.cc | 19 ++++
hbase-native-client/core/get-result.h | 32 ++++++
hbase-native-client/core/location-cache-test.cc | 25 ++++-
hbase-native-client/core/location-cache.cc | 62 +++++++++--
hbase-native-client/core/location-cache.h | 29 ++++-
.../core/native-client-test-env.cc | 6 +-
hbase-native-client/core/pipeline.cc | 42 ++++++++
hbase-native-client/core/pipeline.h | 34 ++++++
hbase-native-client/core/request.h | 33 ++++++
hbase-native-client/core/response.h | 34 ++++++
hbase-native-client/core/service.h | 26 +++++
hbase-native-client/core/simple-client.cc | 57 ++++++++++
.../core/simple-native-client-test.cc | 5 +-
hbase-native-client/core/table-name.cc | 19 ++++
hbase-native-client/core/table-name.h | 32 ++++++
hbase-native-client/if/BUCK | 24 ++++-
hbase-native-client/third-party/BUCK | 105 +++++++++----------
28 files changed, 948 insertions(+), 96 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index d1e89d1..2b00d66 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -16,35 +16,52 @@
# limitations under the License.
cxx_library(name="core",
- headers=[
+ exported_headers=[
"admin.h",
+ "client-dispatcher.h",
+ "client-serialize-handler.h",
"client.h",
+ "connection-factory.h",
"connection.h",
"connection_attr.h",
"delete.h",
+ "get-request.h",
+ "get-result.h",
"get.h",
"hbase_macros.h",
+ "location-cache.h",
"mutation.h",
+ "pipeline.h",
"put.h",
+ "request.h",
+ "response.h",
"scanner.h",
- "location-cache.h",
+ "service.h",
+ "table-name.h",
],
srcs=[
"admin.cc",
+ "client-dispatcher.cc",
+ "client-serialize-handler.cc",
"client.cc",
+ "connection-factory.cc",
"connection.cc",
+ "delete.cc",
+ "get-request.cc",
+ "get-result.cc",
"get.cc",
+ "location-cache.cc",
"mutation.cc",
+ "pipeline.cc",
"put.cc",
- "delete.cc",
"scanner.cc",
- "location-cache.cc",
+ "table-name.cc",
],
deps=[
"//if:if",
- "//third-party:zookeeper_mt",
"//third-party:folly",
"//third-party:wangle",
+ "//third-party:zookeeper_mt",
],
visibility=[
'PUBLIC',
@@ -68,3 +85,6 @@ cxx_test(name="location-cache-test",
":core",
],
run_test_separately=True, )
+cxx_binary(name="simple-client",
+ srcs=["simple-client.cc", ],
+ deps=[":core", ], )
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/client-dispatcher.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client-dispatcher.cc b/hbase-native-client/core/client-dispatcher.cc
new file mode 100644
index 0000000..d356759
--- /dev/null
+++ b/hbase-native-client/core/client-dispatcher.cc
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include "core/client-dispatcher.h"
+
+using namespace folly;
+using namespace hbase;
+using namespace wangle;
+
+void ClientDispatcher::read(Context *ctx, Response in) {
+ auto call_id = in.call_id();
+ auto search = requests_.find(call_id);
+ CHECK(search != requests_.end());
+ auto p = std::move(search->second);
+ requests_.erase(call_id);
+
+ // TODO(eclark): check if the response
+ // is an exception. If it is then set that.
+ p.setValue(in);
+}
+
+Future<Response> ClientDispatcher::operator()(Request arg) {
+ auto call_id = ++current_call_id_;
+ arg.set_call_id(call_id);
+ auto &p = requests_[call_id];
+ auto f = p.getFuture();
+ p.setInterruptHandler([call_id, this](const folly::exception_wrapper &e) {
+ this->requests_.erase(call_id);
+ });
+ this->pipeline_->write(arg);
+
+ return f;
+}
+
+Future<Unit> ClientDispatcher::close() { return ClientDispatcherBase::close(); }
+
+Future<Unit> ClientDispatcher::close(Context *ctx) {
+ return ClientDispatcherBase::close(ctx);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/client-dispatcher.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client-dispatcher.h b/hbase-native-client/core/client-dispatcher.h
new file mode 100644
index 0000000..4b9d35a
--- /dev/null
+++ b/hbase-native-client/core/client-dispatcher.h
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <wangle/service/ClientDispatcher.h>
+
+#include "core/pipeline.h"
+#include "core/request.h"
+#include "core/response.h"
+
+namespace hbase {
+class ClientDispatcher
+ : public wangle::ClientDispatcherBase<SerializePipeline, Request,
+ Response> {
+public:
+ void read(Context *ctx, Response in) override;
+ folly::Future<Response> operator()(Request arg) override;
+ folly::Future<folly::Unit> close(Context *ctx) override;
+ folly::Future<folly::Unit> close() override;
+
+private:
+ std::unordered_map<int32_t, folly::Promise<Response>> requests_;
+ uint32_t current_call_id_ = 1;
+};
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/client-serialize-handler.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client-serialize-handler.cc b/hbase-native-client/core/client-serialize-handler.cc
new file mode 100644
index 0000000..cad1308
--- /dev/null
+++ b/hbase-native-client/core/client-serialize-handler.cc
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/client-serialize-handler.h"
+
+#include <string>
+
+using namespace hbase;
+using namespace folly;
+using namespace wangle;
+
+static const std::string PREAMBLE = "HBas";
+static const std::string INTERFACE = "ClientService";
+static const uint8_t RPC_VERSION = 0;
+static const uint8_t AUTH_TYPE = 80;
+
+// TODO(eclark): Make this actually do ANYTHING.
+void ClientSerializeHandler::read(Context *ctx, std::unique_ptr<IOBuf> msg) {
+ Response received;
+ ctx->fireRead(received);
+}
+
+Future<Unit> ClientSerializeHandler::write(Context *ctx, Request r) {
+ // Keep track of if we have sent the header.
+ if (need_send_header_) {
+ need_send_header_ = false;
+
+ // Should this be replacing the IOBuf rather than
+ // sending several different calls?
+ write_preamble(ctx);
+ write_header(ctx);
+ }
+
+ // Send out the actual request and not just a test string.
+ std::string out{"test"};
+ return ctx->fireWrite(prepend_length(IOBuf::copyBuffer(out)));
+}
+
+Future<Unit> ClientSerializeHandler::write_preamble(Context *ctx) {
+ auto magic = IOBuf::copyBuffer(PREAMBLE);
+ auto buf = IOBuf::create(2);
+ buf->append(2);
+ folly::io::RWPrivateCursor c(buf.get());
+
+ // Version
+ c.write(RPC_VERSION);
+ // Standard security aka Please don't lie to me.
+ c.write(AUTH_TYPE);
+ magic->appendChain(std::move(buf));
+ return ctx->fireWrite(std::move(magic));
+}
+
+Future<Unit> ClientSerializeHandler::write_header(Context *ctx) {
+ pb::ConnectionHeader h;
+
+ // TODO(eclark): Make this not a total lie.
+ h.mutable_user_info()->set_effective_user("elliott");
+ // The service name that we want to talk to.
+ //
+ // Right now we're completely ignoring the service interface.
+ // That may or may not be the correct thing to do.
+ // It worked for a while with the java client; until it
+ // didn't.
+ h.set_service_name(INTERFACE);
+ // TODO(eclark): Make this 1 copy.
+ auto msg = IOBuf::copyBuffer(h.SerializeAsString());
+ return ctx->fireWrite(prepend_length(std::move(msg)));
+}
+
+// Our own simple version of LengthFieldPrepender
+std::unique_ptr<IOBuf>
+ClientSerializeHandler::prepend_length(std::unique_ptr<IOBuf> msg) {
+ // Java ints are 4 long. So create a buffer that large
+ auto len_buf = IOBuf::create(4);
+ // Then make those bytes visible.
+ len_buf->append(4);
+
+ io::RWPrivateCursor c(len_buf.get());
+ // Get the size of the data to be pushed out the network.
+ auto size = msg->computeChainDataLength();
+
+ // Write the length to this IOBuf.
+ c.writeBE(static_cast<uint32_t>(size));
+
+ // Then attach the origional to the back of len_buf
+ len_buf->appendChain(std::move(msg));
+ return len_buf;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/client-serialize-handler.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client-serialize-handler.h b/hbase-native-client/core/client-serialize-handler.h
new file mode 100644
index 0000000..961a03b
--- /dev/null
+++ b/hbase-native-client/core/client-serialize-handler.h
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <wangle/channel/Handler.h>
+
+#include "if/HBase.pb.h"
+#include "if/RPC.pb.h"
+#include "core/request.h"
+#include "core/response.h"
+
+namespace hbase {
+class ClientSerializeHandler
+ : public wangle::Handler<std::unique_ptr<folly::IOBuf>, Response, Request,
+ std::unique_ptr<folly::IOBuf>> {
+public:
+ void read(Context *ctx, std::unique_ptr<folly::IOBuf> msg) override;
+ folly::Future<folly::Unit> write(Context *ctx, Request r) override;
+
+private:
+ folly::Future<folly::Unit> write_preamble(Context *ctx);
+ folly::Future<folly::Unit> write_header(Context *ctx);
+ // Our own simple version of LengthFieldPrepender
+ std::unique_ptr<folly::IOBuf>
+ prepend_length(std::unique_ptr<folly::IOBuf> msg);
+ bool need_send_header_ = true;
+};
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/client.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client.cc b/hbase-native-client/core/client.cc
index a04daee..893894f 100644
--- a/hbase-native-client/core/client.cc
+++ b/hbase-native-client/core/client.cc
@@ -23,20 +23,18 @@
#include <folly/Random.h>
#include <glog/logging.h>
#include <gflags/gflags.h>
+#include <wangle/concurrent/GlobalExecutor.h>
+
+#include <string>
#include "if/ZooKeeper.pb.h"
using namespace folly;
+using namespace std;
using namespace hbase::pb;
-int main(int argc, char *argv[]) {
- MetaRegionServer mrs;
- google::ParseCommandLineFlags(&argc, &argv, true);
- google::InitGoogleLogging(argv[0]);
+namespace hbase {
- FB_LOG_EVERY_MS(INFO, 10000) << "Hello";
- for (long i = 0; i < 10000000; i++) {
- FB_LOG_EVERY_MS(INFO, 1) << Random::rand32();
- }
- return 0;
+Client::Client(string quorum_spec)
+ : location_cache(quorum_spec, wangle::getCPUExecutor()) {}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/client.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client.h b/hbase-native-client/core/client.h
index 35a3bd8..818bc6b 100644
--- a/hbase-native-client/core/client.h
+++ b/hbase-native-client/core/client.h
@@ -20,7 +20,23 @@
#pragma once
#include <folly/io/IOBuf.h>
+#include <folly/futures/Future.h>
+#include <string>
+
+#include "core/get-request.h"
+#include "core/get-result.h"
+#include "core/location-cache.h"
#include "if/Cell.pb.h"
-class Client {};
+namespace hbase {
+class Client {
+public:
+ explicit Client(std::string quorum_spec);
+ folly::Future<GetResult> get(const GetRequest &get_request);
+
+private:
+ LocationCache location_cache;
+};
+
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/connection-factory.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/connection-factory.cc b/hbase-native-client/core/connection-factory.cc
new file mode 100644
index 0000000..785b239
--- /dev/null
+++ b/hbase-native-client/core/connection-factory.cc
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/connection-factory.h"
+
+#include <wangle/channel/AsyncSocketHandler.h>
+#include <wangle/channel/EventBaseHandler.h>
+#include <wangle/channel/OutputBufferingHandler.h>
+#include <wangle/service/ClientDispatcher.h>
+#include <wangle/service/ExpiringFilter.h>
+#include <folly/futures/Future.h>
+
+#include <string>
+
+#include "core/client-dispatcher.h"
+#include "core/pipeline.h"
+#include "core/request.h"
+#include "core/response.h"
+#include "core/service.h"
+
+using namespace folly;
+using namespace hbase;
+using namespace wangle;
+
+ConnectionFactory::ConnectionFactory() {
+ bootstrap_.group(std::make_shared<wangle::IOThreadPoolExecutor>(2));
+ bootstrap_.pipelineFactory(std::make_shared<RpcPipelineFactory>());
+}
+
+Future<ClientDispatcher> ConnectionFactory::make_connection(std::string host,
+ int port) {
+ // Connect to a given server
+ // Then when connected create a ClientDispactcher.
+ auto srv = bootstrap_.connect(SocketAddress(host, port, true))
+ .then([](SerializePipeline *pipeline) {
+ ClientDispatcher dispatcher;
+ dispatcher.setPipeline(pipeline);
+ return dispatcher;
+ });
+ return srv;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/connection-factory.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/connection-factory.h b/hbase-native-client/core/connection-factory.h
new file mode 100644
index 0000000..6f450c2
--- /dev/null
+++ b/hbase-native-client/core/connection-factory.h
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <wangle/bootstrap/ClientBootstrap.h>
+#include <wangle/service/Service.h>
+
+#include <string>
+
+#include "core/service.h"
+#include "core/pipeline.h"
+#include "core/client-dispatcher.h"
+#include "core/request.h"
+#include "core/response.h"
+
+namespace hbase {
+class ConnectionFactory {
+public:
+ ConnectionFactory();
+ folly::Future<ClientDispatcher> make_connection(std::string host, int port);
+
+private:
+ wangle::ClientBootstrap<SerializePipeline> bootstrap_;
+};
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/get-request.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/get-request.cc b/hbase-native-client/core/get-request.cc
new file mode 100644
index 0000000..e927ccc
--- /dev/null
+++ b/hbase-native-client/core/get-request.cc
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include "core/get-request.h"
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/get-request.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/get-request.h b/hbase-native-client/core/get-request.h
new file mode 100644
index 0000000..c9113ad
--- /dev/null
+++ b/hbase-native-client/core/get-request.h
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <string>
+
+#include "core/table-name.h"
+
+namespace hbase {
+
+class GetRequest {
+public:
+ GetRequest(TableName table_name, std::string key);
+
+private:
+ TableName table_name_;
+ std::string key_;
+};
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/get-result.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/get-result.cc b/hbase-native-client/core/get-result.cc
new file mode 100644
index 0000000..7eea483
--- /dev/null
+++ b/hbase-native-client/core/get-result.cc
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include "core/get-result.h"
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/get-result.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/get-result.h b/hbase-native-client/core/get-result.h
new file mode 100644
index 0000000..e021316
--- /dev/null
+++ b/hbase-native-client/core/get-result.h
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <string>
+
+namespace hbase {
+
+class GetResult {
+public:
+ explicit GetResult(std::string key);
+
+private:
+ std::string key_;
+};
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/location-cache-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache-test.cc b/hbase-native-client/core/location-cache-test.cc
index 3106e36..70ca6f1 100644
--- a/hbase-native-client/core/location-cache-test.cc
+++ b/hbase-native-client/core/location-cache-test.cc
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
#include <gtest/gtest.h>
#include <folly/Memory.h>
#include <wangle/concurrent/GlobalExecutor.h>
@@ -8,7 +26,8 @@ using namespace hbase;
TEST(LocationCacheTest, TestGetMetaNodeContents) {
// TODO(elliott): need to make a test utility for this.
LocationCache cache{"localhost:2181", wangle::getCPUExecutor()};
- auto result = cache.LocateMeta();
- result.wait();
- ASSERT_FALSE(result.hasException());
+ auto f = cache.LocateMeta();
+ auto result = f.get();
+ ASSERT_FALSE(f.hasException());
+ ASSERT_TRUE(result.has_port());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/location-cache.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
index cf61e24..34e3236 100644
--- a/hbase-native-client/core/location-cache.cc
+++ b/hbase-native-client/core/location-cache.cc
@@ -1,3 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
#include "location-cache.h"
#include <folly/Logging.h>
@@ -7,11 +25,10 @@
using namespace std;
using namespace folly;
using namespace hbase::pb;
+using namespace hbase;
-namespace hbase {
-
-// TODO(elliott): make this configurable on client creation
-const static string META_LOCATION = "/hbase/meta-region-server";
+// TODO(eclark): make this configurable on client creation
+static const char META_LOCATION[] = "/hbase/meta-region-server";
LocationCache::LocationCache(string quorum_spec,
shared_ptr<folly::Executor> executor)
@@ -50,18 +67,45 @@ void LocationCache::RefreshMetaLocation() {
ServerName LocationCache::ReadMetaLocation() {
char contents[4096];
+ // This needs to be int rather than size_t as that's what ZK expects.
int len = sizeof(contents);
// TODO(elliott): handle disconnects/reconntion as needed.
int zk_result =
- zoo_get(this->zk_, META_LOCATION.c_str(), 0, contents, &len, nullptr);
-
- if (zk_result != ZOK) {
+ zoo_get(this->zk_, META_LOCATION, 0, contents, &len, nullptr);
+ if (zk_result != ZOK || len < 9) {
LOG(ERROR) << "Error getting meta location.";
throw runtime_error("Error getting meta location");
}
+ // There should be a magic number for recoverable zk
+ if (static_cast<uint8_t>(contents[0]) != 255) {
+ LOG(ERROR) << "Magic number not in ZK znode data expected 255 got ="
+ << unsigned(static_cast<uint8_t>(contents[0]));
+ throw runtime_error("Magic number not in znode data");
+ }
+ // pos will keep track of skipped bytes.
+ int pos = 1;
+ // How long is the id?
+ int id_len = 0;
+ for (int i = 0; i < 4; ++i) {
+ id_len = id_len << 8;
+ id_len = id_len | static_cast<uint8_t>(contents[pos]);
+ ++pos;
+ }
+ // Skip the id
+ pos += id_len;
+ // Then all protobuf's for HBase are prefixed with a magic string.
+ // PBUF, so we skip that.
+ // TODO(eclark): check to make sure that the magic string is correct
+ // though I am not sure that will get us much.
+ pos += 4;
MetaRegionServer mrs;
- mrs.ParseFromArray(contents, len);
+ // Try to decode the protobuf.
+ // If there's an error bail out.
+ if (mrs.ParseFromArray(contents + pos, len - pos) == false) {
+ LOG(ERROR) << "Error parsing Protobuf Message";
+ throw runtime_error("Error parsing protobuf");
+ }
+
return mrs.server();
}
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/location-cache.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h
index 8dc2760..efcfde5 100644
--- a/hbase-native-client/core/location-cache.h
+++ b/hbase-native-client/core/location-cache.h
@@ -1,13 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
#pragma once
-#include <memory>
-#include <mutex>
-
#include <zookeeper/zookeeper.h>
#include <folly/futures/Future.h>
#include <folly/futures/SharedPromise.h>
-
#include <folly/Executor.h>
+
+#include <memory>
+#include <mutex>
+#include <string>
+
#include "if/HBase.pb.h"
namespace hbase {
@@ -32,4 +51,4 @@ private:
zhandle_t *zk_;
};
-} // hbase
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/native-client-test-env.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/native-client-test-env.cc b/hbase-native-client/core/native-client-test-env.cc
index 07f30a6..0269a43 100644
--- a/hbase-native-client/core/native-client-test-env.cc
+++ b/hbase-native-client/core/native-client-test-env.cc
@@ -22,7 +22,7 @@
namespace {
class NativeClientTestEnv : public ::testing::Environment {
- public:
+public:
void SetUp() override {
// start local HBase cluster to be reused by all tests
auto result = system("bin/start_local_hbase_and_wait.sh");
@@ -36,9 +36,9 @@ class NativeClientTestEnv : public ::testing::Environment {
}
};
-} // anonymous
+} // anonymous
-int main(int argc, char** argv) {
+int main(int argc, char **argv) {
testing::InitGoogleTest(&argc, argv);
::testing::AddGlobalTestEnvironment(new NativeClientTestEnv());
return RUN_ALL_TESTS();
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/pipeline.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/pipeline.cc b/hbase-native-client/core/pipeline.cc
new file mode 100644
index 0000000..30d14ff
--- /dev/null
+++ b/hbase-native-client/core/pipeline.cc
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include "core/pipeline.h"
+
+#include <folly/Logging.h>
+#include <wangle/channel/AsyncSocketHandler.h>
+#include <wangle/channel/EventBaseHandler.h>
+#include <wangle/channel/OutputBufferingHandler.h>
+#include <wangle/codec/LengthFieldBasedFrameDecoder.h>
+
+#include "core/client-serialize-handler.h"
+
+using namespace folly;
+using namespace hbase;
+using namespace wangle;
+
+SerializePipeline::Ptr
+RpcPipelineFactory::newPipeline(std::shared_ptr<AsyncTransportWrapper> sock) {
+ auto pipeline = SerializePipeline::create();
+ pipeline->addBack(AsyncSocketHandler(sock));
+ pipeline->addBack(EventBaseHandler());
+ pipeline->addBack(LengthFieldBasedFrameDecoder());
+ pipeline->addBack(ClientSerializeHandler());
+ pipeline->finalize();
+ return pipeline;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/pipeline.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/pipeline.h b/hbase-native-client/core/pipeline.h
new file mode 100644
index 0000000..d199d08
--- /dev/null
+++ b/hbase-native-client/core/pipeline.h
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <wangle/service/Service.h>
+#include <folly/io/IOBufQueue.h>
+#include "core/request.h"
+#include "core/response.h"
+
+namespace hbase {
+using SerializePipeline = wangle::Pipeline<folly::IOBufQueue &, Request>;
+
+class RpcPipelineFactory : public wangle::PipelineFactory<SerializePipeline> {
+public:
+ SerializePipeline::Ptr
+ newPipeline(std::shared_ptr<folly::AsyncTransportWrapper> sock) override;
+};
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/request.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/request.h b/hbase-native-client/core/request.h
new file mode 100644
index 0000000..39083ed
--- /dev/null
+++ b/hbase-native-client/core/request.h
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <cstdint>
+
+namespace hbase {
+class Request {
+public:
+ Request() : call_id_(0) {}
+ uint32_t call_id() { return call_id_; }
+ void set_call_id(uint32_t call_id) { call_id_ = call_id; }
+
+private:
+ uint32_t call_id_;
+};
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/response.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/response.h b/hbase-native-client/core/response.h
new file mode 100644
index 0000000..34a284d
--- /dev/null
+++ b/hbase-native-client/core/response.h
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <cstdint>
+
+namespace hbase {
+
+class Response {
+public:
+ Response() : call_id_(0) {}
+ uint32_t call_id() { return call_id_; }
+ void set_call_id(uint32_t call_id) { call_id_ = call_id; }
+
+private:
+ uint32_t call_id_;
+};
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/service.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/service.h b/hbase-native-client/core/service.h
new file mode 100644
index 0000000..880e65f
--- /dev/null
+++ b/hbase-native-client/core/service.h
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include "core/request.h"
+#include "core/response.h"
+
+namespace hbase {
+using HBaseService = wangle::Service<Request, Response>;
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/simple-client.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/simple-client.cc b/hbase-native-client/core/simple-client.cc
new file mode 100644
index 0000000..08e886a
--- /dev/null
+++ b/hbase-native-client/core/simple-client.cc
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <folly/Logging.h>
+#include <folly/Random.h>
+#include <gflags/gflags.h>
+#include <glog/logging.h>
+#include <wangle/concurrent/GlobalExecutor.h>
+
+#include <iostream>
+
+#include "core/client.h"
+#include "core/connection-factory.h"
+#include "if/ZooKeeper.pb.h"
+
+using namespace folly;
+using namespace std;
+using namespace hbase;
+using namespace hbase::pb;
+
+int main(int argc, char *argv[]) {
+ google::ParseCommandLineFlags(&argc, &argv, true);
+ google::InitGoogleLogging(argv[0]);
+
+ // Create a connection factory
+ ConnectionFactory cf;
+
+ LocationCache cache{"localhost:2181", wangle::getCPUExecutor()};
+
+ auto result = cache.LocateMeta().get();
+ cout << "ServerName = " << result.host_name() << ":" << result.port() << endl;
+
+ // Create a connection to the local host
+ auto conn = cf.make_connection(result.host_name(), result.port()).get();
+
+ // Send the request
+ Request r;
+ conn(r).get();
+
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/simple-native-client-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/simple-native-client-test.cc b/hbase-native-client/core/simple-native-client-test.cc
index ef564f7..ee39986 100644
--- a/hbase-native-client/core/simple-native-client-test.cc
+++ b/hbase-native-client/core/simple-native-client-test.cc
@@ -22,7 +22,4 @@
/**
* Sample test.
*/
-TEST(SampleTest, sample) {
- EXPECT_TRUE(true);
-}
-
+TEST(SampleTest, sample) { EXPECT_TRUE(true); }
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/table-name.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/table-name.cc b/hbase-native-client/core/table-name.cc
new file mode 100644
index 0000000..ffaaed0
--- /dev/null
+++ b/hbase-native-client/core/table-name.cc
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include "core/table-name.h"
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/core/table-name.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/table-name.h b/hbase-native-client/core/table-name.h
new file mode 100644
index 0000000..796115b
--- /dev/null
+++ b/hbase-native-client/core/table-name.h
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <memory>
+#include <string>
+
+namespace hbase {
+
+// This is the core class of a HBase client.
+class TableName {
+public:
+ explicit TableName(std::string tableName);
+ explicit TableName(std::string namespaceName, std::string tableName);
+};
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/if/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/BUCK b/hbase-native-client/if/BUCK
index 9b989b5..5ff617d 100644
--- a/hbase-native-client/if/BUCK
+++ b/hbase-native-client/if/BUCK
@@ -1,3 +1,21 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
PROTO_SRCS = glob(['*.proto'])
HEADER_FILENAMES = [ x.replace('.proto','.pb.h') for x in PROTO_SRCS]
CC_FILENAMES = [ x.replace('.proto', '.pb.cc') for x in PROTO_SRCS]
@@ -20,7 +38,7 @@ for cc_filename in CC_FILENAMES:
genrule(
name = cc_filename,
cmd = 'mkdir -p `dirname $OUT` '
- ' && cp $(location :generate-proto-sources)/{} $OUT '
+ ' && cp $(location :generate-proto-sources)/*.cc `dirname $OUT` '
' && cp $(location :generate-proto-sources)/*.h `dirname $OUT`'.format(cc_filename),
out = cc_filename,
)
@@ -29,9 +47,7 @@ cxx_library(
name = 'if',
exported_headers = [':' + x for x in HEADER_FILENAMES],
srcs = [':' + x for x in CC_FILENAMES],
- deps = [ '//third-party:protobuf']
- + [':' + x for x in CC_FILENAMES]
- + [ ':' + x for x in HEADER_FILENAMES ],
+ deps = [ '//third-party:protobuf'],
visibility = [ 'PUBLIC', ],
exported_deps = ['//third-party:protobuf']
)
http://git-wip-us.apache.org/repos/asf/hbase/blob/b23e8630/hbase-native-client/third-party/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/third-party/BUCK b/hbase-native-client/third-party/BUCK
index 6548695..4327530 100644
--- a/hbase-native-client/third-party/BUCK
+++ b/hbase-native-client/third-party/BUCK
@@ -23,19 +23,30 @@ def add_system_libs(names=[],
exported_linker_flags=[]):
rules = []
for name in names:
+ rule_visibility = ['PUBLIC']
gen_rule_name = "gen_lib{}".format(name)
- genrule(name=gen_rule_name,
- out=gen_rule_name,
- bash="mkdir -p $OUT && cp {}/lib{}.a $OUT".format(lib_dir,
- name), )
+ genrule(
+ name=gen_rule_name,
+ out=gen_rule_name,
+ bash="mkdir -p $OUT && cp {}/lib{}.a $OUT".format(lib_dir, name), )
prebuilt_cxx_library(name=name,
lib_name=name,
lib_dir='$(location :{})'.format(gen_rule_name),
deps=deps,
- force_static = True,
+ force_static=True,
exported_deps=exported_deps,
- visibility=['PUBLIC'],
- exported_linker_flags=exported_linker_flags, )
+ visibility=rule_visibility, )
+ rules.append(":" + name)
+ return rules
+
+
+def add_dynamic_libs(names=[]):
+ rules = []
+ for name in names:
+ prebuilt_cxx_library(name=name,
+ header_only=True,
+ exported_linker_flags=["-l" + name],
+ visibility=["PUBLIC"], )
rules.append(":" + name)
return rules
@@ -54,58 +65,46 @@ local_libs = [
"glog",
"protobuf",
]
+dynamic_libs = ["stdc++", "pthread", "ssl", "crypto", "dl", "atomic", ]
+dynamic_rules = add_dynamic_libs(dynamic_libs)
+tp_dep_rules = add_system_libs(system_libs,) \
+ + add_system_libs(local_libs, lib_dir = "/usr/local/lib") \
+ + dynamic_rules
-
-
-tp_dep_rules = add_system_libs(system_libs) \
- + add_system_libs(local_libs, lib_dir = "/usr/local/lib")
-
-zookeeper = add_system_libs(["zookeeper_mt"], lib_dir = "/usr/local/lib")
+zookeeper = add_system_libs(["zookeeper_mt"], lib_dir="/usr/local/lib")
folly = add_system_libs(['folly'],
lib_dir='/usr/local/lib',
- exported_deps=tp_dep_rules,
- exported_linker_flags=["-pthread",
- "-lstdc++", ])
+ exported_deps=tp_dep_rules, )
folly_bench = add_system_libs(['follybenchmark'],
lib_dir='/usr/local/lib',
- exported_deps=tp_dep_rules + folly,
- exported_linker_flags=["-pthread",
- "-lstdc++", ])
+ exported_deps=folly + tp_dep_rules, )
wangle = add_system_libs(['wangle'],
lib_dir='/usr/local/lib',
- exported_deps=tp_dep_rules + folly,
- exported_linker_flags=["-pthread",
- "-lstdc++", ])
-
+ exported_deps=folly + tp_dep_rules)
genrule(
-name = "gen_zk",
-out = "gen_zk",
-bash = "mkdir -p $OUT && wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && tar zxf zookeeper-3.4.8.tar.gz && rm -rf zookeeper-3.4.8.tar.gz && cd zookeeper-3.4.8 && cd src/c && ./configure --prefix=$OUT && make && make install && cd $OUT && rm -rf zookeeper-3.4.8*"
-)
-cxx_library(
- name = 'google-test',
- srcs = [
- 'googletest/googletest/src/gtest-all.cc',
- 'googletest/googlemock/src/gmock-all.cc',
- 'googletest/googlemock/src/gmock_main.cc',
- ],
- header_namespace = '',
- exported_headers = subdir_glob([
- ('googletest/googletest/include', '**/*.h'),
- ('googletest/googlemock/include', '**/*.h'),
- ]),
- headers = subdir_glob([
- ('googletest/googletest', 'src/*.h'),
- ('googletest/googletest', 'src/*.cc'),
- ('googletest/googlemock', 'src/*.h'),
- ('googletest/googlemock', 'src/*.cc'),
- ]),
- exported_linker_flags = [
- "-pthread",
- "-lstdc++",
- ],
- visibility = [
- 'PUBLIC',
- ],
-)
+ name="gen_zk",
+ out="gen_zk",
+ bash=
+ "mkdir -p $OUT && wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && tar zxf zookeeper-3.4.8.tar.gz && rm -rf zookeeper-3.4.8.tar.gz && cd zookeeper-3.4.8 && cd src/c && ./configure --prefix=$OUT && make && make install && cd $OUT && rm -rf zookeeper-3.4.8*")
+cxx_library(name='google-test',
+ srcs=[
+ 'googletest/googletest/src/gtest-all.cc',
+ 'googletest/googlemock/src/gmock-all.cc',
+ 'googletest/googlemock/src/gmock_main.cc',
+ ],
+ header_namespace='',
+ exported_headers=subdir_glob([
+ ('googletest/googletest/include', '**/*.h'),
+ ('googletest/googlemock/include', '**/*.h'),
+ ]),
+ headers=subdir_glob([
+ ('googletest/googletest', 'src/*.h'),
+ ('googletest/googletest', 'src/*.cc'),
+ ('googletest/googlemock', 'src/*.h'),
+ ('googletest/googlemock', 'src/*.cc'),
+ ]),
+ exported_deps=dynamic_rules,
+ visibility=[
+ 'PUBLIC',
+ ], )
[24/50] [abbrv] hbase git commit: HBASE-15418 Clean up un-used
warning in test util
Posted by ec...@apache.org.
HBASE-15418 Clean up un-used warning in test util
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/45cf5657
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/45cf5657
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/45cf5657
Branch: refs/heads/HBASE-14850
Commit: 45cf5657acc4cc7a61655fa6b76eb90910fabb21
Parents: 26ba4ed
Author: Elliott Clark <ec...@apache.org>
Authored: Fri Mar 25 15:44:06 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
.../bin/start_local_hbase_and_wait.sh | 9 +++++-
.../bin/stop_local_hbase_and_wait.sh | 2 +-
hbase-native-client/core/BUCK | 6 ----
.../core/native-client-test-env.cc | 9 ++++--
hbase-native-client/core/test_env.h | 32 --------------------
5 files changed, 15 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/45cf5657/hbase-native-client/bin/start_local_hbase_and_wait.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/start_local_hbase_and_wait.sh b/hbase-native-client/bin/start_local_hbase_and_wait.sh
index 64d0b68..cfc71f9 100755
--- a/hbase-native-client/bin/start_local_hbase_and_wait.sh
+++ b/hbase-native-client/bin/start_local_hbase_and_wait.sh
@@ -17,10 +17,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# Clean up from any other tests.
+rm -rf /tmp/hbase-*
+
+# Start the master/regionservers.
$PWD/../bin/start-hbase.sh
-until [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010) == "200" ]
+until [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010/jmx) == "200" ]
do
printf "Waiting for local HBase cluster to start\n"
sleep 1
done
+
+# This sucks, but master can easily be up and meta not be assigned yet.
+sleep 30
http://git-wip-us.apache.org/repos/asf/hbase/blob/45cf5657/hbase-native-client/bin/stop_local_hbase_and_wait.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/stop_local_hbase_and_wait.sh b/hbase-native-client/bin/stop_local_hbase_and_wait.sh
index 4e89334..761412a 100755
--- a/hbase-native-client/bin/stop_local_hbase_and_wait.sh
+++ b/hbase-native-client/bin/stop_local_hbase_and_wait.sh
@@ -17,7 +17,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-$PWD/../bin/stop-hbase.sh
+ps aux | grep proc_master | awk '{print $2}' | xargs kill -9
while [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010) == "200" ]
do
http://git-wip-us.apache.org/repos/asf/hbase/blob/45cf5657/hbase-native-client/core/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index 817b5a0..d1e89d1 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -51,9 +51,6 @@ cxx_library(name="core",
], )
cxx_test(name="simple-test",
- headers=[
- "test_env.h",
- ],
srcs=[
"native-client-test-env.cc",
"simple-native-client-test.cc",
@@ -63,9 +60,6 @@ cxx_test(name="simple-test",
],
run_test_separately=True, )
cxx_test(name="location-cache-test",
- headers=[
- "test_env.h",
- ],
srcs=[
"native-client-test-env.cc",
"location-cache-test.cc",
http://git-wip-us.apache.org/repos/asf/hbase/blob/45cf5657/hbase-native-client/core/native-client-test-env.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/native-client-test-env.cc b/hbase-native-client/core/native-client-test-env.cc
index a86961f..07f30a6 100644
--- a/hbase-native-client/core/native-client-test-env.cc
+++ b/hbase-native-client/core/native-client-test-env.cc
@@ -18,18 +18,21 @@
*/
#include <gtest/gtest.h>
-#include <core/test_env.h>
namespace {
class NativeClientTestEnv : public ::testing::Environment {
public:
void SetUp() override {
- init_test_env();
+ // start local HBase cluster to be reused by all tests
+ auto result = system("bin/start_local_hbase_and_wait.sh");
+ ASSERT_EQ(0, result);
}
void TearDown() override {
- clean_test_env();
+ // shutdown local HBase cluster
+ auto result = system("bin/stop_local_hbase_and_wait.sh");
+ ASSERT_EQ(0, result);
}
};
http://git-wip-us.apache.org/repos/asf/hbase/blob/45cf5657/hbase-native-client/core/test_env.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/test_env.h b/hbase-native-client/core/test_env.h
deleted file mode 100644
index 79bdbec..0000000
--- a/hbase-native-client/core/test_env.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#pragma once
-
-#include <cstdlib>
-
-inline void init_test_env() {
- // start local HBase cluster to be reused by all tests
- system("scripts/start_local_hbase_and_wait.sh");
-}
-
-inline void clean_test_env() {
- // shutdown local HBase cluster
- system("scripts/stop_local_hbase_and_wait.sh");
-}
[05/50] [abbrv] hbase git commit: HBASE-14548 Expand how table
coprocessor jar and dependency path can be specified (Xiang Li)
Posted by ec...@apache.org.
HBASE-14548 Expand how table coprocessor jar and dependency path can be specified (Xiang Li)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/63296978
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/63296978
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/63296978
Branch: refs/heads/HBASE-14850
Commit: 632969787aa9e9e1d73b83449b93e21f91110b73
Parents: 496fd98
Author: Jerry He <je...@apache.org>
Authored: Sat Jul 9 18:00:41 2016 -0700
Committer: Jerry He <je...@apache.org>
Committed: Sat Jul 9 18:01:49 2016 -0700
----------------------------------------------------------------------
.../hbase/util/CoprocessorClassLoader.java | 72 +++++++++++++-------
.../hbase/util/TestCoprocessorClassLoader.java | 43 ++++++++++++
src/main/asciidoc/_chapters/cp.adoc | 9 ++-
3 files changed, 97 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/63296978/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
index 11016c3..c3635cb 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/CoprocessorClassLoader.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hbase.util;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URL;
@@ -38,6 +39,8 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.io.IOUtils;
@@ -142,7 +145,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
super(parent);
}
- private void init(Path path, String pathPrefix,
+ private void init(Path pathPattern, String pathPrefix,
Configuration conf) throws IOException {
// Copy the jar to the local filesystem
String parentDirStr =
@@ -160,33 +163,50 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
}
}
- FileSystem fs = path.getFileSystem(conf);
- File dst = new File(parentDirStr, "." + pathPrefix + "."
- + path.getName() + "." + System.currentTimeMillis() + ".jar");
- fs.copyToLocalFile(path, new Path(dst.toString()));
- dst.deleteOnExit();
-
- addURL(dst.getCanonicalFile().toURI().toURL());
-
- JarFile jarFile = new JarFile(dst.toString());
- try {
- Enumeration<JarEntry> entries = jarFile.entries();
- while (entries.hasMoreElements()) {
- JarEntry entry = entries.nextElement();
- Matcher m = libJarPattern.matcher(entry.getName());
- if (m.matches()) {
- File file = new File(parentDirStr, "." + pathPrefix + "."
- + path.getName() + "." + System.currentTimeMillis() + "." + m.group(1));
- try (FileOutputStream outStream = new FileOutputStream(file)) {
- IOUtils.copyBytes(jarFile.getInputStream(entry),
- outStream, conf, true);
+ FileSystem fs = pathPattern.getFileSystem(conf);
+ Path pathPattern1 = fs.isDirectory(pathPattern) ?
+ new Path(pathPattern, "*.jar") : pathPattern; // append "*.jar" if a directory is specified
+ FileStatus[] fileStatuses = fs.globStatus(pathPattern1); // return all files that match the pattern
+ if (fileStatuses == null || fileStatuses.length == 0) { // if no one matches
+ throw new FileNotFoundException(pathPattern1.toString());
+ } else {
+ boolean validFileEncountered = false;
+ for (Path path : FileUtil.stat2Paths(fileStatuses)) { // for each file that match the pattern
+ if (fs.isFile(path)) { // only process files, skip for directories
+ File dst = new File(parentDirStr, "." + pathPrefix + "."
+ + path.getName() + "." + System.currentTimeMillis() + ".jar");
+ fs.copyToLocalFile(path, new Path(dst.toString()));
+ dst.deleteOnExit();
+
+ addURL(dst.getCanonicalFile().toURI().toURL());
+
+ JarFile jarFile = new JarFile(dst.toString());
+ try {
+ Enumeration<JarEntry> entries = jarFile.entries(); // get entries inside a jar file
+ while (entries.hasMoreElements()) {
+ JarEntry entry = entries.nextElement();
+ Matcher m = libJarPattern.matcher(entry.getName());
+ if (m.matches()) {
+ File file = new File(parentDirStr, "." + pathPrefix + "."
+ + path.getName() + "." + System.currentTimeMillis() + "." + m.group(1));
+ try (FileOutputStream outStream = new FileOutputStream(file)) {
+ IOUtils.copyBytes(jarFile.getInputStream(entry),
+ outStream, conf, true);
+ }
+ file.deleteOnExit();
+ addURL(file.toURI().toURL());
+ }
+ }
+ } finally {
+ jarFile.close();
}
- file.deleteOnExit();
- addURL(file.toURI().toURL());
+
+ validFileEncountered = true; // Set to true when encountering a file
}
}
- } finally {
- jarFile.close();
+ if (validFileEncountered == false) { // all items returned by globStatus() are directories
+ throw new FileNotFoundException("No file found matching " + pathPattern1.toString());
+ }
}
}
@@ -227,7 +247,7 @@ public class CoprocessorClassLoader extends ClassLoaderBase {
return cl;
}
- if (!pathStr.endsWith(".jar")) {
+ if (path.getFileSystem(conf).isFile(path) && !pathStr.endsWith(".jar")) {
throw new IOException(pathStr + ": not a jar file?");
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/63296978/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
index daba459..967d272 100644
--- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
+++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestCoprocessorClassLoader.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.util;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import java.io.File;
@@ -109,4 +110,46 @@ public class TestCoprocessorClassLoader {
}
fail("Could not find the expected lib jar file");
}
+
+ // HBASE-14548
+ @Test
+ public void testDirectoryAndWildcard() throws Exception {
+ String testClassName = "TestClass";
+ String dataTestDir = TEST_UTIL.getDataTestDir().toString();
+ System.out.println(dataTestDir);
+ String localDirContainingJar = ClassLoaderTestHelper.localDirPath(conf);
+ ClassLoaderTestHelper.buildJar(dataTestDir, testClassName, null, localDirContainingJar);
+ ClassLoader parent = TestCoprocessorClassLoader.class.getClassLoader();
+ CoprocessorClassLoader.parentDirLockSet.clear(); // So that clean up can be triggered
+
+ CoprocessorClassLoader coprocessorClassLoader = null;
+ Path testPath = null;
+
+ // Directory
+ testPath = new Path(localDirContainingJar);
+ coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_1", conf);
+ verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
+
+ // Wildcard - *.jar
+ testPath = new Path(localDirContainingJar, "*.jar");
+ coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_2", conf);
+ verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
+
+ // Wildcard - *.j*
+ testPath = new Path(localDirContainingJar, "*.j*");
+ coprocessorClassLoader = CoprocessorClassLoader.getClassLoader(testPath, parent, "113_3", conf);
+ verifyCoprocessorClassLoader(coprocessorClassLoader, testClassName);
+ }
+
+ /**
+ * Verify the coprocessorClassLoader is not null and the expected class can be loaded successfully
+ * @param coprocessorClassLoader the CoprocessorClassLoader to verify
+ * @param className the expected class to be loaded by the coprocessorClassLoader
+ * @throws ClassNotFoundException
+ */
+ private void verifyCoprocessorClassLoader(CoprocessorClassLoader coprocessorClassLoader, String className)
+ throws ClassNotFoundException{
+ assertNotNull("Classloader should be created and not null", coprocessorClassLoader);
+ assertEquals(className, coprocessorClassLoader.loadClass(className).getName());
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/63296978/src/main/asciidoc/_chapters/cp.adoc
----------------------------------------------------------------------
diff --git a/src/main/asciidoc/_chapters/cp.adoc b/src/main/asciidoc/_chapters/cp.adoc
index 793bb6f..376899a 100644
--- a/src/main/asciidoc/_chapters/cp.adoc
+++ b/src/main/asciidoc/_chapters/cp.adoc
@@ -320,7 +320,14 @@ The value contains four pieces of information which are separated by the pipe (`
* File path: The jar file containing the Coprocessor implementation must be in a location where
all region servers can read it. +
You could copy the file onto the local disk on each region server, but it is recommended to store
-it in HDFS.
+it in HDFS. +
+https://issues.apache.org/jira/browse/HBASE-14548[HBASE-14548] allows a directory containing the jars
+or some wildcards to be specified, such as: hdfs://<namenode>:<port>/user/<hadoop-user>/ or
+hdfs://<namenode>:<port>/user/<hadoop-user>/*.jar. Please note that if a directory is specified,
+all jar files(.jar) directly in the directory are added,
+but it does not search files in the subtree rooted in the directory.
+And do not contain any wildcard if you would like to specify a directory.
+This enhancement applies to the ways of using the JAVA API as well.
* Class name: The full class name of the Coprocessor.
* Priority: An integer. The framework will determine the execution sequence of all configured
observers registered at the same hook using priorities. This field can be left blank. In that
[42/50] [abbrv] hbase git commit: HBASE-15926 Added ASF headers which
were missed in last commit
Posted by ec...@apache.org.
HBASE-15926 Added ASF headers which were missed in last commit
Signed-off-by: Elliott Clark <ec...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2b6341d4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2b6341d4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2b6341d4
Branch: refs/heads/HBASE-14850
Commit: 2b6341d4db3f3cf045b99fb6f71d431098ce7bf1
Parents: 5bb7fa8
Author: Sudeep Sunthankar <su...@hashmapinc.com>
Authored: Thu Jun 2 20:22:37 2016 +1000
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/Makefile | 17 +++++++++++++++++
hbase-native-client/Makefile.protos | 17 +++++++++++++++++
2 files changed, 34 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2b6341d4/hbase-native-client/Makefile
----------------------------------------------------------------------
diff --git a/hbase-native-client/Makefile b/hbase-native-client/Makefile
index 7e68b6a..64cef06 100644
--- a/hbase-native-client/Makefile
+++ b/hbase-native-client/Makefile
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
#use "gcc" to compile source files
CC:=g++
LD:=g++
http://git-wip-us.apache.org/repos/asf/hbase/blob/2b6341d4/hbase-native-client/Makefile.protos
----------------------------------------------------------------------
diff --git a/hbase-native-client/Makefile.protos b/hbase-native-client/Makefile.protos
index 4f1727a..2c6316b 100644
--- a/hbase-native-client/Makefile.protos
+++ b/hbase-native-client/Makefile.protos
@@ -1,3 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
#use "gcc" to compile source files
CC:=g++
LD:=g++
[47/50] [abbrv] hbase git commit: HBASE-15826 Clean up ASF license
issues
Posted by ec...@apache.org.
HBASE-15826 Clean up ASF license issues
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e718c7dc
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e718c7dc
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e718c7dc
Branch: refs/heads/HBASE-14850
Commit: e718c7dc9a4c80015ef7a005714322a2cbb9f6ea
Parents: f6ea493
Author: Elliott Clark <ec...@apache.org>
Authored: Fri May 13 14:53:47 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/Makefile | 17 +++++++++++++++++
hbase-native-client/bin/copy-protobuf.sh | 16 ++++++++++++++++
hbase-native-client/bin/format-code.sh | 16 ++++++++++++++++
hbase-native-client/bin/start-local-hbase.sh | 1 -
hbase-native-client/hbase.doxygen | 22 +++++++++++++++++++---
hbase-native-client/serde/server-name.h | 19 +++++++++++++++++++
6 files changed, 87 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e718c7dc/hbase-native-client/Makefile
----------------------------------------------------------------------
diff --git a/hbase-native-client/Makefile b/hbase-native-client/Makefile
index 90ec6ea..826233f 100644
--- a/hbase-native-client/Makefile
+++ b/hbase-native-client/Makefile
@@ -1,3 +1,20 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
build:
$(shell buck build core/... )
http://git-wip-us.apache.org/repos/asf/hbase/blob/e718c7dc/hbase-native-client/bin/copy-protobuf.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/copy-protobuf.sh b/hbase-native-client/bin/copy-protobuf.sh
index c9e70f2..36541cf 100755
--- a/hbase-native-client/bin/copy-protobuf.sh
+++ b/hbase-native-client/bin/copy-protobuf.sh
@@ -1,4 +1,20 @@
#!/usr/bin/env bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
set -euo pipefail
IFS=$'\n\t'
http://git-wip-us.apache.org/repos/asf/hbase/blob/e718c7dc/hbase-native-client/bin/format-code.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/format-code.sh b/hbase-native-client/bin/format-code.sh
index 24d6515..36a114d 100755
--- a/hbase-native-client/bin/format-code.sh
+++ b/hbase-native-client/bin/format-code.sh
@@ -1,4 +1,20 @@
#!/usr/bin/env bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
set -euo pipefail
IFS=$'\n\t'
http://git-wip-us.apache.org/repos/asf/hbase/blob/e718c7dc/hbase-native-client/bin/start-local-hbase.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/start-local-hbase.sh b/hbase-native-client/bin/start-local-hbase.sh
index 49637d6..12069c2 100755
--- a/hbase-native-client/bin/start-local-hbase.sh
+++ b/hbase-native-client/bin/start-local-hbase.sh
@@ -1,5 +1,4 @@
#!/usr/bin/env bash
-
##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
http://git-wip-us.apache.org/repos/asf/hbase/blob/e718c7dc/hbase-native-client/hbase.doxygen
----------------------------------------------------------------------
diff --git a/hbase-native-client/hbase.doxygen b/hbase-native-client/hbase.doxygen
index 22e2d21..fef404c 100644
--- a/hbase-native-client/hbase.doxygen
+++ b/hbase-native-client/hbase.doxygen
@@ -1,4 +1,20 @@
# Doxyfile 1.8.11
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
#---------------------------------------------------------------------------
# Project related configuration options
@@ -103,14 +119,14 @@ WARN_LOGFILE =
#---------------------------------------------------------------------------
INPUT =
INPUT_ENCODING = UTF-8
-FILE_PATTERNS =
+FILE_PATTERNS =
RECURSIVE = YES
EXCLUDE = third-party/ buck-out/
EXCLUDE_SYMLINKS = NO
EXCLUDE_PATTERNS = /*-test.cc
EXCLUDE_SYMBOLS =
EXAMPLE_PATH =
-EXAMPLE_PATTERNS =
+EXAMPLE_PATTERNS =
EXAMPLE_RECURSIVE = NO
IMAGE_PATH =
INPUT_FILTER =
@@ -142,7 +158,7 @@ IGNORE_PREFIX =
# Configuration options related to the HTML output
#---------------------------------------------------------------------------
GENERATE_HTML = YES
-HTML_OUTPUT = html
+HTML_OUTPUT = html
HTML_FILE_EXTENSION = .html
HTML_HEADER =
HTML_FOOTER =
http://git-wip-us.apache.org/repos/asf/hbase/blob/e718c7dc/hbase-native-client/serde/server-name.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/server-name.h b/hbase-native-client/serde/server-name.h
index 9844465..6551fdc 100644
--- a/hbase-native-client/serde/server-name.h
+++ b/hbase-native-client/serde/server-name.h
@@ -1,3 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
#pragma once
#include "if/HBase.pb.h"
[07/50] [abbrv] hbase git commit: HBASE-16185
TestReplicationSmallTests fails in master branch (Phil Yang)
Posted by ec...@apache.org.
HBASE-16185 TestReplicationSmallTests fails in master branch (Phil Yang)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0875c357
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0875c357
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0875c357
Branch: refs/heads/HBASE-14850
Commit: 0875c35781f1fcc06d38308b915a01791c7a6a20
Parents: 9cf012c
Author: tedyu <yu...@gmail.com>
Authored: Sun Jul 10 23:28:32 2016 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Sun Jul 10 23:28:32 2016 -0700
----------------------------------------------------------------------
.../hadoop/hbase/replication/TestReplicationSmallTests.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0875c357/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
index 8efa67e..e3cc6e9 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/replication/TestReplicationSmallTests.java
@@ -732,7 +732,9 @@ public class TestReplicationSmallTests extends TestReplicationBase {
ClusterStatus status = admin.getClusterStatus();
- for (ServerName server : status.getServers()) {
+ for (JVMClusterUtil.RegionServerThread thread :
+ utility1.getHBaseCluster().getRegionServerThreads()) {
+ ServerName server = thread.getRegionServer().getServerName();
ServerLoad sl = status.getLoad(server);
List<ReplicationLoadSource> rLoadSourceList = sl.getReplicationLoadSourceList();
ReplicationLoadSink rLoadSink = sl.getReplicationLoadSink();
[34/50] [abbrv] hbase git commit: HBASE-15792 Add on a test for
locating a region
Posted by ec...@apache.org.
HBASE-15792 Add on a test for locating a region
Summary:
Add on a TestUtil class that will start a cluster.
Add on the ability to run a shell command in a cluster.
Throw an exception if the table isn't the correct table when looking up a region.
Add a test to test that getting a region throws when the table doesn't exist.
Add a test to make sure that getting a region location works when the table does exist.
Test Plan:
Added unit tests.
Changed unit tests.
Differential Revision: https://reviews.facebook.net/D57891
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/eb4cde49
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/eb4cde49
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/eb4cde49
Branch: refs/heads/HBASE-14850
Commit: eb4cde498f0fed8fd1e7ff35b71c69cabe09768d
Parents: d3a6efd
Author: Elliott Clark <ec...@apache.org>
Authored: Mon May 9 12:06:54 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/bin/start-local-hbase.sh | 5 +-
.../connection/client-handler.cc | 3 +-
hbase-native-client/core/BUCK | 4 +-
hbase-native-client/core/client.cc | 1 +
hbase-native-client/core/location-cache-test.cc | 29 ++++++++++-
hbase-native-client/core/location-cache.cc | 10 +++-
hbase-native-client/core/location-cache.h | 10 +++-
hbase-native-client/core/test-env.cc | 45 ----------------
hbase-native-client/test-util/BUCK | 29 +++++++++++
hbase-native-client/test-util/test-util.cc | 54 ++++++++++++++++++++
hbase-native-client/test-util/test-util.h | 51 ++++++++++++++++++
11 files changed, 187 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/eb4cde49/hbase-native-client/bin/start-local-hbase.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/start-local-hbase.sh b/hbase-native-client/bin/start-local-hbase.sh
index cfc71f9..49637d6 100755
--- a/hbase-native-client/bin/start-local-hbase.sh
+++ b/hbase-native-client/bin/start-local-hbase.sh
@@ -21,7 +21,8 @@
rm -rf /tmp/hbase-*
# Start the master/regionservers.
-$PWD/../bin/start-hbase.sh
+T_DIR=${1:-"/tmp/hbase-testing"}
+$PWD/../bin/start-hbase.sh -Dhbase.tmp.dir="${T_DIR}"
until [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010/jmx) == "200" ]
do
@@ -30,4 +31,4 @@ do
done
# This sucks, but master can easily be up and meta not be assigned yet.
-sleep 30
+sleep 10
http://git-wip-us.apache.org/repos/asf/hbase/blob/eb4cde49/hbase-native-client/connection/client-handler.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-handler.cc b/hbase-native-client/connection/client-handler.cc
index 2e3fcd3..3409b80 100644
--- a/hbase-native-client/connection/client-handler.cc
+++ b/hbase-native-client/connection/client-handler.cc
@@ -37,7 +37,8 @@ using hbase::pb::GetResponse;
using google::protobuf::Message;
ClientHandler::ClientHandler(std::string user_name)
- : user_name_(user_name), serde_(), once_flag_(std::make_unique<std::once_flag>()),
+ : user_name_(user_name), serde_(),
+ once_flag_(std::make_unique<std::once_flag>()),
resp_msgs_(
make_unique<folly::AtomicHashMap<
uint32_t, std::shared_ptr<google::protobuf::Message>>>(5000)) {}
http://git-wip-us.apache.org/repos/asf/hbase/blob/eb4cde49/hbase-native-client/core/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index 485f9ba..1c926e3 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -46,10 +46,10 @@ cxx_library(
], )
cxx_test(name="location-cache-test",
srcs=[
- "test-env.cc",
"location-cache-test.cc",
],
- deps=[":core", ],
+ deps=[":core",
+ "//test-util:test-util", ],
run_test_separately=True, )
cxx_binary(name="simple-client",
srcs=["simple-client.cc", ],
http://git-wip-us.apache.org/repos/asf/hbase/blob/eb4cde49/hbase-native-client/core/client.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client.cc b/hbase-native-client/core/client.cc
index 1e80998..82ecd22 100644
--- a/hbase-native-client/core/client.cc
+++ b/hbase-native-client/core/client.cc
@@ -22,6 +22,7 @@
#include <glog/logging.h>
#include <string>
+#include <unistd.h>
using namespace folly;
using namespace std;
http://git-wip-us.apache.org/repos/asf/hbase/blob/eb4cde49/hbase-native-client/core/location-cache-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache-test.cc b/hbase-native-client/core/location-cache-test.cc
index 8bc6383..53db6fc 100644
--- a/hbase-native-client/core/location-cache-test.cc
+++ b/hbase-native-client/core/location-cache-test.cc
@@ -16,14 +16,22 @@
* limitations under the License.
*
*/
+#include "core/location-cache.h"
+
#include <folly/Memory.h>
#include <gtest/gtest.h>
-#include "location-cache.h"
+#include <chrono>
+
+#include "if/HBase.pb.h"
+#include "serde/table-name.h"
+#include "test-util/test-util.h"
using namespace hbase;
+using namespace std::chrono;
TEST(LocationCacheTest, TestGetMetaNodeContents) {
- // TODO(elliott): need to make a test utility for this.
+ TestUtil test_util{};
+
auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
LocationCache cache{"localhost:2181", cpu, io};
@@ -33,3 +41,20 @@ TEST(LocationCacheTest, TestGetMetaNodeContents) {
ASSERT_TRUE(result.has_port());
ASSERT_TRUE(result.has_host_name());
}
+
+TEST(LocationCacheTest, TestGetRegionLocation) {
+ TestUtil test_util{};
+ auto cpu = std::make_shared<wangle::CPUThreadPoolExecutor>(4);
+ auto io = std::make_shared<wangle::IOThreadPoolExecutor>(4);
+ LocationCache cache{"localhost:2181", cpu, io};
+
+ // If there is no table this should throw an exception
+ auto tn = folly::to<hbase::pb::TableName>("t");
+ auto row = "test";
+ ASSERT_ANY_THROW(cache.LocateFromMeta(tn, row).get(milliseconds(1000)));
+ test_util.RunShellCmd("create 't', 'd'");
+ auto loc = cache.LocateFromMeta(tn, row).get(milliseconds(1000));
+ ASSERT_TRUE(loc != nullptr);
+ cpu->stop();
+ io->stop();
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/eb4cde49/hbase-native-client/core/location-cache.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
index 6c018f9..9f2a0ef 100644
--- a/hbase-native-client/core/location-cache.cc
+++ b/hbase-native-client/core/location-cache.cc
@@ -116,7 +116,7 @@ LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
return this->LocateMeta()
.via(cpu_executor_.get())
.then([this](ServerName sn) { return this->cp_.get(sn); })
- .then([&](std::shared_ptr<HBaseService> service) {
+ .then([tn, row, this](std::shared_ptr<HBaseService> service) {
return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
})
.then([this](Response resp) {
@@ -124,6 +124,14 @@ LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
// a region location.
return this->CreateLocation(std::move(resp));
})
+ .then([tn, this](std::shared_ptr<RegionLocation> rl) {
+ // Make sure that the correct location was found.
+ if (rl->region_info().table_name().namespace_() != tn.namespace_() ||
+ rl->region_info().table_name().qualifier() != tn.qualifier()) {
+ throw std::runtime_error("Doesn't look like table exists.");
+ }
+ return rl;
+ })
.then([this](std::shared_ptr<RegionLocation> rl) {
// Now fill out the connection.
rl->set_service(cp_.get(rl->server_name()));
http://git-wip-us.apache.org/repos/asf/hbase/blob/eb4cde49/hbase-native-client/core/location-cache.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h
index e077750..d435530 100644
--- a/hbase-native-client/core/location-cache.h
+++ b/hbase-native-client/core/location-cache.h
@@ -50,7 +50,9 @@ public:
/**
* Constructor.
* @param quorum_spec Where to connect for Zookeeper.
- * @param executor The cpu executor to run on.
+ * @param cpu_executor executor used to run non network IO based
+ * continuations.
+ * @param io_executor executor used to talk to the network
*/
LocationCache(std::string quorum_spec,
std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_exector,
@@ -70,6 +72,12 @@ public:
/**
* Go read meta and find out where a region is located.
+ *
+ * @param tn Table name of the table to look up. This object must live until
+ * after the future is returned
+ *
+ * @param row of the table to look up. This object must live until after the
+ * future is returned
*/
folly::Future<std::shared_ptr<RegionLocation>>
LocateFromMeta(const hbase::pb::TableName &tn, const std::string &row);
http://git-wip-us.apache.org/repos/asf/hbase/blob/eb4cde49/hbase-native-client/core/test-env.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/test-env.cc b/hbase-native-client/core/test-env.cc
deleted file mode 100644
index 277abd9..0000000
--- a/hbase-native-client/core/test-env.cc
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include <gtest/gtest.h>
-
-namespace {
-
-class NativeClientTestEnv : public ::testing::Environment {
-public:
- void SetUp() override {
- // start local HBase cluster to be reused by all tests
- auto result = system("bin/start-local-hbase.sh");
- ASSERT_EQ(0, result);
- }
-
- void TearDown() override {
- // shutdown local HBase cluster
- auto result = system("bin/stop-local-hbase.sh");
- ASSERT_EQ(0, result);
- }
-};
-
-} // anonymous
-
-int main(int argc, char **argv) {
- testing::InitGoogleTest(&argc, argv);
- ::testing::AddGlobalTestEnvironment(new NativeClientTestEnv());
- return RUN_ALL_TESTS();
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/eb4cde49/hbase-native-client/test-util/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/test-util/BUCK b/hbase-native-client/test-util/BUCK
new file mode 100644
index 0000000..e5e6ea2
--- /dev/null
+++ b/hbase-native-client/test-util/BUCK
@@ -0,0 +1,29 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cxx_library(name="test-util",
+ exported_headers=[
+ "test-util.h",
+ ],
+ srcs=["test-util.cc"],
+ deps=[
+ "//third-party:folly",
+ ],
+ visibility=[
+ 'PUBLIC',
+ ],
+ )
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/eb4cde49/hbase-native-client/test-util/test-util.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/test-util/test-util.cc b/hbase-native-client/test-util/test-util.cc
new file mode 100644
index 0000000..e5fba48
--- /dev/null
+++ b/hbase-native-client/test-util/test-util.cc
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "test-util/test-util.h"
+
+#include <folly/Format.h>
+
+using hbase::TestUtil;
+using folly::Random;
+
+const static int STR_LEN = 32;
+
+std::string TestUtil::RandString() {
+ auto s = std::string(STR_LEN, 'z');
+
+ for (int i = 0; i < STR_LEN; i++) {
+ auto r = Random::rand32('a', 'z');
+ s[i] = static_cast<char>(r);
+ }
+ return s;
+}
+
+TestUtil::TestUtil() : temp_dir_(TestUtil::RandString()) {
+ auto p = temp_dir_.path().string();
+ auto cmd = std::string{"bin/start-local-hbase.sh " + p};
+ auto res_code = std::system(cmd.c_str());
+ CHECK(res_code == 0);
+}
+TestUtil::~TestUtil() {
+ auto res_code = std::system("bin/stop-local-hbase.sh");
+ CHECK(res_code == 0);
+}
+
+void TestUtil::RunShellCmd(const std::string& command) {
+ auto cmd_string = folly::sformat("echo \"{}\" | ../bin/hbase shell", command);
+ auto res_code = std::system(cmd_string.c_str());
+ CHECK(res_code == 0);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/eb4cde49/hbase-native-client/test-util/test-util.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/test-util/test-util.h b/hbase-native-client/test-util/test-util.h
new file mode 100644
index 0000000..395b157
--- /dev/null
+++ b/hbase-native-client/test-util/test-util.h
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <folly/experimental/TestUtil.h>
+#include <folly/Random.h>
+
+#include <cstdlib>
+#include <string>
+
+namespace hbase {
+/**
+ * @brief Class to deal with a local instance cluster for testing.
+ */
+class TestUtil {
+public:
+
+ /**
+ * Creating a TestUtil will spin up a cluster.
+ */
+ TestUtil();
+ /**
+ * Destroying a TestUtil will spin down a cluster.
+ */
+ ~TestUtil();
+ /**
+ * Run a command in the hbase shell.
+ */
+ void RunShellCmd(const std::string& command);
+ static std::string RandString();
+
+private:
+ folly::test::TemporaryDirectory temp_dir_;
+};
+} // namespace hbase
[16/50] [abbrv] hbase git commit: HBASE-14852 Update build env
Posted by ec...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/hbase_result.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/async/hbase_result.h b/hbase-native-client/src/async/hbase_result.h
deleted file mode 100644
index eecbbb3..0000000
--- a/hbase-native-client/src/async/hbase_result.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef ASYNC_HBASE_RESULT_H_
-#define ASYNC_HBASE_RESULT_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "core/hbase_macros.h"
-#include "core/hbase_types.h"
-
-HBASE_API int32_t hb_result_destroy(hb_result_t result);
-
-HBASE_API int32_t hb_result_get_cells(hb_result_t result,
- hb_cell_t ** cell_ptr, size_t * num_cells);
-
-HBASE_API int32_t hb_result_get_table(hb_result_t result,
- char ** table, size_t * table_length);
-HBASE_API int32_t hb_result_get_namespace(hb_result_t result,
- char ** name_space, size_t * name_space_length);
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // ASYNC_HBASE_RESULT_H_
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/hbase_scanner.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/async/hbase_scanner.cc b/hbase-native-client/src/async/hbase_scanner.cc
deleted file mode 100644
index 5a8e555..0000000
--- a/hbase-native-client/src/async/hbase_scanner.cc
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#include "async/hbase_scanner.h"
-
-#include <stdlib.h>
-
-#include "core/hbase_types.h"
-#include "core/scanner.h"
-
-int32_t hb_scanner_create(hb_scanner_t * scanner_ptr) {
- (*scanner_ptr) = reinterpret_cast<hb_scanner_t>(new Scanner());
- return (*scanner_ptr != NULL)?0:1;
-}
-
-HBASE_API int32_t hb_scanner_set_table(hb_scanner_t scanner,
- char * table, size_t table_length) {
- return 0;
-}
-
-HBASE_API int32_t hb_scanner_set_namespace(hb_scanner_t scanner,
- char * name_space, size_t name_space_length) {
- return 0;
-}
-
-int32_t hb_scanner_set_start_row(hb_scanner_t scanner,
- unsigned char * start_row, size_t start_row_length) {
- return 0;
-}
-
-int32_t hb_scanner_set_end_row(hb_scanner_t scanner,
- unsigned char * end_row, size_t end_row_length) {
- return 0;
-}
-
-int32_t hb_scanner_set_cache_size(hb_scanner_t scanner,
- size_t cache_size) {
- return 0;
-}
-
-int32_t hb_scanner_set_num_versions(hb_scanner_t scanner,
- int8_t num_versions) {
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/hbase_scanner.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/async/hbase_scanner.h b/hbase-native-client/src/async/hbase_scanner.h
deleted file mode 100644
index cd3f544..0000000
--- a/hbase-native-client/src/async/hbase_scanner.h
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef ASYNC_HBASE_SCANNER_H_
-#define ASYNC_HBASE_SCANNER_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "async/hbase_result.h"
-#include "core/hbase_types.h"
-
-HBASE_API int32_t hb_scanner_create(hb_scanner_t * scanner_ptr);
-
-HBASE_API int32_t hb_scanner_set_table(hb_scanner_t scanner,
- char * table, size_t table_length);
-HBASE_API int32_t hb_scanner_set_namespace(hb_scanner_t scanner,
- char * name_space, size_t name_space_length);
-
-HBASE_API int32_t hb_scanner_set_start_row(hb_scanner_t scanner,
- unsigned char * start_row, size_t start_row_length);
-HBASE_API int32_t hb_scanner_set_end_row(hb_scanner_t scanner,
- unsigned char * end_row, size_t end_row_length);
-
-HBASE_API int32_t hb_scanner_set_cache_size(hb_scanner_t scanner,
- size_t cache_size);
-HBASE_API int32_t hb_scanner_set_batch_size(hb_scanner_t scanner,
- size_t batch_size);
-HBASE_API int32_t hb_scanner_set_num_versions(hb_scanner_t scanner,
- int8_t num_versions);
-
-/*
- * Scanner call back typedef.
- *
- * This will be called when initinalization of the scanner
- * is complete. It will also be called when scanner next
- * returns results.
- */
-typedef void (* hb_scanner_cb)(int32_t status,
- hb_client_t client,
- hb_scanner_t scanner,
- hb_result_t results,
- size_t num_results,
- void * extra);
-/*
- * Get the next results from the scanner
- */
-HBASE_API int32_t hb_scanner_next(hb_client_t client,
- hb_scanner_t scanner, hb_scanner_cb cb, void * extra);
-
-/*
- * Close the scanner releasing any local and server side
- * resources held. The call back is fired just before the
- * scanner's memory is freed.
- */
-HBASE_API int32_t hb_scanner_destroy(hb_client_t client,
- hb_scanner_t scanner, hb_scanner_cb cb, void * extra);
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // ASYNC_HBASE_SCANNER_H_
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/mutations-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/async/mutations-test.cc b/hbase-native-client/src/async/mutations-test.cc
deleted file mode 100644
index be5898e..0000000
--- a/hbase-native-client/src/async/mutations-test.cc
+++ /dev/null
@@ -1,102 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include <pthread.h>
-
-#include "gtest/gtest.h"
-#include "async/hbase_mutations.h"
-#include "async/hbase_client.h"
-
-pthread_cond_t cv;
-pthread_mutex_t mutex;
-
-bool sent = false;
-
-TEST(ClientTest, EasyTest) {
- EXPECT_EQ(1, 1);
-}
-
-void mutate_cb(int32_t status,
- hb_client_t client, hb_mutation_t mutation,
- hb_result_t result, void * extra) {
-
- // Test Stuff.
- EXPECT_EQ(status, 0);
- EXPECT_TRUE(client != NULL);
- EXPECT_TRUE(mutation != NULL);
-
- pthread_mutex_lock(&mutex);
- sent = true;
- pthread_cond_signal(&cv);
- pthread_mutex_unlock(&mutex);
-}
-
-void wait_send() {
- pthread_mutex_lock(&mutex);
- while (!sent) {
- pthread_cond_wait(&cv, &mutex);
- }
- pthread_mutex_unlock(&mutex);
-}
-
-TEST(MutationTest, TestPut) {
- char tn[] = "T1";
- hb_byte_t row[] = "ROW";
- char fam[] = "D";
- hb_byte_t qual[] = "QUAL";
- hb_byte_t data[] = "Z";
-
- hb_client_t client = NULL;
- hb_put_t put = NULL;
- hb_cell_t cell;
-
- cell.family = fam;
- cell.family_length = 1;
-
- cell.qual = qual;
- cell.qual_length = 4;
-
- cell.value = data;
- cell.value_length = 1;
-
- int32_t status = -1;
-
- status = hb_client_create(&client, NULL);
- EXPECT_EQ(0, status);
-
- hb_put_create(&put);
- hb_mutation_set_table((hb_mutation_t) put, tn, 2);
- hb_mutation_set_row((hb_mutation_t) put, row, 3);
- hb_put_add_cell(put, &cell);
-
- pthread_cond_init(&cv, NULL);
- pthread_mutex_init(&mutex, NULL);
-
- status = hb_mutation_send(client, (hb_mutation_t) put, &mutate_cb, NULL);
- EXPECT_EQ(0, status);
-
- // Now wait a while for things to send.
- wait_send();
- EXPECT_EQ(true, sent);
-
- hb_mutation_destroy((hb_mutation_t *) put);
- hb_client_destroy(client, NULL, NULL);
-
- EXPECT_EQ(0, status);
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/core/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/core/CMakeLists.txt b/hbase-native-client/src/core/CMakeLists.txt
deleted file mode 100644
index 49f93b9..0000000
--- a/hbase-native-client/src/core/CMakeLists.txt
+++ /dev/null
@@ -1,31 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set( CORE_SRC
- admin.cc
- client.cc
- connection.cc
- get.cc
- mutation.cc
- put.cc
- delete.cc
- scanner.cc
- hbase_connection_attr.cc
-)
-
-
-add_library(hcore OBJECT ${CORE_SRC})
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/core/admin.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/core/admin.cc b/hbase-native-client/src/core/admin.cc
deleted file mode 100644
index 897e6bf..0000000
--- a/hbase-native-client/src/core/admin.cc
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "core/admin.h"
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/core/admin.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/core/admin.h b/hbase-native-client/src/core/admin.h
deleted file mode 100644
index 624c4ac..0000000
--- a/hbase-native-client/src/core/admin.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef CORE_ADMIN_H_
-#define CORE_ADMIN_H_
-
-class Admin {
-};
-#endif // CORE_ADMIN_H_
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/core/client.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/core/client.cc b/hbase-native-client/src/core/client.cc
deleted file mode 100644
index 91c235c..0000000
--- a/hbase-native-client/src/core/client.cc
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "core/client.h"
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/core/client.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/core/client.h b/hbase-native-client/src/core/client.h
deleted file mode 100644
index 68348a8..0000000
--- a/hbase-native-client/src/core/client.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef CORE_CLIENT_H_
-#define CORE_CLIENT_H_
-
-class Client {
-};
-#endif // CORE_CLIENT_H_
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/core/connection.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/core/connection.cc b/hbase-native-client/src/core/connection.cc
deleted file mode 100644
index 099f31d..0000000
--- a/hbase-native-client/src/core/connection.cc
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include "core/connection.h"
-
-void Connection::set_zk_quorum(char * zk_q) {
- this->zk_quorum = zk_q;
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/core/connection.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/core/connection.h b/hbase-native-client/src/core/connection.h
deleted file mode 100644
index 463b7b5..0000000
--- a/hbase-native-client/src/core/connection.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#ifndef CORE_CONNECTION_H_
-#define CORE_CONNECTION_H_
-
-class Connection {
- char * zk_quorum;
- public:
- void set_zk_quorum(char * zk_q);
-};
-#endif // CORE_CONNECTION_H_
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/core/connection_attr.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/core/connection_attr.h b/hbase-native-client/src/core/connection_attr.h
deleted file mode 100644
index a7c229e..0000000
--- a/hbase-native-client/src/core/connection_attr.h
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef CORE_CONNECTION_ATTR_H_
-#define CORE_CONNECTION_ATTR_H_
-
-#include "core/hbase_macros.h"
-#include "core/hbase_types.h"
-
-class ConnectionAttr {
-};
-
-#endif // CORE_CONNECTION_ATTR_H_
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/core/delete.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/core/delete.cc b/hbase-native-client/src/core/delete.cc
deleted file mode 100644
index 695f5bf..0000000
--- a/hbase-native-client/src/core/delete.cc
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "core/delete.h"
-
-Delete::~Delete() {
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/core/delete.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/core/delete.h b/hbase-native-client/src/core/delete.h
deleted file mode 100644
index 5740fd9..0000000
--- a/hbase-native-client/src/core/delete.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef CORE_DELETE_H_
-#define CORE_DELETE_H_
-
-#include "core/mutation.h"
-
-class Delete: public Mutation {
- public:
- ~Delete();
-};
-#endif // CORE_DELETE_H_
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/core/get.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/core/get.cc b/hbase-native-client/src/core/get.cc
deleted file mode 100644
index 9e11332..0000000
--- a/hbase-native-client/src/core/get.cc
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "core/get.h"
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/core/get.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/core/get.h b/hbase-native-client/src/core/get.h
deleted file mode 100644
index 3a9fd28..0000000
--- a/hbase-native-client/src/core/get.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef CORE_GET_H_
-#define CORE_GET_H_
-
-class Get {
-};
-
-#endif // CORE_GET_H_
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/core/hbase_connection_attr.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/core/hbase_connection_attr.cc b/hbase-native-client/src/core/hbase_connection_attr.cc
deleted file mode 100644
index 61940e8..0000000
--- a/hbase-native-client/src/core/hbase_connection_attr.cc
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "core/hbase_connection_attr.h"
-
-#include "core/hbase_macros.h"
-#include "core/hbase_types.h"
-#include "core/connection_attr.h"
-
-extern "C" {
-int32_t hb_connection_attr_create(hb_connection_attr_t * attr_ptr) {
- (*attr_ptr) = new ConnectionAttr();
- return (attr_ptr == NULL)?-1:0;
-}
-
-int32_t hb_connection_attr_set_zk_quorum(hb_connection_t connection,
- char * zk_quorum) {
- return 0;
-}
-
-int32_t hb_connection_attr_set_zk_root(hb_connection_t connection,
- char * zk_root) {
- return 0;
-}
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/core/hbase_connection_attr.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/core/hbase_connection_attr.h b/hbase-native-client/src/core/hbase_connection_attr.h
deleted file mode 100644
index c73c818..0000000
--- a/hbase-native-client/src/core/hbase_connection_attr.h
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef CORE_HBASE_CONNECTION_ATTR_H_
-#define CORE_HBASE_CONNECTION_ATTR_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "core/hbase_macros.h"
-#include "core/hbase_types.h"
-
-#include <stdlib.h>
-
-HBASE_API int32_t hb_connection_attr_create(hb_connection_attr_t * attr_ptr);
-
-/**
- * Set the zk quorum of a connection that will be created.
- */
-HBASE_API int32_t hb_connection_attr_set_zk_quorum(hb_connection_t connection,
- char * zk_quorum);
-
-/**
- * Set the zk root of a connection that will be created.
- */
-HBASE_API int32_t hb_connection_attr_set_zk_root(hb_connection_t connection,
- char * zk_root);
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // CORE_HBASE_CONNECTION_ATTR_H_
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/core/hbase_macros.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/core/hbase_macros.h b/hbase-native-client/src/core/hbase_macros.h
deleted file mode 100644
index 71765c8..0000000
--- a/hbase-native-client/src/core/hbase_macros.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef CORE_HBASE_MACROS_H_
-#define CORE_HBASE_MACROS_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/**
- * The following code block define API as the tag for exported
- * functions. The library should be compiled with symbols visibility
- * set to hidden by default and only the exported functions should be
- * tagged as HBASE_API.
- *
- * When building the library on Windows, compile with compiler flag
- * "-D_LIBHBASE_IMPLEMENTATION_", whereas when linking application with
- * this library, this compiler flag should not be used.
- */
-#if defined _WIN32 || defined __CYGWIN__
- #ifdef _LIBHBASE_IMPLEMENTATION_
- #define API __declspec(dllexport)
- #else
- #ifdef _LIBHBASE_TEST_
- #define HBASE_API
- #else
- #define HBASE_API __declspec(dllimport)
- #endif
- #endif
-#else
- #if __GNUC__ >= 4
- #define HBASE_API __attribute__ ((visibility ("default")))
- #else
- #define HBASE_API
- #endif
-#endif
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // CORE_HBASE_MACROS_H_
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/core/hbase_types.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/core/hbase_types.h b/hbase-native-client/src/core/hbase_types.h
deleted file mode 100644
index 8889b92..0000000
--- a/hbase-native-client/src/core/hbase_types.h
+++ /dev/null
@@ -1,83 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef CORE_HBASE_TYPES_H_
-#define CORE_HBASE_TYPES_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdint.h>
-#include <stddef.h>
-
-typedef unsigned char hb_byte_t;
-
-/*
- * Base kv type.
- */
-typedef struct {
- hb_byte_t* row;
- size_t row_length;
-
- char * family;
- size_t family_length;
-
- hb_byte_t* qual;
- size_t qual_length;
-
- hb_byte_t* value;
- size_t value_length;
-
- uint64_t timestamp;
-} hb_cell_t;
-
-typedef enum {
- DELETE_ONE_VERSION,
- DELETE_MULTIPLE_VERSIONS,
- DELETE_FAMILY,
- DELETE_FAMILY_VERSION
-} hb_delete_type;
-
-typedef enum {
- USE_DEFAULT,
- SKIP_WAL,
- ASYNC_WAL,
- SYNC_WAL,
- HSYNC_WAL
-} hb_durability_type;
-
-typedef void* hb_admin_t;
-typedef void* hb_client_t;
-typedef void* hb_connection_attr_t;
-typedef void* hb_connection_t;
-typedef void* hb_get_t;
-typedef void* hb_mutation_t;
-typedef void* hb_put_t;
-typedef void* hb_delete_t;
-typedef void* hb_increment_t;
-typedef void* hb_append_t;
-typedef void* hb_result_t;
-typedef void* hb_scanner_t;
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // CORE_HBASE_TYPES_H_
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/core/mutation.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/core/mutation.cc b/hbase-native-client/src/core/mutation.cc
deleted file mode 100644
index 4855076..0000000
--- a/hbase-native-client/src/core/mutation.cc
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "core/mutation.h"
-
-void Mutation::set_namespace(char * name_space, size_t name_space_length) {
- this->name_space = name_space;
- this->name_space_length = name_space_length;
-}
-
-void Mutation::set_table(char * table, size_t table_length) {
- this->table = table;
- this->table_length = table_length;
-}
-
-void Mutation::set_row(unsigned char * row, size_t row_length) {
- this->row = row;
- this->row_length = row_length;
-}
-
-void Mutation::set_durability(hb_durability_type durability) {
- this->durability = durability;
-}
-
-Mutation::~Mutation() {
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/core/mutation.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/core/mutation.h b/hbase-native-client/src/core/mutation.h
deleted file mode 100644
index e35d595..0000000
--- a/hbase-native-client/src/core/mutation.h
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef CORE_MUTATION_H_
-#define CORE_MUTATION_H_
-
-#include <stdlib.h>
-
-#include "core/hbase_types.h"
-
-class Mutation {
- char * name_space;
- size_t name_space_length;
-
- char * table;
- size_t table_length;
-
- unsigned char * row;
- size_t row_length;
-
- hb_durability_type durability;
- public:
- void set_namespace(char * name_space, size_t name_space_length);
- void set_table(char * table, size_t table_length);
- void set_row(unsigned char * row, size_t row_length);
- void set_durability(hb_durability_type durability);
-
- virtual ~Mutation();
-};
-#endif // CORE_MUTATION_H_
-
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/core/put.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/core/put.cc b/hbase-native-client/src/core/put.cc
deleted file mode 100644
index 8bedc8b..0000000
--- a/hbase-native-client/src/core/put.cc
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "core/put.h"
-
-Put::~Put() {
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/core/put.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/core/put.h b/hbase-native-client/src/core/put.h
deleted file mode 100644
index 7bca3e3..0000000
--- a/hbase-native-client/src/core/put.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef CORE_PUT_H_
-#define CORE_PUT_H_
-
-#include "core/mutation.h"
-
-class Put: public Mutation {
- public:
- ~Put();
-};
-#endif // CORE_PUT_H_
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/core/scanner.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/core/scanner.cc b/hbase-native-client/src/core/scanner.cc
deleted file mode 100644
index a10e444..0000000
--- a/hbase-native-client/src/core/scanner.cc
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "core/scanner.h"
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/core/scanner.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/core/scanner.h b/hbase-native-client/src/core/scanner.h
deleted file mode 100644
index 257583b..0000000
--- a/hbase-native-client/src/core/scanner.h
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef CORE_SCANNER_H_
-#define CORE_SCANNER_H_
-
-class Scanner {
-};
-#endif // CORE_SCANNER_H_
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/rpc/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/rpc/CMakeLists.txt b/hbase-native-client/src/rpc/CMakeLists.txt
deleted file mode 100644
index 2456923..0000000
--- a/hbase-native-client/src/rpc/CMakeLists.txt
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/sync/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/sync/CMakeLists.txt b/hbase-native-client/src/sync/CMakeLists.txt
deleted file mode 100644
index bfb7e6c..0000000
--- a/hbase-native-client/src/sync/CMakeLists.txt
+++ /dev/null
@@ -1,24 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set( SYNC_SRC
- hbase_connection.cc
- hbase_admin.cc
-)
-
-
-add_library(hsync OBJECT ${SYNC_SRC})
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/sync/hbase_admin.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/sync/hbase_admin.cc b/hbase-native-client/src/sync/hbase_admin.cc
deleted file mode 100644
index d43c8ec..0000000
--- a/hbase-native-client/src/sync/hbase_admin.cc
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "sync/hbase_admin.h"
-
-#include <stdlib.h>
-#include <stdbool.h>
-
-#include "core/admin.h"
-
-int32_t hb_admin_create(hb_admin_t** admin_ptr) {
- (*admin_ptr) = reinterpret_cast<hb_admin_t *>(new Admin());
- return 0;
-}
-
-/*
- * Disconnect the admin releasing any internal objects
- * or connections created in the background.
- */
-int32_t hb_admin_destroy(hb_admin_t * admin) {
- Admin * adm = reinterpret_cast<Admin *>(admin);
- delete adm;
- return 0;
-}
-
-/*
- * See if a table exists.
- */
-int32_t hb_admin_table_exists(hb_admin_t * admin,
- char * name_space, size_t name_space_length,
- char * table, size_t table_length,
- bool * exists) {
- *exists = true;
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/sync/hbase_admin.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/sync/hbase_admin.h b/hbase-native-client/src/sync/hbase_admin.h
deleted file mode 100644
index 860bfba..0000000
--- a/hbase-native-client/src/sync/hbase_admin.h
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef SYNC_HBASE_ADMIN_H_
-#define SYNC_HBASE_ADMIN_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdlib.h>
-#include <stdbool.h>
-
-#include "core/hbase_macros.h"
-#include "core/hbase_types.h"
-#include "sync/hbase_connection.h"
-
-
-/**
- * Create a new hb_admin.
- * All fields are initialized to the defaults. If you want to set
- * connection or other properties, set those before calling any
- * RPC functions.
- */
-HBASE_API int32_t hb_admin_create(hb_admin_t* admin_ptr,
- hb_connection_t connection);
-
-/*
- * Disconnect the admin releasing any internal objects
- * or connections created in the background.
- */
-HBASE_API int32_t hb_admin_destroy(hb_admin_t admin);
-
-/*
- * See if a table exists.
- */
-HBASE_API int32_t hb_admin_table_exists(hb_admin_t admin,
- char * name_space, size_t name_space_length,
- char * table, size_t table_length, bool * exists);
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // SYNC_HBASE_ADMIN_H_
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/sync/hbase_connection.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/sync/hbase_connection.cc b/hbase-native-client/src/sync/hbase_connection.cc
deleted file mode 100644
index df05376..0000000
--- a/hbase-native-client/src/sync/hbase_connection.cc
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "sync/hbase_connection.h"
-
-#include "core/connection.h"
-#include "core/hbase_types.h"
-
-extern "C" {
-int32_t hb_connection_create(hb_connection_t * connection_ptr,
- hb_connection_attr_t connection_attr) {
- (*connection_ptr) = reinterpret_cast<hb_connection_t>(new Connection());
- if ((*connection_ptr) == NULL)
- return -1;
- return 0;
-}
-int32_t hb_connection_destroy(hb_connection_t connection) {
- free(connection);
- return 0;
-}
-} // extern "C"
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/sync/hbase_connection.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/sync/hbase_connection.h b/hbase-native-client/src/sync/hbase_connection.h
deleted file mode 100644
index af9284d..0000000
--- a/hbase-native-client/src/sync/hbase_connection.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef SYNC_HBASE_CONNECTION_H_
-#define SYNC_HBASE_CONNECTION_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "core/hbase_macros.h"
-#include "core/hbase_types.h"
-#include "core/hbase_connection_attr.h"
-
-#include <stdlib.h>
-
-/**
- * Create an hb_connection.
- *
- * if connection_attr is null everything will be left as default
- */
-HBASE_API int32_t hb_connection_create(hb_connection_t * connection_ptr,
- hb_connection_attr_t connection_attr);
-
-/**
- * Destroy the connection and free all resources allocated at creation
- * time.
- */
-HBASE_API int32_t hb_connection_destroy(hb_connection_t connection);
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // SYNC_HBASE_CONNECTION_H_
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/third-party/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/third-party/BUCK b/hbase-native-client/third-party/BUCK
new file mode 100644
index 0000000..89d2c93
--- /dev/null
+++ b/hbase-native-client/third-party/BUCK
@@ -0,0 +1,105 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+def add_system_libs(names = []):
+ rules = []
+ for name in names:
+ prebuilt_cxx_library(
+ name = name,
+ lib_name = name,
+ lib_dir = "/usr/lib/x86_64-linux-gnu"
+ )
+ rules.append(":" + name)
+
+ return rules
+
+system_libs = [
+ "double-conversion",
+ "glog",
+ "gflags",
+ "unwind",
+ "lzma",
+ "boost_regex",
+]
+tp_dep_rules = add_system_libs(system_libs)
+prebuilt_cxx_library(
+ name = "folly",
+ lib_name = "folly",
+ lib_dir = "/usr/local/lib",
+ deps = tp_dep_rules,
+ exported_linker_flags = [
+ "-pthread",
+ "-lstdc++",
+ ],
+ visibility = [
+ 'PUBLIC',
+ ]
+)
+prebuilt_cxx_library(
+ name = "follybenchmark",
+ lib_name = "follybenchmark",
+ lib_dir = "/usr/local/lib",
+ deps = tp_dep_rules + [":folly"],
+ exported_linker_flags = [
+ "-pthread",
+ "-lstdc++",
+ ],
+ visibility = [
+ 'PUBLIC',
+ ]
+)
+prebuilt_cxx_library(
+ name = "wangle",
+ lib_name = "wangle",
+ lib_dir = "/usr/local/lib",
+ deps = tp_dep_rules + [":folly"],
+ exported_linker_flags = [
+ "-pthread",
+ "-lstdc++",
+ ],
+ visibility = [
+ 'PUBLIC',
+ ]
+)
+cxx_library(
+ name = 'google-test',
+ srcs = [
+ 'googletest/googletest/src/gtest-all.cc',
+ 'googletest/googlemock/src/gmock-all.cc',
+ 'googletest/googlemock/src/gmock_main.cc',
+ ],
+ header_namespace = '',
+ exported_headers = subdir_glob([
+ ('googletest/googletest/include', '**/*.h'),
+ ('googletest/googlemock/include', '**/*.h'),
+ ]),
+ headers = subdir_glob([
+ ('googletest/googletest', 'src/*.h'),
+ ('googletest/googletest', 'src/*.cc'),
+ ('googletest/googlemock', 'src/*.h'),
+ ('googletest/googlemock', 'src/*.cc'),
+ ]),
+ exported_linker_flags = [
+ "-pthread",
+ "-lstdc++",
+ ],
+ visibility = [
+ 'PUBLIC',
+ ],
+ deps = [
+ ]
+)
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ea7743f..7dea57b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -849,6 +849,10 @@
<exclude>**/CHANGES.txt</exclude>
<exclude>**/generated/**</exclude>
<exclude>**/gen-*/**</exclude>
+ <!-- native build leftovers -->
+ <exclude>**/buck-out/**</exclude>
+ <exclude>**/.buckd/**</exclude>
+ <exclude>hbase-native-client/third-party/**</exclude>
<!-- No material contents -->
<exclude>conf/regionservers</exclude>
<exclude>**/*.avpr</exclude>
[03/50] [abbrv] hbase git commit: HBASE-16201 NPE in RpcServer
causing intermittent UT failure of
TestMasterReplication#testHFileCyclicReplication
Posted by ec...@apache.org.
HBASE-16201 NPE in RpcServer causing intermittent UT failure of TestMasterReplication#testHFileCyclicReplication
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3c39cbd9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3c39cbd9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3c39cbd9
Branch: refs/heads/HBASE-14850
Commit: 3c39cbd92c3f309c98ca01bfb70ca89bc046a228
Parents: 1578a04
Author: Yu Li <li...@apache.org>
Authored: Sat Jul 9 01:05:10 2016 +0800
Committer: Yu Li <li...@apache.org>
Committed: Sat Jul 9 02:02:00 2016 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/3c39cbd9/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
index ad88e8e..73226aa 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
@@ -2246,7 +2246,13 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
// The above callBlockingMethod will always return a SE. Strip the SE wrapper before
// putting it on the wire. Its needed to adhere to the pb Service Interface but we don't
// need to pass it over the wire.
- if (e instanceof ServiceException) e = e.getCause();
+ if (e instanceof ServiceException) {
+ if (e.getCause() == null) {
+ LOG.debug("Caught a ServiceException with null cause", e);
+ } else {
+ e = e.getCause();
+ }
+ }
// increment the number of requests that were exceptions.
metrics.exception(e);
[38/50] [abbrv] hbase git commit: HBASE-15750 Add on meta
deserialization
Posted by ec...@apache.org.
HBASE-15750 Add on meta deserialization
Summary: Add on meta region info deserialization
Test Plan:
Unit tests.
Simple client connects.
Differential Revision: https://reviews.facebook.net/D57555
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/190fc7aa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/190fc7aa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/190fc7aa
Branch: refs/heads/HBASE-14850
Commit: 190fc7aae288c97cb23e8cc130dbe7774917cedb
Parents: 50d1672
Author: Elliott Clark <ec...@apache.org>
Authored: Tue May 3 12:17:07 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
.../connection/client-dispatcher.cc | 1 +
.../connection/client-handler.cc | 13 +-
hbase-native-client/connection/client-handler.h | 6 +-
.../connection/connection-pool.cc | 36 +---
.../connection/connection-pool.h | 8 +-
hbase-native-client/core/BUCK | 4 -
hbase-native-client/core/location-cache-test.cc | 1 +
hbase-native-client/core/location-cache.cc | 73 +++++++-
hbase-native-client/core/location-cache.h | 7 +-
hbase-native-client/core/meta-utils.cc | 9 +-
hbase-native-client/core/meta-utils.h | 7 +-
hbase-native-client/core/region-location.h | 5 +-
hbase-native-client/core/simple-client.cc | 6 +-
hbase-native-client/core/table-name-test.cc | 54 ------
hbase-native-client/core/table-name.h | 50 -----
hbase-native-client/serde/BUCK | 56 +++---
.../serde/client-deserializer-test.cc | 25 ++-
.../serde/client-deserializer.cc | 68 -------
hbase-native-client/serde/client-deserializer.h | 36 ----
.../serde/client-serializer-test.cc | 26 +--
hbase-native-client/serde/client-serializer.cc | 139 --------------
hbase-native-client/serde/client-serializer.h | 55 ------
.../serde/region-info-deserializer-test.cc | 54 ++++++
hbase-native-client/serde/region-info.h | 41 +++++
hbase-native-client/serde/rpc.cc | 181 +++++++++++++++++++
hbase-native-client/serde/rpc.h | 58 ++++++
hbase-native-client/serde/server-name-test.cc | 32 ++++
hbase-native-client/serde/server-name.h | 21 +++
hbase-native-client/serde/table-name-test.cc | 54 ++++++
hbase-native-client/serde/table-name.h | 54 ++++++
.../serde/zk-deserializer-test.cc | 8 +-
hbase-native-client/serde/zk-deserializer.cc | 78 --------
hbase-native-client/serde/zk-deserializer.h | 35 ----
hbase-native-client/serde/zk.cc | 78 ++++++++
hbase-native-client/serde/zk.h | 35 ++++
35 files changed, 761 insertions(+), 653 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/connection/client-dispatcher.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-dispatcher.cc b/hbase-native-client/connection/client-dispatcher.cc
index 817adc1..6e2dc54 100644
--- a/hbase-native-client/connection/client-dispatcher.cc
+++ b/hbase-native-client/connection/client-dispatcher.cc
@@ -44,6 +44,7 @@ Future<Response> ClientDispatcher::operator()(std::unique_ptr<Request> arg) {
auto &p = requests_[call_id];
auto f = p.getFuture();
p.setInterruptHandler([call_id, this](const folly::exception_wrapper &e) {
+ LOG(ERROR) << "e = " << call_id;
this->requests_.erase(call_id);
});
this->pipeline_->write(std::move(arg));
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/connection/client-handler.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-handler.cc b/hbase-native-client/connection/client-handler.cc
index 3180f4e..496e4f2 100644
--- a/hbase-native-client/connection/client-handler.cc
+++ b/hbase-native-client/connection/client-handler.cc
@@ -37,8 +37,7 @@ using hbase::pb::GetResponse;
using google::protobuf::Message;
ClientHandler::ClientHandler(std::string user_name)
- : user_name_(user_name), need_send_header_(true), ser_(), deser_(),
- resp_msgs_() {}
+ : user_name_(user_name), need_send_header_(true), serde_(), resp_msgs_() {}
void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
if (LIKELY(buf != nullptr)) {
@@ -46,7 +45,7 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
Response received;
ResponseHeader header;
- int used_bytes = deser_.parse_delimited(buf.get(), &header);
+ int used_bytes = serde_.ParseDelimited(buf.get(), &header);
LOG(INFO) << "Read ResponseHeader size=" << used_bytes
<< " call_id=" << header.call_id()
<< " has_exception=" << header.has_exception();
@@ -70,7 +69,7 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
// data left on the wire.
if (header.has_exception() == false) {
buf->trimStart(used_bytes);
- used_bytes = deser_.parse_delimited(buf.get(), resp_msg.get());
+ used_bytes = serde_.ParseDelimited(buf.get(), resp_msg.get());
// Make sure that bytes were parsed.
CHECK(used_bytes == buf->length());
received.set_response(resp_msg);
@@ -91,13 +90,13 @@ Future<Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
// and one for the request.
//
// That doesn't seem like too bad, but who knows.
- auto pre = ser_.preamble();
- auto header = ser_.header(user_name_);
+ auto pre = serde_.Preamble();
+ auto header = serde_.Header(user_name_);
pre->appendChain(std::move(header));
ctx->fireWrite(std::move(pre));
}
resp_msgs_[r->call_id()] = r->resp_msg();
return ctx->fireWrite(
- ser_.request(r->call_id(), r->method(), r->req_msg().get()));
+ serde_.Request(r->call_id(), r->method(), r->req_msg().get()));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/connection/client-handler.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-handler.h b/hbase-native-client/connection/client-handler.h
index 68513de..ce99c9e 100644
--- a/hbase-native-client/connection/client-handler.h
+++ b/hbase-native-client/connection/client-handler.h
@@ -22,8 +22,7 @@
#include <string>
-#include "serde/client-deserializer.h"
-#include "serde/client-serializer.h"
+#include "serde/rpc.h"
// Forward decs.
namespace hbase {
@@ -49,8 +48,7 @@ public:
private:
bool need_send_header_;
std::string user_name_;
- ClientSerializer ser_;
- ClientDeserializer deser_;
+ RpcSerde serde_;
// in flight requests
std::unordered_map<uint32_t, std::shared_ptr<google::protobuf::Message>>
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/connection/connection-pool.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-pool.cc b/hbase-native-client/connection/connection-pool.cc
index 72c1306..eafe60a 100644
--- a/hbase-native-client/connection/connection-pool.cc
+++ b/hbase-native-client/connection/connection-pool.cc
@@ -25,41 +25,10 @@ using std::mutex;
using std::unique_ptr;
using std::shared_ptr;
using hbase::pb::ServerName;
-using wangle::ServiceFilter;
using folly::SharedMutexWritePriority;
namespace hbase {
-class RemoveServiceFilter
- : public ServiceFilter<unique_ptr<Request>, Response> {
-
-public:
- RemoveServiceFilter(std::shared_ptr<HBaseService> service, ServerName sn,
- ConnectionPool &cp)
- : ServiceFilter<unique_ptr<Request>, Response>(service), sn_(sn),
- cp_(cp) {}
-
- folly::Future<folly::Unit> close() override {
- if (!released.exchange(true)) {
- return this->service_->close().then(
- [this]() { this->cp_.close(this->sn_); });
- } else {
- return folly::makeFuture();
- }
- }
-
- virtual bool isAvailable() override { return service_->isAvailable(); }
-
- folly::Future<Response> operator()(unique_ptr<Request> req) override {
- return (*this->service_)(std::move(req));
- }
-
-private:
- std::atomic<bool> released{false};
- hbase::pb::ServerName sn_;
- ConnectionPool &cp_;
-};
-
ConnectionPool::ConnectionPool()
: cf_(std::make_shared<ConnectionFactory>()), connections_(), map_mutex_() {
}
@@ -72,13 +41,12 @@ std::shared_ptr<HBaseService> ConnectionPool::get(const ServerName &sn) {
if (found == connections_.end() || found->second == nullptr) {
SharedMutexWritePriority::WriteHolder holder(std::move(holder));
auto new_con = cf_->make_connection(sn.host_name(), sn.port());
- auto wrapped = std::make_shared<RemoveServiceFilter>(new_con, sn, *this);
- connections_[sn] = wrapped;
+ connections_[sn] = new_con;
return new_con;
}
return found->second;
}
-void ConnectionPool::close(ServerName sn) {
+void ConnectionPool::close(const ServerName &sn) {
SharedMutexWritePriority::WriteHolder holder(map_mutex_);
auto found = connections_.find(sn);
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/connection/connection-pool.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-pool.h b/hbase-native-client/connection/connection-pool.h
index 394cd71..b8330e3 100644
--- a/hbase-native-client/connection/connection-pool.h
+++ b/hbase-native-client/connection/connection-pool.h
@@ -27,13 +27,13 @@
#include "if/HBase.pb.h"
namespace hbase {
-struct MyServerNameEquals {
+struct ServerNameEquals {
bool operator()(const hbase::pb::ServerName &lhs,
const hbase::pb::ServerName &rhs) const {
return lhs.host_name() == rhs.host_name() && lhs.port() == rhs.port();
}
};
-struct MyServerNameHash {
+struct ServerNameHash {
std::size_t operator()(hbase::pb::ServerName const &s) const {
std::size_t h1 = std::hash<std::string>()(s.host_name());
std::size_t h2 = std::hash<uint32_t>()(s.port());
@@ -46,12 +46,12 @@ public:
ConnectionPool();
explicit ConnectionPool(std::shared_ptr<ConnectionFactory> cf);
std::shared_ptr<HBaseService> get(const hbase::pb::ServerName &sn);
- void close(hbase::pb::ServerName sn);
+ void close(const hbase::pb::ServerName &sn);
private:
std::shared_ptr<ConnectionFactory> cf_;
std::unordered_map<hbase::pb::ServerName, std::shared_ptr<HBaseService>,
- MyServerNameHash, MyServerNameEquals>
+ ServerNameHash, ServerNameEquals>
connections_;
folly::SharedMutexWritePriority map_mutex_;
};
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/core/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index 447248b..ef8c2f8 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -24,7 +24,6 @@ cxx_library(
"hbase_macros.h",
"region-location.h",
"location-cache.h",
- "table-name.h",
# TODO: move this out of exported
# Once meta lookup works
"meta-utils.h",
@@ -53,9 +52,6 @@ cxx_test(name="location-cache-test",
],
deps=[":core", ],
run_test_separately=True, )
-cxx_test(name="table-name-test",
- srcs=["table-name-test.cc", ],
- deps=[":core", ], )
cxx_binary(name="simple-client",
srcs=["simple-client.cc", ],
deps=[":core", "//connection:connection"], )
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/core/location-cache-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache-test.cc b/hbase-native-client/core/location-cache-test.cc
index f3166fb..172799d 100644
--- a/hbase-native-client/core/location-cache-test.cc
+++ b/hbase-native-client/core/location-cache-test.cc
@@ -30,4 +30,5 @@ TEST(LocationCacheTest, TestGetMetaNodeContents) {
auto result = f.get();
ASSERT_FALSE(f.hasException());
ASSERT_TRUE(result.has_port());
+ ASSERT_TRUE(result.has_host_name());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/core/location-cache.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
index 539051a..2667f11 100644
--- a/hbase-native-client/core/location-cache.cc
+++ b/hbase-native-client/core/location-cache.cc
@@ -20,19 +20,25 @@
#include <folly/Logging.h>
#include <folly/io/IOBuf.h>
+#include <wangle/concurrent/GlobalExecutor.h>
#include "connection/response.h"
#include "if/Client.pb.h"
#include "if/ZooKeeper.pb.h"
-#include "serde/zk-deserializer.h"
+#include "serde/server-name.h"
+#include "serde/region-info.h"
+#include "serde/zk.h"
using namespace std;
using namespace folly;
+using wangle::ServiceFilter;
+using hbase::Request;
using hbase::Response;
using hbase::LocationCache;
using hbase::RegionLocation;
using hbase::HBaseService;
+using hbase::ConnectionPool;
using hbase::pb::ScanResponse;
using hbase::pb::TableName;
using hbase::pb::ServerName;
@@ -45,7 +51,7 @@ static const char META_ZNODE_NAME[] = "/hbase/meta-region-server";
LocationCache::LocationCache(string quorum_spec,
shared_ptr<folly::Executor> executor)
: quorum_spec_(quorum_spec), executor_(executor), meta_promise_(nullptr),
- meta_lock_(), cp_(), meta_util_() {
+ meta_lock_(), cp_(), meta_util_(), zk_(nullptr) {
zk_ = zookeeper_init(quorum_spec.c_str(), nullptr, 1000, 0, 0, 0);
}
@@ -95,28 +101,77 @@ ServerName LocationCache::ReadMetaLocation() {
buf->append(len);
MetaRegionServer mrs;
- if (derser.parse(buf.get(), &mrs) == false) {
+ if (derser.Parse(buf.get(), &mrs) == false) {
LOG(ERROR) << "Unable to decode";
}
return mrs.server();
}
-Future<RegionLocation> LocationCache::locateFromMeta(const TableName &tn,
- const string &row) {
+Future<std::shared_ptr<RegionLocation>>
+LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
+ auto exc = wangle::getIOExecutor();
return this->LocateMeta()
.then([&](ServerName sn) { return this->cp_.get(sn); })
+ .via(exc.get()) // Need to handle all rpc's on the IOExecutor.
.then([&](std::shared_ptr<HBaseService> service) {
- return (*service)(std::move(meta_util_.make_meta_request(tn, row)));
+ return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
})
.then([&](Response resp) {
// take the protobuf response and make it into
// a region location.
- return this->parse_response(std::move(resp));
+ return this->CreateLocation(std::move(resp));
});
}
-RegionLocation LocationCache::parse_response(const Response &resp) {
+class RemoveServiceFilter
+ : public ServiceFilter<std::unique_ptr<Request>, Response> {
+
+public:
+ RemoveServiceFilter(std::shared_ptr<HBaseService> service, ServerName sn,
+ ConnectionPool &cp)
+ : ServiceFilter<unique_ptr<Request>, Response>(service), sn_(sn),
+ cp_(cp) {}
+
+ folly::Future<folly::Unit> close() override {
+ if (!released.exchange(true)) {
+ return this->service_->close().then([this]() {
+ // TODO(eclark): remove the service from the meta cache.
+ this->cp_.close(this->sn_);
+ });
+ } else {
+ return folly::makeFuture();
+ }
+ }
+
+ virtual bool isAvailable() override {
+ return !released && service_->isAvailable();
+ }
+
+ folly::Future<Response> operator()(unique_ptr<Request> req) override {
+ // TODO(eclark): add in an on error handler that will
+ // remove the region location from the cache if needed.
+ // Also close the connection if this is likely to be an error
+ // that needs to get a new connection.
+ return (*this->service_)(std::move(req));
+ }
+
+private:
+ std::atomic<bool> released{false};
+ hbase::pb::ServerName sn_;
+ ConnectionPool &cp_;
+};
+
+std::shared_ptr<RegionLocation>
+LocationCache::CreateLocation(const Response &resp){
auto resp_msg = static_pointer_cast<ScanResponse>(resp.response());
+ auto &results = resp_msg->results().Get(0);
+ auto &cells = results.cell();
LOG(ERROR) << "resp_msg = " << resp_msg->DebugString();
- return RegionLocation{RegionInfo{}, ServerName{}, nullptr};
+ auto ri = folly::to<RegionInfo>(cells.Get(0).value());
+ auto sn = folly::to<ServerName>(cells.Get(1).value());
+
+ LOG(ERROR) << "RegionInfo = " << ri.DebugString();
+ LOG(ERROR) << "ServerName = " << sn.DebugString();
+ auto wrapped = make_shared<RemoveServiceFilter>(cp_.get(sn), sn, this->cp_);
+ return std::make_shared<RegionLocation>(std::move(ri), std::move(sn), wrapped);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/core/location-cache.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h
index cfd6838..99b5e5e 100644
--- a/hbase-native-client/core/location-cache.h
+++ b/hbase-native-client/core/location-cache.h
@@ -29,8 +29,8 @@
#include "connection/connection-pool.h"
#include "core/meta-utils.h"
-#include "core/table-name.h"
#include "core/region-location.h"
+#include "serde/table-name.h"
namespace hbase {
@@ -48,14 +48,14 @@ public:
// Meta Related Methods.
// These are only public until testing is complete
folly::Future<hbase::pb::ServerName> LocateMeta();
- folly::Future<RegionLocation> locateFromMeta(const hbase::pb::TableName &tn,
+ folly::Future<std::shared_ptr<RegionLocation>> LocateFromMeta(const hbase::pb::TableName &tn,
const std::string &row);
- RegionLocation parse_response(const Response &resp);
void InvalidateMeta();
private:
void RefreshMetaLocation();
hbase::pb::ServerName ReadMetaLocation();
+ std::shared_ptr<RegionLocation> CreateLocation(const Response &resp);
std::string quorum_spec_;
std::shared_ptr<folly::Executor> executor_;
@@ -64,7 +64,6 @@ private:
ConnectionPool cp_;
MetaUtil meta_util_;
-
// TODO: migrate this to a smart pointer with a deleter.
zhandle_t *zk_;
};
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/core/meta-utils.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/meta-utils.cc b/hbase-native-client/core/meta-utils.cc
index d2fdd88..1325d83 100644
--- a/hbase-native-client/core/meta-utils.cc
+++ b/hbase-native-client/core/meta-utils.cc
@@ -23,25 +23,26 @@
#include "connection/request.h"
#include "connection/response.h"
-#include "core/table-name.h"
#include "if/Client.pb.h"
+#include "serde/table-name.h"
using hbase::pb::TableName;
using hbase::MetaUtil;
using hbase::Request;
using hbase::Response;
using hbase::pb::ScanRequest;
+using hbase::pb::ServerName;
using hbase::pb::RegionSpecifier_RegionSpecifierType;
static const std::string META_REGION = "1588230740";
-std::string MetaUtil::region_lookup_rowkey(const TableName &tn,
+std::string MetaUtil::RegionLookupRowkey(const TableName &tn,
const std::string &row) const {
return folly::to<std::string>(tn, ",", row, ",", "999999999999999999");
}
std::unique_ptr<Request>
-MetaUtil::make_meta_request(const TableName tn, const std::string &row) const {
+MetaUtil::MetaRequest(const TableName tn, const std::string &row) const {
auto request = Request::scan();
auto msg = std::static_pointer_cast<ScanRequest>(request->req_msg());
@@ -76,6 +77,6 @@ MetaUtil::make_meta_request(const TableName tn, const std::string &row) const {
info_col->add_qualifier("server");
info_col->add_qualifier("regioninfo");
- scan->set_start_row(region_lookup_rowkey(tn, row));
+ scan->set_start_row(RegionLookupRowkey(tn, row));
return request;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/core/meta-utils.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/meta-utils.h b/hbase-native-client/core/meta-utils.h
index e007d02..5a659f3 100644
--- a/hbase-native-client/core/meta-utils.h
+++ b/hbase-native-client/core/meta-utils.h
@@ -21,15 +21,16 @@
#include <string>
#include "connection/Request.h"
-#include "core/table-name.h"
+#include "if/HBase.pb.h"
+#include "serde/table-name.h"
namespace hbase {
class MetaUtil {
public:
- std::string region_lookup_rowkey(const hbase::pb::TableName &tn,
+ std::string RegionLookupRowkey(const hbase::pb::TableName &tn,
const std::string &row) const;
- std::unique_ptr<Request> make_meta_request(const hbase::pb::TableName tn,
+ std::unique_ptr<Request> MetaRequest(const hbase::pb::TableName tn,
const std::string &row) const;
};
} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/core/region-location.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/region-location.h b/hbase-native-client/core/region-location.h
index a46b8e2..7922c95 100644
--- a/hbase-native-client/core/region-location.h
+++ b/hbase-native-client/core/region-location.h
@@ -23,7 +23,6 @@
#include "connection/service.h"
#include "if/HBase.pb.h"
-
namespace hbase {
class RegionLocation {
@@ -32,8 +31,8 @@ public:
std::shared_ptr<HBaseService> service)
: ri_(ri), sn_(sn), service_(service) {}
- const hbase::pb::RegionInfo& region_info() { return ri_; }
- const hbase::pb::ServerName& server_name() { return sn_; }
+ const hbase::pb::RegionInfo ®ion_info() { return ri_; }
+ const hbase::pb::ServerName &server_name() { return sn_; }
std::shared_ptr<HBaseService> service() { return service_; }
private:
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/core/simple-client.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/simple-client.cc b/hbase-native-client/core/simple-client.cc
index ab614e4..00e3369 100644
--- a/hbase-native-client/core/simple-client.cc
+++ b/hbase-native-client/core/simple-client.cc
@@ -27,9 +27,9 @@
#include "connection/connection-pool.h"
#include "core/client.h"
-#include "core/table-name.h"
#include "if/Client.pb.h"
#include "if/ZooKeeper.pb.h"
+#include "serde/table-name.h"
using namespace folly;
using namespace std;
@@ -39,7 +39,7 @@ using hbase::Request;
using hbase::HBaseService;
using hbase::LocationCache;
using hbase::ConnectionPool;
-using hbase::TableNameUtil;
+using hbase::pb::TableName;
using hbase::pb::ServerName;
using hbase::pb::RegionSpecifier_RegionSpecifierType;
using hbase::pb::GetRequest;
@@ -61,7 +61,7 @@ int main(int argc, char *argv[]) {
auto cpu_ex = wangle::getCPUExecutor();
LocationCache cache{FLAGS_zookeeper, cpu_ex};
auto result =
- cache.locateFromMeta(TableNameUtil::create(FLAGS_table), FLAGS_row)
+ cache.LocateFromMeta(folly::to<TableName>(FLAGS_table), FLAGS_row)
.get(milliseconds(5000));
return 0;
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/core/table-name-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/table-name-test.cc b/hbase-native-client/core/table-name-test.cc
deleted file mode 100644
index 7bad3f1..0000000
--- a/hbase-native-client/core/table-name-test.cc
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include <folly/Conv.h>
-#include <gtest/gtest.h>
-
-#include <string>
-
-#include "core/table-name.h"
-
-using namespace hbase;
-using hbase::pb::TableName;
-
-TEST(TestTableName, TestToStringNoDefault) {
- TableName tn;
- tn.set_qualifier("TestTableName");
- std::string result = folly::to<std::string>(tn);
- ASSERT_EQ(result.find("default"), std::string::npos);
- ASSERT_EQ("TestTableName", result);
-}
-
-TEST(TestTableName, TestToStringNoDefaltWhenSet) {
- TableName tn;
- tn.set_namespace_("default");
- tn.set_qualifier("TestTableName");
- std::string result = folly::to<std::string>(tn);
- ASSERT_EQ(result.find("default"), std::string::npos);
- ASSERT_EQ("TestTableName", result);
-}
-
-TEST(TestTableName, TestToStringIncludeNS) {
- TableName tn;
- tn.set_namespace_("hbase");
- tn.set_qualifier("acl");
- std::string result = folly::to<std::string>(tn);
- ASSERT_EQ(result.find("hbase"), 0);
- ASSERT_EQ("hbase:acl", result);
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/core/table-name.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/table-name.h b/hbase-native-client/core/table-name.h
deleted file mode 100644
index 1612667..0000000
--- a/hbase-native-client/core/table-name.h
+++ /dev/null
@@ -1,50 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#pragma once
-
-#include <memory>
-#include <string>
-
-#include "if/HBase.pb.h"
-#include <folly/Conv.h>
-
-namespace hbase {
-namespace pb {
-
-// Provide folly::to<std::string>(TableName);
-template <class String> void toAppend(const TableName &in, String *result) {
- if (!in.has_namespace_() || in.namespace_() == "default") {
- folly::toAppend(in.qualifier(), result);
- } else {
- folly::toAppend(in.namespace_(), ':', in.qualifier(), result);
- }
-}
-
-} // namespace pb
-
-class TableNameUtil {
-public:
- static ::hbase::pb::TableName create(std::string table_name) {
- ::hbase::pb::TableName tn;
- tn.set_namespace_("default");
- tn.set_qualifier(table_name);
- return tn;
- }
-};
-} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/serde/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/BUCK b/hbase-native-client/serde/BUCK
index 539a221..db15026 100644
--- a/hbase-native-client/serde/BUCK
+++ b/hbase-native-client/serde/BUCK
@@ -17,49 +17,47 @@
cxx_library(name="serde",
exported_headers=[
- "client-serializer.h",
- "client-deserializer.h",
- "zk-deserializer.h",
+ "region-info.h",
+ "rpc.h",
+ "server-name.h",
+ "table-name.h",
+ "zk.h",
],
srcs=[
- "client-serializer.cc",
- "client-deserializer.cc",
- "zk-deserializer.cc",
+ "rpc.cc",
+ "zk.cc",
],
deps=[
"//if:if",
"//third-party:folly",
],
tests=[
- ":client-serializer-test",
":client-deserializer-test",
+ ":client-serializer-test",
+ ":server-name-test",
+ ":table-name-test",
+ ":zk-deserializer-test",
+ ":region-info-deserializer-test",
],
compiler_flags=['-Weffc++'],
visibility=[
'PUBLIC',
], )
-
+cxx_test(name="table-name-test",
+ srcs=["table-name-test.cc", ],
+ deps=[":serde", ], )
+cxx_test(name="server-name-test",
+ srcs=["server-name-test.cc", ],
+ deps=[":serde", ], )
cxx_test(name="client-serializer-test",
- srcs=[
- "client-serializer-test.cc",
- ],
- deps=[
- ":serde",
- "//if:if",
- ], )
+ srcs=["client-serializer-test.cc", ],
+ deps=[":serde", ], )
cxx_test(name="client-deserializer-test",
- srcs=[
- "client-deserializer-test.cc",
- ],
- deps=[
- ":serde",
- "//if:if",
- ], )
+ srcs=["client-deserializer-test.cc", ],
+ deps=[":serde", ], )
cxx_test(name="zk-deserializer-test",
- srcs=[
- "zk-deserializer-test.cc",
- ],
- deps=[
- ":serde",
- "//if:if",
- ], )
+ srcs=["zk-deserializer-test.cc", ],
+ deps=[":serde", ], )
+cxx_test(name="region-info-deserializer-test",
+ srcs=["region-info-deserializer-test.cc", ],
+ deps=[":serde", ], )
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/serde/client-deserializer-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/client-deserializer-test.cc b/hbase-native-client/serde/client-deserializer-test.cc
index 9fef093..8c571b1 100644
--- a/hbase-native-client/serde/client-deserializer-test.cc
+++ b/hbase-native-client/serde/client-deserializer-test.cc
@@ -16,13 +16,12 @@
* limitations under the License.
*
*/
+#include "serde/rpc.h"
#include <folly/io/IOBuf.h>
#include <gtest/gtest.h>
#include "if/Client.pb.h"
-#include "serde/client-deserializer.h"
-#include "serde/client-serializer.h"
using namespace hbase;
using folly::IOBuf;
@@ -30,23 +29,23 @@ using hbase::pb::GetRequest;
using hbase::pb::RegionSpecifier;
using hbase::pb::RegionSpecifier_RegionSpecifierType;
-TEST(TestClientDeserializer, TestReturnFalseOnNullPtr) {
- ClientDeserializer deser;
- ASSERT_LT(deser.parse_delimited(nullptr, nullptr), 0);
+TEST(TestRpcSerde, TestReturnFalseOnNullPtr) {
+ RpcSerde deser;
+ ASSERT_LT(deser.ParseDelimited(nullptr, nullptr), 0);
}
-TEST(TestClientDeserializer, TestReturnFalseOnBadInput) {
- ClientDeserializer deser;
+TEST(TestRpcSerde, TestReturnFalseOnBadInput) {
+ RpcSerde deser;
auto buf = IOBuf::copyBuffer("test");
GetRequest gr;
- ASSERT_LT(deser.parse_delimited(buf.get(), &gr), 0);
+ ASSERT_LT(deser.ParseDelimited(buf.get(), &gr), 0);
}
-TEST(TestClientDeserializer, TestGoodGetRequestFullRoundTrip) {
+TEST(TestRpcSerde, TestGoodGetRequestFullRoundTrip) {
GetRequest in;
- ClientSerializer ser;
- ClientDeserializer deser;
+ RpcSerde ser;
+ RpcSerde deser;
// fill up the GetRequest.
in.mutable_region()->set_value("test_region_id");
@@ -56,11 +55,11 @@ TEST(TestClientDeserializer, TestGoodGetRequestFullRoundTrip) {
in.mutable_get()->set_row("test_row");
// Create the buffer
- auto buf = ser.serialize_delimited(in);
+ auto buf = ser.SerializeDelimited(in);
GetRequest out;
- int used_bytes = deser.parse_delimited(buf.get(), &out);
+ int used_bytes = deser.ParseDelimited(buf.get(), &out);
ASSERT_GT(used_bytes, 0);
ASSERT_EQ(used_bytes, buf->length());
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/serde/client-deserializer.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/client-deserializer.cc b/hbase-native-client/serde/client-deserializer.cc
deleted file mode 100644
index acca7ea..0000000
--- a/hbase-native-client/serde/client-deserializer.cc
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "serde/client-deserializer.h"
-
-#include <folly/Logging.h>
-#include <google/protobuf/io/coded_stream.h>
-#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
-#include <google/protobuf/message.h>
-
-using namespace hbase;
-
-using folly::IOBuf;
-using google::protobuf::Message;
-using google::protobuf::io::ArrayInputStream;
-using google::protobuf::io::CodedInputStream;
-
-int ClientDeserializer::parse_delimited(const IOBuf *buf, Message *msg) {
- if (buf == nullptr || msg == nullptr) {
- return -2;
- }
-
- DCHECK(!buf->isChained());
-
- ArrayInputStream ais{buf->data(), static_cast<int>(buf->length())};
- CodedInputStream coded_stream{&ais};
-
- uint32_t msg_size;
-
- // Try and read the varint.
- if (coded_stream.ReadVarint32(&msg_size) == false) {
- FB_LOG_EVERY_MS(ERROR, 1000) << "Unable to read a var uint32_t";
- return -3;
- }
-
- coded_stream.PushLimit(msg_size);
- // Parse the message.
- if (msg->MergeFromCodedStream(&coded_stream) == false) {
- FB_LOG_EVERY_MS(ERROR, 1000)
- << "Unable to read a protobuf message from data.";
- return -4;
- }
-
- // Make sure all the data was consumed.
- if (coded_stream.ConsumedEntireMessage() == false) {
- FB_LOG_EVERY_MS(ERROR, 1000)
- << "Orphaned data left after reading protobuf message";
- return -5;
- }
-
- return coded_stream.CurrentPosition();
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/serde/client-deserializer.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/client-deserializer.h b/hbase-native-client/serde/client-deserializer.h
deleted file mode 100644
index b9664b0..0000000
--- a/hbase-native-client/serde/client-deserializer.h
+++ /dev/null
@@ -1,36 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#pragma once
-
-#include <folly/io/IOBuf.h>
-
-// Forward
-namespace google {
-namespace protobuf {
-class Message;
-}
-}
-
-namespace hbase {
-class ClientDeserializer {
-public:
- int parse_delimited(const folly::IOBuf *buf, google::protobuf::Message *msg);
-};
-
-} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/serde/client-serializer-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/client-serializer-test.cc b/hbase-native-client/serde/client-serializer-test.cc
index 9bf38af..2bd17fb 100644
--- a/hbase-native-client/serde/client-serializer-test.cc
+++ b/hbase-native-client/serde/client-serializer-test.cc
@@ -24,16 +24,16 @@
#include "if/HBase.pb.h"
#include "if/RPC.pb.h"
-#include "serde/client-serializer.h"
+#include "serde/rpc.h"
using namespace hbase;
using namespace hbase::pb;
using namespace folly;
using namespace folly::io;
-TEST(ClientSerializerTest, PreambleIncludesHBas) {
- ClientSerializer ser;
- auto buf = ser.preamble();
+TEST(RpcSerdeTest, PreambleIncludesHBas) {
+ RpcSerde ser;
+ auto buf = ser.Preamble();
const char *p = reinterpret_cast<const char *>(buf->data());
// Take the first for chars and make sure they are the
// magic string
@@ -42,16 +42,16 @@ TEST(ClientSerializerTest, PreambleIncludesHBas) {
EXPECT_EQ(6, buf->computeChainDataLength());
}
-TEST(ClientSerializerTest, PreambleIncludesVersion) {
- ClientSerializer ser;
- auto buf = ser.preamble();
+TEST(RpcSerdeTest, PreambleIncludesVersion) {
+ RpcSerde ser;
+ auto buf = ser.Preamble();
EXPECT_EQ(0, static_cast<const uint8_t *>(buf->data())[4]);
EXPECT_EQ(80, static_cast<const uint8_t *>(buf->data())[5]);
}
-TEST(ClientSerializerTest, TestHeaderLengthPrefixed) {
- ClientSerializer ser;
- auto header = ser.header("elliott");
+TEST(RpcSerdeTest, TestHeaderLengthPrefixed) {
+ RpcSerde ser;
+ auto header = ser.Header("elliott");
// The header should be prefixed by 4 bytes of length.
EXPECT_EQ(4, header->length());
@@ -64,9 +64,9 @@ TEST(ClientSerializerTest, TestHeaderLengthPrefixed) {
EXPECT_EQ(prefixed_len, header->next()->length());
}
-TEST(ClientSerializerTest, TestHeaderDecode) {
- ClientSerializer ser;
- auto buf = ser.header("elliott");
+TEST(RpcSerdeTest, TestHeaderDecode) {
+ RpcSerde ser;
+ auto buf = ser.Header("elliott");
auto header_buf = buf->next();
ConnectionHeader h;
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/serde/client-serializer.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/client-serializer.cc b/hbase-native-client/serde/client-serializer.cc
deleted file mode 100644
index 09b81c8..0000000
--- a/hbase-native-client/serde/client-serializer.cc
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#include "serde/client-serializer.h"
-
-#include <folly/Logging.h>
-#include <folly/io/Cursor.h>
-#include <google/protobuf/io/coded_stream.h>
-#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
-
-#include "if/HBase.pb.h"
-#include "if/RPC.pb.h"
-
-using namespace hbase;
-
-using folly::IOBuf;
-using folly::io::RWPrivateCursor;
-using google::protobuf::Message;
-using google::protobuf::io::ArrayOutputStream;
-using google::protobuf::io::CodedOutputStream;
-using google::protobuf::io::ZeroCopyOutputStream;
-using std::string;
-using std::unique_ptr;
-
-static const std::string PREAMBLE = "HBas";
-static const std::string INTERFACE = "ClientService";
-static const uint8_t RPC_VERSION = 0;
-static const uint8_t DEFAULT_AUTH_TYPE = 80;
-
-ClientSerializer::ClientSerializer() : auth_type_(DEFAULT_AUTH_TYPE) {}
-
-unique_ptr<IOBuf> ClientSerializer::preamble() {
- auto magic = IOBuf::copyBuffer(PREAMBLE, 0, 2);
- magic->append(2);
- RWPrivateCursor c(magic.get());
- c.skip(4);
- // Version
- c.write(RPC_VERSION);
- // Standard security aka Please don't lie to me.
- c.write(auth_type_);
- return magic;
-}
-
-unique_ptr<IOBuf> ClientSerializer::header(const string &user) {
- pb::ConnectionHeader h;
-
- // TODO(eclark): Make this not a total lie.
- h.mutable_user_info()->set_effective_user(user);
- // The service name that we want to talk to.
- //
- // Right now we're completely ignoring the service interface.
- // That may or may not be the correct thing to do.
- // It worked for a while with the java client; until it
- // didn't.
- h.set_service_name(INTERFACE);
- return prepend_length(serialize_message(h));
-}
-
-unique_ptr<IOBuf> ClientSerializer::request(const uint32_t call_id,
- const string &method,
- const Message *msg) {
- pb::RequestHeader rq;
- rq.set_method_name(method);
- rq.set_call_id(call_id);
- rq.set_request_param(msg != nullptr);
- auto ser_header = serialize_delimited(rq);
- if (msg != nullptr) {
- auto ser_req = serialize_delimited(*msg);
- ser_header->appendChain(std::move(ser_req));
- }
-
- return prepend_length(std::move(ser_header));
-}
-
-unique_ptr<IOBuf> ClientSerializer::prepend_length(unique_ptr<IOBuf> msg) {
- // Java ints are 4 long. So create a buffer that large
- auto len_buf = IOBuf::create(4);
- // Then make those bytes visible.
- len_buf->append(4);
-
- RWPrivateCursor c(len_buf.get());
- // Get the size of the data to be pushed out the network.
- auto size = msg->computeChainDataLength();
-
- // Write the length to this IOBuf.
- c.writeBE(static_cast<uint32_t>(size));
-
- // Then attach the origional to the back of len_buf
- len_buf->appendChain(std::move(msg));
- return len_buf;
-}
-
-unique_ptr<IOBuf> ClientSerializer::serialize_delimited(const Message &msg) {
- // Get the buffer size needed for just the message.
- int msg_size = msg.ByteSize();
- int buf_size = CodedOutputStream::VarintSize32(msg_size) + msg_size;
-
- // Create a buffer big enough to hold the varint and the object.
- auto buf = IOBuf::create(buf_size);
- buf->append(buf_size);
-
- // Create the array output stream.
- ArrayOutputStream aos{buf->writableData(), static_cast<int>(buf->length())};
- // Wrap the ArrayOuputStream in the coded output stream to allow writing
- // Varint32
- CodedOutputStream cos{&aos};
-
- // Write out the size.
- cos.WriteVarint32(msg_size);
-
- // Now write the rest out.
- // We're using the protobuf output streams here to keep track
- // of where in the output array we are rather than IOBuf.
- msg.SerializeWithCachedSizesToArray(
- cos.GetDirectBufferForNBytesAndAdvance(msg_size));
-
- // Return the buffer.
- return buf;
-}
-// TODO(eclark): Make this 1 copy.
-unique_ptr<IOBuf> ClientSerializer::serialize_message(const Message &msg) {
- auto buf = IOBuf::copyBuffer(msg.SerializeAsString());
- return buf;
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/serde/client-serializer.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/client-serializer.h b/hbase-native-client/serde/client-serializer.h
deleted file mode 100644
index 9c819fe..0000000
--- a/hbase-native-client/serde/client-serializer.h
+++ /dev/null
@@ -1,55 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#pragma once
-
-#include <cstdint>
-#include <folly/io/IOBuf.h>
-#include <string>
-
-// Forward
-namespace google {
-namespace protobuf {
-class Message;
-}
-}
-namespace hbase {
-class Request;
-}
-
-namespace hbase {
-class ClientSerializer {
-public:
- ClientSerializer();
- std::unique_ptr<folly::IOBuf> preamble();
- std::unique_ptr<folly::IOBuf> header(const std::string &user);
- std::unique_ptr<folly::IOBuf> request(const uint32_t call_id,
- const std::string &method,
- const google::protobuf::Message *msg);
- std::unique_ptr<folly::IOBuf>
- serialize_delimited(const google::protobuf::Message &msg);
-
- std::unique_ptr<folly::IOBuf>
- serialize_message(const google::protobuf::Message &msg);
-
- std::unique_ptr<folly::IOBuf>
- prepend_length(std::unique_ptr<folly::IOBuf> msg);
-
- uint8_t auth_type_;
-};
-} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/serde/region-info-deserializer-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/region-info-deserializer-test.cc b/hbase-native-client/serde/region-info-deserializer-test.cc
new file mode 100644
index 0000000..ce8dedf
--- /dev/null
+++ b/hbase-native-client/serde/region-info-deserializer-test.cc
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "serde/region-info.h"
+
+#include <gtest/gtest.h>
+
+#include <string>
+
+#include "if/HBase.pb.h"
+#include "serde/table-name.h"
+
+using std::string;
+using hbase::pb::RegionInfo;
+using hbase::pb::TableName;
+
+TEST(TestRegionInfoDesializer, TestDeserialize) {
+ string ns{"test_ns"};
+ string tn{"table_name"};
+ string start_row{"AAAAAA"};
+ string stop_row{"BBBBBBBBBBBB"};
+ uint64_t region_id = 2345678;
+
+ RegionInfo ri_out;
+ ri_out.set_region_id(region_id);
+ ri_out.mutable_table_name()->set_namespace_(ns);
+ ri_out.mutable_table_name()->set_qualifier(tn);
+ ri_out.set_start_key(start_row);
+ ri_out.set_end_key(stop_row);
+
+
+ string header{"PBUF"};
+ string ser = header + ri_out.SerializeAsString();
+
+ auto out = folly::to<RegionInfo>(ser);
+
+ EXPECT_EQ(region_id, out.region_id());
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/serde/region-info.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/region-info.h b/hbase-native-client/serde/region-info.h
new file mode 100644
index 0000000..6af351c
--- /dev/null
+++ b/hbase-native-client/serde/region-info.h
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include "if/HBase.pb.h"
+
+#include <folly/Conv.h>
+#include <boost/algorithm/string/predicate.hpp>
+
+namespace hbase {
+namespace pb {
+template <class String> void parseTo(String in, RegionInfo& out) {
+ // TODO(eclark): there has to be something better.
+ std::string s = folly::to<std::string>(in);
+
+ if (!boost::starts_with(s, "PBUF") ) {
+ throw std::runtime_error("Region Info field doesn't contain preamble");
+ }
+ if (!out.ParseFromArray(s.data() + 4, s.size() - 4)) {
+ throw std::runtime_error("Bad protobuf for RegionInfo");
+ }
+}
+} // namespace pb
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/serde/rpc.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/rpc.cc b/hbase-native-client/serde/rpc.cc
new file mode 100644
index 0000000..4c3c999
--- /dev/null
+++ b/hbase-native-client/serde/rpc.cc
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "serde/rpc.h"
+
+#include <folly/Logging.h>
+#include <folly/Logging.h>
+#include <folly/io/Cursor.h>
+#include <google/protobuf/io/coded_stream.h>
+#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
+#include <google/protobuf/message.h>
+
+#include "if/HBase.pb.h"
+#include "if/RPC.pb.h"
+
+using namespace hbase;
+
+using folly::IOBuf;
+using folly::io::RWPrivateCursor;
+using google::protobuf::Message;
+using google::protobuf::Message;
+using google::protobuf::io::ArrayInputStream;
+using google::protobuf::io::ArrayOutputStream;
+using google::protobuf::io::CodedInputStream;
+using google::protobuf::io::CodedOutputStream;
+using google::protobuf::io::ZeroCopyOutputStream;
+using std::string;
+using std::unique_ptr;
+
+static const std::string PREAMBLE = "HBas";
+static const std::string INTERFACE = "ClientService";
+static const uint8_t RPC_VERSION = 0;
+static const uint8_t DEFAULT_AUTH_TYPE = 80;
+
+int RpcSerde::ParseDelimited(const IOBuf *buf, Message *msg) {
+ if (buf == nullptr || msg == nullptr) {
+ return -2;
+ }
+
+ DCHECK(!buf->isChained());
+
+ ArrayInputStream ais{buf->data(), static_cast<int>(buf->length())};
+ CodedInputStream coded_stream{&ais};
+
+ uint32_t msg_size;
+
+ // Try and read the varint.
+ if (coded_stream.ReadVarint32(&msg_size) == false) {
+ FB_LOG_EVERY_MS(ERROR, 1000) << "Unable to read a var uint32_t";
+ return -3;
+ }
+
+ coded_stream.PushLimit(msg_size);
+ // Parse the message.
+ if (msg->MergeFromCodedStream(&coded_stream) == false) {
+ FB_LOG_EVERY_MS(ERROR, 1000)
+ << "Unable to read a protobuf message from data.";
+ return -4;
+ }
+
+ // Make sure all the data was consumed.
+ if (coded_stream.ConsumedEntireMessage() == false) {
+ FB_LOG_EVERY_MS(ERROR, 1000)
+ << "Orphaned data left after reading protobuf message";
+ return -5;
+ }
+
+ return coded_stream.CurrentPosition();
+}
+
+RpcSerde::RpcSerde() : auth_type_(DEFAULT_AUTH_TYPE) {}
+RpcSerde::~RpcSerde() {}
+
+unique_ptr<IOBuf> RpcSerde::Preamble() {
+ auto magic = IOBuf::copyBuffer(PREAMBLE, 0, 2);
+ magic->append(2);
+ RWPrivateCursor c(magic.get());
+ c.skip(4);
+ // Version
+ c.write(RPC_VERSION);
+ // Standard security aka Please don't lie to me.
+ c.write(auth_type_);
+ return magic;
+}
+
+unique_ptr<IOBuf> RpcSerde::Header(const string &user) {
+ pb::ConnectionHeader h;
+
+ // TODO(eclark): Make this not a total lie.
+ h.mutable_user_info()->set_effective_user(user);
+ // The service name that we want to talk to.
+ //
+ // Right now we're completely ignoring the service interface.
+ // That may or may not be the correct thing to do.
+ // It worked for a while with the java client; until it
+ // didn't.
+ h.set_service_name(INTERFACE);
+ return PrependLength(SerializeMessage(h));
+}
+
+unique_ptr<IOBuf> RpcSerde::Request(const uint32_t call_id,
+ const string &method, const Message *msg) {
+ pb::RequestHeader rq;
+ rq.set_method_name(method);
+ rq.set_call_id(call_id);
+ rq.set_request_param(msg != nullptr);
+ auto ser_header = SerializeDelimited(rq);
+ if (msg != nullptr) {
+ auto ser_req = SerializeDelimited(*msg);
+ ser_header->appendChain(std::move(ser_req));
+ }
+
+ return PrependLength(std::move(ser_header));
+}
+
+unique_ptr<IOBuf> RpcSerde::PrependLength(unique_ptr<IOBuf> msg) {
+ // Java ints are 4 long. So create a buffer that large
+ auto len_buf = IOBuf::create(4);
+ // Then make those bytes visible.
+ len_buf->append(4);
+
+ RWPrivateCursor c(len_buf.get());
+ // Get the size of the data to be pushed out the network.
+ auto size = msg->computeChainDataLength();
+
+ // Write the length to this IOBuf.
+ c.writeBE(static_cast<uint32_t>(size));
+
+ // Then attach the origional to the back of len_buf
+ len_buf->appendChain(std::move(msg));
+ return len_buf;
+}
+
+unique_ptr<IOBuf> RpcSerde::SerializeDelimited(const Message &msg) {
+ // Get the buffer size needed for just the message.
+ int msg_size = msg.ByteSize();
+ int buf_size = CodedOutputStream::VarintSize32(msg_size) + msg_size;
+
+ // Create a buffer big enough to hold the varint and the object.
+ auto buf = IOBuf::create(buf_size);
+ buf->append(buf_size);
+
+ // Create the array output stream.
+ ArrayOutputStream aos{buf->writableData(), static_cast<int>(buf->length())};
+ // Wrap the ArrayOuputStream in the coded output stream to allow writing
+ // Varint32
+ CodedOutputStream cos{&aos};
+
+ // Write out the size.
+ cos.WriteVarint32(msg_size);
+
+ // Now write the rest out.
+ // We're using the protobuf output streams here to keep track
+ // of where in the output array we are rather than IOBuf.
+ msg.SerializeWithCachedSizesToArray(
+ cos.GetDirectBufferForNBytesAndAdvance(msg_size));
+
+ // Return the buffer.
+ return buf;
+}
+// TODO(eclark): Make this 1 copy.
+unique_ptr<IOBuf> RpcSerde::SerializeMessage(const Message &msg) {
+ auto buf = IOBuf::copyBuffer(msg.SerializeAsString());
+ return buf;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/serde/rpc.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/rpc.h b/hbase-native-client/serde/rpc.h
new file mode 100644
index 0000000..cefb583
--- /dev/null
+++ b/hbase-native-client/serde/rpc.h
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <memory>
+#include <string>
+
+// Forward
+namespace folly {
+class IOBuf;
+}
+namespace google {
+namespace protobuf {
+class Message;
+}
+}
+
+namespace hbase {
+class RpcSerde {
+public:
+ RpcSerde();
+ virtual ~RpcSerde();
+ int ParseDelimited(const folly::IOBuf *buf, google::protobuf::Message *msg);
+ std::unique_ptr<folly::IOBuf> Preamble();
+ std::unique_ptr<folly::IOBuf> Header(const std::string &user);
+ std::unique_ptr<folly::IOBuf> Request(const uint32_t call_id,
+ const std::string &method,
+ const google::protobuf::Message *msg);
+ std::unique_ptr<folly::IOBuf>
+ SerializeDelimited(const google::protobuf::Message &msg);
+
+ std::unique_ptr<folly::IOBuf>
+ SerializeMessage(const google::protobuf::Message &msg);
+
+ std::unique_ptr<folly::IOBuf>
+ PrependLength(std::unique_ptr<folly::IOBuf> msg);
+
+private:
+ /* data */
+ uint8_t auth_type_;
+};
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/serde/server-name-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/server-name-test.cc b/hbase-native-client/serde/server-name-test.cc
new file mode 100644
index 0000000..35dcbc1
--- /dev/null
+++ b/hbase-native-client/serde/server-name-test.cc
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "serde/server-name.h"
+
+#include <gtest/gtest.h>
+#include <string>
+
+using hbase::pb::ServerName;
+
+TEST(TestServerName, TestMakeServerName) {
+ auto sn = folly::to<ServerName>("test:123");
+
+ ASSERT_EQ("test", sn.host_name());
+ ASSERT_EQ(123, sn.port());
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/serde/server-name.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/server-name.h b/hbase-native-client/serde/server-name.h
new file mode 100644
index 0000000..bdba087
--- /dev/null
+++ b/hbase-native-client/serde/server-name.h
@@ -0,0 +1,21 @@
+#pragma once
+
+#include "if/HBase.pb.h"
+#include <folly/Conv.h>
+#include <folly/String.h>
+
+namespace hbase {
+namespace pb {
+
+template <class String> void parseTo(String in, ServerName &out) {
+ // TODO see about getting rsplit into folly.
+ std::string s = folly::to<std::string>(in);
+
+ auto delim = s.rfind(":");
+ DCHECK(delim != std::string::npos);
+ out.set_host_name(s.substr(0, delim));
+ // Now keep everything after the : (delim + 1) to the end.
+ out.set_port(folly::to<int>(s.substr(delim + 1)));
+}
+}
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/serde/table-name-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/table-name-test.cc b/hbase-native-client/serde/table-name-test.cc
new file mode 100644
index 0000000..877d522
--- /dev/null
+++ b/hbase-native-client/serde/table-name-test.cc
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <folly/Conv.h>
+#include <gtest/gtest.h>
+
+#include <string>
+
+#include "serde/table-name.h"
+
+using namespace hbase;
+using hbase::pb::TableName;
+
+TEST(TestTableName, TestToStringNoDefault) {
+ TableName tn;
+ tn.set_qualifier("TestTableName");
+ std::string result = folly::to<std::string>(tn);
+ ASSERT_EQ(result.find("default"), std::string::npos);
+ ASSERT_EQ("TestTableName", result);
+}
+
+TEST(TestTableName, TestToStringNoDefaltWhenSet) {
+ TableName tn;
+ tn.set_namespace_("default");
+ tn.set_qualifier("TestTableName");
+ std::string result = folly::to<std::string>(tn);
+ ASSERT_EQ(result.find("default"), std::string::npos);
+ ASSERT_EQ("TestTableName", result);
+}
+
+TEST(TestTableName, TestToStringIncludeNS) {
+ TableName tn;
+ tn.set_namespace_("hbase");
+ tn.set_qualifier("acl");
+ std::string result = folly::to<std::string>(tn);
+ ASSERT_EQ(result.find("hbase"), 0);
+ ASSERT_EQ("hbase:acl", result);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/serde/table-name.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/table-name.h b/hbase-native-client/serde/table-name.h
new file mode 100644
index 0000000..c81e166
--- /dev/null
+++ b/hbase-native-client/serde/table-name.h
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <memory>
+#include <string>
+
+#include "if/HBase.pb.h"
+#include <folly/Conv.h>
+#include <folly/String.h>
+
+namespace hbase {
+namespace pb {
+
+// Provide folly::to<std::string>(TableName);
+template <class String> void toAppend(const TableName &in, String *result) {
+ if (!in.has_namespace_() || in.namespace_() == "default") {
+ folly::toAppend(in.qualifier(), result);
+ } else {
+ folly::toAppend(in.namespace_(), ':', in.qualifier(), result);
+ }
+}
+
+template <class String> void parseTo(String in, TableName &out) {
+ std::vector<std::string> v;
+ folly::split(":", in, v);
+
+ if (v.size() == 1) {
+ out.set_namespace_("default");
+ out.set_qualifier(v[0]);
+ } else {
+ out.set_namespace_(v[0]);
+ out.set_qualifier(v[1]);
+ }
+}
+
+} // namespace pb
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/serde/zk-deserializer-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/zk-deserializer-test.cc b/hbase-native-client/serde/zk-deserializer-test.cc
index 92d85a0..f07eecf 100644
--- a/hbase-native-client/serde/zk-deserializer-test.cc
+++ b/hbase-native-client/serde/zk-deserializer-test.cc
@@ -17,7 +17,7 @@
*
*/
-#include "serde/zk-deserializer.h"
+#include "serde/zk.h"
#include <folly/Logging.h>
#include <folly/io/Cursor.h>
@@ -41,7 +41,7 @@ TEST(TestZkDesializer, TestThrowNoMagicNum) {
buf->append(100);
RWPrivateCursor c{buf.get()};
c.write<uint8_t>(99);
- ASSERT_THROW(deser.parse(buf.get(), &mrs), runtime_error);
+ ASSERT_THROW(deser.Parse(buf.get(), &mrs), runtime_error);
}
// Test if the protobuf is in a format that we can't decode
@@ -78,7 +78,7 @@ TEST(TestZkDesializer, TestBadProtoThrow) {
// Create the protobuf
MetaRegionServer out;
- ASSERT_THROW(deser.parse(buf.get(), &out), runtime_error);
+ ASSERT_THROW(deser.Parse(buf.get(), &out), runtime_error);
}
// Test to make sure the whole thing works.
@@ -118,6 +118,6 @@ TEST(TestZkDesializer, TestNoThrow) {
// Create the protobuf
MetaRegionServer out;
- ASSERT_TRUE(deser.parse(buf.get(), &out));
+ ASSERT_TRUE(deser.Parse(buf.get(), &out));
ASSERT_EQ(mrs.server().host_name(), out.server().host_name());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/serde/zk-deserializer.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/zk-deserializer.cc b/hbase-native-client/serde/zk-deserializer.cc
deleted file mode 100644
index 33cf809..0000000
--- a/hbase-native-client/serde/zk-deserializer.cc
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "serde/zk-deserializer.h"
-
-#include <folly/io/Cursor.h>
-#include <folly/io/IOBuf.h>
-#include <google/protobuf/message.h>
-
-using hbase::ZkDeserializer;
-using std::runtime_error;
-using folly::IOBuf;
-using folly::io::Cursor;
-using google::protobuf::Message;
-
-static const std::string MAGIC_STRING = "PBUF";
-
-bool ZkDeserializer::parse(IOBuf *buf, Message *out) {
-
- // The format is like this
- // 1 byte of magic number. 255
- // 4 bytes of id length.
- // id_length number of bytes for the id of who put up the znode
- // 4 bytes of a magic string PBUF
- // Then the protobuf serialized without a varint header.
-
- Cursor c{buf};
-
- // There should be a magic number for recoverable zk
- uint8_t magic_num = c.read<uint8_t>();
- if (magic_num != 255) {
- LOG(ERROR) << "Magic number not in ZK znode data expected 255 got ="
- << unsigned(magic_num);
- throw runtime_error("Magic number not in znode data");
- }
- // How long is the id?
- uint32_t id_len = c.readBE<uint32_t>();
-
- if (id_len >= c.length()) {
- LOG(ERROR) << "After skiping the if from zookeeper data there's not enough "
- "left to read anything else";
- throw runtime_error("Not enough bytes to decode from zookeeper");
- }
-
- // Skip the id
- c.skip(id_len);
-
- // Make sure that the magic string is there.
- if (MAGIC_STRING != c.readFixedString(4)) {
- LOG(ERROR) << "There was no PBUF magic string.";
- throw runtime_error("No PBUF magic string in the zookpeeper data.");
- }
-
- // Try to decode the protobuf.
- // If there's an error bail out.
- if (out->ParseFromArray(c.data(), c.length()) == false) {
- LOG(ERROR) << "Error parsing Protobuf Message";
- throw runtime_error("Error parsing protobuf");
- }
-
- return true;
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/serde/zk-deserializer.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/zk-deserializer.h b/hbase-native-client/serde/zk-deserializer.h
deleted file mode 100644
index aa91661..0000000
--- a/hbase-native-client/serde/zk-deserializer.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#pragma once
-
-namespace google {
-namespace protobuf {
-class Message;
-}
-}
-namespace folly {
-class IOBuf;
-}
-
-namespace hbase {
-class ZkDeserializer {
-public:
- bool parse(folly::IOBuf *buf, google::protobuf::Message *out);
-};
-} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/serde/zk.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/zk.cc b/hbase-native-client/serde/zk.cc
new file mode 100644
index 0000000..59871a5
--- /dev/null
+++ b/hbase-native-client/serde/zk.cc
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "serde/zk.h"
+
+#include <folly/io/Cursor.h>
+#include <folly/io/IOBuf.h>
+#include <google/protobuf/message.h>
+
+using hbase::ZkDeserializer;
+using std::runtime_error;
+using folly::IOBuf;
+using folly::io::Cursor;
+using google::protobuf::Message;
+
+static const std::string MAGIC_STRING = "PBUF";
+
+bool ZkDeserializer::Parse(IOBuf *buf, Message *out) {
+
+ // The format is like this
+ // 1 byte of magic number. 255
+ // 4 bytes of id length.
+ // id_length number of bytes for the id of who put up the znode
+ // 4 bytes of a magic string PBUF
+ // Then the protobuf serialized without a varint header.
+
+ Cursor c{buf};
+
+ // There should be a magic number for recoverable zk
+ uint8_t magic_num = c.read<uint8_t>();
+ if (magic_num != 255) {
+ LOG(ERROR) << "Magic number not in ZK znode data expected 255 got ="
+ << unsigned(magic_num);
+ throw runtime_error("Magic number not in znode data");
+ }
+ // How long is the id?
+ uint32_t id_len = c.readBE<uint32_t>();
+
+ if (id_len >= c.length()) {
+ LOG(ERROR) << "After skiping the if from zookeeper data there's not enough "
+ "left to read anything else";
+ throw runtime_error("Not enough bytes to decode from zookeeper");
+ }
+
+ // Skip the id
+ c.skip(id_len);
+
+ // Make sure that the magic string is there.
+ if (MAGIC_STRING != c.readFixedString(4)) {
+ LOG(ERROR) << "There was no PBUF magic string.";
+ throw runtime_error("No PBUF magic string in the zookpeeper data.");
+ }
+
+ // Try to decode the protobuf.
+ // If there's an error bail out.
+ if (out->ParseFromArray(c.data(), c.length()) == false) {
+ LOG(ERROR) << "Error parsing Protobuf Message";
+ throw runtime_error("Error parsing protobuf");
+ }
+
+ return true;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/190fc7aa/hbase-native-client/serde/zk.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/zk.h b/hbase-native-client/serde/zk.h
new file mode 100644
index 0000000..b672bf4
--- /dev/null
+++ b/hbase-native-client/serde/zk.h
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+namespace google {
+namespace protobuf {
+class Message;
+}
+}
+namespace folly {
+class IOBuf;
+}
+
+namespace hbase {
+class ZkDeserializer {
+public:
+ bool Parse(folly::IOBuf *buf, google::protobuf::Message *out);
+};
+} // namespace hbase
[44/50] [abbrv] hbase git commit: HBASE-15730 Add on script to format
all .h, .cc, and BUCK files.
Posted by ec...@apache.org.
HBASE-15730 Add on script to format all .h,.cc, and BUCK files.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/c1140d74
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/c1140d74
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/c1140d74
Branch: refs/heads/HBASE-14850
Commit: c1140d742b5fa0ec6270479f580d58b0a45c2a17
Parents: 14a33d7
Author: Elliott Clark <ec...@apache.org>
Authored: Wed Apr 27 15:52:27 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/bin/format-code.sh | 7 +++++++
hbase-native-client/connection/request.cc | 14 +++++++-------
hbase-native-client/utils/BUCK | 8 ++++----
hbase-native-client/utils/user-util-test.cc | 2 +-
hbase-native-client/utils/user-util.cc | 4 ++--
hbase-native-client/utils/user-util.h | 2 +-
6 files changed, 22 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/c1140d74/hbase-native-client/bin/format-code.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/format-code.sh b/hbase-native-client/bin/format-code.sh
new file mode 100755
index 0000000..24d6515
--- /dev/null
+++ b/hbase-native-client/bin/format-code.sh
@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+
+
+find core connection serde utils -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i
+find core connection serde utils third-party -name "BUCK" | xargs -P8 yapf -i
http://git-wip-us.apache.org/repos/asf/hbase/blob/c1140d74/hbase-native-client/connection/request.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/request.cc b/hbase-native-client/connection/request.cc
index 50ea029..d4a4603 100644
--- a/hbase-native-client/connection/request.cc
+++ b/hbase-native-client/connection/request.cc
@@ -30,16 +30,16 @@ Request::Request(std::shared_ptr<google::protobuf::Message> req,
std::unique_ptr<Request> Request::get() {
return std::make_unique<Request>(std::make_shared<hbase::pb::GetRequest>(),
- std::make_shared<hbase::pb::GetResponse>(),
- "Get");
+ std::make_shared<hbase::pb::GetResponse>(),
+ "Get");
}
std::unique_ptr<Request> Request::mutate() {
- return std::make_unique<Request>(std::make_shared<hbase::pb::MutateRequest>(),
- std::make_shared<hbase::pb::MutateResponse>(),
- "Mutate");
+ return std::make_unique<Request>(
+ std::make_shared<hbase::pb::MutateRequest>(),
+ std::make_shared<hbase::pb::MutateResponse>(), "Mutate");
}
std::unique_ptr<Request> Request::scan() {
return std::make_unique<Request>(std::make_shared<hbase::pb::ScanRequest>(),
- std::make_shared<hbase::pb::ScanResponse>(),
- "Scan");
+ std::make_shared<hbase::pb::ScanResponse>(),
+ "Scan");
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/c1140d74/hbase-native-client/utils/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/utils/BUCK b/hbase-native-client/utils/BUCK
index 2b65b12..ad12f7d 100644
--- a/hbase-native-client/utils/BUCK
+++ b/hbase-native-client/utils/BUCK
@@ -23,18 +23,18 @@ cxx_library(name="utils",
"user-util.cc",
],
deps=[
- '//third-party:folly',
+ '//third-party:folly',
],
visibility=[
'PUBLIC',
],
tests=[
- ":user-util-test"
- ],)
+ ":user-util-test"
+ ], )
cxx_test(name="user-util-test",
srcs=[
"user-util-test.cc",
],
deps=[
":utils",
- ],)
+ ], )
http://git-wip-us.apache.org/repos/asf/hbase/blob/c1140d74/hbase-native-client/utils/user-util-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/utils/user-util-test.cc b/hbase-native-client/utils/user-util-test.cc
index 1e72bcc..7c11d8c 100644
--- a/hbase-native-client/utils/user-util-test.cc
+++ b/hbase-native-client/utils/user-util-test.cc
@@ -17,9 +17,9 @@
*
*/
+#include <folly/Logging.h>
#include <gtest/gtest.h>
#include <string>
-#include <folly/Logging.h>
#include "utils/user-util.h"
http://git-wip-us.apache.org/repos/asf/hbase/blob/c1140d74/hbase-native-client/utils/user-util.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/utils/user-util.cc b/hbase-native-client/utils/user-util.cc
index c4427e3..b801537 100644
--- a/hbase-native-client/utils/user-util.cc
+++ b/hbase-native-client/utils/user-util.cc
@@ -19,10 +19,10 @@
#include "utils/user-util.h"
+#include <folly/Logging.h>
#include <pwd.h>
-#include <unistd.h>
#include <sys/types.h>
-#include <folly/Logging.h>
+#include <unistd.h>
using namespace hbase;
using namespace std;
http://git-wip-us.apache.org/repos/asf/hbase/blob/c1140d74/hbase-native-client/utils/user-util.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/utils/user-util.h b/hbase-native-client/utils/user-util.h
index 4f74948..9ca28e3 100644
--- a/hbase-native-client/utils/user-util.h
+++ b/hbase-native-client/utils/user-util.h
@@ -19,8 +19,8 @@
#pragma once
-#include <memory>
#include <atomic>
+#include <memory>
#include <mutex>
namespace hbase {
[50/50] [abbrv] hbase git commit: HBASE-16174 Hook cell test up,
and fix broken cell test.
Posted by ec...@apache.org.
HBASE-16174 Hook cell test up, and fix broken cell test.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0eb150f9
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0eb150f9
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0eb150f9
Branch: refs/heads/HBASE-14850
Commit: 0eb150f9dce5c0dc3dae67a9da7b28f912853952
Parents: fa3ab42
Author: Elliott Clark <ec...@apache.org>
Authored: Tue Jul 5 11:13:01 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/core/BUCK | 8 ++++++++
hbase-native-client/core/cell-test.cc | 2 +-
2 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/0eb150f9/hbase-native-client/core/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index 1c926e3..c615426 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -20,6 +20,7 @@ cxx_library(
name="core",
exported_headers=[
"client.h",
+ "cell.h",
"hbase_macros.h",
"region-location.h",
"location-cache.h",
@@ -28,6 +29,7 @@ cxx_library(
"meta-utils.h",
],
srcs=[
+ "cell.cc",
"client.cc",
"location-cache.cc",
"meta-utils.cc",
@@ -51,6 +53,12 @@ cxx_test(name="location-cache-test",
deps=[":core",
"//test-util:test-util", ],
run_test_separately=True, )
+cxx_test(name="cell-test",
+ srcs=[
+ "cell-test.cc",
+ ],
+ deps=[":core", ],
+ run_test_separately=True, )
cxx_binary(name="simple-client",
srcs=["simple-client.cc", ],
deps=[":core", "//connection:connection"], )
http://git-wip-us.apache.org/repos/asf/hbase/blob/0eb150f9/hbase-native-client/core/cell-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/cell-test.cc b/hbase-native-client/core/cell-test.cc
index cbe50eb..49f567b 100644
--- a/hbase-native-client/core/cell-test.cc
+++ b/hbase-native-client/core/cell-test.cc
@@ -100,7 +100,7 @@ TEST(CellTest, MultipleCellsTest) {
TEST(CellTest, CellRowTest) {
std::string row = "only-row";
- std::string family = "";
+ std::string family = "D";
std::string column = "";
std::string value = "";
long timestamp = std::numeric_limits<long>::max();
[37/50] [abbrv] hbase git commit: HBASE-15748 Don't link in static
libunwind.
Posted by ec...@apache.org.
HBASE-15748 Don't link in static libunwind.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/50d16726
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/50d16726
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/50d16726
Branch: refs/heads/HBASE-14850
Commit: 50d16726ccf5b93a318417ab5b69d083c6f6ca8d
Parents: f74379e
Author: Elliott Clark <ec...@apache.org>
Authored: Mon May 2 11:41:02 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/third-party/BUCK | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/50d16726/hbase-native-client/third-party/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/third-party/BUCK b/hbase-native-client/third-party/BUCK
index 4327530..6561b2a 100644
--- a/hbase-native-client/third-party/BUCK
+++ b/hbase-native-client/third-party/BUCK
@@ -51,7 +51,7 @@ def add_dynamic_libs(names=[]):
return rules
-system_libs = ["unwind", "lzma", "event", ]
+system_libs = ["lzma", "event", ]
local_libs = [
"double-conversion",
"boost_regex",
@@ -65,7 +65,7 @@ local_libs = [
"glog",
"protobuf",
]
-dynamic_libs = ["stdc++", "pthread", "ssl", "crypto", "dl", "atomic", ]
+dynamic_libs = ["stdc++", "pthread", "ssl", "crypto", "dl", "atomic", "unwind"]
dynamic_rules = add_dynamic_libs(dynamic_libs)
tp_dep_rules = add_system_libs(system_libs,) \
+ add_system_libs(local_libs, lib_dir = "/usr/local/lib") \
[31/50] [abbrv] hbase git commit: HBASE-15821 Document TestUtil
Posted by ec...@apache.org.
HBASE-15821 Document TestUtil
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8014cdfe
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8014cdfe
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8014cdfe
Branch: refs/heads/HBASE-14850
Commit: 8014cdfe9d716eae92a91a7f376c002c05446ff9
Parents: dea1eb6
Author: Elliott Clark <ec...@apache.org>
Authored: Fri May 13 09:17:49 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/bin/format-code.sh | 2 +-
.../connection/connection-pool.h | 4 ++--
hbase-native-client/core/location-cache.cc | 2 +-
hbase-native-client/core/location-cache.h | 2 +-
hbase-native-client/test-util/test-util.cc | 20 +++++++++++++-------
hbase-native-client/test-util/test-util.h | 20 +++++++++++++++-----
6 files changed, 33 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/8014cdfe/hbase-native-client/bin/format-code.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/format-code.sh b/hbase-native-client/bin/format-code.sh
index 36a114d..cc8b368 100755
--- a/hbase-native-client/bin/format-code.sh
+++ b/hbase-native-client/bin/format-code.sh
@@ -19,5 +19,5 @@ set -euo pipefail
IFS=$'\n\t'
-find core connection serde utils -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i
+find core connection serde utils test-util -name "*.h" -or -name "*.cc" | xargs -P8 clang-format -i
find core connection serde utils third-party -name "BUCK" | xargs -P8 yapf -i
http://git-wip-us.apache.org/repos/asf/hbase/blob/8014cdfe/hbase-native-client/connection/connection-pool.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-pool.h b/hbase-native-client/connection/connection-pool.h
index 605a81b..b8c950b 100644
--- a/hbase-native-client/connection/connection-pool.h
+++ b/hbase-native-client/connection/connection-pool.h
@@ -88,8 +88,8 @@ public:
void Close(const hbase::pb::ServerName &sn);
private:
- std::shared_ptr<HBaseService> GetCached(const hbase::pb::ServerName& sn);
- std::shared_ptr<HBaseService> GetNew(const hbase::pb::ServerName& sn);
+ std::shared_ptr<HBaseService> GetCached(const hbase::pb::ServerName &sn);
+ std::shared_ptr<HBaseService> GetNew(const hbase::pb::ServerName &sn);
std::unordered_map<hbase::pb::ServerName, std::shared_ptr<HBaseService>,
ServerNameHash, ServerNameEquals>
connections_;
http://git-wip-us.apache.org/repos/asf/hbase/blob/8014cdfe/hbase-native-client/core/location-cache.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
index 6ba8add..efd2210 100644
--- a/hbase-native-client/core/location-cache.cc
+++ b/hbase-native-client/core/location-cache.cc
@@ -116,7 +116,7 @@ LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
return this->LocateMeta()
.via(cpu_executor_.get())
.then([this](ServerName sn) { return this->cp_.Get(sn); })
- .then([tn, row, this](std::shared_ptr<HBaseService> service) {
+ .then([tn, row, this](std::shared_ptr<HBaseService> service) {
return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
})
.then([this](Response resp) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/8014cdfe/hbase-native-client/core/location-cache.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h
index d435530..830cd96 100644
--- a/hbase-native-client/core/location-cache.h
+++ b/hbase-native-client/core/location-cache.h
@@ -55,7 +55,7 @@ public:
* @param io_executor executor used to talk to the network
*/
LocationCache(std::string quorum_spec,
- std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_exector,
+ std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor,
std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor);
/**
* Destructor.
http://git-wip-us.apache.org/repos/asf/hbase/blob/8014cdfe/hbase-native-client/test-util/test-util.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/test-util/test-util.cc b/hbase-native-client/test-util/test-util.cc
index e5fba48..88ce7c8 100644
--- a/hbase-native-client/test-util/test-util.cc
+++ b/hbase-native-client/test-util/test-util.cc
@@ -24,13 +24,18 @@
using hbase::TestUtil;
using folly::Random;
-const static int STR_LEN = 32;
-
-std::string TestUtil::RandString() {
- auto s = std::string(STR_LEN, 'z');
-
- for (int i = 0; i < STR_LEN; i++) {
+std::string TestUtil::RandString(int len) {
+ // Create the whole string.
+ // Filling everything with z's
+ auto s = std::string(len, 'z');
+
+ // Now pick a bunch of random numbers
+ for (int i = 0; i < len; i++) {
+ // use Folly's random to get the numbers
+ // as I don't want to have to learn
+ // all the cpp rand invocation magic.
auto r = Random::rand32('a', 'z');
+ // Cast that to ascii.
s[i] = static_cast<char>(r);
}
return s;
@@ -42,12 +47,13 @@ TestUtil::TestUtil() : temp_dir_(TestUtil::RandString()) {
auto res_code = std::system(cmd.c_str());
CHECK(res_code == 0);
}
+
TestUtil::~TestUtil() {
auto res_code = std::system("bin/stop-local-hbase.sh");
CHECK(res_code == 0);
}
-void TestUtil::RunShellCmd(const std::string& command) {
+void TestUtil::RunShellCmd(const std::string &command) {
auto cmd_string = folly::sformat("echo \"{}\" | ../bin/hbase shell", command);
auto res_code = std::system(cmd_string.c_str());
CHECK(res_code == 0);
http://git-wip-us.apache.org/repos/asf/hbase/blob/8014cdfe/hbase-native-client/test-util/test-util.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/test-util/test-util.h b/hbase-native-client/test-util/test-util.h
index 395b157..20e4981 100644
--- a/hbase-native-client/test-util/test-util.h
+++ b/hbase-native-client/test-util/test-util.h
@@ -30,20 +30,30 @@ namespace hbase {
*/
class TestUtil {
public:
-
/**
* Creating a TestUtil will spin up a cluster.
*/
TestUtil();
+
/**
- * Destroying a TestUtil will spin down a cluster.
+ * Destroying a TestUtil will spin down a cluster and remove the test dir.
*/
~TestUtil();
+
+ /**
+ * Run a command in the hbase shell. Command should not include any double
+ * quotes.
+ *
+ * This should only be used until there is a good Admin support from the
+ * native client
+ */
+ void RunShellCmd(const std::string &command);
+
/**
- * Run a command in the hbase shell.
+ * Create a random string. This random string is all letters, as such it is
+ * very good for use as a directory name.
*/
- void RunShellCmd(const std::string& command);
- static std::string RandString();
+ static std::string RandString(int len = 32);
private:
folly::test::TemporaryDirectory temp_dir_;
[35/50] [abbrv] hbase git commit: HBASE-15851 Makefile update for
build env
Posted by ec...@apache.org.
HBASE-15851 Makefile update for build env
1) Makefile to compile protobuf sources which are extracted in build
2) Added -O2 and -D_GLIBCXX_USE_CXX11_ABI=0 compilation flags
3) Header files added in Makefile
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/df14a6a7
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/df14a6a7
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/df14a6a7
Branch: refs/heads/HBASE-14850
Commit: df14a6a7a680dfd15642ab5c420b5da2c8153f88
Parents: 8014cdf
Author: sudeeps <su...@sudeepsunthankar.net>
Authored: Sun May 22 09:38:47 2016 +1000
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/Makefile | 132 +++++++++++++++++++++++------
hbase-native-client/Makefile.protos | 53 ++++++++++++
hbase-native-client/core/meta-utils.h | 2 +-
3 files changed, 159 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/df14a6a7/hbase-native-client/Makefile
----------------------------------------------------------------------
diff --git a/hbase-native-client/Makefile b/hbase-native-client/Makefile
index 826233f..7e68b6a 100644
--- a/hbase-native-client/Makefile
+++ b/hbase-native-client/Makefile
@@ -1,37 +1,115 @@
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-build:
- $(shell buck build core/... )
+#use "gcc" to compile source files
+CC:=g++
+LD:=g++
+
+DEBUG_PATH = build/debug
+RELEASE_PATH = build/release
+PROTO_SRC_DIR = build/if
+MODULES = connection core serde test-util utils
+SRC_DIR = $(MODULES)
+DEBUG_BUILD_DIR = $(addprefix $(DEBUG_PATH)/,$(MODULES))
+RELEASE_BUILD_DIR = $(addprefix $(RELEASE_PATH)/,$(MODULES))
+INCLUDE_DIR = . build/
-check:
- $(shell buck test --all --no-results-cache )
+#flags to pass to the CPP compiler & linker
+CPPFLAGS_DEBUG = -D_GLIBCXX_USE_CXX11_ABI=0 -g -Wall -std=c++14 -pedantic -fPIC
+CPPFLAGS_RELEASE = -D_GLIBCXX_USE_CXX11_ABI=0 -DNDEBUG -O2 -Wall -std=c++14 -pedantic -fPIC
+LDFLAGS = -lprotobuf -lzookeeper_mt -lsasl2 -lfolly -lwangle
+LINKFLAG = -shared
-doc:
- $(shell doxygen hbase.doxygen > /dev/null )
+#define list of source files and object files
+SRC = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc))
+PROTOSRC = $(patsubst %.proto, $(addprefix build/,%.pb.cc),$(wildcard if/*.proto))
+DEPS = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.h))
+PROTODEPS = $(patsubst %.proto, $(addprefix build/,%.pb.h),$(wildcard if/*.proto))
+DEBUG_OBJ = $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(SRC))
+DEBUG_OBJ += $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(PROTOSRC))
+RELEASE_OBJ = $(patsubst %.cc,$(RELEASE_PATH)/%.o,$(SRC))
+INCLUDES = $(addprefix -I,$(INCLUDE_DIR))
+
+LIB_DIR = /usr/local
+LIB_LIBDIR = $(LIB_DIR)/lib
+LIB_INCDIR = $(LIB_DIR)/include
+LIB_RELEASE=$(RELEASE_PATH)/libHbaseClient.so
+ARC_RELEASE=$(RELEASE_PATH)/libHbaseClient.a
+LIB_DEBUG=$(DEBUG_PATH)/libHbaseClient_d.so
+ARC_DEBUG=$(DEBUG_PATH)/libHbaseClient_d.a
+
+vpath %.cc $(SRC_DIR)
+
+$(LIB_DEBUG): $(DEBUG_BUILD_DIR)
+define make-goal-dbg
+$1/%.o: %.cc $(DEPS) $(PROTODEPS) $(PROTOSRC)
+ $(CC) -c $$< -o $$@ $(CPPFLAGS_DEBUG) $(INCLUDES)
+endef
+
+$(LIB_RELEASE): $(RELEASE_BUILD_DIR)
+define make-goal-rel
+$1/%.o: %.cc $(DEPS) $(PROTODEPS) $(PROTOSRC)
+ $(CC) -c $$< -o $$@ $(CPPFLAGS_RELEASE) $(INCLUDES)
+endef
+
+.PHONY: all clean install
+
+build: checkdirs protos $(LIB_DEBUG) $(LIB_RELEASE) $(ARC_DEBUG) $(ARC_RELEASE)
+
+checkdirs: $(DEBUG_BUILD_DIR) $(RELEASE_BUILD_DIR) $(PROTO_SRC_DIR)
+
+protos: createprotosrc
+ @make all -f Makefile.protos
+
+createprotosrc: $(PROTO_SRC_DIR)
+ @protoc --proto_path=if --cpp_out=$(PROTO_SRC_DIR) if/*.proto
+
+install:
+ cp $(LIB_RELEASE) $(LIB_LIBDIR)/libHbaseClient.so
+ cp $(ARC_RELEASE) $(LIB_LIBDIR)/libHbaseClient.a
+ cp $(LIB_DEBUG) $(LIB_LIBDIR)/libHbaseClient_d.so
+ cp $(ARC_DEBUG) $(LIB_LIBDIR)/libHbaseClient_d.a
+
+$(PROTO_SRC_DIR):
+ @mkdir -p $@
+
+$(DEBUG_BUILD_DIR):
+ @mkdir -p $@
+$(RELEASE_BUILD_DIR):
+ @mkdir -p $@
+
+$(ARC_DEBUG): $(DEBUG_OBJ)
+ ar rcs $@ $^
+
+$(ARC_RELEASE): $(RELEASE_OBJ)
+ ar rcs $@ $^
+
+$(LIB_RELEASE): $(RELEASE_OBJ)
+ $(LD) $(LINKFLAG) -o $@ $(LDFLAGS) $(RELEASE_OBJ)
+
+$(LIB_DEBUG): $(DEBUG_OBJ)
+ $(LD) $(LINKFLAG) -o $@ $(LDFLAGS) $(DEBUG_OBJ)
+
+#to clean re-compilable files
clean:
- $(shell rm -rf docs )
+ @rm -rf $(DEBUG_BUILD_DIR) $(RELEASE_BUILD_DIR) $(LIB_RELEASE) $(LIB_DEBUG) $(ARC_RELEASE) $(ARC_DEBUG) docs buck-out build
+
+$(foreach bdir,$(DEBUG_BUILD_DIR), $(eval $(call make-goal-dbg,$(bdir))))
+
+$(foreach bdir,$(RELEASE_BUILD_DIR),$(eval $(call make-goal-rel,$(bdir))))
+
+check:
+ $(shell buck test --all --no-results-cache)
+
+doc:
+ $(shell doxygen hbase.doxygen > /dev/null)
help:
@echo "Available targets:"
@echo ""
- @echo " build : will build everything."
- @echo " clean : will remove the docs folder"
- @echo " check : will test everything."
+ @echo " build : will build everything."
+ @echo " clean : will remove the docs folder, object files and local libraries"
+ @echo " install : will copy the libs to $(LIB_LIBDIR). super user priviliege would be required."
+ @echo " check : will test everything."
+ @echo " protos : will build the corresponding sources for protobufs present in if/ directory."
all: build doc check
+
http://git-wip-us.apache.org/repos/asf/hbase/blob/df14a6a7/hbase-native-client/Makefile.protos
----------------------------------------------------------------------
diff --git a/hbase-native-client/Makefile.protos b/hbase-native-client/Makefile.protos
new file mode 100644
index 0000000..4f1727a
--- /dev/null
+++ b/hbase-native-client/Makefile.protos
@@ -0,0 +1,53 @@
+#use "gcc" to compile source files
+CC:=g++
+LD:=g++
+
+DEBUG_PATH = build/debug
+RELEASE_PATH = build/release
+MODULES = build/if
+SRC_DIR = $(MODULES)
+DEBUG_BUILD_DIR = $(addprefix $(DEBUG_PATH)/,$(MODULES))
+RELEASE_BUILD_DIR = $(addprefix $(RELEASE_PATH)/,$(MODULES))
+INCLUDE_DIR = .
+
+#flags to pass to the CPP compiler & linker
+CPPFLAGS_DEBUG = -D_GLIBCXX_USE_CXX11_ABI=0 -g -Wall -std=c++14 -pedantic -fPIC
+CPPFLAGS_RELEASE = -D_GLIBCXX_USE_CXX11_ABI=0 -DNDEBUG -O2 -Wall -std=c++14 -pedantic -fPIC
+
+#define list of source files and object files
+SRC = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.cc))
+DEPS = $(foreach sdir,$(SRC_DIR),$(wildcard $(sdir)/*.h))
+DEBUG_OBJ = $(patsubst %.cc,$(DEBUG_PATH)/%.o,$(SRC))
+RELEASE_OBJ = $(patsubst %.cc,$(RELEASE_PATH)/%.o,$(SRC))
+INCLUDES = $(addprefix -I,$(INCLUDE_DIR))
+
+vpath %.cc $(SRC_DIR)
+
+$(DEBUG_OBJ): $(DEBUG_BUILD_DIR)
+define make-goal-dbg
+$1/%.o: %.cc $(DEPS)
+ $(CC) -c $$< -o $$@ $(CPPFLAGS_DEBUG) $(INCLUDES)
+endef
+
+$(RELEASE_OBJ): $(RELEASE_BUILD_DIR)
+define make-goal-rel
+$1/%.o: %.cc $(DEPS)
+ $(CC) -c $$< -o $$@ $(CPPFLAGS_RELEASE) $(INCLUDES)
+endef
+
+$(DEBUG_BUILD_DIR):
+ @mkdir -p $@
+
+$(RELEASE_BUILD_DIR):
+ @mkdir -p $@
+
+.PHONY: all clean
+
+all: $(DEBUG_OBJ) $(RELEASE_OBJ)
+
+clean:
+ @rm -rf $(DEBUG_OBJ) $(RELEASE_OBJ)
+
+$(foreach bdir,$(DEBUG_BUILD_DIR), $(eval $(call make-goal-dbg,$(bdir))))
+
+$(foreach bdir,$(RELEASE_BUILD_DIR),$(eval $(call make-goal-rel,$(bdir))))
http://git-wip-us.apache.org/repos/asf/hbase/blob/df14a6a7/hbase-native-client/core/meta-utils.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/meta-utils.h b/hbase-native-client/core/meta-utils.h
index f6cc84f..294e17d 100644
--- a/hbase-native-client/core/meta-utils.h
+++ b/hbase-native-client/core/meta-utils.h
@@ -20,7 +20,7 @@
#include <string>
-#include "connection/Request.h"
+#include "connection/request.h"
#include "if/HBase.pb.h"
#include "serde/table-name.h"
[49/50] [abbrv] hbase git commit: HBASE-15724 Use explicit docker
image
Posted by ec...@apache.org.
HBASE-15724 Use explicit docker image
Summary:
Set the tag for cpp docker image
Don't set the env on command line
Test Plan: Build it.
Differential Revision: https://reviews.facebook.net/D57315
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a1069eb1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a1069eb1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a1069eb1
Branch: refs/heads/HBASE-14850
Commit: a1069eb13978213027d34a09e4a15f3fdea83f5d
Parents: 2aaadab
Author: Elliott Clark <ec...@apache.org>
Authored: Wed Apr 27 10:51:05 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/Dockerfile | 4 +++-
hbase-native-client/bin/start-docker.sh | 1 -
2 files changed, 3 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a1069eb1/hbase-native-client/Dockerfile
----------------------------------------------------------------------
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index 1524c56..8e33e74 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -15,13 +15,15 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM pjameson/buck-folly-watchman
+FROM pjameson/buck-folly-watchman:20160425
ARG CC=/usr/bin/gcc-5
ARG CXX=/usr/bin/g++-5
ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -pthread"
ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -pthread"
+ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
+
RUN apt-get install -y vim maven inetutils-ping python-pip && \
pip install yapf && \
apt-get -qq clean && \
http://git-wip-us.apache.org/repos/asf/hbase/blob/a1069eb1/hbase-native-client/bin/start-docker.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/start-docker.sh b/hbase-native-client/bin/start-docker.sh
index 9cbd8b7..5c03eb9 100755
--- a/hbase-native-client/bin/start-docker.sh
+++ b/hbase-native-client/bin/start-docker.sh
@@ -53,7 +53,6 @@ docker build -t hbase_native .
# After the image is built run the thing
docker run -p 16010:16010/tcp \
- -e "JAVA_HOME=/usr/lib/jvm/java-8-oracle" \
-v ${BASE_DIR}/..:/usr/src/hbase \
-v ~/.m2:/root/.m2 \
-it hbase_native /bin/bash
[08/50] [abbrv] hbase git commit: HBASE-16194 Should count in MSLAB
chunk allocation into heap size change when adding duplicate cells, addendum
Posted by ec...@apache.org.
HBASE-16194 Should count in MSLAB chunk allocation into heap size change when adding duplicate cells, addendum
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/356c1307
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/356c1307
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/356c1307
Branch: refs/heads/HBASE-14850
Commit: 356c1307021e931e871307077f71aab79dc43def
Parents: 0875c35
Author: Yu Li <li...@apache.org>
Authored: Mon Jul 11 22:43:26 2016 +0800
Committer: Yu Li <li...@apache.org>
Committed: Mon Jul 11 22:44:32 2016 +0800
----------------------------------------------------------------------
.../hadoop/hbase/regionserver/AbstractMemStore.java | 14 +++++++-------
.../hadoop/hbase/regionserver/MemStoreCompactor.java | 4 ++--
.../hadoop/hbase/regionserver/MutableSegment.java | 8 ++++----
.../org/apache/hadoop/hbase/regionserver/Segment.java | 6 +++---
4 files changed, 16 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/356c1307/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
index 79e95af..2b9910f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
@@ -109,8 +109,8 @@ public abstract class AbstractMemStore implements MemStore {
@Override
public long add(Cell cell) {
Cell toAdd = maybeCloneWithAllocator(cell);
- boolean useMSLAB = (toAdd != cell);
- return internalAdd(toAdd, useMSLAB);
+ boolean mslabUsed = (toAdd != cell);
+ return internalAdd(toAdd, mslabUsed);
}
/**
@@ -157,8 +157,8 @@ public abstract class AbstractMemStore implements MemStore {
@Override
public long delete(Cell deleteCell) {
Cell toAdd = maybeCloneWithAllocator(deleteCell);
- boolean useMSLAB = (toAdd != deleteCell);
- long s = internalAdd(toAdd, useMSLAB);
+ boolean mslabUsed = (toAdd != deleteCell);
+ long s = internalAdd(toAdd, mslabUsed);
return s;
}
@@ -390,11 +390,11 @@ public abstract class AbstractMemStore implements MemStore {
*
* Callers should ensure they already have the read lock taken
* @param toAdd the cell to add
- * @param useMSLAB whether using MSLAB
+ * @param mslabUsed whether using MSLAB
* @return the heap size change in bytes
*/
- private long internalAdd(final Cell toAdd, final boolean useMSLAB) {
- long s = active.add(toAdd, useMSLAB);
+ private long internalAdd(final Cell toAdd, final boolean mslabUsed) {
+ long s = active.add(toAdd, mslabUsed);
setOldestEditTimeToNow();
checkActiveSize();
return s;
http://git-wip-us.apache.org/repos/asf/hbase/blob/356c1307/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
index 23f792b..042de0a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
@@ -197,8 +197,8 @@ class MemStoreCompactor {
// The scanner is doing all the elimination logic
// now we just copy it to the new segment
Cell newKV = result.maybeCloneWithAllocator(c);
- boolean useMSLAB = (newKV != c);
- result.internalAdd(newKV, useMSLAB);
+ boolean mslabUsed = (newKV != c);
+ result.internalAdd(newKV, mslabUsed);
}
kvs.clear();
http://git-wip-us.apache.org/repos/asf/hbase/blob/356c1307/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
index 6e54060..e62249a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
@@ -36,11 +36,11 @@ public class MutableSegment extends Segment {
/**
* Adds the given cell into the segment
* @param cell the cell to add
- * @param useMSLAB whether using MSLAB
+ * @param mslabUsed whether using MSLAB
* @return the change in the heap size
*/
- public long add(Cell cell, boolean useMSLAB) {
- return internalAdd(cell, useMSLAB);
+ public long add(Cell cell, boolean mslabUsed) {
+ return internalAdd(cell, mslabUsed);
}
/**
@@ -91,4 +91,4 @@ public class MutableSegment extends Segment {
tagsPresent = true;
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/356c1307/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
index adb101e..33c3bfb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
@@ -250,13 +250,13 @@ public abstract class Segment {
return comparator;
}
- protected long internalAdd(Cell cell, boolean useMSLAB) {
+ protected long internalAdd(Cell cell, boolean mslabUsed) {
boolean succ = getCellSet().add(cell);
long s = AbstractMemStore.heapSizeChange(cell, succ);
// If there's already a same cell in the CellSet and we are using MSLAB, we must count in the
// MSLAB allocation size as well, or else there will be memory leak (occupied heap size larger
// than the counted number)
- if (!succ && useMSLAB) {
+ if (!succ && mslabUsed) {
s += getCellLength(cell);
}
updateMetaInfo(cell, s);
@@ -309,4 +309,4 @@ public abstract class Segment {
res += "Min ts "+getMinTimestamp()+"; ";
return res;
}
-}
\ No newline at end of file
+}
[21/50] [abbrv] hbase git commit: HBASE-15401 Add Zookeeper to third
party
Posted by ec...@apache.org.
HBASE-15401 Add Zookeeper to third party
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/27ee8baa
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/27ee8baa
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/27ee8baa
Branch: refs/heads/HBASE-14850
Commit: 27ee8baaafecfaf4709ecefd1ef2099a82ca3d39
Parents: e00e3a0
Author: Elliott Clark <ec...@apache.org>
Authored: Mon Mar 7 14:58:21 2016 -0800
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/Dockerfile | 26 ++++++++-
hbase-native-client/third-party/BUCK | 96 ++++++++++++++-----------------
2 files changed, 68 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/27ee8baa/hbase-native-client/Dockerfile
----------------------------------------------------------------------
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index 5f17f04..1364d22 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -15,8 +15,30 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM pjameson/buck-folly-watchman
+FROM cpp_update
-RUN apt-get install -y libprotobuf-dev protobuf-compiler clang-format-3.7 vim maven inetutils-ping
+ARG CC=/usr/bin/gcc-5
+ARG CXX=/usr/bin/g++-5
+ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 -pthread"
+ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 -pthread"
+
+RUN apt-get install -y clang-format-3.7 vim maven inetutils-ping
+RUN git clone --depth 1 --branch v2.6.1 https://github.com/google/protobuf.git /usr/src/protobuf && \
+ cd /usr/src/protobuf/ && \
+ ./autogen.sh && \
+ ./configure --disable-shared && \
+ make && \
+ make check && \
+ make install
+RUN cd /usr/src && \
+ wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && \
+ tar zxf zookeeper-3.4.8.tar.gz && \
+ rm -rf zookeeper-3.4.8.tar.gz && \
+ cd zookeeper-3.4.8 && \
+ cd src/c && \
+ ./configure --disable-shared && \
+ make && \
+ make install && \
+ make clean
WORKDIR /usr/local/src/hbase/hbase-native-client
http://git-wip-us.apache.org/repos/asf/hbase/blob/27ee8baa/hbase-native-client/third-party/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/third-party/BUCK b/hbase-native-client/third-party/BUCK
index b7baa86..e577a5f 100644
--- a/hbase-native-client/third-party/BUCK
+++ b/hbase-native-client/third-party/BUCK
@@ -15,68 +15,60 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-def add_system_libs(names = []):
+def add_system_libs(names = [], lib_dir = "/usr/lib/x86_64-linux-gnu", deps = [], exported_linker_flags = []):
rules = []
for name in names:
- prebuilt_cxx_library(
- name = name,
- lib_name = name,
- lib_dir = "/usr/lib/x86_64-linux-gnu",
- visibility = [ 'PUBLIC', ],
- )
- rules.append(":" + name)
-
+ gen_rule_name = "gen_lib{}".format(name)
+ genrule(
+ name = gen_rule_name,
+ out = gen_rule_name,
+ bash = "mkdir -p $OUT && cp {}/lib{}.* $OUT".format(lib_dir,name),
+ )
+ prebuilt_cxx_library(
+ name = name,
+ lib_name = name,
+ lib_dir = '$(location :{})'.format(gen_rule_name),
+ force_static = True,
+ deps = deps,
+ visibility = [ 'PUBLIC' ],
+ exported_linker_flags = exported_linker_flags,
+ )
+ rules.append(":" + name)
return rules
system_libs = [
+ "unwind",
+ "lzma",
+]
+local_libs = [
"double-conversion",
"glog",
- "protobuf",
"gflags",
- "unwind",
- "lzma",
+ "protobuf",
+ "zookeeper_mt",
"boost_regex",
]
-tp_dep_rules = add_system_libs(system_libs)
-prebuilt_cxx_library(
- name = "folly",
- lib_name = "folly",
- lib_dir = "/usr/local/lib",
- deps = tp_dep_rules,
- exported_linker_flags = [
- "-pthread",
- "-lstdc++",
- ],
- visibility = [
- 'PUBLIC',
- ]
-)
-prebuilt_cxx_library(
- name = "follybenchmark",
- lib_name = "follybenchmark",
- lib_dir = "/usr/local/lib",
- deps = tp_dep_rules + [":folly"],
- exported_linker_flags = [
- "-pthread",
- "-lstdc++",
- ],
- visibility = [
- 'PUBLIC',
- ]
-)
-prebuilt_cxx_library(
- name = "wangle",
- lib_name = "wangle",
- lib_dir = "/usr/local/lib",
- deps = tp_dep_rules + [":folly"],
- exported_linker_flags = [
- "-pthread",
- "-lstdc++",
- ],
- visibility = [
- 'PUBLIC',
- ]
-)
+
+tp_dep_rules = add_system_libs(system_libs) \
+ + add_system_libs(local_libs, lib_dir = "/usr/local/lib")
+folly = add_system_libs(
+ ['folly'],
+ lib_dir = '/usr/local/lib',
+ deps = tp_dep_rules,
+ exported_linker_flags = [ "-pthread", "-lstdc++",]
+ )
+folly_bench = add_system_libs(
+ ['follybenchmark'],
+ lib_dir = '/usr/local/lib',
+ deps = tp_dep_rules + folly,
+ exported_linker_flags = [ "-pthread", "-lstdc++",]
+ )
+wangle = add_system_libs(
+ ['wangle'],
+ lib_dir = '/usr/local/lib',
+ deps = tp_dep_rules + folly,
+ exported_linker_flags = [ "-pthread", "-lstdc++",]
+ )
cxx_library(
name = 'google-test',
srcs = [
[09/50] [abbrv] hbase git commit: HBASE-16160 Support RPC encryption
with direct ByteBuffers (Colin Ma via garyh)
Posted by ec...@apache.org.
HBASE-16160 Support RPC encryption with direct ByteBuffers (Colin Ma via garyh)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/3b5fbf8d
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/3b5fbf8d
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/3b5fbf8d
Branch: refs/heads/HBASE-14850
Commit: 3b5fbf8d73b0ee50a0b04b044f93c1af39e5d79d
Parents: 356c130
Author: Gary Helmling <ga...@apache.org>
Authored: Mon Jul 11 10:33:22 2016 -0700
Committer: Gary Helmling <ga...@apache.org>
Committed: Mon Jul 11 10:33:22 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hbase/ipc/BufferChain.java | 5 +-
.../hbase/security/token/SecureTestCluster.java | 123 +++++++++++++++++++
.../TestDelegationTokenWithEncryption.java | 91 ++++++++++++++
.../token/TestGenerateDelegationToken.java | 93 +-------------
4 files changed, 218 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b5fbf8d/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
index 39efa40..7adc94d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/BufferChain.java
@@ -63,8 +63,9 @@ class BufferChain {
byte [] bytes = new byte [this.remaining];
int offset = 0;
for (ByteBuffer bb: this.buffers) {
- System.arraycopy(bb.array(), bb.arrayOffset(), bytes, offset, bb.limit());
- offset += bb.capacity();
+ int length = bb.remaining();
+ bb.get(bytes, offset, length);
+ offset += length;
}
return bytes;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b5fbf8d/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
new file mode 100644
index 0000000..a469537
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/SecureTestCluster.java
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.security.token;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.LocalHBaseCluster;
+import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
+import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil;
+import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.http.HttpConfig;
+import org.apache.hadoop.minikdc.MiniKdc;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import java.io.File;
+import java.util.Properties;
+
+/**
+ * The class for set up a security cluster with kerberos, hdfs, hbase.
+ */
+public class SecureTestCluster {
+ protected static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+
+ protected static String USERNAME;
+
+ private static LocalHBaseCluster CLUSTER;
+
+ private static final File KEYTAB_FILE = new File(TEST_UTIL.getDataTestDir("keytab").toUri()
+ .getPath());
+ private static MiniKdc KDC;
+
+ private static String HOST = "localhost";
+
+ private static String PRINCIPAL;
+
+ private static String HTTP_PRINCIPAL;
+
+ /**
+ * Setup the security configuration for hdfs.
+ */
+ private static void setHdfsSecuredConfiguration(Configuration conf) throws Exception {
+ // change XXX_USER_NAME_KEY to XXX_KERBEROS_PRINCIPAL_KEY after we drop support for hadoop-2.4.1
+ conf.set(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, PRINCIPAL + "@" + KDC.getRealm());
+ conf.set(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY, KEYTAB_FILE.getAbsolutePath());
+ conf.set(DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY, PRINCIPAL + "@" + KDC.getRealm());
+ conf.set(DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY, KEYTAB_FILE.getAbsolutePath());
+ conf.set(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, HTTP_PRINCIPAL + "@"
+ + KDC.getRealm());
+ conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
+ conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name());
+ conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0");
+ conf.set(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0");
+
+ File keystoresDir = new File(TEST_UTIL.getDataTestDir("keystore").toUri().getPath());
+ keystoresDir.mkdirs();
+ String sslConfDir = KeyStoreTestUtil.getClasspathDir(TestGenerateDelegationToken.class);
+ KeyStoreTestUtil.setupSSLConfig(keystoresDir.getAbsolutePath(), sslConfDir, conf, false);
+
+ conf.setBoolean("ignore.secure.ports.for.testing", true);
+ }
+
+ /**
+ * Setup and start kerberos, hbase
+ */
+ @BeforeClass
+ public static void setUp() throws Exception {
+ Properties conf = MiniKdc.createConf();
+ conf.put(MiniKdc.DEBUG, true);
+ KDC = new MiniKdc(conf, new File(TEST_UTIL.getDataTestDir("kdc").toUri().getPath()));
+ KDC.start();
+ USERNAME = UserGroupInformation.getLoginUser().getShortUserName();
+ PRINCIPAL = USERNAME + "/" + HOST;
+ HTTP_PRINCIPAL = "HTTP/" + HOST;
+ KDC.createPrincipal(KEYTAB_FILE, PRINCIPAL, HTTP_PRINCIPAL);
+ TEST_UTIL.startMiniZKCluster();
+
+ HBaseKerberosUtils.setKeytabFileForTesting(KEYTAB_FILE.getAbsolutePath());
+ HBaseKerberosUtils.setPrincipalForTesting(PRINCIPAL + "@" + KDC.getRealm());
+ HBaseKerberosUtils.setSecuredConfiguration(TEST_UTIL.getConfiguration());
+ setHdfsSecuredConfiguration(TEST_UTIL.getConfiguration());
+ UserGroupInformation.setConfiguration(TEST_UTIL.getConfiguration());
+ TEST_UTIL.getConfiguration().setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
+ TokenProvider.class.getName());
+ TEST_UTIL.startMiniDFSCluster(1);
+ Path rootdir = TEST_UTIL.getDataTestDirOnTestFS("TestGenerateDelegationToken");
+ FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootdir);
+ CLUSTER = new LocalHBaseCluster(TEST_UTIL.getConfiguration(), 1);
+ CLUSTER.startup();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ if (CLUSTER != null) {
+ CLUSTER.shutdown();
+ }
+ CLUSTER.join();
+ if (KDC != null) {
+ KDC.stop();
+ }
+ TEST_UTIL.shutdownMiniCluster();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b5fbf8d/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestDelegationTokenWithEncryption.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestDelegationTokenWithEncryption.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestDelegationTokenWithEncryption.java
new file mode 100644
index 0000000..60d4a6a
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestDelegationTokenWithEncryption.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.security.token;
+
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.ipc.AsyncRpcClient;
+import org.apache.hadoop.hbase.ipc.RpcClient;
+import org.apache.hadoop.hbase.ipc.RpcClientFactory;
+import org.apache.hadoop.hbase.ipc.RpcClientImpl;
+import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.SecurityTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import java.io.IOException;
+
+@Category({ SecurityTests.class, MediumTests.class })
+public class TestDelegationTokenWithEncryption extends SecureTestCluster {
+
+ @BeforeClass
+ public static void setUp() throws Exception {
+ // enable rpc encryption
+ TEST_UTIL.getConfiguration().set("hbase.rpc.protection", "privacy");
+ SecureTestCluster.setUp();
+ }
+
+ private void testPutGetWithDelegationToken(Class<? extends RpcClient> rpcImplClass)
+ throws IOException, ServiceException {
+ TEST_UTIL.getConfiguration().set(RpcClientFactory.CUSTOM_RPC_CLIENT_IMPL_CONF_KEY,
+ rpcImplClass.getName());
+ try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
+ Table table = conn.getTable(TableName.valueOf("testtable"));) {
+ Put p = new Put(Bytes.toBytes("row"));
+ p.addColumn(Bytes.toBytes("family"),
+ Bytes.toBytes("data"), Bytes.toBytes("testdata"));
+ table.put(p);
+ Get g = new Get(Bytes.toBytes("row"));
+ Result result = table.get(g);
+ Assert.assertArrayEquals(Bytes.toBytes("testdata"),
+ result.getValue(Bytes.toBytes("family"), Bytes.toBytes("data")));
+ }
+ }
+
+ @Test
+ public void testPutGetWithDelegationToken() throws Exception {
+ try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) {
+ Token<? extends TokenIdentifier> token = TokenUtil.obtainToken(conn);
+ UserGroupInformation.getCurrentUser().addToken(token);
+ // create the table for test
+ Admin admin = conn.getAdmin();
+ HTableDescriptor tableDescriptor = new
+ HTableDescriptor(new HTableDescriptor(TableName.valueOf("testtable")));
+ tableDescriptor.addFamily(new HColumnDescriptor("family"));
+ admin.createTable(tableDescriptor);
+
+ testPutGetWithDelegationToken(RpcClientImpl.class);
+ testPutGetWithDelegationToken(AsyncRpcClient.class);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/3b5fbf8d/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestGenerateDelegationToken.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestGenerateDelegationToken.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestGenerateDelegationToken.java
index e3d3c27..1d7c676 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestGenerateDelegationToken.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/security/token/TestGenerateDelegationToken.java
@@ -20,21 +20,13 @@ package org.apache.hadoop.hbase.security.token;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import java.io.File;
import java.io.IOException;
-import java.util.Properties;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.LocalHBaseCluster;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.coprocessor.CoprocessorHost;
-import org.apache.hadoop.hbase.http.ssl.KeyStoreTestUtil;
import org.apache.hadoop.hbase.ipc.AsyncRpcClient;
import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel;
import org.apache.hadoop.hbase.ipc.RpcClient;
@@ -46,101 +38,19 @@ import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos.GetAuthen
import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos.WhoAmIRequest;
import org.apache.hadoop.hbase.protobuf.generated.AuthenticationProtos.WhoAmIResponse;
import org.apache.hadoop.hbase.security.AccessDeniedException;
-import org.apache.hadoop.hbase.security.HBaseKerberosUtils;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.SecurityTests;
-import org.apache.hadoop.hbase.util.FSUtils;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.http.HttpConfig;
-import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import com.google.protobuf.ServiceException;
@Category({ SecurityTests.class, MediumTests.class })
-public class TestGenerateDelegationToken {
-
- private static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-
- private static LocalHBaseCluster CLUSTER;
-
- private static final File KEYTAB_FILE = new File(TEST_UTIL.getDataTestDir("keytab").toUri()
- .getPath());
- private static MiniKdc KDC;
-
- private static String HOST = "localhost";
-
- private static String USERNAME;
-
- private static String PRINCIPAL;
-
- private static String HTTP_PRINCIPAL;
-
- private static void setHdfsSecuredConfiguration(Configuration conf) throws Exception {
- // change XXX_USER_NAME_KEY to XXX_KERBEROS_PRINCIPAL_KEY after we drop support for hadoop-2.4.1
- conf.set(DFSConfigKeys.DFS_NAMENODE_USER_NAME_KEY, PRINCIPAL + "@" + KDC.getRealm());
- conf.set(DFSConfigKeys.DFS_NAMENODE_KEYTAB_FILE_KEY, KEYTAB_FILE.getAbsolutePath());
- conf.set(DFSConfigKeys.DFS_DATANODE_USER_NAME_KEY, PRINCIPAL + "@" + KDC.getRealm());
- conf.set(DFSConfigKeys.DFS_DATANODE_KEYTAB_FILE_KEY, KEYTAB_FILE.getAbsolutePath());
- conf.set(DFSConfigKeys.DFS_WEB_AUTHENTICATION_KERBEROS_PRINCIPAL_KEY, HTTP_PRINCIPAL + "@"
- + KDC.getRealm());
- conf.setBoolean(DFSConfigKeys.DFS_BLOCK_ACCESS_TOKEN_ENABLE_KEY, true);
- conf.set(DFSConfigKeys.DFS_HTTP_POLICY_KEY, HttpConfig.Policy.HTTPS_ONLY.name());
- conf.set(DFSConfigKeys.DFS_NAMENODE_HTTPS_ADDRESS_KEY, "localhost:0");
- conf.set(DFSConfigKeys.DFS_DATANODE_HTTPS_ADDRESS_KEY, "localhost:0");
-
- File keystoresDir = new File(TEST_UTIL.getDataTestDir("keystore").toUri().getPath());
- keystoresDir.mkdirs();
- String sslConfDir = KeyStoreTestUtil.getClasspathDir(TestGenerateDelegationToken.class);
- KeyStoreTestUtil.setupSSLConfig(keystoresDir.getAbsolutePath(), sslConfDir, conf, false);
-
- conf.setBoolean("ignore.secure.ports.for.testing", true);
- }
-
- @BeforeClass
- public static void setUp() throws Exception {
- Properties conf = MiniKdc.createConf();
- conf.put(MiniKdc.DEBUG, true);
- KDC = new MiniKdc(conf, new File(TEST_UTIL.getDataTestDir("kdc").toUri().getPath()));
- KDC.start();
- USERNAME = UserGroupInformation.getLoginUser().getShortUserName();
- PRINCIPAL = USERNAME + "/" + HOST;
- HTTP_PRINCIPAL = "HTTP/" + HOST;
- KDC.createPrincipal(KEYTAB_FILE, PRINCIPAL, HTTP_PRINCIPAL);
- TEST_UTIL.startMiniZKCluster();
-
- HBaseKerberosUtils.setKeytabFileForTesting(KEYTAB_FILE.getAbsolutePath());
- HBaseKerberosUtils.setPrincipalForTesting(PRINCIPAL + "@" + KDC.getRealm());
- HBaseKerberosUtils.setSecuredConfiguration(TEST_UTIL.getConfiguration());
- setHdfsSecuredConfiguration(TEST_UTIL.getConfiguration());
- UserGroupInformation.setConfiguration(TEST_UTIL.getConfiguration());
- TEST_UTIL.getConfiguration().setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY,
- TokenProvider.class.getName());
- TEST_UTIL.startMiniDFSCluster(1);
- Path rootdir = TEST_UTIL.getDataTestDirOnTestFS("TestGenerateDelegationToken");
- FSUtils.setRootDir(TEST_UTIL.getConfiguration(), rootdir);
- CLUSTER = new LocalHBaseCluster(TEST_UTIL.getConfiguration(), 1);
- CLUSTER.startup();
- }
-
- @AfterClass
- public static void tearDown() throws Exception {
- if (CLUSTER != null) {
- CLUSTER.shutdown();
- }
- CLUSTER.join();
- if (KDC != null) {
- KDC.stop();
- }
- TEST_UTIL.shutdownMiniCluster();
- }
+public class TestGenerateDelegationToken extends SecureTestCluster {
private void testTokenAuth(Class<? extends RpcClient> rpcImplClass) throws IOException,
ServiceException {
@@ -172,6 +82,5 @@ public class TestGenerateDelegationToken {
testTokenAuth(RpcClientImpl.class);
testTokenAuth(AsyncRpcClient.class);
}
-
}
}
[04/50] [abbrv] hbase git commit: HBASE-16176 Bug fixes/improvements
on HBASE-15650 Remove TimeRangeTracker as point of contention when many
threads reading a StoreFile Fixes HBASE-16074 ITBLL fails,
reports lost big or tiny families broken scanning b
Posted by ec...@apache.org.
HBASE-16176 Bug fixes/improvements on HBASE-15650 Remove
TimeRangeTracker as point of contention when many threads reading a StoreFile
Fixes HBASE-16074 ITBLL fails, reports lost big or tiny families broken
scanning because of a side effect of a clean up in HBASE-15650 to make
TimeRange construction consistent exposed a latent issue in
TimeRange#compare. See HBASE-16074 for more detail.
Also change HFile Writer constructor so we pass in the TimeRangeTracker, if one,
on construction rather than set later (the flag and reference were not volatile
so could have made for issues in concurrent case). And make sure the construction
of a TimeRange from a TimeRangeTracer on open of an HFile Reader never makes a
bad minimum value, one that would preclude us reading any values from a file
(set min to 0)
M hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
Call through to next constructor (if minStamp was 0, we'd skip setting
allTime=true). Add asserts that timestamps are not < 0 cos it messes
us up if they are (we already were checking for < 0 on construction but
assert passed in timestamps are not < 0).
M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
Add constructor override that takes a TimeRangeTracker (set when flushing
but not when compacting)
M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
Add override creating an HFile in tmp that takes a TimeRangeTracker
M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
Add override for HFile Writer that takes a TimeRangeTracker Take it on
construction instead of having it passed by a setter later (flags and
reference set by the setter were not volatile... could have been prob
in concurrent case)
M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
Log WARN if bad initial TimeRange value (and then 'fix' it)
M hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTimeRangeTracker.java
A few tests to prove serialization works as expected and that we'll get a bad min if not constructed properly.
M hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
Handle OLDEST_TIMESTAMP explictly. Don't expect TimeRange to do it.
M hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
Refactor from junit3 to junit4 and add test for this weird case.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/496fd983
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/496fd983
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/496fd983
Branch: refs/heads/HBASE-14850
Commit: 496fd9837a0fb199a516758a632fecfe59b0b480
Parents: 3c39cbd
Author: stack <st...@apache.org>
Authored: Fri Jul 8 18:18:01 2016 -0700
Committer: stack <st...@apache.org>
Committed: Fri Jul 8 18:20:31 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/client/Query.java | 4 +-
.../org/apache/hadoop/hbase/HConstants.java | 5 +
.../org/apache/hadoop/hbase/KeyValueUtil.java | 2 +-
.../org/apache/hadoop/hbase/io/TimeRange.java | 99 ++++++++++++--------
.../hbase/mob/DefaultMobStoreFlusher.java | 5 +-
.../hbase/regionserver/DefaultStoreFlusher.java | 4 +-
.../regionserver/ExplicitColumnTracker.java | 4 +-
.../hadoop/hbase/regionserver/HStore.java | 27 +++++-
.../hbase/regionserver/ScanQueryMatcher.java | 11 ++-
.../apache/hadoop/hbase/regionserver/Store.java | 20 +++-
.../hadoop/hbase/regionserver/StoreFile.java | 1 -
.../hbase/regionserver/StoreFileReader.java | 2 +-
.../hbase/regionserver/StoreFileWriter.java | 61 ++++++++----
.../hadoop/hbase/regionserver/StoreScanner.java | 3 +-
.../hbase/regionserver/StripeStoreFlusher.java | 4 +-
.../hbase/regionserver/TimeRangeTracker.java | 23 ++---
.../hbase/regionserver/TestQueryMatcher.java | 35 +++++--
.../regionserver/TestTimeRangeTracker.java | 40 +++++++-
18 files changed, 250 insertions(+), 100 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/496fd983/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java
index 53062a0..53dd2c1 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Query.java
@@ -203,6 +203,4 @@ public abstract class Query extends OperationWithAttributes {
public Map<byte[], TimeRange> getColumnFamilyTimeRange() {
return this.colFamTimeRangeMap;
}
-
-
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/496fd983/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
index 0bc0a07..256c374 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
@@ -544,7 +544,12 @@ public final class HConstants {
/**
* Timestamp to use when we want to refer to the oldest cell.
+ * Special! Used in fake Cells only. Should never be the timestamp on an actual Cell returned to
+ * a client.
+ * @deprecated Should not be public since hbase-1.3.0. For internal use only. Move internal to
+ * Scanners flagged as special timestamp value never to be returned as timestamp on a Cell.
*/
+ @Deprecated
public static final long OLDEST_TIMESTAMP = Long.MIN_VALUE;
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/496fd983/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java
index c9da738..6b740a7 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java
@@ -237,7 +237,7 @@ public class KeyValueUtil {
/**
* Create a KeyValue for the specified row, family and qualifier that would be
* larger than or equal to all other possible KeyValues that have the same
- * row, family, qualifier. Used for reseeking.
+ * row, family, qualifier. Used for reseeking. Should NEVER be returned to a client.
*
* @param row
* row key
http://git-wip-us.apache.org/repos/asf/hbase/blob/496fd983/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
index d5b2509..fed20c4 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/TimeRange.java
@@ -1,5 +1,4 @@
/*
- *
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -26,57 +25,86 @@ import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.util.Bytes;
/**
- * Represents an interval of version timestamps.
+ * Represents an interval of version timestamps. Presumes timestamps between
+ * {@link #INITIAL_MIN_TIMESTAMP} and {@link #INITIAL_MAX_TIMESTAMP} only. Gets freaked out if
+ * passed a timestamp that is < {@link #INITIAL_MIN_TIMESTAMP},
* <p>
* Evaluated according to minStamp <= timestamp < maxStamp
* or [minStamp,maxStamp) in interval notation.
* <p>
* Only used internally; should not be accessed directly by clients.
+ *<p>Immutable. Thread-safe.
*/
@InterfaceAudience.Public
@InterfaceStability.Stable
public class TimeRange {
- static final long INITIAL_MIN_TIMESTAMP = 0l;
- private static final long MIN_TIME = INITIAL_MIN_TIMESTAMP;
- static final long INITIAL_MAX_TIMESTAMP = Long.MAX_VALUE;
- static final long MAX_TIME = INITIAL_MAX_TIMESTAMP;
- private long minStamp = MIN_TIME;
- private long maxStamp = MAX_TIME;
+ public static final long INITIAL_MIN_TIMESTAMP = 0L;
+ public static final long INITIAL_MAX_TIMESTAMP = Long.MAX_VALUE;
+ private final long minStamp;
+ private final long maxStamp;
private final boolean allTime;
/**
* Default constructor.
* Represents interval [0, Long.MAX_VALUE) (allTime)
+ * @deprecated This is made @InterfaceAudience.Private in the 2.0 line and above
*/
+ @Deprecated
public TimeRange() {
- allTime = true;
+ this(INITIAL_MIN_TIMESTAMP, INITIAL_MAX_TIMESTAMP);
}
/**
* Represents interval [minStamp, Long.MAX_VALUE)
* @param minStamp the minimum timestamp value, inclusive
+ * @deprecated This is made @InterfaceAudience.Private in the 2.0 line and above
*/
+ @Deprecated
public TimeRange(long minStamp) {
- this.minStamp = minStamp;
- this.allTime = this.minStamp == MIN_TIME;
+ this(minStamp, INITIAL_MAX_TIMESTAMP);
}
/**
* Represents interval [minStamp, Long.MAX_VALUE)
* @param minStamp the minimum timestamp value, inclusive
+ * @deprecated This is made @InterfaceAudience.Private in the 2.0 line and above
*/
+ @Deprecated
public TimeRange(byte [] minStamp) {
- this.minStamp = Bytes.toLong(minStamp);
- this.allTime = false;
+ this(Bytes.toLong(minStamp));
+ }
+
+ /**
+ * Represents interval [minStamp, maxStamp)
+ * @param minStamp the minimum timestamp, inclusive
+ * @param maxStamp the maximum timestamp, exclusive
+ * @deprecated This is made @InterfaceAudience.Private in the 2.0 line and above
+ */
+ @Deprecated
+ public TimeRange(byte [] minStamp, byte [] maxStamp) {
+ this(Bytes.toLong(minStamp), Bytes.toLong(maxStamp));
}
/**
* Represents interval [minStamp, maxStamp)
* @param minStamp the minimum timestamp, inclusive
* @param maxStamp the maximum timestamp, exclusive
- * @throws IllegalArgumentException
+ * @throws IllegalArgumentException if either <0,
+ * @deprecated This is made @InterfaceAudience.Private in the 2.0 line and above
*/
+ @Deprecated
public TimeRange(long minStamp, long maxStamp) {
+ check(minStamp, maxStamp);
+ this.minStamp = minStamp;
+ this.maxStamp = maxStamp;
+ this.allTime = isAllTime(minStamp, maxStamp);
+ }
+
+ private static boolean isAllTime(long minStamp, long maxStamp) {
+ return minStamp == INITIAL_MIN_TIMESTAMP && maxStamp == INITIAL_MAX_TIMESTAMP;
+ }
+
+ private static void check(long minStamp, long maxStamp) {
if (minStamp < 0 || maxStamp < 0) {
throw new IllegalArgumentException("Timestamp cannot be negative. minStamp:" + minStamp
+ ", maxStamp:" + maxStamp);
@@ -84,20 +112,6 @@ public class TimeRange {
if (maxStamp < minStamp) {
throw new IllegalArgumentException("maxStamp is smaller than minStamp");
}
- this.minStamp = minStamp;
- this.maxStamp = maxStamp;
- this.allTime = this.minStamp == MIN_TIME && this.maxStamp == MAX_TIME;
- }
-
- /**
- * Represents interval [minStamp, maxStamp)
- * @param minStamp the minimum timestamp, inclusive
- * @param maxStamp the maximum timestamp, exclusive
- * @throws IOException
- */
- public TimeRange(byte [] minStamp, byte [] maxStamp)
- throws IOException {
- this(Bytes.toLong(minStamp), Bytes.toLong(maxStamp));
}
/**
@@ -125,14 +139,15 @@ public class TimeRange {
/**
* Check if the specified timestamp is within this TimeRange.
* <p>
- * Returns true if within interval [minStamp, maxStamp), false
- * if not.
+ * Returns true if within interval [minStamp, maxStamp), false if not.
* @param bytes timestamp to check
* @param offset offset into the bytes
* @return true if within TimeRange, false if not
*/
public boolean withinTimeRange(byte [] bytes, int offset) {
- if(allTime) return true;
+ if (allTime) {
+ return true;
+ }
return withinTimeRange(Bytes.toLong(bytes, offset));
}
@@ -145,6 +160,7 @@ public class TimeRange {
* @return true if within TimeRange, false if not
*/
public boolean withinTimeRange(long timestamp) {
+ assert timestamp >= 0;
if (this.allTime) {
return true;
}
@@ -163,6 +179,7 @@ public class TimeRange {
if (this.allTime) {
return true;
}
+ assert tr.getMin() >= 0;
return getMin() < tr.getMax() && getMax() >= tr.getMin();
}
@@ -175,27 +192,29 @@ public class TimeRange {
* @return true if within TimeRange, false if not
*/
public boolean withinOrAfterTimeRange(long timestamp) {
- if(allTime) return true;
+ assert timestamp >= 0;
+ if (allTime) {
+ return true;
+ }
// check if >= minStamp
- return (timestamp >= minStamp);
+ return timestamp >= minStamp;
}
/**
- * Compare the timestamp to timerange
- * @param timestamp
+ * Compare the timestamp to timerange.
* @return -1 if timestamp is less than timerange,
* 0 if timestamp is within timerange,
* 1 if timestamp is greater than timerange
*/
public int compare(long timestamp) {
- if (allTime) return 0;
+ assert timestamp >= 0;
+ if (this.allTime) {
+ return 0;
+ }
if (timestamp < minStamp) {
return -1;
- } else if (timestamp >= maxStamp) {
- return 1;
- } else {
- return 0;
}
+ return timestamp >= maxStamp? 1: 0;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/496fd983/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
index 93fa327..eb2564d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mob/DefaultMobStoreFlusher.java
@@ -99,7 +99,7 @@ public class DefaultMobStoreFlusher extends DefaultStoreFlusher {
public List<Path> flushSnapshot(MemStoreSnapshot snapshot, long cacheFlushId,
MonitoredTask status, ThroughputController throughputController) throws IOException {
ArrayList<Path> result = new ArrayList<Path>();
- int cellsCount = snapshot.getCellsCount();
+ long cellsCount = snapshot.getCellsCount();
if (cellsCount == 0) return result; // don't flush if there are no entries
// Use a store scanner to find which rows to flush.
@@ -116,8 +116,7 @@ public class DefaultMobStoreFlusher extends DefaultStoreFlusher {
status.setStatus("Flushing " + store + ": creating writer");
// Write the map out to the disk
writer = store.createWriterInTmp(cellsCount, store.getFamily().getCompression(),
- false, true, true);
- writer.setTimeRangeTracker(snapshot.getTimeRangeTracker());
+ false, true, true, false/*default for dropbehind*/, snapshot.getTimeRangeTracker());
try {
// It's a mob store, flush the cells in a mob way. This is the difference of flushing
// between a normal and a mob store.
http://git-wip-us.apache.org/repos/asf/hbase/blob/496fd983/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java
index bf8794a..079501e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultStoreFlusher.java
@@ -68,8 +68,8 @@ public class DefaultStoreFlusher extends StoreFlusher {
/* isCompaction = */ false,
/* includeMVCCReadpoint = */ true,
/* includesTags = */ snapshot.isTagsPresent(),
- /* shouldDropBehind = */ false);
- writer.setTimeRangeTracker(snapshot.getTimeRangeTracker());
+ /* shouldDropBehind = */ false,
+ snapshot.getTimeRangeTracker());
IOException e = null;
try {
performFlush(scanner, writer, smallestReadPoint, throughputController);
http://git-wip-us.apache.org/repos/asf/hbase/blob/496fd983/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
index 86f2fbc..86c8b48 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ExplicitColumnTracker.java
@@ -46,7 +46,7 @@ import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode;
* believes that the current column should be skipped (by timestamp, filter etc.)</li>
* </ul>
* <p>
- * These two methods returns a
+ * These two methods returns a
* {@link org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode}
* to define what action should be taken.
* <p>
@@ -77,7 +77,7 @@ public class ExplicitColumnTracker implements ColumnTracker {
* @param minVersions minimum number of versions to keep
* @param maxVersions maximum versions to return per column
* @param oldestUnexpiredTS the oldest timestamp we are interested in,
- * based on TTL
+ * based on TTL
*/
public ExplicitColumnTracker(NavigableSet<byte[]> columns, int minVersions,
int maxVersions, long oldestUnexpiredTS) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/496fd983/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
index 2d1b9a6..4b79153 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java
@@ -958,6 +958,23 @@ public class HStore implements Store {
boolean isCompaction, boolean includeMVCCReadpoint, boolean includesTag,
boolean shouldDropBehind)
throws IOException {
+ return createWriterInTmp(maxKeyCount, compression, isCompaction, includeMVCCReadpoint,
+ includesTag, shouldDropBehind, null);
+ }
+
+ /*
+ * @param maxKeyCount
+ * @param compression Compression algorithm to use
+ * @param isCompaction whether we are creating a new file in a compaction
+ * @param includesMVCCReadPoint - whether to include MVCC or not
+ * @param includesTag - includesTag or not
+ * @return Writer for a new StoreFile in the tmp dir.
+ */
+ @Override
+ public StoreFileWriter createWriterInTmp(long maxKeyCount, Compression.Algorithm compression,
+ boolean isCompaction, boolean includeMVCCReadpoint, boolean includesTag,
+ boolean shouldDropBehind, final TimeRangeTracker trt)
+ throws IOException {
final CacheConfig writerCacheConf;
if (isCompaction) {
// Don't cache data on write on compactions.
@@ -973,7 +990,7 @@ public class HStore implements Store {
}
HFileContext hFileContext = createFileContext(compression, includeMVCCReadpoint, includesTag,
cryptoContext);
- StoreFileWriter w = new StoreFileWriter.Builder(conf, writerCacheConf,
+ StoreFileWriter.Builder builder = new StoreFileWriter.Builder(conf, writerCacheConf,
this.getFileSystem())
.withFilePath(fs.createTempName())
.withComparator(comparator)
@@ -981,9 +998,11 @@ public class HStore implements Store {
.withMaxKeyCount(maxKeyCount)
.withFavoredNodes(favoredNodes)
.withFileContext(hFileContext)
- .withShouldDropCacheBehind(shouldDropBehind)
- .build();
- return w;
+ .withShouldDropCacheBehind(shouldDropBehind);
+ if (trt != null) {
+ builder.withTimeRangeTracker(trt);
+ }
+ return builder.build();
}
private HFileContext createFileContext(Compression.Algorithm compression,
http://git-wip-us.apache.org/repos/asf/hbase/blob/496fd983/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
index bfd399b..d2d0ccb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java
@@ -404,7 +404,16 @@ public class ScanQueryMatcher {
}
}
- int timestampComparison = tr.compare(timestamp);
+ // NOTE: Cryptic stuff!
+ // if the timestamp is HConstants.OLDEST_TIMESTAMP, then this is a fake cell made to prime a
+ // Scanner; See KeyValueUTil#createLastOnRow. This Cell should never end up returning out of
+ // here a matchcode of INCLUDE else we will return to the client a fake Cell. If we call
+ // TimeRange, it will return 0 because it doesn't deal in OLDEST_TIMESTAMP and we will fall
+ // into the later code where we could return a matchcode of INCLUDE. See HBASE-16074 "ITBLL
+ // fails, reports lost big or tiny families" for a horror story. Check here for
+ // OLDEST_TIMESTAMP. TimeRange#compare is about more generic timestamps, between 0L and
+ // Long.MAX_LONG. It doesn't do OLDEST_TIMESTAMP weird handling.
+ int timestampComparison = timestamp == HConstants.OLDEST_TIMESTAMP? -1: tr.compare(timestamp);
if (timestampComparison >= 1) {
return MatchCode.SKIP;
} else if (timestampComparison <= -1) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/496fd983/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
index 3419937..5dec59a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java
@@ -189,8 +189,24 @@ public interface Store extends HeapSize, StoreConfigInformation, PropagatingConf
boolean shouldDropBehind
) throws IOException;
-
-
+ /**
+ * @param maxKeyCount
+ * @param compression Compression algorithm to use
+ * @param isCompaction whether we are creating a new file in a compaction
+ * @param includeMVCCReadpoint whether we should out the MVCC readpoint
+ * @param shouldDropBehind should the writer drop caches behind writes
+ * @param trt Ready-made timetracker to use.
+ * @return Writer for a new StoreFile in the tmp dir.
+ */
+ StoreFileWriter createWriterInTmp(
+ long maxKeyCount,
+ Compression.Algorithm compression,
+ boolean isCompaction,
+ boolean includeMVCCReadpoint,
+ boolean includesTags,
+ boolean shouldDropBehind,
+ final TimeRangeTracker trt
+ ) throws IOException;
// Compaction oriented methods
http://git-wip-us.apache.org/repos/asf/hbase/blob/496fd983/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
index 46f0cb8..589d844 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java
@@ -198,7 +198,6 @@ public class StoreFile {
this(fs, new StoreFileInfo(conf, fs, p), conf, cacheConf, cfBloomType);
}
-
/**
* Constructor, loads a reader and it's indices, etc. May allocate a
* substantial amount of ram depending on the underlying files (10-20MB?).
http://git-wip-us.apache.org/repos/asf/hbase/blob/496fd983/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
index e890381..1582237 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileReader.java
@@ -639,7 +639,7 @@ public class StoreFileReader {
}
public long getMaxTimestamp() {
- return timeRange == null ? Long.MAX_VALUE : timeRange.getMax();
+ return timeRange == null ? TimeRange.INITIAL_MAX_TIMESTAMP: timeRange.getMax();
}
boolean isSkipResetSeqId() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/496fd983/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java
index 4a42b7f..442b90d 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileWriter.java
@@ -62,7 +62,6 @@ public class StoreFileWriter implements Compactor.CellSink {
private Cell lastDeleteFamilyCell = null;
private long deleteFamilyCnt = 0;
- TimeRangeTracker timeRangeTracker = new TimeRangeTracker();
/**
* timeRangeTrackerSet is used to figure if we were passed a filled-out TimeRangeTracker or not.
* When flushing a memstore, we set the TimeRangeTracker that it accumulated during updates to
@@ -70,7 +69,8 @@ public class StoreFileWriter implements Compactor.CellSink {
* recalculate the timeRangeTracker bounds; it was done already as part of add-to-memstore.
* A completed TimeRangeTracker is not set in cases of compactions when it is recalculated.
*/
- boolean timeRangeTrackerSet = false;
+ private final boolean timeRangeTrackerSet;
+ final TimeRangeTracker timeRangeTracker;
protected HFile.Writer writer;
private KeyValue.KeyOnlyKeyValue lastBloomKeyOnlyKV = null;
@@ -91,6 +91,36 @@ public class StoreFileWriter implements Compactor.CellSink {
final CellComparator comparator, BloomType bloomType, long maxKeys,
InetSocketAddress[] favoredNodes, HFileContext fileContext)
throws IOException {
+ this(fs, path, conf, cacheConf, comparator, bloomType, maxKeys, favoredNodes, fileContext,
+ null);
+ }
+
+ /**
+ * Creates an HFile.Writer that also write helpful meta data.
+ * @param fs file system to write to
+ * @param path file name to create
+ * @param conf user configuration
+ * @param comparator key comparator
+ * @param bloomType bloom filter setting
+ * @param maxKeys the expected maximum number of keys to be added. Was used
+ * for Bloom filter size in {@link HFile} format version 1.
+ * @param favoredNodes
+ * @param fileContext - The HFile context
+ * @param trt Ready-made timetracker to use.
+ * @throws IOException problem writing to FS
+ */
+ private StoreFileWriter(FileSystem fs, Path path,
+ final Configuration conf,
+ CacheConfig cacheConf,
+ final CellComparator comparator, BloomType bloomType, long maxKeys,
+ InetSocketAddress[] favoredNodes, HFileContext fileContext,
+ final TimeRangeTracker trt)
+ throws IOException {
+ // If passed a TimeRangeTracker, use it. Set timeRangeTrackerSet so we don't destroy it.
+ // TODO: put the state of the TRT on the TRT; i.e. make a read-only version (TimeRange) when
+ // it no longer writable.
+ this.timeRangeTrackerSet = trt != null;
+ this.timeRangeTracker = this.timeRangeTrackerSet? trt: new TimeRangeTracker();
writer = HFile.getWriterFactory(conf, cacheConf)
.withPath(fs, path)
.withComparator(comparator)
@@ -171,19 +201,6 @@ public class StoreFileWriter implements Compactor.CellSink {
}
/**
- * Set TimeRangeTracker.
- * Called when flushing to pass us a pre-calculated TimeRangeTracker, one made during updates
- * to memstore so we don't have to make one ourselves as Cells get appended. Call before first
- * append. If this method is not called, we will calculate our own range of the Cells that
- * comprise this StoreFile (and write them on the end as metadata). It is good to have this stuff
- * passed because it is expensive to make.
- */
- public void setTimeRangeTracker(final TimeRangeTracker trt) {
- this.timeRangeTracker = trt;
- timeRangeTrackerSet = true;
- }
-
- /**
* Record the earlest Put timestamp.
*
* If the timeRangeTracker is not set,
@@ -413,6 +430,7 @@ public class StoreFileWriter implements Compactor.CellSink {
private Path filePath;
private InetSocketAddress[] favoredNodes;
private HFileContext fileContext;
+ private TimeRangeTracker trt;
public Builder(Configuration conf, CacheConfig cacheConf,
FileSystem fs) {
@@ -422,6 +440,17 @@ public class StoreFileWriter implements Compactor.CellSink {
}
/**
+ * @param trt A premade TimeRangeTracker to use rather than build one per append (building one
+ * of these is expensive so good to pass one in if you have one).
+ * @return this (for chained invocation)
+ */
+ public Builder withTimeRangeTracker(final TimeRangeTracker trt) {
+ Preconditions.checkNotNull(trt);
+ this.trt = trt;
+ return this;
+ }
+
+ /**
* Use either this method or {@link #withFilePath}, but not both.
* @param dir Path to column family directory. The directory is created if
* does not exist. The file is given a unique name within this
@@ -514,7 +543,7 @@ public class StoreFileWriter implements Compactor.CellSink {
comparator = CellComparator.COMPARATOR;
}
return new StoreFileWriter(fs, filePath,
- conf, cacheConf, comparator, bloomType, maxKeyCount, favoredNodes, fileContext);
+ conf, cacheConf, comparator, bloomType, maxKeyCount, favoredNodes, fileContext, trt);
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/496fd983/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
index ceb4b8e..080bb95 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java
@@ -402,8 +402,7 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner
// We can only exclude store files based on TTL if minVersions is set to 0.
// Otherwise, we might have to return KVs that have technically expired.
- long expiredTimestampCutoff = minVersions == 0 ? oldestUnexpiredTS :
- Long.MIN_VALUE;
+ long expiredTimestampCutoff = minVersions == 0 ? oldestUnexpiredTS: Long.MIN_VALUE;
// include only those scan files which pass all filters
for (KeyValueScanner kvs : allScanners) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/496fd983/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFlusher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFlusher.java
index cf0de1f..22c3ce7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFlusher.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StripeStoreFlusher.java
@@ -114,8 +114,8 @@ public class StripeStoreFlusher extends StoreFlusher {
/* isCompaction = */ false,
/* includeMVCCReadpoint = */ true,
/* includesTags = */ true,
- /* shouldDropBehind = */ false);
- writer.setTimeRangeTracker(tracker);
+ /* shouldDropBehind = */ false,
+ tracker);
return writer;
}
};
http://git-wip-us.apache.org/repos/asf/hbase/blob/496fd983/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
index 48adc9d..1ea3c70 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/TimeRangeTracker.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.io.Writable;
/**
- * Stores minimum and maximum timestamp values. Both timestamps are inclusive.
+ * Stores minimum and maximum timestamp values.
* Use this class at write-time ONLY. Too much synchronization to use at read time
* (TODO: there are two scenarios writing, once when lots of concurrency as part of memstore
* updates but then later we can make one as part of a compaction when there is only one thread
@@ -45,8 +45,8 @@ import org.apache.hadoop.io.Writable;
@InterfaceAudience.Private
public class TimeRangeTracker implements Writable {
static final long INITIAL_MIN_TIMESTAMP = Long.MAX_VALUE;
+ static final long INITIAL_MAX_TIMESTAMP = -1L;
long minimumTimestamp = INITIAL_MIN_TIMESTAMP;
- static final long INITIAL_MAX_TIMESTAMP = -1;
long maximumTimestamp = INITIAL_MAX_TIMESTAMP;
/**
@@ -125,7 +125,7 @@ public class TimeRangeTracker implements Writable {
}
/**
- * Check if the range has any overlap with TimeRange
+ * Check if the range has ANY overlap with TimeRange
* @param tr TimeRange
* @return True if there is overlap, false otherwise
*/
@@ -185,22 +185,19 @@ public class TimeRangeTracker implements Writable {
return trt == null? null: trt.toTimeRange();
}
- private boolean isFreshInstance() {
- return getMin() == INITIAL_MIN_TIMESTAMP && getMax() == INITIAL_MAX_TIMESTAMP;
- }
-
/**
* @return Make a TimeRange from current state of <code>this</code>.
*/
TimeRange toTimeRange() {
long min = getMin();
long max = getMax();
- // Check for the case where the TimeRangeTracker is fresh. In that case it has
- // initial values that are antithetical to a TimeRange... Return an uninitialized TimeRange
- // if passed an uninitialized TimeRangeTracker.
- if (isFreshInstance()) {
- return new TimeRange();
+ // Initial TimeRangeTracker timestamps are the opposite of what you want for a TimeRange. Fix!
+ if (min == INITIAL_MIN_TIMESTAMP) {
+ min = TimeRange.INITIAL_MIN_TIMESTAMP;
+ }
+ if (max == INITIAL_MAX_TIMESTAMP) {
+ max = TimeRange.INITIAL_MAX_TIMESTAMP;
}
return new TimeRange(min, max);
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/496fd983/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
index d8de178..db01ee0 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestQueryMatcher.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.hbase.regionserver;
import static org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode.INCLUDE;
import static org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode.SKIP;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import java.io.IOException;
import java.util.ArrayList;
@@ -28,14 +30,17 @@ import java.util.List;
import java.util.NavigableSet;
import org.apache.hadoop.hbase.CellComparator;
-import org.apache.hadoop.hbase.HBaseTestCase;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.KeyValue.Type;
-import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode;
@@ -46,8 +51,9 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
@Category({RegionServerTests.class, SmallTests.class})
-public class TestQueryMatcher extends HBaseTestCase {
+public class TestQueryMatcher {
private static final boolean PRINT = false;
+ private Configuration conf;
private byte[] row1;
private byte[] row2;
@@ -70,7 +76,7 @@ public class TestQueryMatcher extends HBaseTestCase {
@Before
public void setUp() throws Exception {
- super.setUp();
+ this.conf = HBaseConfiguration.create();
row1 = Bytes.toBytes("row1");
row2 = Bytes.toBytes("row2");
row3 = Bytes.toBytes("row3");
@@ -130,6 +136,25 @@ public class TestQueryMatcher extends HBaseTestCase {
}
}
+ /**
+ * This is a cryptic test. It is checking that we don't include a fake cell, one that has a
+ * timestamp of {@link HConstants#OLDEST_TIMESTAMP}. See HBASE-16074 for background.
+ * @throws IOException
+ */
+ @Test
+ public void testNeverIncludeFakeCell() throws IOException {
+ long now = EnvironmentEdgeManager.currentTime();
+ // Do with fam2 which has a col2 qualifier.
+ ScanQueryMatcher qm = new ScanQueryMatcher(scan,
+ new ScanInfo(this.conf, fam2, 10, 1, ttl, KeepDeletedCells.FALSE, 0, rowComparator),
+ get.getFamilyMap().get(fam2), now - ttl, now);
+ Cell kv = new KeyValue(row1, fam2, col2, 1, data);
+ Cell cell = CellUtil.createLastOnRowCol(kv);
+ qm.setToNewRow(kv);
+ MatchCode code = qm.match(cell);
+ assertFalse(code.compareTo(MatchCode.SEEK_NEXT_COL) != 0);
+ }
+
@Test
public void testMatch_ExplicitColumns()
throws IOException {
@@ -251,7 +276,6 @@ public class TestQueryMatcher extends HBaseTestCase {
}
}
-
/**
* Verify that {@link ScanQueryMatcher} only skips expired KeyValue
* instances and does not exit early from the row (skipping
@@ -356,4 +380,3 @@ public class TestQueryMatcher extends HBaseTestCase {
}
}
}
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/496fd983/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTimeRangeTracker.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTimeRangeTracker.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTimeRangeTracker.java
index 3b93bb9..2803baf 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTimeRangeTracker.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestTimeRangeTracker.java
@@ -20,9 +20,11 @@ package org.apache.hadoop.hbase.regionserver;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import java.io.IOException;
import org.apache.hadoop.hbase.io.TimeRange;
import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.apache.hadoop.hbase.util.Writables;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -32,6 +34,42 @@ import java.io.IOException;
@Category({RegionServerTests.class, SmallTests.class})
public class TestTimeRangeTracker {
@Test
+ public void testExtreme() {
+ TimeRange tr = new TimeRange();
+ assertTrue(tr.includesTimeRange(new TimeRange()));
+ TimeRangeTracker trt = new TimeRangeTracker();
+ assertFalse(trt.includesTimeRange(new TimeRange()));
+ trt.includeTimestamp(1);
+ trt.includeTimestamp(10);
+ assertTrue(trt.includesTimeRange(new TimeRange()));
+ }
+
+ @Test
+ public void testTimeRangeInitialized() {
+ TimeRangeTracker src = new TimeRangeTracker();
+ TimeRange tr = new TimeRange(System.currentTimeMillis());
+ assertFalse(src.includesTimeRange(tr));
+ }
+
+ @Test
+ public void testTimeRangeTrackerNullIsSameAsTimeRangeNull() throws IOException {
+ TimeRangeTracker src = new TimeRangeTracker(1, 2);
+ byte [] bytes = Writables.getBytes(src);
+ TimeRange tgt = TimeRangeTracker.getTimeRange(bytes);
+ assertEquals(src.getMin(), tgt.getMin());
+ assertEquals(src.getMax(), tgt.getMax());
+ }
+
+ @Test
+ public void testSerialization() throws IOException {
+ TimeRangeTracker src = new TimeRangeTracker(1, 2);
+ TimeRangeTracker tgt = new TimeRangeTracker();
+ Writables.copyWritable(src, tgt);
+ assertEquals(src.getMin(), tgt.getMin());
+ assertEquals(src.getMax(), tgt.getMax());
+ }
+
+ @Test
public void testAlwaysDecrementingSetsMaximum() {
TimeRangeTracker trr = new TimeRangeTracker();
trr.includeTimestamp(3);
@@ -145,4 +183,4 @@ public class TestTimeRangeTracker {
System.out.println(trr.getMin() + " " + trr.getMax() + " " +
(System.currentTimeMillis() - start));
}
-}
\ No newline at end of file
+}
[41/50] [abbrv] hbase git commit: HBASE-15690 Add utility to get
current username
Posted by ec...@apache.org.
HBASE-15690 Add utility to get current username
Summary:
Add a class to fine the username of the current process.
It will only call out once and is multithread safe
Test Plan: Unit Test
Differential Revision: https://reviews.facebook.net/D57081
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8b2d65f1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8b2d65f1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8b2d65f1
Branch: refs/heads/HBASE-14850
Commit: 8b2d65f1aa7c7dddcde4be03133f8c35e1fbcba2
Parents: b23e863
Author: Elliott Clark <ec...@apache.org>
Authored: Thu Apr 21 21:17:11 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/utils/BUCK | 40 +++++++++++++++++
hbase-native-client/utils/user-util-test.cc | 34 +++++++++++++++
hbase-native-client/utils/user-util.cc | 55 ++++++++++++++++++++++++
hbase-native-client/utils/user-util.h | 37 ++++++++++++++++
4 files changed, 166 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/8b2d65f1/hbase-native-client/utils/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/utils/BUCK b/hbase-native-client/utils/BUCK
new file mode 100644
index 0000000..2b65b12
--- /dev/null
+++ b/hbase-native-client/utils/BUCK
@@ -0,0 +1,40 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cxx_library(name="utils",
+ exported_headers=[
+ "user-util.h",
+ ],
+ srcs=[
+ "user-util.cc",
+ ],
+ deps=[
+ '//third-party:folly',
+ ],
+ visibility=[
+ 'PUBLIC',
+ ],
+ tests=[
+ ":user-util-test"
+ ],)
+cxx_test(name="user-util-test",
+ srcs=[
+ "user-util-test.cc",
+ ],
+ deps=[
+ ":utils",
+ ],)
http://git-wip-us.apache.org/repos/asf/hbase/blob/8b2d65f1/hbase-native-client/utils/user-util-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/utils/user-util-test.cc b/hbase-native-client/utils/user-util-test.cc
new file mode 100644
index 0000000..2a7434f
--- /dev/null
+++ b/hbase-native-client/utils/user-util-test.cc
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <gtest/gtest.h>
+#include <string>
+#include <folly/Logging.h>
+
+#include "utils/user-util.h"
+
+using namespace std;
+using namespace hbase;
+
+TEST(TestUserUtil, TestGetSomething) {
+ UserUtil u_util;
+ string name = u_util.user_name();
+
+ ASSERT_GT(name.length(), 0);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8b2d65f1/hbase-native-client/utils/user-util.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/utils/user-util.cc b/hbase-native-client/utils/user-util.cc
new file mode 100644
index 0000000..c4427e3
--- /dev/null
+++ b/hbase-native-client/utils/user-util.cc
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "utils/user-util.h"
+
+#include <pwd.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <folly/Logging.h>
+
+using namespace hbase;
+using namespace std;
+
+string UserUtil::user_name() {
+ if (!init_) {
+ compute_user_name();
+ }
+ return user_name_;
+}
+
+void UserUtil::compute_user_name() {
+ lock_guard<mutex> lock(m_);
+
+ if (init_) {
+ return;
+ }
+
+ // According to the man page of getpwuid
+ // this should never be free'd
+ //
+ // So yeah a raw pointer with no ownership....
+ struct passwd *passwd = getpwuid(getuid());
+
+ // make sure that we got something.
+ if (passwd && passwd->pw_name) {
+ user_name_ = string{passwd->pw_name};
+ init_ = true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8b2d65f1/hbase-native-client/utils/user-util.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/utils/user-util.h b/hbase-native-client/utils/user-util.h
new file mode 100644
index 0000000..4f74948
--- /dev/null
+++ b/hbase-native-client/utils/user-util.h
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <memory>
+#include <atomic>
+#include <mutex>
+
+namespace hbase {
+class UserUtil {
+public:
+ std::string user_name();
+
+private:
+ void compute_user_name();
+ std::atomic<bool> init_{false};
+ std::string user_name_;
+ std::mutex m_;
+};
+} // namespace hbase
[06/50] [abbrv] hbase git commit: HBASE-16194 Should count in MSLAB
chunk allocation into heap size change when adding duplicate cells
Posted by ec...@apache.org.
HBASE-16194 Should count in MSLAB chunk allocation into heap size change when adding duplicate cells
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9cf012cd
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9cf012cd
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9cf012cd
Branch: refs/heads/HBASE-14850
Commit: 9cf012cd0086e054a66f54cb54771af2381fade0
Parents: 6329697
Author: Yu Li <li...@apache.org>
Authored: Sun Jul 10 13:45:17 2016 +0800
Committer: Yu Li <li...@apache.org>
Committed: Sun Jul 10 13:45:17 2016 +0800
----------------------------------------------------------------------
.../hbase/regionserver/AbstractMemStore.java | 15 +++++++----
.../hbase/regionserver/HeapMemStoreLAB.java | 11 +++++++++
.../hbase/regionserver/MemStoreCompactor.java | 3 ++-
.../hbase/regionserver/MutableSegment.java | 6 +++--
.../hadoop/hbase/regionserver/Segment.java | 23 ++++++++++++++---
.../hbase/regionserver/TestDefaultMemStore.java | 26 ++++++++++++++++++++
6 files changed, 73 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/9cf012cd/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
index d25f960..79e95af 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java
@@ -109,7 +109,8 @@ public abstract class AbstractMemStore implements MemStore {
@Override
public long add(Cell cell) {
Cell toAdd = maybeCloneWithAllocator(cell);
- return internalAdd(toAdd);
+ boolean useMSLAB = (toAdd != cell);
+ return internalAdd(toAdd, useMSLAB);
}
/**
@@ -156,7 +157,8 @@ public abstract class AbstractMemStore implements MemStore {
@Override
public long delete(Cell deleteCell) {
Cell toAdd = maybeCloneWithAllocator(deleteCell);
- long s = internalAdd(toAdd);
+ boolean useMSLAB = (toAdd != deleteCell);
+ long s = internalAdd(toAdd, useMSLAB);
return s;
}
@@ -243,7 +245,7 @@ public abstract class AbstractMemStore implements MemStore {
// hitting OOME - see TestMemStore.testUpsertMSLAB for a
// test that triggers the pathological case if we don't avoid MSLAB
// here.
- long addedSize = internalAdd(cell);
+ long addedSize = internalAdd(cell, false);
// Get the Cells for the row/family/qualifier regardless of timestamp.
// For this case we want to clean up any other puts
@@ -387,9 +389,12 @@ public abstract class AbstractMemStore implements MemStore {
* allocator, and doesn't take the lock.
*
* Callers should ensure they already have the read lock taken
+ * @param toAdd the cell to add
+ * @param useMSLAB whether using MSLAB
+ * @return the heap size change in bytes
*/
- private long internalAdd(final Cell toAdd) {
- long s = active.add(toAdd);
+ private long internalAdd(final Cell toAdd, final boolean useMSLAB) {
+ long s = active.add(toAdd, useMSLAB);
setOldestEditTimeToNow();
checkActiveSize();
return s;
http://git-wip-us.apache.org/repos/asf/hbase/blob/9cf012cd/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java
index f22a6e5..625811a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemStoreLAB.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.util.ByteRange;
import org.apache.hadoop.hbase.util.SimpleMutableByteRange;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
/**
@@ -206,6 +207,11 @@ public class HeapMemStoreLAB implements MemStoreLAB {
}
}
+ @VisibleForTesting
+ Chunk getCurrentChunk() {
+ return this.curChunk.get();
+ }
+
/**
* A chunk of memory out of which allocations are sliced.
*/
@@ -311,5 +317,10 @@ public class HeapMemStoreLAB implements MemStoreLAB {
" allocs=" + allocCount.get() + "waste=" +
(data.length - nextFreeOffset.get());
}
+
+ @VisibleForTesting
+ int getNextFreeOffset() {
+ return this.nextFreeOffset.get();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/9cf012cd/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
index 65d3af6..23f792b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreCompactor.java
@@ -197,7 +197,8 @@ class MemStoreCompactor {
// The scanner is doing all the elimination logic
// now we just copy it to the new segment
Cell newKV = result.maybeCloneWithAllocator(c);
- result.internalAdd(newKV);
+ boolean useMSLAB = (newKV != c);
+ result.internalAdd(newKV, useMSLAB);
}
kvs.clear();
http://git-wip-us.apache.org/repos/asf/hbase/blob/9cf012cd/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
index 6337657..6e54060 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java
@@ -35,10 +35,12 @@ public class MutableSegment extends Segment {
/**
* Adds the given cell into the segment
+ * @param cell the cell to add
+ * @param useMSLAB whether using MSLAB
* @return the change in the heap size
*/
- public long add(Cell cell) {
- return internalAdd(cell);
+ public long add(Cell cell, boolean useMSLAB) {
+ return internalAdd(cell, useMSLAB);
}
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/9cf012cd/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
index dd824c1..adb101e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java
@@ -31,6 +31,8 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.ByteRange;
+import com.google.common.annotations.VisibleForTesting;
+
/**
* This is an abstraction of a segment maintained in a memstore, e.g., the active
* cell set or its snapshot.
@@ -136,7 +138,7 @@ public abstract class Segment {
return cell;
}
- int len = KeyValueUtil.length(cell);
+ int len = getCellLength(cell);
ByteRange alloc = getMemStoreLAB().allocateBytes(len);
if (alloc == null) {
// The allocation was too large, allocator decided
@@ -150,6 +152,14 @@ public abstract class Segment {
return newKv;
}
+ /**
+ * Get cell length after serialized in {@link KeyValue}
+ */
+ @VisibleForTesting
+ int getCellLength(Cell cell) {
+ return KeyValueUtil.length(cell);
+ }
+
public abstract boolean shouldSeek(Scan scan, long oldestUnexpiredTS);
public abstract long getMinTimestamp();
@@ -240,9 +250,15 @@ public abstract class Segment {
return comparator;
}
- protected long internalAdd(Cell cell) {
+ protected long internalAdd(Cell cell, boolean useMSLAB) {
boolean succ = getCellSet().add(cell);
long s = AbstractMemStore.heapSizeChange(cell, succ);
+ // If there's already a same cell in the CellSet and we are using MSLAB, we must count in the
+ // MSLAB allocation size as well, or else there will be memory leak (occupied heap size larger
+ // than the counted number)
+ if (!succ && useMSLAB) {
+ s += getCellLength(cell);
+ }
updateMetaInfo(cell, s);
return s;
}
@@ -269,7 +285,8 @@ public abstract class Segment {
return getCellSet().tailSet(firstCell);
}
- private MemStoreLAB getMemStoreLAB() {
+ @VisibleForTesting
+ public MemStoreLAB getMemStoreLAB() {
return memStoreLAB;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/9cf012cd/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
index 1614462..7285c67 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.regionserver;
import com.google.common.base.Joiner;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -56,6 +57,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertNotNull;
+
import org.junit.experimental.categories.Category;
import org.junit.rules.TestName;
import org.junit.rules.TestRule;
@@ -110,6 +112,30 @@ public class TestDefaultMemStore {
assertTrue(Bytes.toString(found.getValueArray()), CellUtil.matchingValue(samekey, found));
}
+ @Test
+ public void testPutSameCell() {
+ byte[] bytes = Bytes.toBytes(getName());
+ KeyValue kv = new KeyValue(bytes, bytes, bytes, bytes);
+ long sizeChangeForFirstCell = this.memstore.add(kv);
+ long sizeChangeForSecondCell = this.memstore.add(kv);
+ // make sure memstore size increase won't double-count MSLAB chunk size
+ assertEquals(AbstractMemStore.heapSizeChange(kv, true), sizeChangeForFirstCell);
+ Segment segment = this.memstore.getActive();
+ MemStoreLAB msLab = segment.getMemStoreLAB();
+ if (msLab != null) {
+ // make sure memstore size increased even when writing the same cell, if using MSLAB
+ assertEquals(segment.getCellLength(kv), sizeChangeForSecondCell);
+ // make sure chunk size increased even when writing the same cell, if using MSLAB
+ if (msLab instanceof HeapMemStoreLAB) {
+ assertEquals(2 * segment.getCellLength(kv),
+ ((HeapMemStoreLAB) msLab).getCurrentChunk().getNextFreeOffset());
+ }
+ } else {
+ // make sure no memstore size change w/o MSLAB
+ assertEquals(0, sizeChangeForSecondCell);
+ }
+ }
+
/**
* Test memstore snapshot happening while scanning.
* @throws IOException
[20/50] [abbrv] hbase git commit: HBASE-14853 Add on protobuf to c++
chain
Posted by ec...@apache.org.
HBASE-14853 Add on protobuf to c++ chain
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e00e3a04
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e00e3a04
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e00e3a04
Branch: refs/heads/HBASE-14850
Commit: e00e3a046dddf46cef62d1e5ff3c2dda10b9e559
Parents: 54aaaa3
Author: Elliott Clark <ec...@apache.org>
Authored: Mon Dec 28 15:33:52 2015 -0800
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/bin/start-docker.sh | 1 +
hbase-native-client/core/BUCK | 1 +
hbase-native-client/core/client.cc | 4 +
hbase-native-client/core/client.h | 2 +
hbase-native-client/if/AccessControl.proto | 123 +++
hbase-native-client/if/Admin.proto | 309 ++++++++
hbase-native-client/if/Aggregate.proto | 63 ++
hbase-native-client/if/Authentication.proto | 82 ++
hbase-native-client/if/BUCK | 36 +
hbase-native-client/if/Cell.proto | 67 ++
hbase-native-client/if/Client.proto | 472 +++++++++++
hbase-native-client/if/ClusterId.proto | 34 +
hbase-native-client/if/ClusterStatus.proto | 224 ++++++
hbase-native-client/if/Comparator.proto | 74 ++
hbase-native-client/if/Encryption.proto | 33 +
hbase-native-client/if/ErrorHandling.proto | 58 ++
hbase-native-client/if/FS.proto | 45 ++
hbase-native-client/if/Filter.proto | 170 ++++
hbase-native-client/if/HBase.proto | 258 ++++++
hbase-native-client/if/HFile.proto | 49 ++
hbase-native-client/if/LoadBalancer.proto | 29 +
hbase-native-client/if/MapReduce.proto | 37 +
hbase-native-client/if/Master.proto | 778 +++++++++++++++++++
hbase-native-client/if/MasterProcedure.proto | 245 ++++++
hbase-native-client/if/MultiRowMutation.proto | 45 ++
hbase-native-client/if/Procedure.proto | 119 +++
hbase-native-client/if/Quota.proto | 76 ++
hbase-native-client/if/RPC.proto | 136 ++++
hbase-native-client/if/RegionNormalizer.proto | 28 +
hbase-native-client/if/RegionServerStatus.proto | 158 ++++
hbase-native-client/if/RowProcessor.proto | 45 ++
hbase-native-client/if/SecureBulkLoad.proto | 72 ++
hbase-native-client/if/Snapshot.proto | 66 ++
hbase-native-client/if/Tracing.proto | 33 +
hbase-native-client/if/VisibilityLabels.proto | 83 ++
hbase-native-client/if/WAL.proto | 172 ++++
hbase-native-client/if/ZooKeeper.proto | 155 ++++
hbase-native-client/rpc/CMakeLists.txt | 17 -
hbase-native-client/third-party/BUCK | 4 +-
39 files changed, 4385 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/bin/start-docker.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/start-docker.sh b/hbase-native-client/bin/start-docker.sh
index bf38912..4426705 100755
--- a/hbase-native-client/bin/start-docker.sh
+++ b/hbase-native-client/bin/start-docker.sh
@@ -20,6 +20,7 @@ set -e
set -x
eval "$(docker-machine env docker-vm)"
+eval "$(docker-machine env dinghy)"
docker build -t hbase_native .
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/core/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index 2e4e755..ef027a1 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -41,6 +41,7 @@ cxx_binary(
"scanner.cc",
],
deps = [
+ "//if:if",
"//third-party:folly",
"//third-party:wangle",
],
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/core/client.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client.cc b/hbase-native-client/core/client.cc
index 98cf38a..a04daee 100644
--- a/hbase-native-client/core/client.cc
+++ b/hbase-native-client/core/client.cc
@@ -24,9 +24,13 @@
#include <glog/logging.h>
#include <gflags/gflags.h>
+#include "if/ZooKeeper.pb.h"
+
using namespace folly;
+using namespace hbase::pb;
int main(int argc, char *argv[]) {
+ MetaRegionServer mrs;
google::ParseCommandLineFlags(&argc, &argv, true);
google::InitGoogleLogging(argv[0]);
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/core/client.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client.h b/hbase-native-client/core/client.h
index 921cecc..35a3bd8 100644
--- a/hbase-native-client/core/client.h
+++ b/hbase-native-client/core/client.h
@@ -21,4 +21,6 @@
#include <folly/io/IOBuf.h>
+#include "if/Cell.pb.h"
+
class Client {};
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/AccessControl.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/AccessControl.proto b/hbase-native-client/if/AccessControl.proto
new file mode 100644
index 0000000..e67540b
--- /dev/null
+++ b/hbase-native-client/if/AccessControl.proto
@@ -0,0 +1,123 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "AccessControlProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+
+message Permission {
+ enum Action {
+ READ = 0;
+ WRITE = 1;
+ EXEC = 2;
+ CREATE = 3;
+ ADMIN = 4;
+ }
+ enum Type {
+ Global = 1;
+ Namespace = 2;
+ Table = 3;
+ }
+ required Type type = 1;
+ optional GlobalPermission global_permission = 2;
+ optional NamespacePermission namespace_permission = 3;
+ optional TablePermission table_permission = 4;
+}
+
+message TablePermission {
+ optional TableName table_name = 1;
+ optional bytes family = 2;
+ optional bytes qualifier = 3;
+ repeated Permission.Action action = 4;
+}
+
+message NamespacePermission {
+ optional bytes namespace_name = 1;
+ repeated Permission.Action action = 2;
+}
+
+message GlobalPermission {
+ repeated Permission.Action action = 1;
+}
+
+message UserPermission {
+ required bytes user = 1;
+ required Permission permission = 3;
+}
+
+/**
+ * Content of the /hbase/acl/<table or namespace> znode.
+ */
+message UsersAndPermissions {
+ message UserPermissions {
+ required bytes user = 1;
+ repeated Permission permissions = 2;
+ }
+
+ repeated UserPermissions user_permissions = 1;
+}
+
+message GrantRequest {
+ required UserPermission user_permission = 1;
+}
+
+message GrantResponse {
+}
+
+message RevokeRequest {
+ required UserPermission user_permission = 1;
+}
+
+message RevokeResponse {
+}
+
+message GetUserPermissionsRequest {
+ optional Permission.Type type = 1;
+ optional TableName table_name = 2;
+ optional bytes namespace_name = 3;
+}
+
+message GetUserPermissionsResponse {
+ repeated UserPermission user_permission = 1;
+}
+
+message CheckPermissionsRequest {
+ repeated Permission permission = 1;
+}
+
+message CheckPermissionsResponse {
+}
+
+service AccessControlService {
+ rpc Grant(GrantRequest)
+ returns (GrantResponse);
+
+ rpc Revoke(RevokeRequest)
+ returns (RevokeResponse);
+
+ rpc GetUserPermissions(GetUserPermissionsRequest)
+ returns (GetUserPermissionsResponse);
+
+ rpc CheckPermissions(CheckPermissionsRequest)
+ returns (CheckPermissionsResponse);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/Admin.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/Admin.proto b/hbase-native-client/if/Admin.proto
new file mode 100644
index 0000000..e905340
--- /dev/null
+++ b/hbase-native-client/if/Admin.proto
@@ -0,0 +1,309 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers that are used for Admin service.
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "AdminProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "WAL.proto";
+
+message GetRegionInfoRequest {
+ required RegionSpecifier region = 1;
+ optional bool compaction_state = 2;
+}
+
+message GetRegionInfoResponse {
+ required RegionInfo region_info = 1;
+ optional CompactionState compaction_state = 2;
+ optional bool isRecovering = 3;
+
+ enum CompactionState {
+ NONE = 0;
+ MINOR = 1;
+ MAJOR = 2;
+ MAJOR_AND_MINOR = 3;
+ }
+}
+
+/**
+ * Get a list of store files for a set of column families in a particular region.
+ * If no column family is specified, get the store files for all column families.
+ */
+message GetStoreFileRequest {
+ required RegionSpecifier region = 1;
+ repeated bytes family = 2;
+}
+
+message GetStoreFileResponse {
+ repeated string store_file = 1;
+}
+
+message GetOnlineRegionRequest {
+}
+
+message GetOnlineRegionResponse {
+ repeated RegionInfo region_info = 1;
+}
+
+message OpenRegionRequest {
+ repeated RegionOpenInfo open_info = 1;
+ // the intended server for this RPC.
+ optional uint64 serverStartCode = 2;
+ // wall clock time from master
+ optional uint64 master_system_time = 5;
+
+ message RegionOpenInfo {
+ required RegionInfo region = 1;
+ optional uint32 version_of_offline_node = 2;
+ repeated ServerName favored_nodes = 3;
+ // open region for distributedLogReplay
+ optional bool openForDistributedLogReplay = 4;
+ }
+}
+
+message OpenRegionResponse {
+ repeated RegionOpeningState opening_state = 1;
+
+ enum RegionOpeningState {
+ OPENED = 0;
+ ALREADY_OPENED = 1;
+ FAILED_OPENING = 2;
+ }
+}
+
+message WarmupRegionRequest {
+
+ required RegionInfo regionInfo = 1;
+}
+
+message WarmupRegionResponse {
+}
+
+/**
+ * Closes the specified region and will use or not use ZK during the close
+ * according to the specified flag.
+ */
+message CloseRegionRequest {
+ required RegionSpecifier region = 1;
+ optional uint32 version_of_closing_node = 2;
+ optional bool transition_in_ZK = 3 [default = true];
+ optional ServerName destination_server = 4;
+ // the intended server for this RPC.
+ optional uint64 serverStartCode = 5;
+}
+
+message CloseRegionResponse {
+ required bool closed = 1;
+}
+
+/**
+ * Flushes the MemStore of the specified region.
+ * <p>
+ * This method is synchronous.
+ */
+message FlushRegionRequest {
+ required RegionSpecifier region = 1;
+ optional uint64 if_older_than_ts = 2;
+ optional bool write_flush_wal_marker = 3; // whether to write a marker to WAL even if not flushed
+}
+
+message FlushRegionResponse {
+ required uint64 last_flush_time = 1;
+ optional bool flushed = 2;
+ optional bool wrote_flush_wal_marker = 3;
+}
+
+/**
+ * Splits the specified region.
+ * <p>
+ * This method currently flushes the region and then forces a compaction which
+ * will then trigger a split. The flush is done synchronously but the
+ * compaction is asynchronous.
+ */
+message SplitRegionRequest {
+ required RegionSpecifier region = 1;
+ optional bytes split_point = 2;
+}
+
+message SplitRegionResponse {
+}
+
+/**
+ * Compacts the specified region. Performs a major compaction if specified.
+ * <p>
+ * This method is asynchronous.
+ */
+message CompactRegionRequest {
+ required RegionSpecifier region = 1;
+ optional bool major = 2;
+ optional bytes family = 3;
+}
+
+message CompactRegionResponse {
+}
+
+message UpdateFavoredNodesRequest {
+ repeated RegionUpdateInfo update_info = 1;
+
+ message RegionUpdateInfo {
+ required RegionInfo region = 1;
+ repeated ServerName favored_nodes = 2;
+ }
+}
+
+message UpdateFavoredNodesResponse {
+ optional uint32 response = 1;
+}
+
+/**
+ * Merges the specified regions.
+ * <p>
+ * This method currently closes the regions and then merges them
+ */
+message MergeRegionsRequest {
+ required RegionSpecifier region_a = 1;
+ required RegionSpecifier region_b = 2;
+ optional bool forcible = 3 [default = false];
+ // wall clock time from master
+ optional uint64 master_system_time = 4;
+}
+
+message MergeRegionsResponse {
+}
+
+// Protocol buffer version of WAL for replication
+message WALEntry {
+ required WALKey key = 1;
+ // Following may be null if the KVs/Cells are carried along the side in a cellblock (See
+ // RPC for more on cellblocks). If Cells/KVs are in a cellblock, this next field is null
+ // and associated_cell_count has count of Cells associated w/ this WALEntry
+ repeated bytes key_value_bytes = 2;
+ // If Cell data is carried alongside in a cellblock, this is count of Cells in the cellblock.
+ optional int32 associated_cell_count = 3;
+}
+
+/**
+ * Replicates the given entries. The guarantee is that the given entries
+ * will be durable on the slave cluster if this method returns without
+ * any exception. hbase.replication has to be set to true for this to work.
+ */
+message ReplicateWALEntryRequest {
+ repeated WALEntry entry = 1;
+ optional string replicationClusterId = 2;
+ optional string sourceBaseNamespaceDirPath = 3;
+ optional string sourceHFileArchiveDirPath = 4;
+}
+
+message ReplicateWALEntryResponse {
+}
+
+message RollWALWriterRequest {
+}
+
+/*
+ * Roll request responses no longer include regions to flush
+ * this list will always be empty when talking to a 1.0 server
+ */
+message RollWALWriterResponse {
+ // A list of encoded name of regions to flush
+ repeated bytes region_to_flush = 1;
+}
+
+message StopServerRequest {
+ required string reason = 1;
+}
+
+message StopServerResponse {
+}
+
+message GetServerInfoRequest {
+}
+
+message ServerInfo {
+ required ServerName server_name = 1;
+ optional uint32 webui_port = 2;
+}
+
+message GetServerInfoResponse {
+ required ServerInfo server_info = 1;
+}
+
+message UpdateConfigurationRequest {
+}
+
+message UpdateConfigurationResponse {
+}
+
+service AdminService {
+ rpc GetRegionInfo(GetRegionInfoRequest)
+ returns(GetRegionInfoResponse);
+
+ rpc GetStoreFile(GetStoreFileRequest)
+ returns(GetStoreFileResponse);
+
+ rpc GetOnlineRegion(GetOnlineRegionRequest)
+ returns(GetOnlineRegionResponse);
+
+ rpc OpenRegion(OpenRegionRequest)
+ returns(OpenRegionResponse);
+
+ rpc WarmupRegion(WarmupRegionRequest)
+ returns(WarmupRegionResponse);
+
+ rpc CloseRegion(CloseRegionRequest)
+ returns(CloseRegionResponse);
+
+ rpc FlushRegion(FlushRegionRequest)
+ returns(FlushRegionResponse);
+
+ rpc SplitRegion(SplitRegionRequest)
+ returns(SplitRegionResponse);
+
+ rpc CompactRegion(CompactRegionRequest)
+ returns(CompactRegionResponse);
+
+ rpc MergeRegions(MergeRegionsRequest)
+ returns(MergeRegionsResponse);
+
+ rpc ReplicateWALEntry(ReplicateWALEntryRequest)
+ returns(ReplicateWALEntryResponse);
+
+ rpc Replay(ReplicateWALEntryRequest)
+ returns(ReplicateWALEntryResponse);
+
+ rpc RollWALWriter(RollWALWriterRequest)
+ returns(RollWALWriterResponse);
+
+ rpc GetServerInfo(GetServerInfoRequest)
+ returns(GetServerInfoResponse);
+
+ rpc StopServer(StopServerRequest)
+ returns(StopServerResponse);
+
+ rpc UpdateFavoredNodes(UpdateFavoredNodesRequest)
+ returns(UpdateFavoredNodesResponse);
+
+ rpc UpdateConfiguration(UpdateConfigurationRequest)
+ returns(UpdateConfigurationResponse);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/Aggregate.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/Aggregate.proto b/hbase-native-client/if/Aggregate.proto
new file mode 100644
index 0000000..4d32e70
--- /dev/null
+++ b/hbase-native-client/if/Aggregate.proto
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "AggregateProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "Client.proto";
+
+message AggregateRequest {
+ /** The request passed to the AggregateService consists of three parts
+ * (1) the (canonical) classname of the ColumnInterpreter implementation
+ * (2) the Scan query
+ * (3) any bytes required to construct the ColumnInterpreter object
+ * properly
+ */
+ required string interpreter_class_name = 1;
+ required Scan scan = 2;
+ optional bytes interpreter_specific_bytes = 3;
+}
+
+message AggregateResponse {
+ /**
+ * The AggregateService methods all have a response that either is a Pair
+ * or a simple object. When it is a Pair both first_part and second_part
+ * have defined values (and the second_part is not present in the response
+ * when the response is not a pair). Refer to the AggregateImplementation
+ * class for an overview of the AggregateResponse object constructions.
+ */
+ repeated bytes first_part = 1;
+ optional bytes second_part = 2;
+}
+
+/** Refer to the AggregateImplementation class for an overview of the
+ * AggregateService method implementations and their functionality.
+ */
+service AggregateService {
+ rpc GetMax (AggregateRequest) returns (AggregateResponse);
+ rpc GetMin (AggregateRequest) returns (AggregateResponse);
+ rpc GetSum (AggregateRequest) returns (AggregateResponse);
+ rpc GetRowNum (AggregateRequest) returns (AggregateResponse);
+ rpc GetAvg (AggregateRequest) returns (AggregateResponse);
+ rpc GetStd (AggregateRequest) returns (AggregateResponse);
+ rpc GetMedian (AggregateRequest) returns (AggregateResponse);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/Authentication.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/Authentication.proto b/hbase-native-client/if/Authentication.proto
new file mode 100644
index 0000000..2f64799
--- /dev/null
+++ b/hbase-native-client/if/Authentication.proto
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "AuthenticationProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+message AuthenticationKey {
+ required int32 id = 1;
+ required int64 expiration_date = 2;
+ required bytes key = 3;
+}
+
+
+message TokenIdentifier {
+ enum Kind {
+ HBASE_AUTH_TOKEN = 0;
+ }
+ required Kind kind = 1;
+ required bytes username = 2;
+ required int32 key_id = 3;
+ optional int64 issue_date = 4;
+ optional int64 expiration_date = 5;
+ optional int64 sequence_number = 6;
+}
+
+
+// Serialization of the org.apache.hadoop.security.token.Token class
+// Note that this is a Hadoop class, so fields may change!
+message Token {
+ // the TokenIdentifier in serialized form
+ // Note: we can't use the protobuf directly because the Hadoop Token class
+ // only stores the serialized bytes
+ optional bytes identifier = 1;
+ optional bytes password = 2;
+ optional bytes service = 3;
+}
+
+
+// RPC request & response messages
+message GetAuthenticationTokenRequest {
+}
+
+message GetAuthenticationTokenResponse {
+ optional Token token = 1;
+}
+
+message WhoAmIRequest {
+}
+
+message WhoAmIResponse {
+ optional string username = 1;
+ optional string auth_method = 2;
+}
+
+
+// RPC service
+service AuthenticationService {
+ rpc GetAuthenticationToken(GetAuthenticationTokenRequest)
+ returns (GetAuthenticationTokenResponse);
+
+ rpc WhoAmI(WhoAmIRequest)
+ returns (WhoAmIResponse);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/BUCK b/hbase-native-client/if/BUCK
new file mode 100644
index 0000000..3490a05
--- /dev/null
+++ b/hbase-native-client/if/BUCK
@@ -0,0 +1,36 @@
+PROTO_SRCS = glob(['*.proto'])
+HEADER_FILENAMES = [ x.replace('.proto','.pb.h') for x in PROTO_SRCS]
+CC_FILENAMES = [ x.replace('.proto', '.pb.cc') for x in PROTO_SRCS]
+
+genrule(
+ name = 'generate-proto-sources',
+ srcs = PROTO_SRCS,
+ cmd = 'mkdir -p $OUT && pwd && protoc --proto_path=. --cpp_out=$OUT *.proto',
+ out = 'output',
+)
+
+for header_filename in HEADER_FILENAMES:
+ genrule(
+ name = header_filename,
+ cmd = 'mkdir -p `dirname $OUT` '
+ ' && cp $(location :generate-proto-sources)/{} $OUT'.format(header_filename),
+ out = header_filename,
+ )
+for cc_filename in CC_FILENAMES:
+ genrule(
+ name = cc_filename,
+ cmd = 'mkdir -p `dirname $OUT` '
+ ' && cp $(location :generate-proto-sources)/{} $OUT '
+ ' && cp $(location :generate-proto-sources)/*.h `dirname $OUT`'.format(cc_filename),
+ out = cc_filename,
+ )
+
+cxx_library(
+ name = 'if',
+ exported_headers = [':' + x for x in HEADER_FILENAMES],
+ srcs = [':' + x for x in CC_FILENAMES],
+ deps = [ '//third-party:protobuf']
+ + [':' + x for x in CC_FILENAMES]
+ + [ ':' + x for x in HEADER_FILENAMES ],
+ visibility = [ 'PUBLIC', ],
+)
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/Cell.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/Cell.proto b/hbase-native-client/if/Cell.proto
new file mode 100644
index 0000000..2c61035
--- /dev/null
+++ b/hbase-native-client/if/Cell.proto
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Cell and KeyValue protos
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "CellProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+/**
+ * The type of the key in a Cell
+ */
+enum CellType {
+ MINIMUM = 0;
+ PUT = 4;
+
+ DELETE = 8;
+ DELETE_COLUMN = 12;
+ DELETE_FAMILY = 14;
+
+ // MAXIMUM is used when searching; you look from maximum on down.
+ MAXIMUM = 255;
+}
+
+/**
+ * Protocol buffer version of Cell.
+ */
+message Cell {
+ optional bytes row = 1;
+ optional bytes family = 2;
+ optional bytes qualifier = 3;
+ optional uint64 timestamp = 4;
+ optional CellType cell_type = 5;
+ optional bytes value = 6;
+ optional bytes tags = 7;
+}
+
+/**
+ * Protocol buffer version of KeyValue.
+ * It doesn't have those transient parameters
+ */
+message KeyValue {
+ required bytes row = 1;
+ required bytes family = 2;
+ required bytes qualifier = 3;
+ optional uint64 timestamp = 4;
+ optional CellType key_type = 5;
+ optional bytes value = 6;
+ optional bytes tags = 7;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/Client.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/Client.proto b/hbase-native-client/if/Client.proto
new file mode 100644
index 0000000..ca9abdc
--- /dev/null
+++ b/hbase-native-client/if/Client.proto
@@ -0,0 +1,472 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers that are used for Client service.
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "ClientProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "Filter.proto";
+import "Cell.proto";
+import "Comparator.proto";
+import "MapReduce.proto";
+
+/**
+ * The protocol buffer version of Authorizations.
+ */
+message Authorizations {
+ repeated string label = 1;
+}
+
+/**
+ * The protocol buffer version of CellVisibility.
+ */
+message CellVisibility {
+ required string expression = 1;
+}
+
+/**
+ * Container for a list of column qualifier names of a family.
+ */
+message Column {
+ required bytes family = 1;
+ repeated bytes qualifier = 2;
+}
+
+/**
+ * Consistency defines the expected consistency level for an operation.
+ */
+enum Consistency {
+ STRONG = 0;
+ TIMELINE = 1;
+}
+
+/**
+ * The protocol buffer version of Get.
+ * Unless existence_only is specified, return all the requested data
+ * for the row that matches exactly.
+ */
+message Get {
+ required bytes row = 1;
+ repeated Column column = 2;
+ repeated NameBytesPair attribute = 3;
+ optional Filter filter = 4;
+ optional TimeRange time_range = 5;
+ optional uint32 max_versions = 6 [default = 1];
+ optional bool cache_blocks = 7 [default = true];
+ optional uint32 store_limit = 8;
+ optional uint32 store_offset = 9;
+
+ // The result isn't asked for, just check for
+ // the existence.
+ optional bool existence_only = 10 [default = false];
+
+ optional Consistency consistency = 12 [default = STRONG];
+ repeated ColumnFamilyTimeRange cf_time_range = 13;
+}
+
+message Result {
+ // Result includes the Cells or else it just has a count of Cells
+ // that are carried otherwise.
+ repeated Cell cell = 1;
+ // The below count is set when the associated cells are
+ // not part of this protobuf message; they are passed alongside
+ // and then this Message is just a placeholder with metadata.
+ // The count is needed to know how many to peel off the block of Cells as
+ // ours. NOTE: This is different from the pb managed cell_count of the
+ // 'cell' field above which is non-null when the cells are pb'd.
+ optional int32 associated_cell_count = 2;
+
+ // used for Get to check existence only. Not set if existence_only was not set to true
+ // in the query.
+ optional bool exists = 3;
+
+ // Whether or not the results are coming from possibly stale data
+ optional bool stale = 4 [default = false];
+
+ // Whether or not the entire result could be returned. Results will be split when
+ // the RPC chunk size limit is reached. Partial results contain only a subset of the
+ // cells for a row and must be combined with a result containing the remaining cells
+ // to form a complete result
+ optional bool partial = 5 [default = false];
+}
+
+/**
+ * The get request. Perform a single Get operation.
+ */
+message GetRequest {
+ required RegionSpecifier region = 1;
+ required Get get = 2;
+}
+
+message GetResponse {
+ optional Result result = 1;
+}
+
+/**
+ * Condition to check if the value of a given cell (row,
+ * family, qualifier) matches a value via a given comparator.
+ *
+ * Condition is used in check and mutate operations.
+ */
+message Condition {
+ required bytes row = 1;
+ required bytes family = 2;
+ required bytes qualifier = 3;
+ required CompareType compare_type = 4;
+ required Comparator comparator = 5;
+}
+
+
+/**
+ * A specific mutation inside a mutate request.
+ * It can be an append, increment, put or delete based
+ * on the mutation type. It can be fully filled in or
+ * only metadata present because data is being carried
+ * elsewhere outside of pb.
+ */
+message MutationProto {
+ optional bytes row = 1;
+ optional MutationType mutate_type = 2;
+ repeated ColumnValue column_value = 3;
+ optional uint64 timestamp = 4;
+ repeated NameBytesPair attribute = 5;
+ optional Durability durability = 6 [default = USE_DEFAULT];
+
+ // For some mutations, a result may be returned, in which case,
+ // time range can be specified for potential performance gain
+ optional TimeRange time_range = 7;
+ // The below count is set when the associated cells are NOT
+ // part of this protobuf message; they are passed alongside
+ // and then this Message is a placeholder with metadata. The
+ // count is needed to know how many to peel off the block of Cells as
+ // ours. NOTE: This is different from the pb managed cell_count of the
+ // 'cell' field above which is non-null when the cells are pb'd.
+ optional int32 associated_cell_count = 8;
+
+ optional uint64 nonce = 9;
+
+ enum Durability {
+ USE_DEFAULT = 0;
+ SKIP_WAL = 1;
+ ASYNC_WAL = 2;
+ SYNC_WAL = 3;
+ FSYNC_WAL = 4;
+ }
+
+ enum MutationType {
+ APPEND = 0;
+ INCREMENT = 1;
+ PUT = 2;
+ DELETE = 3;
+ }
+
+ enum DeleteType {
+ DELETE_ONE_VERSION = 0;
+ DELETE_MULTIPLE_VERSIONS = 1;
+ DELETE_FAMILY = 2;
+ DELETE_FAMILY_VERSION = 3;
+ }
+
+ message ColumnValue {
+ required bytes family = 1;
+ repeated QualifierValue qualifier_value = 2;
+
+ message QualifierValue {
+ optional bytes qualifier = 1;
+ optional bytes value = 2;
+ optional uint64 timestamp = 3;
+ optional DeleteType delete_type = 4;
+ optional bytes tags = 5;
+ }
+ }
+}
+
+/**
+ * The mutate request. Perform a single Mutate operation.
+ *
+ * Optionally, you can specify a condition. The mutate
+ * will take place only if the condition is met. Otherwise,
+ * the mutate will be ignored. In the response result,
+ * parameter processed is used to indicate if the mutate
+ * actually happened.
+ */
+message MutateRequest {
+ required RegionSpecifier region = 1;
+ required MutationProto mutation = 2;
+ optional Condition condition = 3;
+ optional uint64 nonce_group = 4;
+}
+
+message MutateResponse {
+ optional Result result = 1;
+
+ // used for mutate to indicate processed only
+ optional bool processed = 2;
+}
+
+/**
+ * Instead of get from a table, you can scan it with optional filters.
+ * You can specify the row key range, time range, the columns/families
+ * to scan and so on.
+ *
+ * This scan is used the first time in a scan request. The response of
+ * the initial scan will return a scanner id, which should be used to
+ * fetch result batches later on before it is closed.
+ */
+message Scan {
+ repeated Column column = 1;
+ repeated NameBytesPair attribute = 2;
+ optional bytes start_row = 3;
+ optional bytes stop_row = 4;
+ optional Filter filter = 5;
+ optional TimeRange time_range = 6;
+ optional uint32 max_versions = 7 [default = 1];
+ optional bool cache_blocks = 8 [default = true];
+ optional uint32 batch_size = 9;
+ optional uint64 max_result_size = 10;
+ optional uint32 store_limit = 11;
+ optional uint32 store_offset = 12;
+ optional bool load_column_families_on_demand = 13; /* DO NOT add defaults to load_column_families_on_demand. */
+ optional bool small = 14;
+ optional bool reversed = 15 [default = false];
+ optional Consistency consistency = 16 [default = STRONG];
+ optional uint32 caching = 17;
+ optional bool allow_partial_results = 18;
+ repeated ColumnFamilyTimeRange cf_time_range = 19;
+}
+
+/**
+ * A scan request. Initially, it should specify a scan. Later on, you
+ * can use the scanner id returned to fetch result batches with a different
+ * scan request.
+ *
+ * The scanner will remain open if there are more results, and it's not
+ * asked to be closed explicitly.
+ *
+ * You can fetch the results and ask the scanner to be closed to save
+ * a trip if you are not interested in remaining results.
+ */
+message ScanRequest {
+ optional RegionSpecifier region = 1;
+ optional Scan scan = 2;
+ optional uint64 scanner_id = 3;
+ optional uint32 number_of_rows = 4;
+ optional bool close_scanner = 5;
+ optional uint64 next_call_seq = 6;
+ optional bool client_handles_partials = 7;
+ optional bool client_handles_heartbeats = 8;
+ optional bool track_scan_metrics = 9;
+ optional bool renew = 10 [default = false];
+}
+
+/**
+ * The scan response. If there are no more results, more_results will
+ * be false. If it is not specified, it means there are more.
+ */
+message ScanResponse {
+ // This field is filled in if we are doing cellblocks. A cellblock is made up
+ // of all Cells serialized out as one cellblock BUT responses from a server
+ // have their Cells grouped by Result. So we can reconstitute the
+ // Results on the client-side, this field is a list of counts of Cells
+ // in each Result that makes up the response. For example, if this field
+ // has 3, 3, 3 in it, then we know that on the client, we are to make
+ // three Results each of three Cells each.
+ repeated uint32 cells_per_result = 1;
+
+ optional uint64 scanner_id = 2;
+ optional bool more_results = 3;
+ optional uint32 ttl = 4;
+ // If cells are not carried in an accompanying cellblock, then they are pb'd here.
+ // This field is mutually exclusive with cells_per_result (since the Cells will
+ // be inside the pb'd Result)
+ repeated Result results = 5;
+ optional bool stale = 6;
+
+ // This field is filled in if we are doing cellblocks. In the event that a row
+ // could not fit all of its cells into a single RPC chunk, the results will be
+ // returned as partials, and reconstructed into a complete result on the client
+ // side. This field is a list of flags indicating whether or not the result
+ // that the cells belong to is a partial result. For example, if this field
+ // has false, false, true in it, then we know that on the client side, we need to
+ // make another RPC request since the last result was only a partial.
+ repeated bool partial_flag_per_result = 7;
+
+ // A server may choose to limit the number of results returned to the client for
+ // reasons such as the size in bytes or quantity of results accumulated. This field
+ // will true when more results exist in the current region.
+ optional bool more_results_in_region = 8;
+
+ // This field is filled in if the server is sending back a heartbeat message.
+ // Heartbeat messages are sent back to the client to prevent the scanner from
+ // timing out. Seeing a heartbeat message communicates to the Client that the
+ // server would have continued to scan had the time limit not been reached.
+ optional bool heartbeat_message = 9;
+
+ // This field is filled in if the client has requested that scan metrics be tracked.
+ // The metrics tracked here are sent back to the client to be tracked together with
+ // the existing client side metrics.
+ optional ScanMetrics scan_metrics = 10;
+}
+
+/**
+ * Atomically bulk load multiple HFiles (say from different column families)
+ * into an open region.
+ */
+message BulkLoadHFileRequest {
+ required RegionSpecifier region = 1;
+ repeated FamilyPath family_path = 2;
+ optional bool assign_seq_num = 3;
+
+ message FamilyPath {
+ required bytes family = 1;
+ required string path = 2;
+ }
+}
+
+message BulkLoadHFileResponse {
+ required bool loaded = 1;
+}
+
+message CoprocessorServiceCall {
+ required bytes row = 1;
+ required string service_name = 2;
+ required string method_name = 3;
+ required bytes request = 4;
+}
+
+message CoprocessorServiceResult {
+ optional NameBytesPair value = 1;
+}
+
+message CoprocessorServiceRequest {
+ required RegionSpecifier region = 1;
+ required CoprocessorServiceCall call = 2;
+}
+
+message CoprocessorServiceResponse {
+ required RegionSpecifier region = 1;
+ required NameBytesPair value = 2;
+}
+
+// Either a Get or a Mutation
+message Action {
+ // If part of a multi action, useful aligning
+ // result with what was originally submitted.
+ optional uint32 index = 1;
+ optional MutationProto mutation = 2;
+ optional Get get = 3;
+ optional CoprocessorServiceCall service_call = 4;
+}
+
+/**
+ * Actions to run against a Region.
+ */
+message RegionAction {
+ required RegionSpecifier region = 1;
+ // When set, run mutations as atomic unit.
+ optional bool atomic = 2;
+ repeated Action action = 3;
+}
+
+/*
+* Statistics about the current load on the region
+*/
+message RegionLoadStats {
+ // Percent load on the memstore. Guaranteed to be positive, between 0 and 100.
+ optional int32 memstoreLoad = 1 [default = 0];
+ // Percent JVM heap occupancy. Guaranteed to be positive, between 0 and 100.
+ // We can move this to "ServerLoadStats" should we develop them.
+ optional int32 heapOccupancy = 2 [default = 0];
+ // Compaction pressure. Guaranteed to be positive, between 0 and 100.
+ optional int32 compactionPressure = 3 [default = 0];
+}
+
+/**
+ * Either a Result or an Exception NameBytesPair (keyed by
+ * exception name whose value is the exception stringified)
+ * or maybe empty if no result and no exception.
+ */
+message ResultOrException {
+ // If part of a multi call, save original index of the list of all
+ // passed so can align this response w/ original request.
+ optional uint32 index = 1;
+ optional Result result = 2;
+ optional NameBytesPair exception = 3;
+ // result if this was a coprocessor service call
+ optional CoprocessorServiceResult service_result = 4;
+ // current load on the region
+ optional RegionLoadStats loadStats = 5;
+}
+
+/**
+ * The result of a RegionAction.
+ */
+message RegionActionResult {
+ repeated ResultOrException resultOrException = 1;
+ // If the operation failed globally for this region, this exception is set
+ optional NameBytesPair exception = 2;
+}
+
+/**
+ * Execute a list of actions on a given region in order.
+ * Nothing prevents a request to contains a set of RegionAction on the same region.
+ * For this reason, the matching between the MultiRequest and the MultiResponse is not
+ * done by the region specifier but by keeping the order of the RegionActionResult vs.
+ * the order of the RegionAction.
+ */
+message MultiRequest {
+ repeated RegionAction regionAction = 1;
+ optional uint64 nonceGroup = 2;
+ optional Condition condition = 3;
+}
+
+message MultiResponse {
+ repeated RegionActionResult regionActionResult = 1;
+ // used for mutate to indicate processed only
+ optional bool processed = 2;
+}
+
+
+service ClientService {
+ rpc Get(GetRequest)
+ returns(GetResponse);
+
+ rpc Mutate(MutateRequest)
+ returns(MutateResponse);
+
+ rpc Scan(ScanRequest)
+ returns(ScanResponse);
+
+ rpc BulkLoadHFile(BulkLoadHFileRequest)
+ returns(BulkLoadHFileResponse);
+
+ rpc ExecService(CoprocessorServiceRequest)
+ returns(CoprocessorServiceResponse);
+
+ rpc ExecRegionServerService(CoprocessorServiceRequest)
+ returns(CoprocessorServiceResponse);
+
+ rpc Multi(MultiRequest)
+ returns(MultiResponse);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/ClusterId.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/ClusterId.proto b/hbase-native-client/if/ClusterId.proto
new file mode 100644
index 0000000..aed8cfc
--- /dev/null
+++ b/hbase-native-client/if/ClusterId.proto
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers that are shared throughout HBase
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "ClusterIdProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+/**
+ * Content of the '/hbase/hbaseid', cluster id, znode.
+ * Also cluster of the ${HBASE_ROOTDIR}/hbase.id file.
+ */
+message ClusterId {
+ // This is the cluster id, a uuid as a String
+ required string cluster_id = 1;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/ClusterStatus.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/ClusterStatus.proto b/hbase-native-client/if/ClusterStatus.proto
new file mode 100644
index 0000000..228be7e
--- /dev/null
+++ b/hbase-native-client/if/ClusterStatus.proto
@@ -0,0 +1,224 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers that are used for ClustStatus
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "ClusterStatusProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "ClusterId.proto";
+import "FS.proto";
+
+message RegionState {
+ required RegionInfo region_info = 1;
+ required State state = 2;
+ optional uint64 stamp = 3;
+ enum State {
+ OFFLINE = 0; // region is in an offline state
+ PENDING_OPEN = 1; // sent rpc to server to open but has not begun
+ OPENING = 2; // server has begun to open but not yet done
+ OPEN = 3; // server opened region and updated meta
+ PENDING_CLOSE = 4; // sent rpc to server to close but has not begun
+ CLOSING = 5; // server has begun to close but not yet done
+ CLOSED = 6; // server closed region and updated meta
+ SPLITTING = 7; // server started split of a region
+ SPLIT = 8; // server completed split of a region
+ FAILED_OPEN = 9; // failed to open, and won't retry any more
+ FAILED_CLOSE = 10; // failed to close, and won't retry any more
+ MERGING = 11; // server started merge a region
+ MERGED = 12; // server completed merge of a region
+ SPLITTING_NEW = 13; // new region to be created when RS splits a parent
+ // region but hasn't be created yet, or master doesn't
+ // know it's already created
+ MERGING_NEW = 14; // new region to be created when RS merges two
+ // daughter regions but hasn't be created yet, or
+ // master doesn't know it's already created
+ }
+}
+
+message RegionInTransition {
+ required RegionSpecifier spec = 1;
+ required RegionState region_state = 2;
+}
+
+/**
+ * sequence Id of a store
+ */
+message StoreSequenceId {
+ required bytes family_name = 1;
+ required uint64 sequence_id = 2;
+}
+
+/**
+ * contains a sequence id of a region which should be the minimum of its store sequence ids and
+ * list of sequence ids of the region's stores
+ */
+message RegionStoreSequenceIds {
+ required uint64 last_flushed_sequence_id = 1;
+ repeated StoreSequenceId store_sequence_id = 2;
+}
+
+message RegionLoad {
+ /** the region specifier */
+ required RegionSpecifier region_specifier = 1;
+
+ /** the number of stores for the region */
+ optional uint32 stores = 2;
+
+ /** the number of storefiles for the region */
+ optional uint32 storefiles = 3;
+
+ /** the total size of the store files for the region, uncompressed, in MB */
+ optional uint32 store_uncompressed_size_MB = 4;
+
+ /** the current total size of the store files for the region, in MB */
+ optional uint32 storefile_size_MB = 5;
+
+ /** the current size of the memstore for the region, in MB */
+ optional uint32 memstore_size_MB = 6;
+
+ /**
+ * The current total size of root-level store file indexes for the region,
+ * in MB. The same as {@link #rootIndexSizeKB} but in MB.
+ */
+ optional uint32 storefile_index_size_MB = 7;
+
+ /** the current total read requests made to region */
+ optional uint64 read_requests_count = 8;
+
+ /** the current total write requests made to region */
+ optional uint64 write_requests_count = 9;
+
+ /** the total compacting key values in currently running compaction */
+ optional uint64 total_compacting_KVs = 10;
+
+ /** the completed count of key values in currently running compaction */
+ optional uint64 current_compacted_KVs = 11;
+
+ /** The current total size of root-level indexes for the region, in KB. */
+ optional uint32 root_index_size_KB = 12;
+
+ /** The total size of all index blocks, not just the root level, in KB. */
+ optional uint32 total_static_index_size_KB = 13;
+
+ /**
+ * The total size of all Bloom filter blocks, not just loaded into the
+ * block cache, in KB.
+ */
+ optional uint32 total_static_bloom_size_KB = 14;
+
+ /** the most recent sequence Id from cache flush */
+ optional uint64 complete_sequence_id = 15;
+
+ /** The current data locality for region in the regionserver */
+ optional float data_locality = 16;
+
+ optional uint64 last_major_compaction_ts = 17 [default = 0];
+
+ /** the most recent sequence Id of store from cache flush */
+ repeated StoreSequenceId store_complete_sequence_id = 18;
+}
+
+/* Server-level protobufs */
+
+message ReplicationLoadSink {
+ required uint64 ageOfLastAppliedOp = 1;
+ required uint64 timeStampsOfLastAppliedOp = 2;
+}
+
+message ReplicationLoadSource {
+ required string peerID = 1;
+ required uint64 ageOfLastShippedOp = 2;
+ required uint32 sizeOfLogQueue = 3;
+ required uint64 timeStampOfLastShippedOp = 4;
+ required uint64 replicationLag = 5;
+}
+
+message ServerLoad {
+ /** Number of requests since last report. */
+ optional uint64 number_of_requests = 1;
+
+ /** Total Number of requests from the start of the region server. */
+ optional uint64 total_number_of_requests = 2;
+
+ /** the amount of used heap, in MB. */
+ optional uint32 used_heap_MB = 3;
+
+ /** the maximum allowable size of the heap, in MB. */
+ optional uint32 max_heap_MB = 4;
+
+ /** Information on the load of individual regions. */
+ repeated RegionLoad region_loads = 5;
+
+ /**
+ * Regionserver-level coprocessors, e.g., WALObserver implementations.
+ * Region-level coprocessors, on the other hand, are stored inside RegionLoad
+ * objects.
+ */
+ repeated Coprocessor coprocessors = 6;
+
+ /**
+ * Time when incremental (non-total) counts began being calculated (e.g. number_of_requests)
+ * time is measured as the difference, measured in milliseconds, between the current time
+ * and midnight, January 1, 1970 UTC.
+ */
+ optional uint64 report_start_time = 7;
+
+ /**
+ * Time when report was generated.
+ * time is measured as the difference, measured in milliseconds, between the current time
+ * and midnight, January 1, 1970 UTC.
+ */
+ optional uint64 report_end_time = 8;
+
+ /**
+ * The port number that this region server is hosing an info server on.
+ */
+ optional uint32 info_server_port = 9;
+
+ /**
+ * The replicationLoadSource for the replication Source status of this region server.
+ */
+ repeated ReplicationLoadSource replLoadSource = 10;
+
+ /**
+ * The replicationLoadSink for the replication Sink status of this region server.
+ */
+ optional ReplicationLoadSink replLoadSink = 11;
+}
+
+message LiveServerInfo {
+ required ServerName server = 1;
+ required ServerLoad server_load = 2;
+}
+
+message ClusterStatus {
+ optional HBaseVersionFileContent hbase_version = 1;
+ repeated LiveServerInfo live_servers = 2;
+ repeated ServerName dead_servers = 3;
+ repeated RegionInTransition regions_in_transition = 4;
+ optional ClusterId cluster_id = 5;
+ repeated Coprocessor master_coprocessors = 6;
+ optional ServerName master = 7;
+ repeated ServerName backup_masters = 8;
+ optional bool balancer_on = 9;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/Comparator.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/Comparator.proto b/hbase-native-client/if/Comparator.proto
new file mode 100644
index 0000000..496b68d
--- /dev/null
+++ b/hbase-native-client/if/Comparator.proto
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers that are used for filters
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "ComparatorProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+// This file contains protocol buffers that are used for comparators (e.g. in filters)
+
+message Comparator {
+ required string name = 1;
+ optional bytes serialized_comparator = 2;
+}
+
+message ByteArrayComparable {
+ optional bytes value = 1;
+}
+
+message BinaryComparator {
+ required ByteArrayComparable comparable = 1;
+}
+
+message LongComparator {
+ required ByteArrayComparable comparable = 1;
+}
+
+message BinaryPrefixComparator {
+ required ByteArrayComparable comparable = 1;
+}
+
+message BitComparator {
+ required ByteArrayComparable comparable = 1;
+ required BitwiseOp bitwise_op = 2;
+
+ enum BitwiseOp {
+ AND = 1;
+ OR = 2;
+ XOR = 3;
+ }
+}
+
+message NullComparator {
+}
+
+message RegexStringComparator {
+ required string pattern = 1;
+ required int32 pattern_flags = 2;
+ required string charset = 3;
+ optional string engine = 4;
+}
+
+message SubstringComparator {
+ required string substr = 1;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/Encryption.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/Encryption.proto b/hbase-native-client/if/Encryption.proto
new file mode 100644
index 0000000..97ab5b2
--- /dev/null
+++ b/hbase-native-client/if/Encryption.proto
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers used for encryption
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "EncryptionProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+message WrappedKey {
+ required string algorithm = 1;
+ required uint32 length = 2;
+ required bytes data = 3;
+ optional bytes iv = 4;
+ optional bytes hash = 5;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/ErrorHandling.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/ErrorHandling.proto b/hbase-native-client/if/ErrorHandling.proto
new file mode 100644
index 0000000..be9a743
--- /dev/null
+++ b/hbase-native-client/if/ErrorHandling.proto
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers that are used for error handling
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "ErrorHandlingProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+/**
+ * Protobuf version of a java.lang.StackTraceElement
+ * so we can serialize exceptions.
+ */
+message StackTraceElementMessage {
+ optional string declaring_class = 1;
+ optional string method_name = 2;
+ optional string file_name = 3;
+ optional int32 line_number = 4;
+}
+
+/**
+ * Cause of a remote failure for a generic exception. Contains
+ * all the information for a generic exception as well as
+ * optional info about the error for generic info passing
+ * (which should be another protobuffed class).
+ */
+message GenericExceptionMessage {
+ optional string class_name = 1;
+ optional string message = 2;
+ optional bytes error_info = 3;
+ repeated StackTraceElementMessage trace = 4;
+}
+
+/**
+ * Exception sent across the wire when a remote task needs
+ * to notify other tasks that it failed and why
+ */
+message ForeignExceptionMessage {
+ optional string source = 1;
+ optional GenericExceptionMessage generic_exception = 2;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/FS.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/FS.proto b/hbase-native-client/if/FS.proto
new file mode 100644
index 0000000..9e93120
--- /dev/null
+++ b/hbase-native-client/if/FS.proto
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers that are written into the filesystem
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "FSProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+/**
+ * The ${HBASE_ROOTDIR}/hbase.version file content
+ */
+message HBaseVersionFileContent {
+ required string version = 1;
+}
+
+/**
+ * Reference file content used when we split an hfile under a region.
+ */
+message Reference {
+ required bytes splitkey = 1;
+ enum Range {
+ TOP = 0;
+ BOTTOM = 1;
+ }
+ required Range range = 2;
+}
+
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/Filter.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/Filter.proto b/hbase-native-client/if/Filter.proto
new file mode 100644
index 0000000..67d5717
--- /dev/null
+++ b/hbase-native-client/if/Filter.proto
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers that are used for filters
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "FilterProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "Comparator.proto";
+
+message Filter {
+ required string name = 1;
+ optional bytes serialized_filter = 2;
+}
+
+message ColumnCountGetFilter {
+ required int32 limit = 1;
+}
+
+message ColumnPaginationFilter {
+ required int32 limit = 1;
+ optional int32 offset = 2;
+ optional bytes column_offset = 3;
+}
+
+message ColumnPrefixFilter {
+ required bytes prefix = 1;
+}
+
+message ColumnRangeFilter {
+ optional bytes min_column = 1;
+ optional bool min_column_inclusive = 2;
+ optional bytes max_column = 3;
+ optional bool max_column_inclusive = 4;
+}
+
+message CompareFilter {
+ required CompareType compare_op = 1;
+ optional Comparator comparator = 2;
+}
+
+message DependentColumnFilter {
+ required CompareFilter compare_filter = 1;
+ optional bytes column_family = 2;
+ optional bytes column_qualifier = 3;
+ optional bool drop_dependent_column = 4;
+}
+
+message FamilyFilter {
+ required CompareFilter compare_filter = 1;
+}
+
+message FilterList {
+ required Operator operator = 1;
+ repeated Filter filters = 2;
+
+ enum Operator {
+ MUST_PASS_ALL = 1;
+ MUST_PASS_ONE = 2;
+ }
+}
+
+message FilterWrapper {
+ required Filter filter = 1;
+}
+
+message FirstKeyOnlyFilter {
+}
+
+message FirstKeyValueMatchingQualifiersFilter {
+ repeated bytes qualifiers = 1;
+}
+
+message FuzzyRowFilter {
+ repeated BytesBytesPair fuzzy_keys_data = 1;
+}
+
+message InclusiveStopFilter {
+ optional bytes stop_row_key = 1;
+}
+
+message KeyOnlyFilter {
+ required bool len_as_val = 1;
+}
+
+message MultipleColumnPrefixFilter {
+ repeated bytes sorted_prefixes = 1;
+}
+
+message PageFilter {
+ required int64 page_size = 1;
+}
+
+message PrefixFilter {
+ optional bytes prefix = 1;
+}
+
+message QualifierFilter {
+ required CompareFilter compare_filter = 1;
+}
+
+message RandomRowFilter {
+ required float chance = 1;
+}
+
+message RowFilter {
+ required CompareFilter compare_filter = 1;
+}
+
+message SingleColumnValueExcludeFilter {
+ required SingleColumnValueFilter single_column_value_filter = 1;
+}
+
+message SingleColumnValueFilter {
+ optional bytes column_family = 1;
+ optional bytes column_qualifier = 2;
+ required CompareType compare_op = 3;
+ required Comparator comparator = 4;
+ optional bool filter_if_missing = 5;
+ optional bool latest_version_only = 6;
+}
+
+message SkipFilter {
+ required Filter filter = 1;
+}
+
+message TimestampsFilter {
+ repeated int64 timestamps = 1 [packed=true];
+}
+
+message ValueFilter {
+ required CompareFilter compare_filter = 1;
+}
+
+message WhileMatchFilter {
+ required Filter filter = 1;
+}
+message FilterAllFilter {
+}
+
+message RowRange {
+ optional bytes start_row = 1;
+ optional bool start_row_inclusive = 2;
+ optional bytes stop_row = 3;
+ optional bool stop_row_inclusive =4;
+}
+
+message MultiRowRangeFilter {
+ repeated RowRange row_range_list = 1;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/HBase.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/HBase.proto b/hbase-native-client/if/HBase.proto
new file mode 100644
index 0000000..e5c967a
--- /dev/null
+++ b/hbase-native-client/if/HBase.proto
@@ -0,0 +1,258 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers that are shared throughout HBase
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "HBaseProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "Cell.proto";
+
+/**
+ * Table Name
+ */
+message TableName {
+ required bytes namespace = 1;
+ required bytes qualifier = 2;
+}
+
+/**
+ * Table Schema
+ * Inspired by the rest TableSchema
+ */
+message TableSchema {
+ optional TableName table_name = 1;
+ repeated BytesBytesPair attributes = 2;
+ repeated ColumnFamilySchema column_families = 3;
+ repeated NameStringPair configuration = 4;
+}
+
+/** Denotes state of the table */
+message TableState {
+ // Table's current state
+ enum State {
+ ENABLED = 0;
+ DISABLED = 1;
+ DISABLING = 2;
+ ENABLING = 3;
+ }
+ // This is the table's state.
+ required State state = 1;
+}
+
+/** On HDFS representation of table state. */
+message TableDescriptor {
+ required TableSchema schema = 1;
+}
+
+/**
+ * Column Family Schema
+ * Inspired by the rest ColumSchemaMessage
+ */
+message ColumnFamilySchema {
+ required bytes name = 1;
+ repeated BytesBytesPair attributes = 2;
+ repeated NameStringPair configuration = 3;
+}
+
+/**
+ * Protocol buffer version of HRegionInfo.
+ */
+message RegionInfo {
+ required uint64 region_id = 1;
+ required TableName table_name = 2;
+ optional bytes start_key = 3;
+ optional bytes end_key = 4;
+ optional bool offline = 5;
+ optional bool split = 6;
+ optional int32 replica_id = 7 [default = 0];
+}
+
+/**
+ * Protocol buffer for favored nodes
+ */
+message FavoredNodes {
+ repeated ServerName favored_node = 1;
+}
+
+/**
+ * Container protocol buffer to specify a region.
+ * You can specify region by region name, or the hash
+ * of the region name, which is known as encoded
+ * region name.
+ */
+message RegionSpecifier {
+ required RegionSpecifierType type = 1;
+ required bytes value = 2;
+
+ enum RegionSpecifierType {
+ // <tablename>,<startkey>,<regionId>.<encodedName>
+ REGION_NAME = 1;
+
+ // hash of <tablename>,<startkey>,<regionId>
+ ENCODED_REGION_NAME = 2;
+ }
+}
+
+/**
+ * A range of time. Both from and to are Java time
+ * stamp in milliseconds. If you don't specify a time
+ * range, it means all time. By default, if not
+ * specified, from = 0, and to = Long.MAX_VALUE
+ */
+message TimeRange {
+ optional uint64 from = 1;
+ optional uint64 to = 2;
+}
+
+/* ColumnFamily Specific TimeRange */
+message ColumnFamilyTimeRange {
+ required bytes column_family = 1;
+ required TimeRange time_range = 2;
+}
+
+/* Comparison operators */
+enum CompareType {
+ LESS = 0;
+ LESS_OR_EQUAL = 1;
+ EQUAL = 2;
+ NOT_EQUAL = 3;
+ GREATER_OR_EQUAL = 4;
+ GREATER = 5;
+ NO_OP = 6;
+}
+
+/**
+ * Protocol buffer version of ServerName
+ */
+message ServerName {
+ required string host_name = 1;
+ optional uint32 port = 2;
+ optional uint64 start_code = 3;
+}
+
+// Comment data structures
+
+message Coprocessor {
+ required string name = 1;
+}
+
+message NameStringPair {
+ required string name = 1;
+ required string value = 2;
+}
+
+message NameBytesPair {
+ required string name = 1;
+ optional bytes value = 2;
+}
+
+message BytesBytesPair {
+ required bytes first = 1;
+ required bytes second = 2;
+}
+
+message NameInt64Pair {
+ optional string name = 1;
+ optional int64 value = 2;
+}
+
+/**
+ * Description of the snapshot to take
+ */
+message SnapshotDescription {
+ required string name = 1;
+ optional string table = 2; // not needed for delete, but checked for in taking snapshot
+ optional int64 creation_time = 3 [default = 0];
+ enum Type {
+ DISABLED = 0;
+ FLUSH = 1;
+ SKIPFLUSH = 2;
+ }
+ optional Type type = 4 [default = FLUSH];
+ optional int32 version = 5;
+ optional string owner = 6;
+}
+
+/**
+ * Description of the distributed procedure to take
+ */
+message ProcedureDescription {
+ required string signature = 1; // the unique signature of the procedure
+ optional string instance = 2; // the procedure instance name
+ optional int64 creation_time = 3 [default = 0];
+ repeated NameStringPair configuration = 4;
+}
+
+message EmptyMsg {
+}
+
+enum TimeUnit {
+ NANOSECONDS = 1;
+ MICROSECONDS = 2;
+ MILLISECONDS = 3;
+ SECONDS = 4;
+ MINUTES = 5;
+ HOURS = 6;
+ DAYS = 7;
+}
+
+message LongMsg {
+ required int64 long_msg = 1;
+}
+
+message DoubleMsg {
+ required double double_msg = 1;
+}
+
+message BigDecimalMsg {
+ required bytes bigdecimal_msg = 1;
+}
+
+message UUID {
+ required uint64 least_sig_bits = 1;
+ required uint64 most_sig_bits = 2;
+}
+
+message NamespaceDescriptor {
+ required bytes name = 1;
+ repeated NameStringPair configuration = 2;
+}
+
+// Rpc client version info proto. Included in ConnectionHeader on connection setup
+message VersionInfo {
+ required string version = 1;
+ required string url = 2;
+ required string revision = 3;
+ required string user = 4;
+ required string date = 5;
+ required string src_checksum = 6;
+ optional uint32 version_major = 7;
+ optional uint32 version_minor = 8;
+}
+
+/**
+ * Description of the region server info
+ */
+message RegionServerInfo {
+ optional int32 infoPort = 1;
+ optional VersionInfo version_info = 2;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/HFile.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/HFile.proto b/hbase-native-client/if/HFile.proto
new file mode 100644
index 0000000..5c5e4f3
--- /dev/null
+++ b/hbase-native-client/if/HFile.proto
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "HFileProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+
+// Map of name/values
+message FileInfoProto {
+ repeated BytesBytesPair map_entry = 1;
+}
+
+// HFile file trailer
+message FileTrailerProto {
+ optional uint64 file_info_offset = 1;
+ optional uint64 load_on_open_data_offset = 2;
+ optional uint64 uncompressed_data_index_size = 3;
+ optional uint64 total_uncompressed_bytes = 4;
+ optional uint32 data_index_count = 5;
+ optional uint32 meta_index_count = 6;
+ optional uint64 entry_count = 7;
+ optional uint32 num_data_index_levels = 8;
+ optional uint64 first_data_block_offset = 9;
+ optional uint64 last_data_block_offset = 10;
+ optional string comparator_class_name = 11;
+ optional uint32 compression_codec = 12;
+ optional bytes encryption_key = 13;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/LoadBalancer.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/LoadBalancer.proto b/hbase-native-client/if/LoadBalancer.proto
new file mode 100644
index 0000000..f9c5d0d
--- /dev/null
+++ b/hbase-native-client/if/LoadBalancer.proto
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers to represent the state of the load balancer.
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "LoadBalancerProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+message LoadBalancerState {
+ optional bool balancer_on = 1;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/MapReduce.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/MapReduce.proto b/hbase-native-client/if/MapReduce.proto
new file mode 100644
index 0000000..f96ffdf
--- /dev/null
+++ b/hbase-native-client/if/MapReduce.proto
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ //This file includes protocol buffers used in MapReduce only.
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "MapReduceProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+
+message ScanMetrics {
+ repeated NameInt64Pair metrics = 1;
+}
+
+message TableSnapshotRegionSplit {
+ repeated string locations = 2;
+ optional TableSchema table = 3;
+ optional RegionInfo region = 4;
+}
[36/50] [abbrv] hbase git commit: HBASE-15822 Move to the latest
docker base image
Posted by ec...@apache.org.
HBASE-15822 Move to the latest docker base image
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/5bb7fa8b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/5bb7fa8b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/5bb7fa8b
Branch: refs/heads/HBASE-14850
Commit: 5bb7fa8b832a3134f83b7d94404494ba07f0bd4d
Parents: df14a6a
Author: Elliott Clark <el...@fb.com>
Authored: Tue May 31 10:42:45 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/Dockerfile | 2 +-
hbase-native-client/if/RPC.proto | 1 +
hbase-native-client/if/WAL.proto | 5 +++--
3 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/5bb7fa8b/hbase-native-client/Dockerfile
----------------------------------------------------------------------
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index be0cbc6..ca87523 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM pjameson/buck-folly-watchman:20160425
+FROM pjameson/buck-folly-watchman:20160511
ARG CC=/usr/bin/gcc-5
ARG CXX=/usr/bin/g++-5
http://git-wip-us.apache.org/repos/asf/hbase/blob/5bb7fa8b/hbase-native-client/if/RPC.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/RPC.proto b/hbase-native-client/if/RPC.proto
index 59bb03d..8413d25 100644
--- a/hbase-native-client/if/RPC.proto
+++ b/hbase-native-client/if/RPC.proto
@@ -125,6 +125,7 @@ message RequestHeader {
// 0 is NORMAL priority. 200 is HIGH. If no priority, treat it as NORMAL.
// See HConstants.
optional uint32 priority = 6;
+ optional uint32 timeout = 7;
}
message ResponseHeader {
http://git-wip-us.apache.org/repos/asf/hbase/blob/5bb7fa8b/hbase-native-client/if/WAL.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/WAL.proto b/hbase-native-client/if/WAL.proto
index cb9bd8f..2061b22 100644
--- a/hbase-native-client/if/WAL.proto
+++ b/hbase-native-client/if/WAL.proto
@@ -44,9 +44,9 @@ message WALKey {
required uint64 log_sequence_number = 3;
required uint64 write_time = 4;
/*
- This parameter is deprecated in favor of clusters which
+ This parameter is deprecated in favor of clusters which
contains the list of clusters that have consumed the change.
- It is retained so that the log created by earlier releases (0.94)
+ It is retained so that the log created by earlier releases (0.94)
can be read by the newer releases.
*/
optional UUID cluster_id = 5 [deprecated=true];
@@ -132,6 +132,7 @@ message StoreDescriptor {
required bytes family_name = 1;
required string store_home_dir = 2; //relative to region dir
repeated string store_file = 3; // relative to store dir
+ optional uint64 store_file_size_bytes = 4; // size of store file
}
/**
[29/50] [abbrv] hbase git commit: HBASE-15620 Add on Call
serialization
Posted by ec...@apache.org.
HBASE-15620 Add on Call serialization
Summary: Add on delimited serialization so that request headers and request payloads can be serialized.
Test Plan: Add a unit test.
Differential Revision: https://reviews.facebook.net/D56757
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dc0f0175
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dc0f0175
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dc0f0175
Branch: refs/heads/HBASE-14850
Commit: dc0f0175a3619618318849cb372f74bc7c25a2d0
Parents: 8b2d65f
Author: Elliott Clark <ec...@apache.org>
Authored: Tue Apr 12 12:03:52 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/Dockerfile | 19 ++-
hbase-native-client/bin/start-local-hbase.sh | 33 +++++
.../bin/start_local_hbase_and_wait.sh | 33 -----
hbase-native-client/bin/stop-local-hbase.sh | 26 ++++
.../bin/stop_local_hbase_and_wait.sh | 26 ----
hbase-native-client/connection/BUCK | 43 ++++++
.../connection/client-dispatcher.cc | 56 ++++++++
.../connection/client-dispatcher.h | 42 ++++++
.../connection/client-handler.cc | 80 +++++++++++
hbase-native-client/connection/client-handler.h | 49 +++++++
.../connection/connection-factory.cc | 58 ++++++++
.../connection/connection-factory.h | 40 ++++++
hbase-native-client/connection/pipeline.cc | 42 ++++++
hbase-native-client/connection/pipeline.h | 39 ++++++
hbase-native-client/connection/request.h | 42 ++++++
hbase-native-client/connection/response.h | 46 ++++++
hbase-native-client/connection/service.h | 26 ++++
hbase-native-client/core/BUCK | 41 +-----
hbase-native-client/core/admin.cc | 20 ---
hbase-native-client/core/admin.h | 22 ---
hbase-native-client/core/client-dispatcher.cc | 54 -------
hbase-native-client/core/client-dispatcher.h | 42 ------
.../core/client-serialize-handler.cc | 104 --------------
.../core/client-serialize-handler.h | 44 ------
hbase-native-client/core/client.h | 2 +-
hbase-native-client/core/connection-factory.cc | 57 --------
hbase-native-client/core/connection-factory.h | 41 ------
hbase-native-client/core/connection_attr.h | 24 ----
hbase-native-client/core/delete.cc | 21 ---
hbase-native-client/core/delete.h | 27 ----
hbase-native-client/core/get-request.h | 2 +-
hbase-native-client/core/get-result.h | 2 +-
hbase-native-client/core/location-cache.cc | 3 +-
hbase-native-client/core/location-cache.h | 2 +-
hbase-native-client/core/mutation.cc | 41 ------
hbase-native-client/core/mutation.h | 58 --------
.../core/native-client-test-env.cc | 45 ------
hbase-native-client/core/pipeline.cc | 42 ------
hbase-native-client/core/pipeline.h | 34 -----
hbase-native-client/core/request.h | 33 -----
hbase-native-client/core/response.h | 34 -----
hbase-native-client/core/scanner.cc | 20 ---
hbase-native-client/core/scanner.h | 22 ---
hbase-native-client/core/service.h | 26 ----
hbase-native-client/core/simple-client.cc | 45 +++++-
.../core/simple-native-client-test.cc | 25 ----
hbase-native-client/core/table-name.h | 2 +-
hbase-native-client/core/test-env.cc | 45 ++++++
hbase-native-client/if/BUCK | 53 ++++---
hbase-native-client/serde/BUCK | 54 +++++++
.../serde/client-deserializer-test.cc | 67 +++++++++
.../serde/client-deserializer.cc | 68 +++++++++
hbase-native-client/serde/client-deserializer.h | 36 +++++
.../serde/client-serializer-test.cc | 75 ++++++++++
hbase-native-client/serde/client-serializer.cc | 139 +++++++++++++++++++
hbase-native-client/serde/client-serializer.h | 55 ++++++++
56 files changed, 1248 insertions(+), 979 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/Dockerfile
----------------------------------------------------------------------
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index 36959a5..9ccc0d3 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -22,7 +22,14 @@ ARG CXX=/usr/bin/g++-5
ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 -pthread"
ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 -pthread"
-RUN apt-get install -y clang-format-3.7 vim maven inetutils-ping
+RUN apt-get install -y clang-format-3.7 vim maven inetutils-ping python-pip && \
+ pip install yapf && \
+ ln -sf /usr/bin/clang-format-3.7 /usr/bin/clang-format && \
+ apt-get -qq clean && \
+ apt-get -y -qq autoremove && \
+ rm -rf /var/lib/{apt,dpkg,cache,log}/ && \
+ rm -rf /tmp/*
+
RUN git clone --depth 1 --branch v2.6.1 https://github.com/google/protobuf.git /usr/src/protobuf && \
cd /usr/src/protobuf/ && \
ldconfig && \
@@ -31,9 +38,8 @@ RUN git clone --depth 1 --branch v2.6.1 https://github.com/google/protobuf.git /
make && \
make install && \
make clean && \
- rm -rf .git
-
-RUN cd /usr/src && \
+ rm -rf .git && \
+ cd /usr/src && \
wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && \
tar zxf zookeeper-3.4.8.tar.gz && \
rm -rf zookeeper-3.4.8.tar.gz && \
@@ -43,8 +49,7 @@ RUN cd /usr/src && \
./configure && \
make && \
make install && \
- make clean
-
-RUN ldconfig
+ make clean && \
+ ldconfig
WORKDIR /usr/src/hbase/hbase-native-client
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/bin/start-local-hbase.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/start-local-hbase.sh b/hbase-native-client/bin/start-local-hbase.sh
new file mode 100755
index 0000000..cfc71f9
--- /dev/null
+++ b/hbase-native-client/bin/start-local-hbase.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Clean up from any other tests.
+rm -rf /tmp/hbase-*
+
+# Start the master/regionservers.
+$PWD/../bin/start-hbase.sh
+
+until [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010/jmx) == "200" ]
+do
+ printf "Waiting for local HBase cluster to start\n"
+ sleep 1
+done
+
+# This sucks, but master can easily be up and meta not be assigned yet.
+sleep 30
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/bin/start_local_hbase_and_wait.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/start_local_hbase_and_wait.sh b/hbase-native-client/bin/start_local_hbase_and_wait.sh
deleted file mode 100755
index cfc71f9..0000000
--- a/hbase-native-client/bin/start_local_hbase_and_wait.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# Clean up from any other tests.
-rm -rf /tmp/hbase-*
-
-# Start the master/regionservers.
-$PWD/../bin/start-hbase.sh
-
-until [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010/jmx) == "200" ]
-do
- printf "Waiting for local HBase cluster to start\n"
- sleep 1
-done
-
-# This sucks, but master can easily be up and meta not be assigned yet.
-sleep 30
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/bin/stop-local-hbase.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/stop-local-hbase.sh b/hbase-native-client/bin/stop-local-hbase.sh
new file mode 100755
index 0000000..761412a
--- /dev/null
+++ b/hbase-native-client/bin/stop-local-hbase.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ps aux | grep proc_master | awk '{print $2}' | xargs kill -9
+
+while [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010) == "200" ]
+do
+ printf "Waiting for local HBase cluster to stop\n"
+ sleep 1
+done
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/bin/stop_local_hbase_and_wait.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/stop_local_hbase_and_wait.sh b/hbase-native-client/bin/stop_local_hbase_and_wait.sh
deleted file mode 100755
index 761412a..0000000
--- a/hbase-native-client/bin/stop_local_hbase_and_wait.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-
-##
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ps aux | grep proc_master | awk '{print $2}' | xargs kill -9
-
-while [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010) == "200" ]
-do
- printf "Waiting for local HBase cluster to stop\n"
- sleep 1
-done
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/connection/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/BUCK b/hbase-native-client/connection/BUCK
new file mode 100644
index 0000000..5067708
--- /dev/null
+++ b/hbase-native-client/connection/BUCK
@@ -0,0 +1,43 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is the library dealing with a single connection
+# to a single server.
+cxx_library(name="connection",
+ exported_headers=[
+ "client-dispatcher.h",
+ "client-handler.h",
+ "connection-factory.h",
+ "pipeline.h",
+ "request.h",
+ "response.h",
+ "service.h",
+ ],
+ srcs=[
+ "client-dispatcher.cc",
+ "client-handler.cc",
+ "connection-factory.cc",
+ "pipeline.cc",
+ ],
+ deps=[
+ "//if:if",
+ "//utils:utils",
+ "//serde:serde",
+ "//third-party:folly",
+ "//third-party:wangle",
+ ],
+ visibility=['//core/...', ], )
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/connection/client-dispatcher.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-dispatcher.cc b/hbase-native-client/connection/client-dispatcher.cc
new file mode 100644
index 0000000..25cff7d
--- /dev/null
+++ b/hbase-native-client/connection/client-dispatcher.cc
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include "connection/client-dispatcher.h"
+
+using namespace folly;
+using namespace hbase;
+using namespace wangle;
+
+void ClientDispatcher::read(Context *ctx, Response in) {
+ auto call_id = in.call_id();
+ auto search = requests_.find(call_id);
+ CHECK(search != requests_.end());
+ auto p = std::move(search->second);
+
+ requests_.erase(call_id);
+
+ // TODO(eclark): check if the response
+ // is an exception. If it is then set that.
+ p.setValue(in);
+}
+
+Future<Response> ClientDispatcher::operator()(Request arg) {
+ auto call_id = ++current_call_id_;
+
+ arg.set_call_id(call_id);
+ auto &p = requests_[call_id];
+ auto f = p.getFuture();
+ p.setInterruptHandler([call_id, this](const folly::exception_wrapper &e) {
+ this->requests_.erase(call_id);
+ });
+ this->pipeline_->write(arg);
+
+ return f;
+}
+
+Future<Unit> ClientDispatcher::close() { return ClientDispatcherBase::close(); }
+
+Future<Unit> ClientDispatcher::close(Context *ctx) {
+ return ClientDispatcherBase::close(ctx);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/connection/client-dispatcher.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-dispatcher.h b/hbase-native-client/connection/client-dispatcher.h
new file mode 100644
index 0000000..c3987c9
--- /dev/null
+++ b/hbase-native-client/connection/client-dispatcher.h
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <wangle/service/ClientDispatcher.h>
+
+#include "connection/pipeline.h"
+#include "connection/request.h"
+#include "connection/response.h"
+
+namespace hbase {
+class ClientDispatcher
+ : public wangle::ClientDispatcherBase<SerializePipeline, Request,
+ Response> {
+public:
+ void read(Context *ctx, Response in) override;
+ folly::Future<Response> operator()(Request arg) override;
+ folly::Future<folly::Unit> close(Context *ctx) override;
+ folly::Future<folly::Unit> close() override;
+
+private:
+ std::unordered_map<uint32_t, folly::Promise<Response>> requests_;
+ uint32_t current_call_id_ = 10;
+};
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/connection/client-handler.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-handler.cc b/hbase-native-client/connection/client-handler.cc
new file mode 100644
index 0000000..74b23ef
--- /dev/null
+++ b/hbase-native-client/connection/client-handler.cc
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "connection/client-handler.h"
+
+#include <glog/logging.h>
+#include <folly/Likely.h>
+
+#include <string>
+
+#include "connection/request.h"
+#include "connection/response.h"
+#include "if/RPC.pb.h"
+#include "if/Client.pb.h"
+
+using namespace hbase;
+using namespace folly;
+using namespace wangle;
+using hbase::pb::ResponseHeader;
+using hbase::pb::GetResponse;
+
+ClientHandler::ClientHandler(std::string user_name) : user_name_(user_name) {}
+
+void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
+ if (LIKELY(buf != nullptr)) {
+ buf->coalesce();
+ Response received;
+ ResponseHeader header;
+
+ int used_bytes = deser_.parse_delimited(buf.get(), &header);
+ LOG(INFO) << "Read ResponseHeader size=" << used_bytes
+ << " call_id=" << header.call_id()
+ << " has_exception=" << header.has_exception();
+ received.set_call_id(header.call_id());
+
+ if (header.has_exception() == false) {
+ buf->trimStart(used_bytes);
+ // For now assume that everything was a get.
+ // We'll need to set this up later.
+ received.set_response(std::make_shared<GetResponse>());
+ used_bytes = deser_.parse_delimited(buf.get(), received.response().get());
+ }
+ ctx->fireRead(std::move(received));
+ }
+}
+
+Future<Unit> ClientHandler::write(Context *ctx, Request r) {
+ // Keep track of if we have sent the header.
+ if (UNLIKELY(need_send_header_)) {
+ need_send_header_ = false;
+
+ // Should we be sending just one fireWrite?
+ // Right now we're sending one for the header
+ // and one for the request.
+ //
+ // That doesn't seem like too bad, but who knows.
+ auto pre = ser_.preamble();
+ auto header = ser_.header(user_name_);
+ pre->appendChain(std::move(header));
+ ctx->fireWrite(std::move(pre));
+ }
+
+ return ctx->fireWrite(ser_.request(r.call_id(), r.method(), r.msg()));
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/connection/client-handler.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-handler.h b/hbase-native-client/connection/client-handler.h
new file mode 100644
index 0000000..38c5725
--- /dev/null
+++ b/hbase-native-client/connection/client-handler.h
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <wangle/channel/Handler.h>
+
+#include <string>
+
+#include "serde/client-serializer.h"
+#include "serde/client-deserializer.h"
+
+// Forward decs.
+namespace hbase {
+class Request;
+class Response;
+}
+
+namespace hbase {
+class ClientHandler
+ : public wangle::Handler<std::unique_ptr<folly::IOBuf>, Response, Request,
+ std::unique_ptr<folly::IOBuf>> {
+public:
+ ClientHandler(std::string user_name);
+ void read(Context *ctx, std::unique_ptr<folly::IOBuf> msg) override;
+ folly::Future<folly::Unit> write(Context *ctx, Request r) override;
+
+private:
+ bool need_send_header_ = true;
+ std::string user_name_;
+ ClientSerializer ser_;
+ ClientDeserializer deser_;
+};
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/connection/connection-factory.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-factory.cc b/hbase-native-client/connection/connection-factory.cc
new file mode 100644
index 0000000..5d1b0da
--- /dev/null
+++ b/hbase-native-client/connection/connection-factory.cc
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "connection/connection-factory.h"
+
+#include <folly/futures/Future.h>
+#include <wangle/bootstrap/ClientBootstrap.h>
+#include <wangle/channel/AsyncSocketHandler.h>
+#include <wangle/channel/EventBaseHandler.h>
+#include <wangle/channel/OutputBufferingHandler.h>
+#include <wangle/service/ClientDispatcher.h>
+#include <wangle/service/CloseOnReleaseFilter.h>
+#include <wangle/service/ExpiringFilter.h>
+
+#include <string>
+
+#include "connection/client-dispatcher.h"
+#include "connection/pipeline.h"
+#include "connection/request.h"
+#include "connection/response.h"
+#include "connection/service.h"
+
+using namespace folly;
+using namespace hbase;
+using namespace wangle;
+
+ConnectionFactory::ConnectionFactory() {
+ bootstrap_.group(std::make_shared<wangle::IOThreadPoolExecutor>(1));
+ bootstrap_.pipelineFactory(std::make_shared<RpcPipelineFactory>());
+}
+
+std::shared_ptr<Service<Request, Response>>
+ConnectionFactory::make_connection(std::string host, int port) {
+ // Connect to a given server
+ // Then when connected create a ClientDispactcher.
+ auto pipeline = bootstrap_.connect(SocketAddress(host, port, true)).get();
+ auto dispatcher = std::make_shared<ClientDispatcher>();
+ dispatcher->setPipeline(pipeline);
+ auto service =
+ std::make_shared<CloseOnReleaseFilter<Request, Response>>(dispatcher);
+ return service;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/connection/connection-factory.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-factory.h b/hbase-native-client/connection/connection-factory.h
new file mode 100644
index 0000000..73ac032
--- /dev/null
+++ b/hbase-native-client/connection/connection-factory.h
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <wangle/service/Service.h>
+
+#include <string>
+
+#include "connection/pipeline.h"
+#include "connection/request.h"
+#include "connection/response.h"
+#include "connection/service.h"
+
+namespace hbase {
+class ConnectionFactory {
+public:
+ ConnectionFactory();
+ std::shared_ptr<wangle::Service<Request, Response>>
+ make_connection(std::string host, int port);
+
+private:
+ wangle::ClientBootstrap<SerializePipeline> bootstrap_;
+};
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/connection/pipeline.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/pipeline.cc b/hbase-native-client/connection/pipeline.cc
new file mode 100644
index 0000000..b9f5e0b
--- /dev/null
+++ b/hbase-native-client/connection/pipeline.cc
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include "connection/pipeline.h"
+
+#include <folly/Logging.h>
+#include <wangle/channel/AsyncSocketHandler.h>
+#include <wangle/channel/EventBaseHandler.h>
+#include <wangle/channel/OutputBufferingHandler.h>
+#include <wangle/codec/LengthFieldBasedFrameDecoder.h>
+
+#include "connection/client-handler.h"
+
+using namespace folly;
+using namespace hbase;
+using namespace wangle;
+
+SerializePipeline::Ptr
+RpcPipelineFactory::newPipeline(std::shared_ptr<AsyncTransportWrapper> sock) {
+ auto pipeline = SerializePipeline::create();
+ pipeline->addBack(AsyncSocketHandler{sock});
+ pipeline->addBack(EventBaseHandler{});
+ pipeline->addBack(LengthFieldBasedFrameDecoder{});
+ pipeline->addBack(ClientHandler{user_util_.user_name()});
+ pipeline->finalize();
+ return pipeline;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/connection/pipeline.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/pipeline.h b/hbase-native-client/connection/pipeline.h
new file mode 100644
index 0000000..68ade48
--- /dev/null
+++ b/hbase-native-client/connection/pipeline.h
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <wangle/service/Service.h>
+#include <folly/io/IOBufQueue.h>
+
+#include "connection/request.h"
+#include "connection/response.h"
+#include "utils/user-util.h"
+
+namespace hbase {
+using SerializePipeline = wangle::Pipeline<folly::IOBufQueue &, Request>;
+
+class RpcPipelineFactory : public wangle::PipelineFactory<SerializePipeline> {
+public:
+ SerializePipeline::Ptr
+ newPipeline(std::shared_ptr<folly::AsyncTransportWrapper> sock) override;
+
+private:
+ UserUtil user_util_;
+};
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/connection/request.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/request.h b/hbase-native-client/connection/request.h
new file mode 100644
index 0000000..e9e3e88
--- /dev/null
+++ b/hbase-native-client/connection/request.h
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <google/protobuf/message.h>
+
+#include <cstdint>
+#include <string>
+
+namespace hbase {
+class Request {
+public:
+ Request() : call_id_(0) {}
+ uint32_t call_id() { return call_id_; }
+ void set_call_id(uint32_t call_id) { call_id_ = call_id; }
+ google::protobuf::Message *msg() { return msg_.get(); }
+ void set_msg(std::shared_ptr<google::protobuf::Message> msg) { msg_ = msg; }
+ std::string method() { return method_; }
+ void set_method(std::string method) { method_ = method; }
+
+private:
+ uint32_t call_id_;
+ std::shared_ptr<google::protobuf::Message> msg_ = nullptr;
+ std::string method_ = "Get";
+};
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/connection/response.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/response.h b/hbase-native-client/connection/response.h
new file mode 100644
index 0000000..a7f7939
--- /dev/null
+++ b/hbase-native-client/connection/response.h
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <cstdint>
+
+// Forward
+namespace google {
+namespace protobuf {
+class Message;
+}
+}
+
+namespace hbase {
+
+class Response {
+public:
+ Response() : call_id_(0) {}
+ uint32_t call_id() { return call_id_; }
+ void set_call_id(uint32_t call_id) { call_id_ = call_id; }
+ std::shared_ptr<google::protobuf::Message> response() { return response_; }
+ void set_response(std::shared_ptr<google::protobuf::Message> response) {
+ response_ = std::move(response);
+ }
+
+private:
+ uint32_t call_id_;
+ std::shared_ptr<google::protobuf::Message> response_;
+};
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/connection/service.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/service.h b/hbase-native-client/connection/service.h
new file mode 100644
index 0000000..feb14ec
--- /dev/null
+++ b/hbase-native-client/connection/service.h
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include "connection/request.h"
+#include "connection/response.h"
+
+namespace hbase {
+using HBaseService = wangle::Service<Request, Response>;
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index 2b00d66..195fc5c 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -15,50 +15,28 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+# This is the main library.
cxx_library(name="core",
exported_headers=[
- "admin.h",
- "client-dispatcher.h",
- "client-serialize-handler.h",
"client.h",
- "connection-factory.h",
"connection.h",
- "connection_attr.h",
- "delete.h",
"get-request.h",
"get-result.h",
- "get.h",
"hbase_macros.h",
"location-cache.h",
- "mutation.h",
- "pipeline.h",
- "put.h",
- "request.h",
- "response.h",
- "scanner.h",
- "service.h",
"table-name.h",
],
srcs=[
- "admin.cc",
- "client-dispatcher.cc",
- "client-serialize-handler.cc",
"client.cc",
- "connection-factory.cc",
- "connection.cc",
- "delete.cc",
"get-request.cc",
"get-result.cc",
- "get.cc",
"location-cache.cc",
- "mutation.cc",
- "pipeline.cc",
- "put.cc",
- "scanner.cc",
"table-name.cc",
],
deps=[
+ "//connection:connection",
"//if:if",
+ "//serde:serde",
"//third-party:folly",
"//third-party:wangle",
"//third-party:zookeeper_mt",
@@ -67,18 +45,9 @@ cxx_library(name="core",
'PUBLIC',
], )
-cxx_test(name="simple-test",
- srcs=[
- "native-client-test-env.cc",
- "simple-native-client-test.cc",
- ],
- deps=[
- ":core",
- ],
- run_test_separately=True, )
cxx_test(name="location-cache-test",
srcs=[
- "native-client-test-env.cc",
+ "test-env.cc",
"location-cache-test.cc",
],
deps=[
@@ -87,4 +56,4 @@ cxx_test(name="location-cache-test",
run_test_separately=True, )
cxx_binary(name="simple-client",
srcs=["simple-client.cc", ],
- deps=[":core", ], )
+ deps=[":core", "//connection:connection"], )
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/admin.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/admin.cc b/hbase-native-client/core/admin.cc
deleted file mode 100644
index 897e6bf..0000000
--- a/hbase-native-client/core/admin.cc
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "core/admin.h"
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/admin.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/admin.h b/hbase-native-client/core/admin.h
deleted file mode 100644
index 775181c..0000000
--- a/hbase-native-client/core/admin.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#pragma once
-
-class Admin {};
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/client-dispatcher.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client-dispatcher.cc b/hbase-native-client/core/client-dispatcher.cc
deleted file mode 100644
index d356759..0000000
--- a/hbase-native-client/core/client-dispatcher.cc
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#include "core/client-dispatcher.h"
-
-using namespace folly;
-using namespace hbase;
-using namespace wangle;
-
-void ClientDispatcher::read(Context *ctx, Response in) {
- auto call_id = in.call_id();
- auto search = requests_.find(call_id);
- CHECK(search != requests_.end());
- auto p = std::move(search->second);
- requests_.erase(call_id);
-
- // TODO(eclark): check if the response
- // is an exception. If it is then set that.
- p.setValue(in);
-}
-
-Future<Response> ClientDispatcher::operator()(Request arg) {
- auto call_id = ++current_call_id_;
- arg.set_call_id(call_id);
- auto &p = requests_[call_id];
- auto f = p.getFuture();
- p.setInterruptHandler([call_id, this](const folly::exception_wrapper &e) {
- this->requests_.erase(call_id);
- });
- this->pipeline_->write(arg);
-
- return f;
-}
-
-Future<Unit> ClientDispatcher::close() { return ClientDispatcherBase::close(); }
-
-Future<Unit> ClientDispatcher::close(Context *ctx) {
- return ClientDispatcherBase::close(ctx);
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/client-dispatcher.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client-dispatcher.h b/hbase-native-client/core/client-dispatcher.h
deleted file mode 100644
index 4b9d35a..0000000
--- a/hbase-native-client/core/client-dispatcher.h
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#pragma once
-
-#include <wangle/service/ClientDispatcher.h>
-
-#include "core/pipeline.h"
-#include "core/request.h"
-#include "core/response.h"
-
-namespace hbase {
-class ClientDispatcher
- : public wangle::ClientDispatcherBase<SerializePipeline, Request,
- Response> {
-public:
- void read(Context *ctx, Response in) override;
- folly::Future<Response> operator()(Request arg) override;
- folly::Future<folly::Unit> close(Context *ctx) override;
- folly::Future<folly::Unit> close() override;
-
-private:
- std::unordered_map<int32_t, folly::Promise<Response>> requests_;
- uint32_t current_call_id_ = 1;
-};
-} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/client-serialize-handler.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client-serialize-handler.cc b/hbase-native-client/core/client-serialize-handler.cc
deleted file mode 100644
index cad1308..0000000
--- a/hbase-native-client/core/client-serialize-handler.cc
+++ /dev/null
@@ -1,104 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "core/client-serialize-handler.h"
-
-#include <string>
-
-using namespace hbase;
-using namespace folly;
-using namespace wangle;
-
-static const std::string PREAMBLE = "HBas";
-static const std::string INTERFACE = "ClientService";
-static const uint8_t RPC_VERSION = 0;
-static const uint8_t AUTH_TYPE = 80;
-
-// TODO(eclark): Make this actually do ANYTHING.
-void ClientSerializeHandler::read(Context *ctx, std::unique_ptr<IOBuf> msg) {
- Response received;
- ctx->fireRead(received);
-}
-
-Future<Unit> ClientSerializeHandler::write(Context *ctx, Request r) {
- // Keep track of if we have sent the header.
- if (need_send_header_) {
- need_send_header_ = false;
-
- // Should this be replacing the IOBuf rather than
- // sending several different calls?
- write_preamble(ctx);
- write_header(ctx);
- }
-
- // Send out the actual request and not just a test string.
- std::string out{"test"};
- return ctx->fireWrite(prepend_length(IOBuf::copyBuffer(out)));
-}
-
-Future<Unit> ClientSerializeHandler::write_preamble(Context *ctx) {
- auto magic = IOBuf::copyBuffer(PREAMBLE);
- auto buf = IOBuf::create(2);
- buf->append(2);
- folly::io::RWPrivateCursor c(buf.get());
-
- // Version
- c.write(RPC_VERSION);
- // Standard security aka Please don't lie to me.
- c.write(AUTH_TYPE);
- magic->appendChain(std::move(buf));
- return ctx->fireWrite(std::move(magic));
-}
-
-Future<Unit> ClientSerializeHandler::write_header(Context *ctx) {
- pb::ConnectionHeader h;
-
- // TODO(eclark): Make this not a total lie.
- h.mutable_user_info()->set_effective_user("elliott");
- // The service name that we want to talk to.
- //
- // Right now we're completely ignoring the service interface.
- // That may or may not be the correct thing to do.
- // It worked for a while with the java client; until it
- // didn't.
- h.set_service_name(INTERFACE);
- // TODO(eclark): Make this 1 copy.
- auto msg = IOBuf::copyBuffer(h.SerializeAsString());
- return ctx->fireWrite(prepend_length(std::move(msg)));
-}
-
-// Our own simple version of LengthFieldPrepender
-std::unique_ptr<IOBuf>
-ClientSerializeHandler::prepend_length(std::unique_ptr<IOBuf> msg) {
- // Java ints are 4 long. So create a buffer that large
- auto len_buf = IOBuf::create(4);
- // Then make those bytes visible.
- len_buf->append(4);
-
- io::RWPrivateCursor c(len_buf.get());
- // Get the size of the data to be pushed out the network.
- auto size = msg->computeChainDataLength();
-
- // Write the length to this IOBuf.
- c.writeBE(static_cast<uint32_t>(size));
-
- // Then attach the origional to the back of len_buf
- len_buf->appendChain(std::move(msg));
- return len_buf;
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/client-serialize-handler.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client-serialize-handler.h b/hbase-native-client/core/client-serialize-handler.h
deleted file mode 100644
index 961a03b..0000000
--- a/hbase-native-client/core/client-serialize-handler.h
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#pragma once
-
-#include <wangle/channel/Handler.h>
-
-#include "if/HBase.pb.h"
-#include "if/RPC.pb.h"
-#include "core/request.h"
-#include "core/response.h"
-
-namespace hbase {
-class ClientSerializeHandler
- : public wangle::Handler<std::unique_ptr<folly::IOBuf>, Response, Request,
- std::unique_ptr<folly::IOBuf>> {
-public:
- void read(Context *ctx, std::unique_ptr<folly::IOBuf> msg) override;
- folly::Future<folly::Unit> write(Context *ctx, Request r) override;
-
-private:
- folly::Future<folly::Unit> write_preamble(Context *ctx);
- folly::Future<folly::Unit> write_header(Context *ctx);
- // Our own simple version of LengthFieldPrepender
- std::unique_ptr<folly::IOBuf>
- prepend_length(std::unique_ptr<folly::IOBuf> msg);
- bool need_send_header_ = true;
-};
-} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/client.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client.h b/hbase-native-client/core/client.h
index 818bc6b..c2dc226 100644
--- a/hbase-native-client/core/client.h
+++ b/hbase-native-client/core/client.h
@@ -39,4 +39,4 @@ private:
LocationCache location_cache;
};
-} // namespace hbase
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/connection-factory.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/connection-factory.cc b/hbase-native-client/core/connection-factory.cc
deleted file mode 100644
index 785b239..0000000
--- a/hbase-native-client/core/connection-factory.cc
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "core/connection-factory.h"
-
-#include <wangle/channel/AsyncSocketHandler.h>
-#include <wangle/channel/EventBaseHandler.h>
-#include <wangle/channel/OutputBufferingHandler.h>
-#include <wangle/service/ClientDispatcher.h>
-#include <wangle/service/ExpiringFilter.h>
-#include <folly/futures/Future.h>
-
-#include <string>
-
-#include "core/client-dispatcher.h"
-#include "core/pipeline.h"
-#include "core/request.h"
-#include "core/response.h"
-#include "core/service.h"
-
-using namespace folly;
-using namespace hbase;
-using namespace wangle;
-
-ConnectionFactory::ConnectionFactory() {
- bootstrap_.group(std::make_shared<wangle::IOThreadPoolExecutor>(2));
- bootstrap_.pipelineFactory(std::make_shared<RpcPipelineFactory>());
-}
-
-Future<ClientDispatcher> ConnectionFactory::make_connection(std::string host,
- int port) {
- // Connect to a given server
- // Then when connected create a ClientDispactcher.
- auto srv = bootstrap_.connect(SocketAddress(host, port, true))
- .then([](SerializePipeline *pipeline) {
- ClientDispatcher dispatcher;
- dispatcher.setPipeline(pipeline);
- return dispatcher;
- });
- return srv;
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/connection-factory.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/connection-factory.h b/hbase-native-client/core/connection-factory.h
deleted file mode 100644
index 6f450c2..0000000
--- a/hbase-native-client/core/connection-factory.h
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#pragma once
-
-#include <wangle/bootstrap/ClientBootstrap.h>
-#include <wangle/service/Service.h>
-
-#include <string>
-
-#include "core/service.h"
-#include "core/pipeline.h"
-#include "core/client-dispatcher.h"
-#include "core/request.h"
-#include "core/response.h"
-
-namespace hbase {
-class ConnectionFactory {
-public:
- ConnectionFactory();
- folly::Future<ClientDispatcher> make_connection(std::string host, int port);
-
-private:
- wangle::ClientBootstrap<SerializePipeline> bootstrap_;
-};
-} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/connection_attr.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/connection_attr.h b/hbase-native-client/core/connection_attr.h
deleted file mode 100644
index a312005..0000000
--- a/hbase-native-client/core/connection_attr.h
+++ /dev/null
@@ -1,24 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#pragma once
-
-#include "core/hbase_macros.h"
-
-class ConnectionAttr {};
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/delete.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/delete.cc b/hbase-native-client/core/delete.cc
deleted file mode 100644
index 57030be..0000000
--- a/hbase-native-client/core/delete.cc
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "core/delete.h"
-
-Delete::~Delete() {}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/delete.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/delete.h b/hbase-native-client/core/delete.h
deleted file mode 100644
index 34f6a6c..0000000
--- a/hbase-native-client/core/delete.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#pragma once
-
-#include "core/mutation.h"
-
-class Delete : public Mutation {
-public:
- ~Delete();
-};
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/get-request.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/get-request.h b/hbase-native-client/core/get-request.h
index c9113ad..bb755c5 100644
--- a/hbase-native-client/core/get-request.h
+++ b/hbase-native-client/core/get-request.h
@@ -32,4 +32,4 @@ private:
TableName table_name_;
std::string key_;
};
-} // namespace hbase
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/get-result.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/get-result.h b/hbase-native-client/core/get-result.h
index e021316..a49ad98 100644
--- a/hbase-native-client/core/get-result.h
+++ b/hbase-native-client/core/get-result.h
@@ -29,4 +29,4 @@ public:
private:
std::string key_;
};
-} // namespace hbase
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/location-cache.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
index 34e3236..52e86e3 100644
--- a/hbase-native-client/core/location-cache.cc
+++ b/hbase-native-client/core/location-cache.cc
@@ -70,8 +70,7 @@ ServerName LocationCache::ReadMetaLocation() {
// This needs to be int rather than size_t as that's what ZK expects.
int len = sizeof(contents);
// TODO(elliott): handle disconnects/reconntion as needed.
- int zk_result =
- zoo_get(this->zk_, META_LOCATION, 0, contents, &len, nullptr);
+ int zk_result = zoo_get(this->zk_, META_LOCATION, 0, contents, &len, nullptr);
if (zk_result != ZOK || len < 9) {
LOG(ERROR) << "Error getting meta location.";
throw runtime_error("Error getting meta location");
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/location-cache.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h
index efcfde5..28a1ee1 100644
--- a/hbase-native-client/core/location-cache.h
+++ b/hbase-native-client/core/location-cache.h
@@ -51,4 +51,4 @@ private:
zhandle_t *zk_;
};
-} // namespace hbase
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/mutation.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/mutation.cc b/hbase-native-client/core/mutation.cc
deleted file mode 100644
index 52910d5..0000000
--- a/hbase-native-client/core/mutation.cc
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "core/mutation.h"
-
-void Mutation::set_namespace(char *name_space, size_t name_space_length) {
- this->name_space = name_space;
- this->name_space_length = name_space_length;
-}
-
-void Mutation::set_table(char *table, size_t table_length) {
- this->table = table;
- this->table_length = table_length;
-}
-
-void Mutation::set_row(unsigned char *row, size_t row_length) {
- this->row = row;
- this->row_length = row_length;
-}
-
-void Mutation::set_durability(durability_type durability) {
- this->durability = durability;
-}
-
-Mutation::~Mutation() {}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/mutation.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/mutation.h b/hbase-native-client/core/mutation.h
deleted file mode 100644
index 1880571..0000000
--- a/hbase-native-client/core/mutation.h
+++ /dev/null
@@ -1,58 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#pragma once
-
-#include <stdlib.h>
-
-typedef enum {
- DELETE_ONE_VERSION,
- DELETE_MULTIPLE_VERSIONS,
- DELETE_FAMILY,
- DELETE_FAMILY_VERSION
-} delete_type;
-
-typedef enum {
- USE_DEFAULT,
- SKIP_WAL,
- ASYNC_WAL,
- SYNC_WAL,
- HSYNC_WAL
-} durability_type;
-
-class Mutation {
- char *name_space;
- size_t name_space_length;
-
- char *table;
- size_t table_length;
-
- unsigned char *row;
- size_t row_length;
-
- durability_type durability;
-
-public:
- void set_namespace(char *name_space, size_t name_space_length);
- void set_table(char *table, size_t table_length);
- void set_row(unsigned char *row, size_t row_length);
- void set_durability(durability_type durability);
-
- virtual ~Mutation();
-};
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/native-client-test-env.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/native-client-test-env.cc b/hbase-native-client/core/native-client-test-env.cc
deleted file mode 100644
index 0269a43..0000000
--- a/hbase-native-client/core/native-client-test-env.cc
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include <gtest/gtest.h>
-
-namespace {
-
-class NativeClientTestEnv : public ::testing::Environment {
-public:
- void SetUp() override {
- // start local HBase cluster to be reused by all tests
- auto result = system("bin/start_local_hbase_and_wait.sh");
- ASSERT_EQ(0, result);
- }
-
- void TearDown() override {
- // shutdown local HBase cluster
- auto result = system("bin/stop_local_hbase_and_wait.sh");
- ASSERT_EQ(0, result);
- }
-};
-
-} // anonymous
-
-int main(int argc, char **argv) {
- testing::InitGoogleTest(&argc, argv);
- ::testing::AddGlobalTestEnvironment(new NativeClientTestEnv());
- return RUN_ALL_TESTS();
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/pipeline.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/pipeline.cc b/hbase-native-client/core/pipeline.cc
deleted file mode 100644
index 30d14ff..0000000
--- a/hbase-native-client/core/pipeline.cc
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#include "core/pipeline.h"
-
-#include <folly/Logging.h>
-#include <wangle/channel/AsyncSocketHandler.h>
-#include <wangle/channel/EventBaseHandler.h>
-#include <wangle/channel/OutputBufferingHandler.h>
-#include <wangle/codec/LengthFieldBasedFrameDecoder.h>
-
-#include "core/client-serialize-handler.h"
-
-using namespace folly;
-using namespace hbase;
-using namespace wangle;
-
-SerializePipeline::Ptr
-RpcPipelineFactory::newPipeline(std::shared_ptr<AsyncTransportWrapper> sock) {
- auto pipeline = SerializePipeline::create();
- pipeline->addBack(AsyncSocketHandler(sock));
- pipeline->addBack(EventBaseHandler());
- pipeline->addBack(LengthFieldBasedFrameDecoder());
- pipeline->addBack(ClientSerializeHandler());
- pipeline->finalize();
- return pipeline;
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/pipeline.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/pipeline.h b/hbase-native-client/core/pipeline.h
deleted file mode 100644
index d199d08..0000000
--- a/hbase-native-client/core/pipeline.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#pragma once
-
-#include <wangle/service/Service.h>
-#include <folly/io/IOBufQueue.h>
-#include "core/request.h"
-#include "core/response.h"
-
-namespace hbase {
-using SerializePipeline = wangle::Pipeline<folly::IOBufQueue &, Request>;
-
-class RpcPipelineFactory : public wangle::PipelineFactory<SerializePipeline> {
-public:
- SerializePipeline::Ptr
- newPipeline(std::shared_ptr<folly::AsyncTransportWrapper> sock) override;
-};
-} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/request.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/request.h b/hbase-native-client/core/request.h
deleted file mode 100644
index 39083ed..0000000
--- a/hbase-native-client/core/request.h
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#pragma once
-
-#include <cstdint>
-
-namespace hbase {
-class Request {
-public:
- Request() : call_id_(0) {}
- uint32_t call_id() { return call_id_; }
- void set_call_id(uint32_t call_id) { call_id_ = call_id; }
-
-private:
- uint32_t call_id_;
-};
-} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/response.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/response.h b/hbase-native-client/core/response.h
deleted file mode 100644
index 34a284d..0000000
--- a/hbase-native-client/core/response.h
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#pragma once
-
-#include <cstdint>
-
-namespace hbase {
-
-class Response {
-public:
- Response() : call_id_(0) {}
- uint32_t call_id() { return call_id_; }
- void set_call_id(uint32_t call_id) { call_id_ = call_id; }
-
-private:
- uint32_t call_id_;
-};
-} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/scanner.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/scanner.cc b/hbase-native-client/core/scanner.cc
deleted file mode 100644
index a10e444..0000000
--- a/hbase-native-client/core/scanner.cc
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "core/scanner.h"
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/scanner.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/scanner.h b/hbase-native-client/core/scanner.h
deleted file mode 100644
index 180865a..0000000
--- a/hbase-native-client/core/scanner.h
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#pragma once
-
-class Scanner {};
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/service.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/service.h b/hbase-native-client/core/service.h
deleted file mode 100644
index 880e65f..0000000
--- a/hbase-native-client/core/service.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#pragma once
-
-#include "core/request.h"
-#include "core/response.h"
-
-namespace hbase {
-using HBaseService = wangle::Service<Request, Response>;
-} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/simple-client.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/simple-client.cc b/hbase-native-client/core/simple-client.cc
index 08e886a..065f70b 100644
--- a/hbase-native-client/core/simple-client.cc
+++ b/hbase-native-client/core/simple-client.cc
@@ -20,21 +20,30 @@
#include <folly/Logging.h>
#include <folly/Random.h>
#include <gflags/gflags.h>
-#include <glog/logging.h>
#include <wangle/concurrent/GlobalExecutor.h>
#include <iostream>
+#include <chrono>
#include "core/client.h"
-#include "core/connection-factory.h"
+#include "connection/connection-factory.h"
#include "if/ZooKeeper.pb.h"
+#include "if/Client.pb.h"
using namespace folly;
using namespace std;
+using namespace std::chrono;
using namespace hbase;
using namespace hbase::pb;
+using namespace google::protobuf;
+
+// TODO(eclark): remove the need for this.
+DEFINE_string(region, "1588230740", "What region to send a get to");
+DEFINE_string(row, "test", "What row to get");
int main(int argc, char *argv[]) {
+ google::SetUsageMessage(
+ "Simple client to get a single row from HBase on the comamnd line");
google::ParseCommandLineFlags(&argc, &argv, true);
google::InitGoogleLogging(argv[0]);
@@ -44,14 +53,40 @@ int main(int argc, char *argv[]) {
LocationCache cache{"localhost:2181", wangle::getCPUExecutor()};
auto result = cache.LocateMeta().get();
- cout << "ServerName = " << result.host_name() << ":" << result.port() << endl;
// Create a connection to the local host
- auto conn = cf.make_connection(result.host_name(), result.port()).get();
+ auto conn = cf.make_connection(result.host_name(), result.port());
// Send the request
Request r;
- conn(r).get();
+
+ // This is a get request so make that
+ auto msg = make_shared<hbase::pb::GetRequest>();
+
+ // Set what region
+ msg->mutable_region()->set_value(FLAGS_region);
+ // It's always this.
+ msg->mutable_region()->set_type(
+ RegionSpecifier_RegionSpecifierType::
+ RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
+ // What row.
+ msg->mutable_get()->set_row(FLAGS_row);
+ // Send it.
+ r.set_msg(std::move(msg));
+ auto resp = (*conn)(r).get(milliseconds(5000));
+
+ auto get_resp = std::static_pointer_cast<GetResponse>(resp.response());
+ cout << "GetResponse has_result = " << get_resp->has_result() << '\n';
+ if (get_resp->has_result()) {
+ auto &r = get_resp->result();
+ cout << "Result cell_size = " << r.cell_size() << endl;
+ for (auto &cell : r.cell()) {
+ cout << "\trow = " << cell.row() << " family = " << cell.family()
+ << " qualifier = " << cell.qualifier()
+ << " timestamp = " << cell.timestamp() << " value = " << cell.value()
+ << endl;
+ }
+ }
return 0;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/simple-native-client-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/simple-native-client-test.cc b/hbase-native-client/core/simple-native-client-test.cc
deleted file mode 100644
index ee39986..0000000
--- a/hbase-native-client/core/simple-native-client-test.cc
+++ /dev/null
@@ -1,25 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "gtest/gtest.h"
-
-/**
- * Sample test.
- */
-TEST(SampleTest, sample) { EXPECT_TRUE(true); }
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/table-name.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/table-name.h b/hbase-native-client/core/table-name.h
index 796115b..37c3461 100644
--- a/hbase-native-client/core/table-name.h
+++ b/hbase-native-client/core/table-name.h
@@ -29,4 +29,4 @@ public:
explicit TableName(std::string tableName);
explicit TableName(std::string namespaceName, std::string tableName);
};
-} // namespace hbase
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/core/test-env.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/test-env.cc b/hbase-native-client/core/test-env.cc
new file mode 100644
index 0000000..277abd9
--- /dev/null
+++ b/hbase-native-client/core/test-env.cc
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <gtest/gtest.h>
+
+namespace {
+
+class NativeClientTestEnv : public ::testing::Environment {
+public:
+ void SetUp() override {
+ // start local HBase cluster to be reused by all tests
+ auto result = system("bin/start-local-hbase.sh");
+ ASSERT_EQ(0, result);
+ }
+
+ void TearDown() override {
+ // shutdown local HBase cluster
+ auto result = system("bin/stop-local-hbase.sh");
+ ASSERT_EQ(0, result);
+ }
+};
+
+} // anonymous
+
+int main(int argc, char **argv) {
+ testing::InitGoogleTest(&argc, argv);
+ ::testing::AddGlobalTestEnvironment(new NativeClientTestEnv());
+ return RUN_ALL_TESTS();
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/if/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/BUCK b/hbase-native-client/if/BUCK
index 5ff617d..1a9721d 100644
--- a/hbase-native-client/if/BUCK
+++ b/hbase-native-client/if/BUCK
@@ -15,39 +15,34 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-
PROTO_SRCS = glob(['*.proto'])
-HEADER_FILENAMES = [ x.replace('.proto','.pb.h') for x in PROTO_SRCS]
-CC_FILENAMES = [ x.replace('.proto', '.pb.cc') for x in PROTO_SRCS]
+HEADER_FILENAMES = [x.replace('.proto', '.pb.h') for x in PROTO_SRCS]
+CC_FILENAMES = [x.replace('.proto', '.pb.cc') for x in PROTO_SRCS]
genrule(
- name = 'generate-proto-sources',
- srcs = PROTO_SRCS,
- cmd = 'mkdir -p $OUT && pwd && protoc --proto_path=. --cpp_out=$OUT *.proto',
- out = 'output',
-)
+ name='generate-proto-sources',
+ srcs=PROTO_SRCS,
+ cmd='mkdir -p $OUT && pwd && protoc --proto_path=. --cpp_out=$OUT *.proto',
+ out='output', )
for header_filename in HEADER_FILENAMES:
- genrule(
- name = header_filename,
- cmd = 'mkdir -p `dirname $OUT` '
- ' && cp $(location :generate-proto-sources)/{} $OUT'.format(header_filename),
- out = header_filename,
- )
+ genrule(name=header_filename,
+ cmd='mkdir -p `dirname $OUT` '
+ ' && cp $(location :generate-proto-sources)/{} $OUT'.format(
+ header_filename),
+ out=header_filename, )
for cc_filename in CC_FILENAMES:
- genrule(
- name = cc_filename,
- cmd = 'mkdir -p `dirname $OUT` '
- ' && cp $(location :generate-proto-sources)/*.cc `dirname $OUT` '
- ' && cp $(location :generate-proto-sources)/*.h `dirname $OUT`'.format(cc_filename),
- out = cc_filename,
- )
+ genrule(
+ name=cc_filename,
+ cmd='mkdir -p `dirname $OUT` '
+ ' && cp $(location :generate-proto-sources)/*.cc `dirname $OUT` '
+ ' && cp $(location :generate-proto-sources)/*.h `dirname $OUT`'.format(
+ cc_filename),
+ out=cc_filename, )
-cxx_library(
- name = 'if',
- exported_headers = [':' + x for x in HEADER_FILENAMES],
- srcs = [':' + x for x in CC_FILENAMES],
- deps = [ '//third-party:protobuf'],
- visibility = [ 'PUBLIC', ],
- exported_deps = ['//third-party:protobuf']
-)
+cxx_library(name='if',
+ exported_headers=[':' + x for x in HEADER_FILENAMES],
+ srcs=[':' + x for x in CC_FILENAMES],
+ deps=['//third-party:protobuf'],
+ visibility=['PUBLIC', ],
+ exported_deps=['//third-party:protobuf'])
[18/50] [abbrv] hbase git commit: HBASE-15078 Added ability to
start/stop hbase local cluster for tests, global test_env for gtest,
small changes to dockerfile and docker run.
Posted by ec...@apache.org.
HBASE-15078 Added ability to start/stop hbase local cluster for tests, global test_env for gtest, small changes to dockerfile and docker run.
Added check ~/.m2 folder exists; moved scripts to ./bin
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/54aaaa31
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/54aaaa31
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/54aaaa31
Branch: refs/heads/HBASE-14850
Commit: 54aaaa314e3f1be9d6b5efcab845b8f321e4b4e2
Parents: 8556e3a
Author: Mikhail Antonov <an...@apache.org>
Authored: Wed Jan 6 15:08:21 2016 -0800
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/.buckconfig | 2 +-
hbase-native-client/Dockerfile | 2 +-
hbase-native-client/bin/start-docker.sh | 10 ++++-
.../bin/start_local_hbase_and_wait.sh | 26 ++++++++++++
.../bin/stop_local_hbase_and_wait.sh | 26 ++++++++++++
hbase-native-client/core/BUCK | 15 +++++++
.../core/HBaseNativeClientTestEnv.cc | 42 ++++++++++++++++++++
.../core/SampleNativeClientTest.cc | 28 +++++++++++++
hbase-native-client/core/test_env.h | 30 ++++++++++++++
9 files changed, 178 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/54aaaa31/hbase-native-client/.buckconfig
----------------------------------------------------------------------
diff --git a/hbase-native-client/.buckconfig b/hbase-native-client/.buckconfig
index 3227a2a..402ef27 100644
--- a/hbase-native-client/.buckconfig
+++ b/hbase-native-client/.buckconfig
@@ -1,2 +1,2 @@
[cxx]
- gtest_dep = //third-party/googletest/googletest:google-test
+ gtest_dep = //third-party:google-test
http://git-wip-us.apache.org/repos/asf/hbase/blob/54aaaa31/hbase-native-client/Dockerfile
----------------------------------------------------------------------
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index 70e823b..5f17f04 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -17,6 +17,6 @@
FROM pjameson/buck-folly-watchman
-RUN apt-get install -y libprotobuf-dev protobuf-compiler clang-format-3.7 vim
+RUN apt-get install -y libprotobuf-dev protobuf-compiler clang-format-3.7 vim maven inetutils-ping
WORKDIR /usr/local/src/hbase/hbase-native-client
http://git-wip-us.apache.org/repos/asf/hbase/blob/54aaaa31/hbase-native-client/bin/start-docker.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/start-docker.sh b/hbase-native-client/bin/start-docker.sh
index 1c9b02e..bf38912 100755
--- a/hbase-native-client/bin/start-docker.sh
+++ b/hbase-native-client/bin/start-docker.sh
@@ -28,5 +28,13 @@ if [[ ! -d third-party/googletest ]]; then
git clone https://github.com/google/googletest.git third-party/googletest
fi
+if [[ ! -d ~/.m2 ]]; then
+ echo "~/.m2 directory doesn't exist. Check Apache Maven is installed."
+ exit 1
+fi;
-docker run -v ${PWD}/..:/usr/local/src/hbase -it hbase_native /bin/bash
+docker run -p 16010:16010/tcp \
+ -e "JAVA_HOME=/usr/lib/jvm/java-8-oracle" \
+ -v ${PWD}/..:/usr/local/src/hbase \
+ -v ~/.m2:/root/.m2 \
+ -it hbase_native /bin/bash
http://git-wip-us.apache.org/repos/asf/hbase/blob/54aaaa31/hbase-native-client/bin/start_local_hbase_and_wait.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/start_local_hbase_and_wait.sh b/hbase-native-client/bin/start_local_hbase_and_wait.sh
new file mode 100755
index 0000000..64d0b68
--- /dev/null
+++ b/hbase-native-client/bin/start_local_hbase_and_wait.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+$PWD/../bin/start-hbase.sh
+
+until [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010) == "200" ]
+do
+ printf "Waiting for local HBase cluster to start\n"
+ sleep 1
+done
http://git-wip-us.apache.org/repos/asf/hbase/blob/54aaaa31/hbase-native-client/bin/stop_local_hbase_and_wait.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/stop_local_hbase_and_wait.sh b/hbase-native-client/bin/stop_local_hbase_and_wait.sh
new file mode 100755
index 0000000..4e89334
--- /dev/null
+++ b/hbase-native-client/bin/stop_local_hbase_and_wait.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+$PWD/../bin/stop-hbase.sh
+
+while [ $(curl -s -o /dev/null -I -w "%{http_code}" http://localhost:16010) == "200" ]
+do
+ printf "Waiting for local HBase cluster to stop\n"
+ sleep 1
+done
http://git-wip-us.apache.org/repos/asf/hbase/blob/54aaaa31/hbase-native-client/core/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index a7eaa9e..2e4e755 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -48,3 +48,18 @@ cxx_binary(
'PUBLIC',
],
)
+
+cxx_test(
+ name = "core_test",
+ headers = [
+ "test_env.h",
+ ],
+ srcs = [
+ "HBaseNativeClientTestEnv.cc",
+ "SampleNativeClientTest.cc",
+ ],
+ deps = [
+ ":core",
+ ],
+ run_test_separately = True,
+)
http://git-wip-us.apache.org/repos/asf/hbase/blob/54aaaa31/hbase-native-client/core/HBaseNativeClientTestEnv.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/HBaseNativeClientTestEnv.cc b/hbase-native-client/core/HBaseNativeClientTestEnv.cc
new file mode 100644
index 0000000..b8cb8db
--- /dev/null
+++ b/hbase-native-client/core/HBaseNativeClientTestEnv.cc
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <gtest/gtest.h>
+#include <core/test_env.h>
+
+namespace {
+
+class HBaseNativeClientTestEnv : public ::testing::Environment {
+ public:
+ void SetUp() override {
+ init_test_env();
+ }
+
+ void TearDown() override {
+ clean_test_env();
+ }
+};
+
+} // anonymous
+
+int main(int argc, char** argv) {
+ testing::InitGoogleTest(&argc, argv);
+ ::testing::AddGlobalTestEnvironment(new HBaseNativeClientTestEnv());
+ return RUN_ALL_TESTS();
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/54aaaa31/hbase-native-client/core/SampleNativeClientTest.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/SampleNativeClientTest.cc b/hbase-native-client/core/SampleNativeClientTest.cc
new file mode 100644
index 0000000..ef564f7
--- /dev/null
+++ b/hbase-native-client/core/SampleNativeClientTest.cc
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "gtest/gtest.h"
+
+/**
+ * Sample test.
+ */
+TEST(SampleTest, sample) {
+ EXPECT_TRUE(true);
+}
+
http://git-wip-us.apache.org/repos/asf/hbase/blob/54aaaa31/hbase-native-client/core/test_env.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/test_env.h b/hbase-native-client/core/test_env.h
new file mode 100644
index 0000000..5796ae1
--- /dev/null
+++ b/hbase-native-client/core/test_env.h
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <cstdlib>
+
+inline void init_test_env() {
+ // start local HBase cluster to be reused by all tests
+ system("scripts/start_local_hbase_and_wait.sh");
+}
+
+inline void clean_test_env() {
+ // shutdown local HBase cluster
+ system("scripts/stop_local_hbase_and_wait.sh");
+}
[39/50] [abbrv] hbase git commit: HBASE-15777 Fix needs header in
client handler
Posted by ec...@apache.org.
HBASE-15777 Fix needs header in client handler
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/16e1de78
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/16e1de78
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/16e1de78
Branch: refs/heads/HBASE-14850
Commit: 16e1de78a6a0fa92adbd330890f213d8763adb83
Parents: 79b5085
Author: Elliott Clark <ec...@apache.org>
Authored: Thu May 5 13:51:18 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
.../connection/client-handler.cc | 31 ++++++++++++--------
hbase-native-client/connection/client-handler.h | 19 +++++++++++-
2 files changed, 36 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/16e1de78/hbase-native-client/connection/client-handler.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-handler.cc b/hbase-native-client/connection/client-handler.cc
index b92ad89..4fdb7ae 100644
--- a/hbase-native-client/connection/client-handler.cc
+++ b/hbase-native-client/connection/client-handler.cc
@@ -37,7 +37,7 @@ using hbase::pb::GetResponse;
using google::protobuf::Message;
ClientHandler::ClientHandler(std::string user_name)
- : user_name_(user_name), need_send_header_(true), serde_(),
+ : user_name_(user_name), serde_(), header_info_(),
resp_msgs_(
make_unique<folly::AtomicHashMap<
uint32_t, std::shared_ptr<google::protobuf::Message>>>(5000)) {}
@@ -81,22 +81,27 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
}
}
-// TODO(eclark): Figure out how to handle the
-// network errors that are going to come.
Future<Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
// Keep track of if we have sent the header.
- if (UNLIKELY(need_send_header_)) {
- need_send_header_ = false;
+ //
+ // even though the bool is atomic we can load it lazily here.
+ if (UNLIKELY(header_info_->need_.load(std::memory_order_relaxed))) {
- // Should we be sending just one fireWrite?
- // Right now we're sending one for the header
- // and one for the request.
+ // Grab the lock.
+ // We need to make sure that no one gets past here without there being a
+ // hearder sent.
+ std::lock_guard<std::mutex> lock(header_info_->mutex_);
+
+ // Now see if we are the first thread to get through.
//
- // That doesn't seem like too bad, but who knows.
- auto pre = serde_.Preamble();
- auto header = serde_.Header(user_name_);
- pre->appendChain(std::move(header));
- ctx->fireWrite(std::move(pre));
+ // If this is the first thread to get through then the
+ // need_send_header will have been true before this.
+ if (header_info_->need_.exchange(false)) {
+ auto pre = serde_.Preamble();
+ auto header = serde_.Header(user_name_);
+ pre->appendChain(std::move(header));
+ ctx->fireWrite(std::move(pre));
+ }
}
resp_msgs_->insert(r->call_id(), r->resp_msg());
http://git-wip-us.apache.org/repos/asf/hbase/blob/16e1de78/hbase-native-client/connection/client-handler.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-handler.h b/hbase-native-client/connection/client-handler.h
index be5143c..1a4275f 100644
--- a/hbase-native-client/connection/client-handler.h
+++ b/hbase-native-client/connection/client-handler.h
@@ -21,6 +21,8 @@
#include <folly/AtomicHashMap.h>
#include <wangle/channel/Handler.h>
+#include <atomic>
+#include <mutex>
#include <string>
#include "serde/rpc.h"
@@ -29,6 +31,7 @@
namespace hbase {
class Request;
class Response;
+class HeaderInfo;
}
namespace google {
namespace protobuf {
@@ -37,6 +40,7 @@ class Message;
}
namespace hbase {
+
class ClientHandler : public wangle::Handler<std::unique_ptr<folly::IOBuf>,
Response, std::unique_ptr<Request>,
std::unique_ptr<folly::IOBuf>> {
@@ -47,7 +51,7 @@ public:
std::unique_ptr<Request> r) override;
private:
- bool need_send_header_;
+ std::unique_ptr<HeaderInfo> header_info_;
std::string user_name_;
RpcSerde serde_;
@@ -56,4 +60,17 @@ private:
uint32_t, std::shared_ptr<google::protobuf::Message>>>
resp_msgs_;
};
+
+/**
+ * Class to contain the info about if the connection header and preamble has
+ * been sent.
+ *
+ * We use a serperate class here so that ClientHandler is relocatable.
+ */
+class HeaderInfo {
+public:
+ HeaderInfo() : need_(true), mutex_() {}
+ std::atomic<bool> need_;
+ std::mutex mutex_;
+};
} // namespace hbase
[02/50] [abbrv] hbase git commit: Revert "NPE in RpcServer causing
intermittent UT failure of TestMasterReplication#testHFileCyclicReplication"
Posted by ec...@apache.org.
Revert "NPE in RpcServer causing intermittent UT failure of TestMasterReplication#testHFileCyclicReplication"
This reverts commit a33097e067b73be8e877b822afa90b89a0c7974f.
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/1578a045
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/1578a045
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/1578a045
Branch: refs/heads/HBASE-14850
Commit: 1578a045b6af6e8cd4c85d7b43397e204345a99d
Parents: 5e0d97e
Author: Yu Li <li...@apache.org>
Authored: Sat Jul 9 02:01:43 2016 +0800
Committer: Yu Li <li...@apache.org>
Committed: Sat Jul 9 02:01:43 2016 +0800
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java | 8 +-------
1 file changed, 1 insertion(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/1578a045/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
index 73226aa..ad88e8e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/ipc/RpcServer.java
@@ -2246,13 +2246,7 @@ public class RpcServer implements RpcServerInterface, ConfigurationObserver {
// The above callBlockingMethod will always return a SE. Strip the SE wrapper before
// putting it on the wire. Its needed to adhere to the pb Service Interface but we don't
// need to pass it over the wire.
- if (e instanceof ServiceException) {
- if (e.getCause() == null) {
- LOG.debug("Caught a ServiceException with null cause", e);
- } else {
- e = e.getCause();
- }
- }
+ if (e instanceof ServiceException) e = e.getCause();
// increment the number of requests that were exceptions.
metrics.exception(e);
[14/50] [abbrv] hbase git commit: HBASE-15718 Add on TableName
implementation and tests
Posted by ec...@apache.org.
HBASE-15718 Add on TableName implementation and tests
Summary:
Table name will be needed to look up a row in meta. So this
patch adds on the implementation for creating that lookup
row. It also adds tests for TableName
Test Plan: Added on a unit test
Differential Revision: https://reviews.facebook.net/D57285
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/2aaadab1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/2aaadab1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/2aaadab1
Branch: refs/heads/HBASE-14850
Commit: 2aaadab1519cdb11dd2c8df6cc8e61c1ecf4e82d
Parents: 7cae46e
Author: Elliott Clark <ec...@apache.org>
Authored: Tue Apr 26 14:51:06 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
.../connection/client-handler.cc | 1 -
hbase-native-client/core/BUCK | 69 +++++++++++---------
hbase-native-client/core/meta-utils.cc | 31 +++++++++
hbase-native-client/core/meta-utils.h | 30 +++++++++
hbase-native-client/core/simple-client.cc | 4 +-
hbase-native-client/core/table-name-test.cc | 50 ++++++++++++++
hbase-native-client/core/table-name.cc | 15 +++++
hbase-native-client/core/table-name.h | 25 ++++++-
hbase-native-client/if/Master.proto | 13 ++++
hbase-native-client/if/ZooKeeper.proto | 10 ++-
10 files changed, 211 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/2aaadab1/hbase-native-client/connection/client-handler.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-handler.cc b/hbase-native-client/connection/client-handler.cc
index 7b7794d..205993a7 100644
--- a/hbase-native-client/connection/client-handler.cc
+++ b/hbase-native-client/connection/client-handler.cc
@@ -60,7 +60,6 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
}
}
-
// TODO(eclark): Figure out how to handle the
// network errors that are going to come.
Future<Unit> ClientHandler::write(Context *ctx, Request r) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/2aaadab1/hbase-native-client/core/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index 195fc5c..9db6fda 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -16,44 +16,49 @@
# limitations under the License.
# This is the main library.
-cxx_library(name="core",
- exported_headers=[
- "client.h",
- "connection.h",
- "get-request.h",
- "get-result.h",
- "hbase_macros.h",
- "location-cache.h",
- "table-name.h",
- ],
- srcs=[
- "client.cc",
- "get-request.cc",
- "get-result.cc",
- "location-cache.cc",
- "table-name.cc",
- ],
- deps=[
- "//connection:connection",
- "//if:if",
- "//serde:serde",
- "//third-party:folly",
- "//third-party:wangle",
- "//third-party:zookeeper_mt",
- ],
- visibility=[
- 'PUBLIC',
- ], )
-
+cxx_library(
+ name="core",
+ exported_headers=[
+ "client.h",
+ "connection.h",
+ "get-request.h",
+ "get-result.h",
+ "hbase_macros.h",
+ "location-cache.h",
+ "table-name.h",
+ # TODO: move this out of exported
+ # Once meta lookup works
+ "meta-utils.h",
+ ],
+ srcs=[
+ "client.cc",
+ "get-request.cc",
+ "get-result.cc",
+ "location-cache.cc",
+ "meta-utils.cc",
+ "table-name.cc",
+ ],
+ deps=[
+ "//connection:connection",
+ "//if:if",
+ "//serde:serde",
+ "//third-party:folly",
+ "//third-party:wangle",
+ "//third-party:zookeeper_mt",
+ ],
+ visibility=[
+ 'PUBLIC',
+ ], )
cxx_test(name="location-cache-test",
srcs=[
"test-env.cc",
"location-cache-test.cc",
],
- deps=[
- ":core",
- ],
+ deps=[":core", ],
run_test_separately=True, )
+cxx_test(name="table-name-test",
+ srcs=["table-name-test.cc", ],
+ deps=[":core", ], )
cxx_binary(name="simple-client",
srcs=["simple-client.cc", ],
deps=[":core", "//connection:connection"], )
http://git-wip-us.apache.org/repos/asf/hbase/blob/2aaadab1/hbase-native-client/core/meta-utils.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/meta-utils.cc b/hbase-native-client/core/meta-utils.cc
new file mode 100644
index 0000000..7c7d45f
--- /dev/null
+++ b/hbase-native-client/core/meta-utils.cc
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/meta-utils.h"
+
+#include <folly/Conv.h>
+
+#include "core/table-name.h"
+
+using namespace hbase;
+
+std::string MetaUtil::region_lookup_rowkey(const TableName &tn,
+ const std::string &row) {
+ return folly::to<std::string>(tn, ",", row, ",", "999999999999999999");
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2aaadab1/hbase-native-client/core/meta-utils.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/meta-utils.h b/hbase-native-client/core/meta-utils.h
new file mode 100644
index 0000000..391d0ea
--- /dev/null
+++ b/hbase-native-client/core/meta-utils.h
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <string>
+
+namespace hbase {
+class TableName;
+
+class MetaUtil {
+public:
+ std::string region_lookup_rowkey(const TableName &tn, const std::string &row);
+};
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/2aaadab1/hbase-native-client/core/simple-client.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/simple-client.cc b/hbase-native-client/core/simple-client.cc
index 5e6f82e..8b2fae5 100644
--- a/hbase-native-client/core/simple-client.cc
+++ b/hbase-native-client/core/simple-client.cc
@@ -69,10 +69,12 @@ int main(int argc, char *argv[]) {
msg->mutable_region()->set_type(
RegionSpecifier_RegionSpecifierType::
RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
+
// What row.
msg->mutable_get()->set_row(FLAGS_row);
+
// Send it.
- r.set_msg(std::move(msg));
+ r.set_msg(msg);
auto resp = (*conn)(r).get(milliseconds(5000));
auto get_resp = std::static_pointer_cast<GetResponse>(resp.response());
http://git-wip-us.apache.org/repos/asf/hbase/blob/2aaadab1/hbase-native-client/core/table-name-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/table-name-test.cc b/hbase-native-client/core/table-name-test.cc
new file mode 100644
index 0000000..8aff375
--- /dev/null
+++ b/hbase-native-client/core/table-name-test.cc
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <folly/Conv.h>
+#include <gtest/gtest.h>
+
+#include <string>
+
+#include "core/table-name.h"
+
+using namespace hbase;
+
+TEST(TestTableName, TestToStringNoDefault) {
+ TableName tn{"TestTableName"};
+ std::string result = folly::to<std::string>(tn);
+ ASSERT_EQ(result.find("default"), std::string::npos);
+ ASSERT_EQ("TestTableName", result);
+}
+TEST(TestTableName, TestToStringIncludeNS) {
+ TableName tn{"hbase", "acl"};
+ std::string result = folly::to<std::string>(tn);
+ ASSERT_EQ(result.find("hbase"), 0);
+ ASSERT_EQ("hbase:acl", result);
+}
+TEST(TestTableName, TestIsDefault) {
+ TableName default_t1{"in_default"};
+ ASSERT_TRUE(default_t1.is_default_name_space());
+
+ TableName default_t2{"default", "in_also"};
+ ASSERT_TRUE(default_t2.is_default_name_space());
+
+ TableName non_default{"testing", "hmm"};
+ ASSERT_FALSE(non_default.is_default_name_space());
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2aaadab1/hbase-native-client/core/table-name.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/table-name.cc b/hbase-native-client/core/table-name.cc
index ffaaed0..33087fd 100644
--- a/hbase-native-client/core/table-name.cc
+++ b/hbase-native-client/core/table-name.cc
@@ -17,3 +17,18 @@
*
*/
#include "core/table-name.h"
+
+#include <folly/Conv.h>
+
+using namespace hbase;
+
+TableName::TableName(std::string table_name)
+ : name_space_("default"), table_(table_name) {}
+TableName::TableName(std::string name_space, std::string table_name)
+ : name_space_(name_space), table_(table_name) {}
+bool TableName::is_default_name_space() const {
+ return name_space_.length() == 0 || name_space_ == "default";
+}
+bool TableName::operator==(const TableName &other) const {
+ return name_space_ == other.name_space_ && table_ == other.table_;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2aaadab1/hbase-native-client/core/table-name.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/table-name.h b/hbase-native-client/core/table-name.h
index 37c3461..816f7da 100644
--- a/hbase-native-client/core/table-name.h
+++ b/hbase-native-client/core/table-name.h
@@ -21,12 +21,33 @@
#include <memory>
#include <string>
+#include <folly/Conv.h>
+
namespace hbase {
// This is the core class of a HBase client.
class TableName {
public:
- explicit TableName(std::string tableName);
- explicit TableName(std::string namespaceName, std::string tableName);
+ explicit TableName(std::string table_name);
+ explicit TableName(std::string name_space, std::string table_name);
+
+ std::string name_space() const { return name_space_; };
+ std::string table() const { return table_; };
+ bool is_default_name_space() const;
+ bool operator==(const TableName &other) const;
+
+private:
+ std::string name_space_;
+ std::string table_;
};
+
+// Provide folly::to<std::string>(TableName);
+template <class String> void toAppend(const TableName &in, String *result) {
+ if (in.is_default_name_space()) {
+ folly::toAppend(in.table(), result);
+ } else {
+ folly::toAppend(in.name_space(), ':', in.table(), result);
+ }
+}
+
} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/2aaadab1/hbase-native-client/if/Master.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/Master.proto b/hbase-native-client/if/Master.proto
index 1f7a3b7..ad8111e 100644
--- a/hbase-native-client/if/Master.proto
+++ b/hbase-native-client/if/Master.proto
@@ -288,6 +288,7 @@ message SetSplitOrMergeEnabledRequest {
required bool enabled = 1;
optional bool synchronous = 2;
repeated MasterSwitchType switch_types = 3;
+ optional bool skip_lock = 4;
}
message SetSplitOrMergeEnabledResponse {
@@ -302,6 +303,12 @@ message IsSplitOrMergeEnabledResponse {
required bool enabled = 1;
}
+message ReleaseSplitOrMergeLockAndRollbackRequest {
+}
+
+message ReleaseSplitOrMergeLockAndRollbackResponse {
+}
+
message NormalizeRequest {
}
@@ -672,6 +679,12 @@ service MasterService {
returns(IsSplitOrMergeEnabledResponse);
/**
+ * Release lock and rollback state.
+ */
+ rpc ReleaseSplitOrMergeLockAndRollback(ReleaseSplitOrMergeLockAndRollbackRequest)
+ returns(ReleaseSplitOrMergeLockAndRollbackResponse);
+
+ /**
* Run region normalizer. Can NOT run for various reasons. Check logs.
*/
rpc Normalize(NormalizeRequest)
http://git-wip-us.apache.org/repos/asf/hbase/blob/2aaadab1/hbase-native-client/if/ZooKeeper.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/ZooKeeper.proto b/hbase-native-client/if/ZooKeeper.proto
index 186d183..41c0e0e 100644
--- a/hbase-native-client/if/ZooKeeper.proto
+++ b/hbase-native-client/if/ZooKeeper.proto
@@ -165,4 +165,12 @@ message TableLock {
*/
message SwitchState {
optional bool enabled = 1;
-}
\ No newline at end of file
+}
+
+/**
+ * State for split and merge, used in hbck
+ */
+message SplitAndMergeState {
+ optional bool split_enabled = 1;
+ optional bool merge_enabled = 2;
+}
[30/50] [abbrv] hbase git commit: HBASE-15696 Move region location
cache serialization into serde
Posted by ec...@apache.org.
HBASE-15696 Move region location cache serialization into serde
Summary: Use IOBuf for zk
Test Plan: Added a unit test
Differential Revision: https://reviews.facebook.net/D57147
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7cae46e1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7cae46e1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7cae46e1
Branch: refs/heads/HBASE-14850
Commit: 7cae46e117586fbb2336be24b71573c7b5b6ac3a
Parents: dc0f017
Author: Elliott Clark <ec...@apache.org>
Authored: Fri Apr 22 22:21:45 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/Dockerfile | 7 +-
.../connection/client-dispatcher.h | 8 ++
.../connection/client-handler.cc | 7 +-
hbase-native-client/connection/client-handler.h | 2 +-
hbase-native-client/connection/pipeline.h | 2 +-
hbase-native-client/core/client.cc | 4 +-
hbase-native-client/core/client.h | 4 +-
hbase-native-client/core/location-cache-test.cc | 2 +-
hbase-native-client/core/location-cache.cc | 48 +++-----
hbase-native-client/core/location-cache.h | 4 +-
hbase-native-client/core/simple-client.cc | 6 +-
hbase-native-client/serde/BUCK | 10 ++
.../serde/client-deserializer-test.cc | 4 +-
.../serde/client-deserializer.cc | 4 +-
.../serde/client-serializer-test.cc | 2 +-
hbase-native-client/serde/client-serializer.cc | 2 +-
hbase-native-client/serde/client-serializer.h | 2 +-
.../serde/zk-deserializer-test.cc | 123 +++++++++++++++++++
hbase-native-client/serde/zk-deserializer.cc | 78 ++++++++++++
hbase-native-client/serde/zk-deserializer.h | 35 ++++++
hbase-native-client/utils/user-util-test.cc | 1 +
21 files changed, 297 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/Dockerfile
----------------------------------------------------------------------
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index 9ccc0d3..1524c56 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -19,12 +19,11 @@ FROM pjameson/buck-folly-watchman
ARG CC=/usr/bin/gcc-5
ARG CXX=/usr/bin/g++-5
-ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 -pthread"
-ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 -pthread"
+ARG CFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -pthread"
+ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -pthread"
-RUN apt-get install -y clang-format-3.7 vim maven inetutils-ping python-pip && \
+RUN apt-get install -y vim maven inetutils-ping python-pip && \
pip install yapf && \
- ln -sf /usr/bin/clang-format-3.7 /usr/bin/clang-format && \
apt-get -qq clean && \
apt-get -y -qq autoremove && \
rm -rf /var/lib/{apt,dpkg,cache,log}/ && \
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/connection/client-dispatcher.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-dispatcher.h b/hbase-native-client/connection/client-dispatcher.h
index c3987c9..89c7119 100644
--- a/hbase-native-client/connection/client-dispatcher.h
+++ b/hbase-native-client/connection/client-dispatcher.h
@@ -37,6 +37,14 @@ public:
private:
std::unordered_map<uint32_t, folly::Promise<Response>> requests_;
+ // Start at some number way above what could
+ // be there for un-initialized call id counters.
+ //
+ // This makes it easier to make sure that the're are
+ // no access to un-initialized variables.
+ //
+ // uint32_t has a max of 4Billion so 10 more or less is
+ // not a big deal.
uint32_t current_call_id_ = 10;
};
} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/connection/client-handler.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-handler.cc b/hbase-native-client/connection/client-handler.cc
index 74b23ef..7b7794d 100644
--- a/hbase-native-client/connection/client-handler.cc
+++ b/hbase-native-client/connection/client-handler.cc
@@ -19,15 +19,15 @@
#include "connection/client-handler.h"
-#include <glog/logging.h>
#include <folly/Likely.h>
+#include <glog/logging.h>
#include <string>
#include "connection/request.h"
#include "connection/response.h"
-#include "if/RPC.pb.h"
#include "if/Client.pb.h"
+#include "if/RPC.pb.h"
using namespace hbase;
using namespace folly;
@@ -60,6 +60,9 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
}
}
+
+// TODO(eclark): Figure out how to handle the
+// network errors that are going to come.
Future<Unit> ClientHandler::write(Context *ctx, Request r) {
// Keep track of if we have sent the header.
if (UNLIKELY(need_send_header_)) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/connection/client-handler.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-handler.h b/hbase-native-client/connection/client-handler.h
index 38c5725..dbaf5a0 100644
--- a/hbase-native-client/connection/client-handler.h
+++ b/hbase-native-client/connection/client-handler.h
@@ -22,8 +22,8 @@
#include <string>
-#include "serde/client-serializer.h"
#include "serde/client-deserializer.h"
+#include "serde/client-serializer.h"
// Forward decs.
namespace hbase {
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/connection/pipeline.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/pipeline.h b/hbase-native-client/connection/pipeline.h
index 68ade48..8114fab 100644
--- a/hbase-native-client/connection/pipeline.h
+++ b/hbase-native-client/connection/pipeline.h
@@ -18,8 +18,8 @@
*/
#pragma once
-#include <wangle/service/Service.h>
#include <folly/io/IOBufQueue.h>
+#include <wangle/service/Service.h>
#include "connection/request.h"
#include "connection/response.h"
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/core/client.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client.cc b/hbase-native-client/core/client.cc
index 893894f..266c239 100644
--- a/hbase-native-client/core/client.cc
+++ b/hbase-native-client/core/client.cc
@@ -21,8 +21,8 @@
#include <folly/Logging.h>
#include <folly/Random.h>
-#include <glog/logging.h>
#include <gflags/gflags.h>
+#include <glog/logging.h>
#include <wangle/concurrent/GlobalExecutor.h>
#include <string>
@@ -36,5 +36,5 @@ using namespace hbase::pb;
namespace hbase {
Client::Client(string quorum_spec)
- : location_cache(quorum_spec, wangle::getCPUExecutor()) {}
+ : location_cache_(quorum_spec, wangle::getCPUExecutor()) {}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/core/client.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client.h b/hbase-native-client/core/client.h
index c2dc226..b583285 100644
--- a/hbase-native-client/core/client.h
+++ b/hbase-native-client/core/client.h
@@ -19,8 +19,8 @@
#pragma once
-#include <folly/io/IOBuf.h>
#include <folly/futures/Future.h>
+#include <folly/io/IOBuf.h>
#include <string>
@@ -36,7 +36,7 @@ public:
folly::Future<GetResult> get(const GetRequest &get_request);
private:
- LocationCache location_cache;
+ LocationCache location_cache_;
};
} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/core/location-cache-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache-test.cc b/hbase-native-client/core/location-cache-test.cc
index 70ca6f1..f3166fb 100644
--- a/hbase-native-client/core/location-cache-test.cc
+++ b/hbase-native-client/core/location-cache-test.cc
@@ -16,8 +16,8 @@
* limitations under the License.
*
*/
-#include <gtest/gtest.h>
#include <folly/Memory.h>
+#include <gtest/gtest.h>
#include <wangle/concurrent/GlobalExecutor.h>
#include "location-cache.h"
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/core/location-cache.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
index 52e86e3..5925f4a 100644
--- a/hbase-native-client/core/location-cache.cc
+++ b/hbase-native-client/core/location-cache.cc
@@ -16,11 +16,13 @@
* limitations under the License.
*
*/
-#include "location-cache.h"
+#include "core/location-cache.h"
#include <folly/Logging.h>
+#include <folly/io/IOBuf.h>
#include "if/ZooKeeper.pb.h"
+#include "serde/zk-deserializer.h"
using namespace std;
using namespace folly;
@@ -28,7 +30,7 @@ using namespace hbase::pb;
using namespace hbase;
// TODO(eclark): make this configurable on client creation
-static const char META_LOCATION[] = "/hbase/meta-region-server";
+static const char META_ZNODE_NAME[] = "/hbase/meta-region-server";
LocationCache::LocationCache(string quorum_spec,
shared_ptr<folly::Executor> executor)
@@ -66,45 +68,25 @@ void LocationCache::RefreshMetaLocation() {
}
ServerName LocationCache::ReadMetaLocation() {
- char contents[4096];
+ auto buf = IOBuf::create(4096);
+ ZkDeserializer derser;
+
// This needs to be int rather than size_t as that's what ZK expects.
- int len = sizeof(contents);
+ int len = buf->capacity();
// TODO(elliott): handle disconnects/reconntion as needed.
- int zk_result = zoo_get(this->zk_, META_LOCATION, 0, contents, &len, nullptr);
+ int zk_result =
+ zoo_get(this->zk_, META_ZNODE_NAME, 0,
+ reinterpret_cast<char *>(buf->writableData()), &len, nullptr);
+ LOG(ERROR) << "len = " << len;
if (zk_result != ZOK || len < 9) {
LOG(ERROR) << "Error getting meta location.";
throw runtime_error("Error getting meta location");
}
- // There should be a magic number for recoverable zk
- if (static_cast<uint8_t>(contents[0]) != 255) {
- LOG(ERROR) << "Magic number not in ZK znode data expected 255 got ="
- << unsigned(static_cast<uint8_t>(contents[0]));
- throw runtime_error("Magic number not in znode data");
- }
- // pos will keep track of skipped bytes.
- int pos = 1;
- // How long is the id?
- int id_len = 0;
- for (int i = 0; i < 4; ++i) {
- id_len = id_len << 8;
- id_len = id_len | static_cast<uint8_t>(contents[pos]);
- ++pos;
- }
- // Skip the id
- pos += id_len;
- // Then all protobuf's for HBase are prefixed with a magic string.
- // PBUF, so we skip that.
- // TODO(eclark): check to make sure that the magic string is correct
- // though I am not sure that will get us much.
- pos += 4;
+ buf->append(len);
MetaRegionServer mrs;
- // Try to decode the protobuf.
- // If there's an error bail out.
- if (mrs.ParseFromArray(contents + pos, len - pos) == false) {
- LOG(ERROR) << "Error parsing Protobuf Message";
- throw runtime_error("Error parsing protobuf");
+ if (derser.parse(buf.get(), &mrs) == false) {
+ LOG(ERROR) << "Unable to decode";
}
-
return mrs.server();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/core/location-cache.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h
index 28a1ee1..ce97c43 100644
--- a/hbase-native-client/core/location-cache.h
+++ b/hbase-native-client/core/location-cache.h
@@ -18,10 +18,10 @@
*/
#pragma once
-#include <zookeeper/zookeeper.h>
+#include <folly/Executor.h>
#include <folly/futures/Future.h>
#include <folly/futures/SharedPromise.h>
-#include <folly/Executor.h>
+#include <zookeeper/zookeeper.h>
#include <memory>
#include <mutex>
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/core/simple-client.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/simple-client.cc b/hbase-native-client/core/simple-client.cc
index 065f70b..5e6f82e 100644
--- a/hbase-native-client/core/simple-client.cc
+++ b/hbase-native-client/core/simple-client.cc
@@ -22,13 +22,13 @@
#include <gflags/gflags.h>
#include <wangle/concurrent/GlobalExecutor.h>
-#include <iostream>
#include <chrono>
+#include <iostream>
-#include "core/client.h"
#include "connection/connection-factory.h"
-#include "if/ZooKeeper.pb.h"
+#include "core/client.h"
#include "if/Client.pb.h"
+#include "if/ZooKeeper.pb.h"
using namespace folly;
using namespace std;
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/serde/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/BUCK b/hbase-native-client/serde/BUCK
index 207607f..0014c0b 100644
--- a/hbase-native-client/serde/BUCK
+++ b/hbase-native-client/serde/BUCK
@@ -19,10 +19,12 @@ cxx_library(name="serde",
exported_headers=[
"client-serializer.h",
"client-deserializer.h",
+ "zk-deserializer.h",
],
srcs=[
"client-serializer.cc",
"client-deserializer.cc",
+ "zk-deserializer.cc",
],
deps=[
"//if:if",
@@ -52,3 +54,11 @@ cxx_test(name="client-deserializer-test",
":serde",
"//if:if",
], )
+cxx_test(name="zk-deserializer-test",
+ srcs=[
+ "zk-deserializer-test.cc",
+ ],
+ deps=[
+ ":serde",
+ "//if:if",
+ ], )
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/serde/client-deserializer-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/client-deserializer-test.cc b/hbase-native-client/serde/client-deserializer-test.cc
index bb57e50..9fef093 100644
--- a/hbase-native-client/serde/client-deserializer-test.cc
+++ b/hbase-native-client/serde/client-deserializer-test.cc
@@ -17,12 +17,12 @@
*
*/
-#include <gtest/gtest.h>
#include <folly/io/IOBuf.h>
+#include <gtest/gtest.h>
+#include "if/Client.pb.h"
#include "serde/client-deserializer.h"
#include "serde/client-serializer.h"
-#include "if/Client.pb.h"
using namespace hbase;
using folly::IOBuf;
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/serde/client-deserializer.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/client-deserializer.cc b/hbase-native-client/serde/client-deserializer.cc
index 118b0d1..acca7ea 100644
--- a/hbase-native-client/serde/client-deserializer.cc
+++ b/hbase-native-client/serde/client-deserializer.cc
@@ -19,10 +19,10 @@
#include "serde/client-deserializer.h"
-#include <google/protobuf/message.h>
+#include <folly/Logging.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
-#include <folly/Logging.h>
+#include <google/protobuf/message.h>
using namespace hbase;
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/serde/client-serializer-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/client-serializer-test.cc b/hbase-native-client/serde/client-serializer-test.cc
index b32b55d..9bf38af 100644
--- a/hbase-native-client/serde/client-serializer-test.cc
+++ b/hbase-native-client/serde/client-serializer-test.cc
@@ -22,9 +22,9 @@
#include <string>
-#include "serde/client-serializer.h"
#include "if/HBase.pb.h"
#include "if/RPC.pb.h"
+#include "serde/client-serializer.h"
using namespace hbase;
using namespace hbase::pb;
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/serde/client-serializer.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/client-serializer.cc b/hbase-native-client/serde/client-serializer.cc
index 881b6e4..09b81c8 100644
--- a/hbase-native-client/serde/client-serializer.cc
+++ b/hbase-native-client/serde/client-serializer.cc
@@ -18,8 +18,8 @@
*/
#include "serde/client-serializer.h"
-#include <folly/io/Cursor.h>
#include <folly/Logging.h>
+#include <folly/io/Cursor.h>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/serde/client-serializer.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/client-serializer.h b/hbase-native-client/serde/client-serializer.h
index 685095d..9c819fe 100644
--- a/hbase-native-client/serde/client-serializer.h
+++ b/hbase-native-client/serde/client-serializer.h
@@ -18,9 +18,9 @@
*/
#pragma once
+#include <cstdint>
#include <folly/io/IOBuf.h>
#include <string>
-#include <cstdint>
// Forward
namespace google {
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/serde/zk-deserializer-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/zk-deserializer-test.cc b/hbase-native-client/serde/zk-deserializer-test.cc
new file mode 100644
index 0000000..92d85a0
--- /dev/null
+++ b/hbase-native-client/serde/zk-deserializer-test.cc
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "serde/zk-deserializer.h"
+
+#include <folly/Logging.h>
+#include <folly/io/Cursor.h>
+#include <folly/io/IOBuf.h>
+#include <gtest/gtest.h>
+
+#include "if/ZooKeeper.pb.h"
+
+using namespace hbase;
+using namespace hbase::pb;
+using namespace folly;
+using namespace std;
+using namespace folly::io;
+
+// Test that would test if there's nothing there.
+TEST(TestZkDesializer, TestThrowNoMagicNum) {
+ ZkDeserializer deser;
+ MetaRegionServer mrs;
+
+ auto buf = IOBuf::create(100);
+ buf->append(100);
+ RWPrivateCursor c{buf.get()};
+ c.write<uint8_t>(99);
+ ASSERT_THROW(deser.parse(buf.get(), &mrs), runtime_error);
+}
+
+// Test if the protobuf is in a format that we can't decode
+TEST(TestZkDesializer, TestBadProtoThrow) {
+ ZkDeserializer deser;
+ MetaRegionServer mrs;
+ string magic{"PBUF"};
+
+ // Set ServerName
+ mrs.mutable_server()->set_host_name("test");
+ mrs.mutable_server()->set_port(567);
+ mrs.mutable_server()->set_start_code(9567);
+
+ // One byte magic number
+ // four bytes for id length
+ // four bytes for id
+ // four bytes for PBUF
+ uint32_t start_len = 1 + 4 + 4 + 4;
+ // How large the protobuf will be
+ uint32_t pbuf_size = mrs.ByteSize();
+
+ auto buf = IOBuf::create(start_len + pbuf_size);
+ buf->append(start_len + pbuf_size);
+ RWPrivateCursor c{buf.get()};
+
+ // Write the magic number
+ c.write<uint8_t>(255);
+ // Write the id len
+ c.writeBE<uint32_t>(4);
+ // Write the id
+ c.write<uint32_t>(13);
+ // Write the PBUF string
+ c.push(reinterpret_cast<const uint8_t *>(magic.c_str()), 4);
+
+ // Create the protobuf
+ MetaRegionServer out;
+ ASSERT_THROW(deser.parse(buf.get(), &out), runtime_error);
+}
+
+// Test to make sure the whole thing works.
+TEST(TestZkDesializer, TestNoThrow) {
+ ZkDeserializer deser;
+ MetaRegionServer mrs;
+ string magic{"PBUF"};
+
+ // Set ServerName
+ mrs.mutable_server()->set_host_name("test");
+ mrs.mutable_server()->set_port(567);
+ mrs.mutable_server()->set_start_code(9567);
+
+ // One byte magic number
+ // four bytes for id length
+ // four bytes for id
+ // four bytes for PBUF
+ uint32_t start_len = 1 + 4 + 4 + 4;
+ // How large the protobuf will be
+ uint32_t pbuf_size = mrs.ByteSize();
+
+ auto buf = IOBuf::create(start_len + pbuf_size);
+ buf->append(start_len + pbuf_size);
+ RWPrivateCursor c{buf.get()};
+
+ // Write the magic number
+ c.write<uint8_t>(255);
+ // Write the id len
+ c.writeBE<uint32_t>(4);
+ // Write the id
+ c.write<uint32_t>(13);
+ // Write the PBUF string
+ c.push(reinterpret_cast<const uint8_t *>(magic.c_str()), 4);
+
+ // Now write the serialized protobuf
+ mrs.SerializeWithCachedSizesToArray(buf->writableData() + start_len);
+
+ // Create the protobuf
+ MetaRegionServer out;
+ ASSERT_TRUE(deser.parse(buf.get(), &out));
+ ASSERT_EQ(mrs.server().host_name(), out.server().host_name());
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/serde/zk-deserializer.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/zk-deserializer.cc b/hbase-native-client/serde/zk-deserializer.cc
new file mode 100644
index 0000000..33cf809
--- /dev/null
+++ b/hbase-native-client/serde/zk-deserializer.cc
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "serde/zk-deserializer.h"
+
+#include <folly/io/Cursor.h>
+#include <folly/io/IOBuf.h>
+#include <google/protobuf/message.h>
+
+using hbase::ZkDeserializer;
+using std::runtime_error;
+using folly::IOBuf;
+using folly::io::Cursor;
+using google::protobuf::Message;
+
+static const std::string MAGIC_STRING = "PBUF";
+
+bool ZkDeserializer::parse(IOBuf *buf, Message *out) {
+
+ // The format is like this
+ // 1 byte of magic number. 255
+ // 4 bytes of id length.
+ // id_length number of bytes for the id of who put up the znode
+ // 4 bytes of a magic string PBUF
+ // Then the protobuf serialized without a varint header.
+
+ Cursor c{buf};
+
+ // There should be a magic number for recoverable zk
+ uint8_t magic_num = c.read<uint8_t>();
+ if (magic_num != 255) {
+ LOG(ERROR) << "Magic number not in ZK znode data expected 255 got ="
+ << unsigned(magic_num);
+ throw runtime_error("Magic number not in znode data");
+ }
+ // How long is the id?
+ uint32_t id_len = c.readBE<uint32_t>();
+
+ if (id_len >= c.length()) {
+ LOG(ERROR) << "After skiping the if from zookeeper data there's not enough "
+ "left to read anything else";
+ throw runtime_error("Not enough bytes to decode from zookeeper");
+ }
+
+ // Skip the id
+ c.skip(id_len);
+
+ // Make sure that the magic string is there.
+ if (MAGIC_STRING != c.readFixedString(4)) {
+ LOG(ERROR) << "There was no PBUF magic string.";
+ throw runtime_error("No PBUF magic string in the zookpeeper data.");
+ }
+
+ // Try to decode the protobuf.
+ // If there's an error bail out.
+ if (out->ParseFromArray(c.data(), c.length()) == false) {
+ LOG(ERROR) << "Error parsing Protobuf Message";
+ throw runtime_error("Error parsing protobuf");
+ }
+
+ return true;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/serde/zk-deserializer.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/zk-deserializer.h b/hbase-native-client/serde/zk-deserializer.h
new file mode 100644
index 0000000..aa91661
--- /dev/null
+++ b/hbase-native-client/serde/zk-deserializer.h
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+namespace google {
+namespace protobuf {
+class Message;
+}
+}
+namespace folly {
+class IOBuf;
+}
+
+namespace hbase {
+class ZkDeserializer {
+public:
+ bool parse(folly::IOBuf *buf, google::protobuf::Message *out);
+};
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/7cae46e1/hbase-native-client/utils/user-util-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/utils/user-util-test.cc b/hbase-native-client/utils/user-util-test.cc
index 2a7434f..1e72bcc 100644
--- a/hbase-native-client/utils/user-util-test.cc
+++ b/hbase-native-client/utils/user-util-test.cc
@@ -30,5 +30,6 @@ TEST(TestUserUtil, TestGetSomething) {
UserUtil u_util;
string name = u_util.user_name();
+ // TODO shell out to whoami to check this.
ASSERT_GT(name.length(), 0);
}
[46/50] [abbrv] hbase git commit: HBASE-15771 Document all the public
classes
Posted by ec...@apache.org.
HBASE-15771 Document all the public classes
Summary: Add on a bunch of documentation around implementation so far.
Test Plan:
make doc
Doxygen has no warnings
Differential Revision: https://reviews.facebook.net/D57753
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fdf00239
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fdf00239
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fdf00239
Branch: refs/heads/HBASE-14850
Commit: fdf00239c691d0de9b1b97745dafe1cea57dbf9c
Parents: 16e1de7
Author: Elliott Clark <ec...@apache.org>
Authored: Thu May 5 13:14:20 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/BUILDING.md | 18 +++++-
.../connection/client-dispatcher.h | 11 +++-
.../connection/client-handler.cc | 37 +++++-------
hbase-native-client/connection/client-handler.h | 36 +++++++-----
.../connection/connection-factory.h | 18 ++++++
.../connection/connection-pool.cc | 5 +-
.../connection/connection-pool.h | 44 +++++++++++++-
hbase-native-client/connection/pipeline.h | 18 ++++++
hbase-native-client/connection/request.h | 32 ++++++----
hbase-native-client/connection/response.h | 33 +++++++++--
hbase-native-client/core/BUCK | 1 -
hbase-native-client/core/client.h | 12 ++++
hbase-native-client/core/connection.cc | 20 -------
hbase-native-client/core/connection.h | 26 ---------
hbase-native-client/core/location-cache.cc | 14 ++++-
hbase-native-client/core/location-cache.h | 36 ++++++++++--
hbase-native-client/core/meta-utils.h | 12 ++++
hbase-native-client/core/put.cc | 21 -------
hbase-native-client/core/put.h | 27 ---------
hbase-native-client/core/region-location.h | 41 +++++++++++++
hbase-native-client/serde/rpc.cc | 1 -
hbase-native-client/serde/rpc.h | 61 +++++++++++++++++++-
hbase-native-client/serde/zk.h | 14 +++++
hbase-native-client/utils/user-util.h | 18 ++++++
24 files changed, 387 insertions(+), 169 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/BUILDING.md
----------------------------------------------------------------------
diff --git a/hbase-native-client/BUILDING.md b/hbase-native-client/BUILDING.md
index 20ef2a0..4c06776 100644
--- a/hbase-native-client/BUILDING.md
+++ b/hbase-native-client/BUILDING.md
@@ -17,7 +17,7 @@ specific language governing permissions and limitations
under the License.
-->
-#Building HBase native client
+# Building HBase native client
The HBase native client build using buck and produces a linux library.
@@ -58,6 +58,20 @@ buck build //core:simple-client
```
That will build the library, then build and test everything, then build
-the simple-client binary. Buck will find all modules used, and compile
+the simple-client binary. Buck will find all modules used, and compile
them in parallel, caching the results. Output from buck is in the buck-out
foulder. Generated binaries are in buck-out/gen logs are in buck-out/logs
+
+
+# Make
+
+If learning buck isn't your thing there is a Makefile wrapper for your
+convenience.
+
+```
+make help
+make check
+make clean
+make all
+make build
+```
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/connection/client-dispatcher.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-dispatcher.h b/hbase-native-client/connection/client-dispatcher.h
index 4435a1b..4bfb35d 100644
--- a/hbase-native-client/connection/client-dispatcher.h
+++ b/hbase-native-client/connection/client-dispatcher.h
@@ -30,17 +30,22 @@
#include "connection/response.h"
namespace hbase {
+/**
+ * Dispatcher that assigns a call_id and then routes the response back to the future.
+ */
class ClientDispatcher
: public wangle::ClientDispatcherBase<SerializePipeline,
std::unique_ptr<Request>, Response> {
public:
+ /** Create a new ClientDispatcher */
ClientDispatcher();
- ~ClientDispatcher() {
- LOG(ERROR) << "Killing ClientDispatcher call_id = " << current_call_id_;
- }
+ /** Read a response off the pipeline. */
void read(Context *ctx, Response in) override;
+ /** Take a request as a call and send it down the pipeline. */
folly::Future<Response> operator()(std::unique_ptr<Request> arg) override;
+ /** Close the dispatcher and the associated pipeline. */
folly::Future<folly::Unit> close(Context *ctx) override;
+ /** Close the dispatcher and the associated pipeline. */
folly::Future<folly::Unit> close() override;
private:
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/connection/client-handler.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-handler.cc b/hbase-native-client/connection/client-handler.cc
index 4fdb7ae..cae03c7 100644
--- a/hbase-native-client/connection/client-handler.cc
+++ b/hbase-native-client/connection/client-handler.cc
@@ -37,7 +37,7 @@ using hbase::pb::GetResponse;
using google::protobuf::Message;
ClientHandler::ClientHandler(std::string user_name)
- : user_name_(user_name), serde_(), header_info_(),
+ : user_name_(user_name), serde_(), once_flag_(),
resp_msgs_(
make_unique<folly::AtomicHashMap<
uint32_t, std::shared_ptr<google::protobuf::Message>>>(5000)) {}
@@ -75,36 +75,25 @@ void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
used_bytes = serde_.ParseDelimited(buf.get(), resp_msg.get());
// Make sure that bytes were parsed.
CHECK(used_bytes == buf->length());
- received.set_response(resp_msg);
+ received.set_resp_msg(resp_msg);
}
ctx->fireRead(std::move(received));
}
}
Future<Unit> ClientHandler::write(Context *ctx, std::unique_ptr<Request> r) {
- // Keep track of if we have sent the header.
- //
- // even though the bool is atomic we can load it lazily here.
- if (UNLIKELY(header_info_->need_.load(std::memory_order_relaxed))) {
-
- // Grab the lock.
- // We need to make sure that no one gets past here without there being a
- // hearder sent.
- std::lock_guard<std::mutex> lock(header_info_->mutex_);
-
- // Now see if we are the first thread to get through.
- //
- // If this is the first thread to get through then the
- // need_send_header will have been true before this.
- if (header_info_->need_.exchange(false)) {
- auto pre = serde_.Preamble();
- auto header = serde_.Header(user_name_);
- pre->appendChain(std::move(header));
- ctx->fireWrite(std::move(pre));
- }
- }
-
+ // We need to send the header once.
+ // So use call_once to make sure that only one thread wins this.
+ std::call_once((*once_flag_), [ctx, this]() {
+ auto pre = serde_.Preamble();
+ auto header = serde_.Header(user_name_);
+ pre->appendChain(std::move(header));
+ ctx->fireWrite(std::move(pre));
+ });
+
+ // Now store the call id to response.
resp_msgs_->insert(r->call_id(), r->resp_msg());
+ // Send the data down the pipeline.
return ctx->fireWrite(
serde_.Request(r->call_id(), r->method(), r->req_msg().get()));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/connection/client-handler.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-handler.h b/hbase-native-client/connection/client-handler.h
index 1a4275f..984a947 100644
--- a/hbase-native-client/connection/client-handler.h
+++ b/hbase-native-client/connection/client-handler.h
@@ -41,17 +41,38 @@ class Message;
namespace hbase {
+/**
+ * wangle::Handler implementation to convert hbase::Request to IOBuf and
+ * convert IOBuf to hbase::Response.
+ *
+ * This class deals with sending the connection header and preamble
+ * on first request.
+ */
class ClientHandler : public wangle::Handler<std::unique_ptr<folly::IOBuf>,
Response, std::unique_ptr<Request>,
std::unique_ptr<folly::IOBuf>> {
public:
+ /**
+ * Create the handler
+ * @param user_name the user name of the user running this process.
+ */
ClientHandler(std::string user_name);
+
+ /**
+ * Get bytes from the wire.
+ * This should be the full message as the length field decoder should be
+ * in the pipeline before this.
+ */
void read(Context *ctx, std::unique_ptr<folly::IOBuf> msg) override;
+
+ /**
+ * Write the data down the wire.
+ */
folly::Future<folly::Unit> write(Context *ctx,
std::unique_ptr<Request> r) override;
private:
- std::unique_ptr<HeaderInfo> header_info_;
+ std::unique_ptr<std::once_flag> once_flag_;
std::string user_name_;
RpcSerde serde_;
@@ -60,17 +81,4 @@ private:
uint32_t, std::shared_ptr<google::protobuf::Message>>>
resp_msgs_;
};
-
-/**
- * Class to contain the info about if the connection header and preamble has
- * been sent.
- *
- * We use a serperate class here so that ClientHandler is relocatable.
- */
-class HeaderInfo {
-public:
- HeaderInfo() : need_(true), mutex_() {}
- std::atomic<bool> need_;
- std::mutex mutex_;
-};
} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/connection/connection-factory.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-factory.h b/hbase-native-client/connection/connection-factory.h
index 2284a7c..8b6d8d8 100644
--- a/hbase-native-client/connection/connection-factory.h
+++ b/hbase-native-client/connection/connection-factory.h
@@ -28,14 +28,32 @@
#include "connection/service.h"
namespace hbase {
+
+/**
+ * Class to create a ClientBootstrap and turn it into a connected
+ * pipeline.
+ */
class ConnectionFactory {
public:
+ /**
+ * Constructor.
+ * There should only be one ConnectionFactory per client.
+ */
ConnectionFactory();
+ /** Default Desctructor */
virtual ~ConnectionFactory() = default;
+ /**
+ * Create a BootStrap from which a connection can be made.
+ */
virtual std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>>
MakeBootstrap();
+ /**
+ * Connect a ClientBootstrap to a server and return the pipeline.
+ *
+ * This is mostly visible so that mocks can override socket connections.
+ */
virtual std::shared_ptr<HBaseService>
Connect(std::shared_ptr<wangle::ClientBootstrap<SerializePipeline>> client,
const std::string &hostname, int port);
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/connection/connection-pool.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-pool.cc b/hbase-native-client/connection/connection-pool.cc
index 6ed5ad9..75f343e 100644
--- a/hbase-native-client/connection/connection-pool.cc
+++ b/hbase-native-client/connection/connection-pool.cc
@@ -26,11 +26,11 @@ using std::mutex;
using std::unique_ptr;
using std::shared_ptr;
using hbase::pb::ServerName;
+using hbase::ConnectionPool;
+using hbase::HBaseService;
using folly::SharedMutexWritePriority;
using folly::SocketAddress;
-namespace hbase {
-
ConnectionPool::ConnectionPool()
: cf_(std::make_shared<ConnectionFactory>()), clients_(), connections_(),
map_mutex_() {}
@@ -75,4 +75,3 @@ void ConnectionPool::close(const ServerName &sn) {
auto service = found->second;
connections_.erase(found);
}
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/connection/connection-pool.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-pool.h b/hbase-native-client/connection/connection-pool.h
index 907afdb..5edd407 100644
--- a/hbase-native-client/connection/connection-pool.h
+++ b/hbase-native-client/connection/connection-pool.h
@@ -18,6 +18,7 @@
*/
#pragma once
+#include <boost/functional/hash.hpp>
#include <folly/SharedMutex.h>
#include <mutex>
#include <unordered_map>
@@ -27,26 +28,63 @@
#include "if/HBase.pb.h"
namespace hbase {
+
+/** Equals function for server name that ignores start time */
struct ServerNameEquals {
+
+ /** equals */
bool operator()(const hbase::pb::ServerName &lhs,
const hbase::pb::ServerName &rhs) const {
return lhs.host_name() == rhs.host_name() && lhs.port() == rhs.port();
}
};
+
+/** Hash for ServerName that ignores the start time. */
struct ServerNameHash {
+ /** hash */
std::size_t operator()(hbase::pb::ServerName const &s) const {
- std::size_t h1 = std::hash<std::string>()(s.host_name());
- std::size_t h2 = std::hash<uint32_t>()(s.port());
- return h1 ^ (h2 << 2);
+ std::size_t h = 0;
+ boost::hash_combine(h, s.host_name());
+ boost::hash_combine(h, s.port());
+ return h;
}
};
+/**
+ * @brief Connection pooling for HBase rpc connection.
+ *
+ * This is a thread safe connection pool. It allows getting
+ * a shared connection to HBase by server name. This is
+ * useful for keeping a single connection no matter how many regions a
+ * regionserver has on it.
+ */
class ConnectionPool {
public:
+ /** Create connection pool wit default connection factory */
ConnectionPool();
+
+ /**
+ * Desctructor.
+ * All connections will be close.
+ * All connections will be released
+ */
~ConnectionPool();
+
+ /**
+ * Constructor that allows specifiying the connetion factory.
+ * This is useful for testing.
+ */
explicit ConnectionPool(std::shared_ptr<ConnectionFactory> cf);
+
+ /**
+ * Get a connection to the server name. Start time is ignored.
+ * This can be a blocking operation for a short time.
+ */
std::shared_ptr<HBaseService> get(const hbase::pb::ServerName &sn);
+
+ /**
+ * Close/remove a connection.
+ */
void close(const hbase::pb::ServerName &sn);
private:
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/connection/pipeline.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/pipeline.h b/hbase-native-client/connection/pipeline.h
index e90cb93..88d7845 100644
--- a/hbase-native-client/connection/pipeline.h
+++ b/hbase-native-client/connection/pipeline.h
@@ -26,12 +26,30 @@
#include "utils/user-util.h"
namespace hbase {
+
+/** Pipeline to turn IOBuf into requests */
using SerializePipeline =
wangle::Pipeline<folly::IOBufQueue &, std::unique_ptr<Request>>;
+/**
+ * Factory to create new pipelines for HBase RPC's.
+ */
class RpcPipelineFactory : public wangle::PipelineFactory<SerializePipeline> {
public:
+ /**
+ * Constructor. This will create user util.
+ */
RpcPipelineFactory();
+
+ /**
+ * Create a new pipeline.
+ * The pipeline will be:
+ *
+ * - Async Socke Handler
+ * - Event Base Handler
+ * - Length Field Based Frame Decoder
+ * - Client Handler
+ */
SerializePipeline::Ptr
newPipeline(std::shared_ptr<folly::AsyncTransportWrapper> sock) override;
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/connection/request.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/request.h b/hbase-native-client/connection/request.h
index 743c469..634d1ba 100644
--- a/hbase-native-client/connection/request.h
+++ b/hbase-native-client/connection/request.h
@@ -25,30 +25,42 @@
#include <string>
namespace hbase {
+
+/**
+ * Main request class.
+ * This holds the request object and the un-filled in approriatley typed
+ * response object.
+ */
class Request {
public:
+ /** Create a request object for a get */
static std::unique_ptr<Request> get();
+ /** Create a request object for a mutate */
static std::unique_ptr<Request> mutate();
+ /** Create a request object for a scan */
static std::unique_ptr<Request> scan();
+ /**
+ * This should be private. Do not use this.
+ *
+ *
+ * Constructor that's public for make_unique. This sets all the messages and
+ * method name.
+ */
Request(std::shared_ptr<google::protobuf::Message> req,
std::shared_ptr<google::protobuf::Message> resp, std::string method);
+ /** Get the call id. */
uint32_t call_id() { return call_id_; }
+ /** Set the call id. This should only be set once. */
void set_call_id(uint32_t call_id) { call_id_ = call_id; }
-
+ /** Get the backing request protobuf message. */
std::shared_ptr<google::protobuf::Message> req_msg() { return req_msg_; }
+ /** Get the backing response protobuf message. */
std::shared_ptr<google::protobuf::Message> resp_msg() { return resp_msg_; }
-
- void set_req_msg(std::shared_ptr<google::protobuf::Message> msg) {
- req_msg_ = msg;
- }
- void set_resp_msg(std::shared_ptr<google::protobuf::Message> msg) {
- resp_msg_ = msg;
- }
-
+ /** Get the method name. This is used to the the receiving rpc server what
+ * method type to decode. */
std::string method() { return method_; }
- void set_method(std::string method) { method_ = method; }
private:
uint32_t call_id_;
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/connection/response.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/response.h b/hbase-native-client/connection/response.h
index d2eb19a..aac57c2 100644
--- a/hbase-native-client/connection/response.h
+++ b/hbase-native-client/connection/response.h
@@ -29,20 +29,41 @@ class Message;
namespace hbase {
+/**
+ * @brief Class representing a rpc response
+ *
+ * This is the class sent to a service.
+ */
class Response {
public:
- Response() : call_id_(0), response_(nullptr) {}
+ /**
+ * Constructor.
+ * Initinalizes the call id to 0. 0 should never be a valid call id.
+ */
+ Response() : call_id_(0), resp_msg_(nullptr) {}
+
+ /** Get the call_id */
uint32_t call_id() { return call_id_; }
+
+ /** Set the call_id */
void set_call_id(uint32_t call_id) { call_id_ = call_id; }
- std::shared_ptr<google::protobuf::Message> response() const {
- return response_;
+
+ /**
+ * Get the response message.
+ * The caller is reponsible for knowing the type. In practice the call id is
+ * used to figure out the type.
+ */
+ std::shared_ptr<google::protobuf::Message> resp_msg() const {
+ return resp_msg_;
}
- void set_response(std::shared_ptr<google::protobuf::Message> response) {
- response_ = std::move(response);
+
+ /** Set the response message. */
+ void set_resp_msg(std::shared_ptr<google::protobuf::Message> response) {
+ resp_msg_ = std::move(response);
}
private:
uint32_t call_id_;
- std::shared_ptr<google::protobuf::Message> response_;
+ std::shared_ptr<google::protobuf::Message> resp_msg_;
};
} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/core/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index ef8c2f8..485f9ba 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -20,7 +20,6 @@ cxx_library(
name="core",
exported_headers=[
"client.h",
- "connection.h",
"hbase_macros.h",
"region-location.h",
"location-cache.h",
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/core/client.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client.h b/hbase-native-client/core/client.h
index 4bed751..4a6d23b 100644
--- a/hbase-native-client/core/client.h
+++ b/hbase-native-client/core/client.h
@@ -28,8 +28,20 @@
#include "if/Cell.pb.h"
namespace hbase {
+
+/**
+ * Client.
+ *
+ * This is the class that provides access to an HBase cluster.
+ * It is thread safe and does connection pooling. Current recommendations are to have only one Client per cluster around.
+ */
class Client {
public:
+
+ /**
+ * Create a new client.
+ * @param quorum_spec Where to connect to get Zookeeper bootstrap information.
+ */
explicit Client(std::string quorum_spec);
private:
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/core/connection.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/connection.cc b/hbase-native-client/core/connection.cc
deleted file mode 100644
index e9a28eb..0000000
--- a/hbase-native-client/core/connection.cc
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#include "core/connection.h"
-
-void Connection::set_zk_quorum(char *zk_q) { this->zk_quorum = zk_q; }
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/core/connection.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/connection.h b/hbase-native-client/core/connection.h
deleted file mode 100644
index 5c9d6b8..0000000
--- a/hbase-native-client/core/connection.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-
-class Connection {
- char *zk_quorum;
-
-public:
- void set_zk_quorum(char *zk_q);
-};
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/core/location-cache.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
index e2a6251..583d305 100644
--- a/hbase-native-client/core/location-cache.cc
+++ b/hbase-native-client/core/location-cache.cc
@@ -128,15 +128,24 @@ LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
});
}
+/**
+ * Filter to remove a service from the location cache and the connection cache on errors
+ * or on cloase.
+ */
class RemoveServiceFilter
: public ServiceFilter<std::unique_ptr<Request>, Response> {
public:
+
+ /** Create a new filter. */
RemoveServiceFilter(std::shared_ptr<HBaseService> service, ServerName sn,
ConnectionPool &cp)
: ServiceFilter<unique_ptr<Request>, Response>(service), sn_(sn),
cp_(cp) {}
+ /**
+ * Close will remove the connection from all caches.
+ */
folly::Future<folly::Unit> close() override {
if (!released.exchange(true)) {
return this->service_->close().then([this]() {
@@ -148,10 +157,13 @@ public:
}
}
+
+ /** Has this been closed */
virtual bool isAvailable() override {
return !released && service_->isAvailable();
}
+ /** Send the message. */
folly::Future<Response> operator()(unique_ptr<Request> req) override {
// TODO(eclark): add in an on error handler that will
// remove the region location from the cache if needed.
@@ -168,7 +180,7 @@ private:
std::shared_ptr<RegionLocation>
LocationCache::CreateLocation(const Response &resp) {
- auto resp_msg = static_pointer_cast<ScanResponse>(resp.response());
+ auto resp_msg = static_pointer_cast<ScanResponse>(resp.resp_msg());
auto &results = resp_msg->results().Get(0);
auto &cells = results.cell();
auto ri = folly::to<RegionInfo>(cells.Get(0).value());
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/core/location-cache.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h
index 7f76428..88bec18 100644
--- a/hbase-native-client/core/location-cache.h
+++ b/hbase-native-client/core/location-cache.h
@@ -33,25 +33,48 @@
#include "serde/table-name.h"
namespace hbase {
-
+// Forward
class Request;
class Response;
namespace pb {
class ServerName;
}
+/**
+ * Class that can look up and cache locations.
+ */
class LocationCache {
public:
- explicit LocationCache(std::string quorum_spec,
- std::shared_ptr<folly::Executor> executor);
+ /**
+ * Constructor.
+ * @param quorum_spec Where to connect for Zookeeper.
+ * @param executor The cpu executor to run on.
+ */
+ LocationCache(std::string quorum_spec,
+ std::shared_ptr<folly::Executor> executor);
+ /**
+ * Destructor.
+ * This will clean up the zookeeper connections.
+ */
~LocationCache();
- // Meta Related Methods.
- // These are only public until testing is complete
+
+ /**
+ * Where is meta hosted.
+ *
+ * TODO: This should be a RegionLocation.
+ */
folly::Future<hbase::pb::ServerName> LocateMeta();
+
+ /**
+ * Go read meta and find out where a region is located.
+ */
folly::Future<std::shared_ptr<RegionLocation>>
LocateFromMeta(const hbase::pb::TableName &tn, const std::string &row);
+
+ /**
+ * Remove the cached location of meta.
+ */
void InvalidateMeta();
- ConnectionPool cp_;
private:
void RefreshMetaLocation();
@@ -63,6 +86,7 @@ private:
std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> meta_promise_;
std::mutex meta_lock_;
MetaUtil meta_util_;
+ ConnectionPool cp_;
// TODO: migrate this to a smart pointer with a deleter.
zhandle_t *zk_;
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/core/meta-utils.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/meta-utils.h b/hbase-native-client/core/meta-utils.h
index dfef065..f6cc84f 100644
--- a/hbase-native-client/core/meta-utils.h
+++ b/hbase-native-client/core/meta-utils.h
@@ -26,10 +26,22 @@
namespace hbase {
+/**
+ * @brief Utility for meta operations.
+ */
class MetaUtil {
public:
+ /**
+ * Given a table and a row give the row key from which to start a scan to find
+ * region locations.
+ */
std::string RegionLookupRowkey(const hbase::pb::TableName &tn,
const std::string &row) const;
+
+ /**
+ * Given a row we're trying to access create a request to look up the
+ * location.
+ */
std::unique_ptr<Request> MetaRequest(const hbase::pb::TableName tn,
const std::string &row) const;
};
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/core/put.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/put.cc b/hbase-native-client/core/put.cc
deleted file mode 100644
index 806a478..0000000
--- a/hbase-native-client/core/put.cc
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include "core/put.h"
-
-Put::~Put() {}
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/core/put.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/put.h b/hbase-native-client/core/put.h
deleted file mode 100644
index d34aa0d..0000000
--- a/hbase-native-client/core/put.h
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#pragma once
-
-#include "core/mutation.h"
-
-class Put : public Mutation {
-public:
- ~Put();
-};
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/core/region-location.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/region-location.h b/hbase-native-client/core/region-location.h
index 7887526..a3d832e 100644
--- a/hbase-native-client/core/region-location.h
+++ b/hbase-native-client/core/region-location.h
@@ -25,19 +25,60 @@
namespace hbase {
+/**
+ * @brief class to hold where a region is located.
+ *
+ * This class holds where a region is located, the information about it, the
+ * region name, and a connection to the service used for connecting to it.
+ */
class RegionLocation {
public:
+ /**
+ * Constructor.
+ * @param region_name The region name of this region.
+ * @param ri The decoded RegionInfo of this region.
+ * @param sn The server name of the HBase regionserver thought to be hosting
+ * this region.
+ * @param service the connected service to the regionserver.
+ */
RegionLocation(std::string region_name, hbase::pb::RegionInfo ri,
hbase::pb::ServerName sn,
std::shared_ptr<HBaseService> service)
: region_name_(region_name), ri_(ri), sn_(sn), service_(service) {}
+ /**
+ * Get a reference to the regio info
+ */
const hbase::pb::RegionInfo ®ion_info() { return ri_; }
+
+ /**
+ * Get a reference to the server name
+ */
const hbase::pb::ServerName &server_name() { return sn_; }
+
+ /**
+ * Get a reference to the region name.
+ */
const std::string ®ion_name() { return region_name_; }
+
+ /**
+ * Get a service. This could be closed or null. It's the caller's
+ * responsibility to check.
+ */
std::shared_ptr<HBaseService> service() { return service_; }
+
+ /**
+ * Set the service.
+ * This should be used if the region moved or if the connection is thought to
+ * be bad and a new tcp connection needs to be made.
+ */
void set_service(std::shared_ptr<HBaseService> s) { service_ = s; }
+ /**
+ * Set the servername if the region has moved.
+ */
+ void set_server_name(hbase::pb::ServerName sn) { sn_ = sn; }
+
private:
std::string region_name_;
hbase::pb::RegionInfo ri_;
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/serde/rpc.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/rpc.cc b/hbase-native-client/serde/rpc.cc
index 4c3c999..b573738 100644
--- a/hbase-native-client/serde/rpc.cc
+++ b/hbase-native-client/serde/rpc.cc
@@ -85,7 +85,6 @@ int RpcSerde::ParseDelimited(const IOBuf *buf, Message *msg) {
}
RpcSerde::RpcSerde() : auth_type_(DEFAULT_AUTH_TYPE) {}
-RpcSerde::~RpcSerde() {}
unique_ptr<IOBuf> RpcSerde::Preamble() {
auto magic = IOBuf::copyBuffer(PREAMBLE, 0, 2);
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/serde/rpc.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/rpc.h b/hbase-native-client/serde/rpc.h
index cefb583..b28b4a3 100644
--- a/hbase-native-client/serde/rpc.h
+++ b/hbase-native-client/serde/rpc.h
@@ -32,22 +32,81 @@ class Message;
}
namespace hbase {
+
+/**
+ * @brief Class for serializing a deserializing rpc formatted data.
+ *
+ * RpcSerde is the one stop shop for reading/writing data to HBase daemons.
+ * It should throw exceptions if anything goes wrong.
+ */
class RpcSerde {
public:
+ /**
+ * Constructor assumes the default auth type.
+ */
RpcSerde();
- virtual ~RpcSerde();
+
+ /**
+ * Destructor. This is provided just for testing purposes.
+ */
+ virtual ~RpcSerde() = default;
+
+ /**
+ * Pase a message in the delimited format.
+ *
+ * A message in delimited format consists of the following:
+ *
+ * - a protobuf var int32.
+ * - A protobuf object serialized.
+ */
int ParseDelimited(const folly::IOBuf *buf, google::protobuf::Message *msg);
+
+ /**
+ * Create a new connection preamble in a new IOBuf.
+ */
std::unique_ptr<folly::IOBuf> Preamble();
+
+ /**
+ * Create the header protobuf object and serialize it to a new IOBuf.
+ * Header is in the following format:
+ *
+ * - Big endian length
+ * - ConnectionHeader object serialized out.
+ */
std::unique_ptr<folly::IOBuf> Header(const std::string &user);
+
+ /**
+ * Serialize a request message into a protobuf.
+ * Request consists of:
+ *
+ * - Big endian length
+ * - RequestHeader object
+ * - The passed in Message object
+ */
std::unique_ptr<folly::IOBuf> Request(const uint32_t call_id,
const std::string &method,
const google::protobuf::Message *msg);
+
+ /**
+ * Serialize a message in the delimited format.
+ * Delimited format consists of the following:
+ *
+ * - A protobuf var int32
+ * - The message object seriailized after that.
+ */
std::unique_ptr<folly::IOBuf>
SerializeDelimited(const google::protobuf::Message &msg);
+ /**
+ * Serilalize a message. This does not add any length prepend.
+ */
std::unique_ptr<folly::IOBuf>
SerializeMessage(const google::protobuf::Message &msg);
+ /**
+ * Prepend a length IOBuf to the given IOBuf chain.
+ * This involves no copies or moves of the passed in data.
+ */
std::unique_ptr<folly::IOBuf>
PrependLength(std::unique_ptr<folly::IOBuf> msg);
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/serde/zk.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/zk.h b/hbase-native-client/serde/zk.h
index b672bf4..fa3de5b 100644
--- a/hbase-native-client/serde/zk.h
+++ b/hbase-native-client/serde/zk.h
@@ -28,8 +28,22 @@ class IOBuf;
}
namespace hbase {
+
+/** @brief A class to convert data from ZooKeeper to other formats.
+ *
+ * This class will convert data to and from Zookeeper into protobuf objects.
+ *
+ */
class ZkDeserializer {
public:
+ /**
+ * Merge the data from a buffer into a given message.
+ *
+ * @param buf Naked pointer to iobuf containing data read from zookeeper.
+ * @param out Naked pointer into which the data will be merged. The message
+ * should be the correct type.
+ * @return returns true if the parsing was successful.
+ */
bool Parse(folly::IOBuf *buf, google::protobuf::Message *out);
};
} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/fdf00239/hbase-native-client/utils/user-util.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/utils/user-util.h b/hbase-native-client/utils/user-util.h
index da95d78..0b4cc73 100644
--- a/hbase-native-client/utils/user-util.h
+++ b/hbase-native-client/utils/user-util.h
@@ -24,12 +24,30 @@
#include <mutex>
namespace hbase {
+
+/**
+ * @brief Class to help with user/group information.
+ *
+ * This class will get the current user, and information about them. It caches
+ * the user information after the first invocation.
+ */
class UserUtil {
public:
+ /**
+ * Constructor.
+ */
UserUtil();
+
+ /**
+ * Get the username of the user owning this process. This is thread safe and
+ * lockless for every invocation other than the first one.
+ */
std::string user_name();
private:
+ /**
+ * Compute the username. This will block.
+ */
void compute_user_name();
std::atomic<bool> init_;
std::string user_name_;
[32/50] [abbrv] hbase git commit: HBASE-15774 Fix Upgrade lock usage
in connection pool.
Posted by ec...@apache.org.
HBASE-15774 Fix Upgrade lock usage in connection pool.
Summary:
Use upgrade lock better.
Fix a clang warning around initializing the UpgradeHolder incorrectly.
Remove dead code. ( We'l need to add it back when there's a better plan)
Add on more comments.
Test Plan: buck test --all
Differential Revision: https://reviews.facebook.net/D58005
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f6ea4937
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f6ea4937
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f6ea4937
Branch: refs/heads/HBASE-14850
Commit: f6ea49374a552bc321e6954fc3e36587cbf4740f
Parents: eb4cde4
Author: Elliott Clark <ec...@apache.org>
Authored: Tue May 10 17:44:41 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
.../connection/connection-pool-test.cc | 12 ++--
.../connection/connection-pool.cc | 47 +++++++++++---
.../connection/connection-pool.h | 6 +-
hbase-native-client/core/client.h | 2 +-
hbase-native-client/core/location-cache.cc | 68 ++++----------------
5 files changed, 62 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f6ea4937/hbase-native-client/connection/connection-pool-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-pool-test.cc b/hbase-native-client/connection/connection-pool-test.cc
index c0c346f..bd2d585 100644
--- a/hbase-native-client/connection/connection-pool-test.cc
+++ b/hbase-native-client/connection/connection-pool-test.cc
@@ -79,9 +79,9 @@ TEST(TestConnectionPool, TestOnlyCreateOnce) {
sn.set_host_name(hostname);
sn.set_port(port);
- auto result = cp.get(sn);
+ auto result = cp.Get(sn);
ASSERT_TRUE(result != nullptr);
- result = cp.get(sn);
+ result = cp.Get(sn);
}
TEST(TestConnectionPool, TestOnlyCreateMultipleDispose) {
@@ -102,13 +102,13 @@ TEST(TestConnectionPool, TestOnlyCreateMultipleDispose) {
ConnectionPool cp{mock_cf};
{
- auto result_one = cp.get(folly::to<ServerName>(
+ auto result_one = cp.Get(folly::to<ServerName>(
hostname_one + ":" + folly::to<std::string>(port)));
- auto result_two = cp.get(folly::to<ServerName>(
+ auto result_two = cp.Get(folly::to<ServerName>(
hostname_two + ":" + folly::to<std::string>(port)));
}
- auto result_one = cp.get(
+ auto result_one = cp.Get(
folly::to<ServerName>(hostname_one + ":" + folly::to<std::string>(port)));
- auto result_two = cp.get(
+ auto result_two = cp.Get(
folly::to<ServerName>(hostname_two + ":" + folly::to<std::string>(port)));
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f6ea4937/hbase-native-client/connection/connection-pool.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-pool.cc b/hbase-native-client/connection/connection-pool.cc
index 90e2056..aa3d094 100644
--- a/hbase-native-client/connection/connection-pool.cc
+++ b/hbase-native-client/connection/connection-pool.cc
@@ -48,25 +48,54 @@ ConnectionPool::~ConnectionPool() {
clients_.clear();
}
-std::shared_ptr<HBaseService> ConnectionPool::get(const ServerName &sn) {
- // Create a read lock.
- SharedMutexWritePriority::UpgradeHolder holder(map_mutex_);
+std::shared_ptr<HBaseService> ConnectionPool::Get(const ServerName &sn) {
+ // Try and get th cached connection.
+ auto found_ptr = GetCached(sn);
+ // If there's no connection then create it.
+ if (found_ptr == nullptr) {
+ found_ptr = GetNew(sn);
+ }
+ return found_ptr;
+}
+
+std::shared_ptr<HBaseService> ConnectionPool::GetCached(const ServerName &sn) {
+ SharedMutexWritePriority::ReadHolder holder(map_mutex_);
auto found = connections_.find(sn);
- if (found == connections_.end() || found->second == nullptr) {
- // Move the upgradable lock into the write lock if the connection
- // hasn't been found.
- SharedMutexWritePriority::WriteHolder holder(std::move(holder));
+ if (found == connections_.end()) {
+ return nullptr;
+ }
+ return found->second;
+}
+
+std::shared_ptr<HBaseService> ConnectionPool::GetNew(const ServerName &sn) {
+ // Grab the upgrade lock. While we are double checking other readers can
+ // continue on
+ SharedMutexWritePriority::UpgradeHolder u_holder{map_mutex_};
+
+ // Now check if someone else created the connection before we got the lock
+ // This is safe since we hold the upgrade lock.
+ // upgrade lock is more power than the reader lock.
+ auto found = connections_.find(sn);
+ if (found != connections_.end() && found->second != nullptr) {
+ return found->second;
+ } else {
+ // Yeah it looks a lot like there's no connection
+ SharedMutexWritePriority::WriteHolder w_holder{std::move(u_holder)};
+
+ // Make double sure there are not stale connections hanging around.
+ connections_.erase(sn);
+
+ // Nope we are the ones who should create the new connection.
auto client = cf_->MakeBootstrap();
auto dispatcher = cf_->Connect(client, sn.host_name(), sn.port());
clients_.insert(std::make_pair(sn, client));
connections_.insert(std::make_pair(sn, dispatcher));
return dispatcher;
}
- return found->second;
}
-void ConnectionPool::close(const ServerName &sn) {
+void ConnectionPool::Close(const ServerName &sn) {
SharedMutexWritePriority::WriteHolder holder{map_mutex_};
auto found = connections_.find(sn);
http://git-wip-us.apache.org/repos/asf/hbase/blob/f6ea4937/hbase-native-client/connection/connection-pool.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-pool.h b/hbase-native-client/connection/connection-pool.h
index 60f00de..605a81b 100644
--- a/hbase-native-client/connection/connection-pool.h
+++ b/hbase-native-client/connection/connection-pool.h
@@ -80,14 +80,16 @@ public:
* Get a connection to the server name. Start time is ignored.
* This can be a blocking operation for a short time.
*/
- std::shared_ptr<HBaseService> get(const hbase::pb::ServerName &sn);
+ std::shared_ptr<HBaseService> Get(const hbase::pb::ServerName &sn);
/**
* Close/remove a connection.
*/
- void close(const hbase::pb::ServerName &sn);
+ void Close(const hbase::pb::ServerName &sn);
private:
+ std::shared_ptr<HBaseService> GetCached(const hbase::pb::ServerName& sn);
+ std::shared_ptr<HBaseService> GetNew(const hbase::pb::ServerName& sn);
std::unordered_map<hbase::pb::ServerName, std::shared_ptr<HBaseService>,
ServerNameHash, ServerNameEquals>
connections_;
http://git-wip-us.apache.org/repos/asf/hbase/blob/f6ea4937/hbase-native-client/core/client.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client.h b/hbase-native-client/core/client.h
index 4db82c4..ba24bb9 100644
--- a/hbase-native-client/core/client.h
+++ b/hbase-native-client/core/client.h
@@ -48,9 +48,9 @@ public:
~Client();
private:
- LocationCache location_cache_;
std::shared_ptr<wangle::CPUThreadPoolExecutor> cpu_executor_;
std::shared_ptr<wangle::IOThreadPoolExecutor> io_executor_;
+ LocationCache location_cache_;
};
} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/f6ea4937/hbase-native-client/core/location-cache.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
index 9f2a0ef..6ba8add 100644
--- a/hbase-native-client/core/location-cache.cc
+++ b/hbase-native-client/core/location-cache.cc
@@ -115,7 +115,7 @@ Future<std::shared_ptr<RegionLocation>>
LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
return this->LocateMeta()
.via(cpu_executor_.get())
- .then([this](ServerName sn) { return this->cp_.get(sn); })
+ .then([this](ServerName sn) { return this->cp_.Get(sn); })
.then([tn, row, this](std::shared_ptr<HBaseService> service) {
return (*service)(std::move(meta_util_.MetaRequest(tn, row)));
})
@@ -134,67 +134,25 @@ LocationCache::LocateFromMeta(const TableName &tn, const string &row) {
})
.then([this](std::shared_ptr<RegionLocation> rl) {
// Now fill out the connection.
- rl->set_service(cp_.get(rl->server_name()));
+ rl->set_service(cp_.Get(rl->server_name()));
return rl;
});
}
-/**
- * Filter to remove a service from the location cache and the connection cache
- * on errors
- * or on cloase.
- */
-class RemoveServiceFilter
- : public ServiceFilter<std::unique_ptr<Request>, Response> {
-
-public:
- /** Create a new filter. */
- RemoveServiceFilter(std::shared_ptr<HBaseService> service, ServerName sn,
- ConnectionPool &cp)
- : ServiceFilter<unique_ptr<Request>, Response>(service), sn_(sn),
- cp_(cp) {}
-
- /**
- * Close will remove the connection from all caches.
- */
- folly::Future<folly::Unit> close() override {
- if (!released.exchange(true)) {
- return this->service_->close().then([this]() {
- // TODO(eclark): remove the service from the meta cache.
- this->cp_.close(this->sn_);
- });
- } else {
- return folly::makeFuture();
- }
- }
-
- /** Has this been closed */
- virtual bool isAvailable() override {
- return !released && service_->isAvailable();
- }
-
- /** Send the message. */
- folly::Future<Response> operator()(unique_ptr<Request> req) override {
- // TODO(eclark): add in an on error handler that will
- // remove the region location from the cache if needed.
- // Also close the connection if this is likely to be an error
- // that needs to get a new connection.
- return (*this->service_)(std::move(req));
- }
-
-private:
- std::atomic<bool> released{false};
- hbase::pb::ServerName sn_;
- ConnectionPool &cp_;
-};
-
std::shared_ptr<RegionLocation>
LocationCache::CreateLocation(const Response &resp) {
auto resp_msg = static_pointer_cast<ScanResponse>(resp.resp_msg());
auto &results = resp_msg->results().Get(0);
auto &cells = results.cell();
- auto ri = folly::to<RegionInfo>(cells.Get(0).value());
- auto sn = folly::to<ServerName>(cells.Get(1).value());
- return std::make_shared<RegionLocation>(cells.Get(0).row(), std::move(ri), sn,
- nullptr);
+
+ // TODO(eclark): There should probably be some better error
+ // handling around this.
+ auto cell_zero = cells.Get(0).value();
+ auto cell_one = cells.Get(1).value();
+ auto row = cells.Get(0).row();
+
+ auto region_info = folly::to<RegionInfo>(cell_zero);
+ auto server_name = folly::to<ServerName>(cell_one);
+ return std::make_shared<RegionLocation>(row, std::move(region_info),
+ server_name, nullptr);
}
[28/50] [abbrv] hbase git commit: HBASE-15620 Add on Call
serialization
Posted by ec...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/serde/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/BUCK b/hbase-native-client/serde/BUCK
new file mode 100644
index 0000000..207607f
--- /dev/null
+++ b/hbase-native-client/serde/BUCK
@@ -0,0 +1,54 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cxx_library(name="serde",
+ exported_headers=[
+ "client-serializer.h",
+ "client-deserializer.h",
+ ],
+ srcs=[
+ "client-serializer.cc",
+ "client-deserializer.cc",
+ ],
+ deps=[
+ "//if:if",
+ "//third-party:folly",
+ ],
+ tests=[
+ ":client-serializer-test",
+ ":client-deserializer-test",
+ ],
+ visibility=[
+ 'PUBLIC',
+ ], )
+
+cxx_test(name="client-serializer-test",
+ srcs=[
+ "client-serializer-test.cc",
+ ],
+ deps=[
+ ":serde",
+ "//if:if",
+ ], )
+cxx_test(name="client-deserializer-test",
+ srcs=[
+ "client-deserializer-test.cc",
+ ],
+ deps=[
+ ":serde",
+ "//if:if",
+ ], )
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/serde/client-deserializer-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/client-deserializer-test.cc b/hbase-native-client/serde/client-deserializer-test.cc
new file mode 100644
index 0000000..bb57e50
--- /dev/null
+++ b/hbase-native-client/serde/client-deserializer-test.cc
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <gtest/gtest.h>
+#include <folly/io/IOBuf.h>
+
+#include "serde/client-deserializer.h"
+#include "serde/client-serializer.h"
+#include "if/Client.pb.h"
+
+using namespace hbase;
+using folly::IOBuf;
+using hbase::pb::GetRequest;
+using hbase::pb::RegionSpecifier;
+using hbase::pb::RegionSpecifier_RegionSpecifierType;
+
+TEST(TestClientDeserializer, TestReturnFalseOnNullPtr) {
+ ClientDeserializer deser;
+ ASSERT_LT(deser.parse_delimited(nullptr, nullptr), 0);
+}
+
+TEST(TestClientDeserializer, TestReturnFalseOnBadInput) {
+ ClientDeserializer deser;
+ auto buf = IOBuf::copyBuffer("test");
+ GetRequest gr;
+
+ ASSERT_LT(deser.parse_delimited(buf.get(), &gr), 0);
+}
+
+TEST(TestClientDeserializer, TestGoodGetRequestFullRoundTrip) {
+ GetRequest in;
+ ClientSerializer ser;
+ ClientDeserializer deser;
+
+ // fill up the GetRequest.
+ in.mutable_region()->set_value("test_region_id");
+ in.mutable_region()->set_type(
+ RegionSpecifier_RegionSpecifierType::
+ RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
+ in.mutable_get()->set_row("test_row");
+
+ // Create the buffer
+ auto buf = ser.serialize_delimited(in);
+
+ GetRequest out;
+
+ int used_bytes = deser.parse_delimited(buf.get(), &out);
+
+ ASSERT_GT(used_bytes, 0);
+ ASSERT_EQ(used_bytes, buf->length());
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/serde/client-deserializer.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/client-deserializer.cc b/hbase-native-client/serde/client-deserializer.cc
new file mode 100644
index 0000000..118b0d1
--- /dev/null
+++ b/hbase-native-client/serde/client-deserializer.cc
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "serde/client-deserializer.h"
+
+#include <google/protobuf/message.h>
+#include <google/protobuf/io/coded_stream.h>
+#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
+#include <folly/Logging.h>
+
+using namespace hbase;
+
+using folly::IOBuf;
+using google::protobuf::Message;
+using google::protobuf::io::ArrayInputStream;
+using google::protobuf::io::CodedInputStream;
+
+int ClientDeserializer::parse_delimited(const IOBuf *buf, Message *msg) {
+ if (buf == nullptr || msg == nullptr) {
+ return -2;
+ }
+
+ DCHECK(!buf->isChained());
+
+ ArrayInputStream ais{buf->data(), static_cast<int>(buf->length())};
+ CodedInputStream coded_stream{&ais};
+
+ uint32_t msg_size;
+
+ // Try and read the varint.
+ if (coded_stream.ReadVarint32(&msg_size) == false) {
+ FB_LOG_EVERY_MS(ERROR, 1000) << "Unable to read a var uint32_t";
+ return -3;
+ }
+
+ coded_stream.PushLimit(msg_size);
+ // Parse the message.
+ if (msg->MergeFromCodedStream(&coded_stream) == false) {
+ FB_LOG_EVERY_MS(ERROR, 1000)
+ << "Unable to read a protobuf message from data.";
+ return -4;
+ }
+
+ // Make sure all the data was consumed.
+ if (coded_stream.ConsumedEntireMessage() == false) {
+ FB_LOG_EVERY_MS(ERROR, 1000)
+ << "Orphaned data left after reading protobuf message";
+ return -5;
+ }
+
+ return coded_stream.CurrentPosition();
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/serde/client-deserializer.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/client-deserializer.h b/hbase-native-client/serde/client-deserializer.h
new file mode 100644
index 0000000..b9664b0
--- /dev/null
+++ b/hbase-native-client/serde/client-deserializer.h
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <folly/io/IOBuf.h>
+
+// Forward
+namespace google {
+namespace protobuf {
+class Message;
+}
+}
+
+namespace hbase {
+class ClientDeserializer {
+public:
+ int parse_delimited(const folly::IOBuf *buf, google::protobuf::Message *msg);
+};
+
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/serde/client-serializer-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/client-serializer-test.cc b/hbase-native-client/serde/client-serializer-test.cc
new file mode 100644
index 0000000..b32b55d
--- /dev/null
+++ b/hbase-native-client/serde/client-serializer-test.cc
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include <gtest/gtest.h>
+
+#include <folly/io/Cursor.h>
+
+#include <string>
+
+#include "serde/client-serializer.h"
+#include "if/HBase.pb.h"
+#include "if/RPC.pb.h"
+
+using namespace hbase;
+using namespace hbase::pb;
+using namespace folly;
+using namespace folly::io;
+
+TEST(ClientSerializerTest, PreambleIncludesHBas) {
+ ClientSerializer ser;
+ auto buf = ser.preamble();
+ const char *p = reinterpret_cast<const char *>(buf->data());
+ // Take the first for chars and make sure they are the
+ // magic string
+ EXPECT_EQ("HBas", std::string(p, 4));
+
+ EXPECT_EQ(6, buf->computeChainDataLength());
+}
+
+TEST(ClientSerializerTest, PreambleIncludesVersion) {
+ ClientSerializer ser;
+ auto buf = ser.preamble();
+ EXPECT_EQ(0, static_cast<const uint8_t *>(buf->data())[4]);
+ EXPECT_EQ(80, static_cast<const uint8_t *>(buf->data())[5]);
+}
+
+TEST(ClientSerializerTest, TestHeaderLengthPrefixed) {
+ ClientSerializer ser;
+ auto header = ser.header("elliott");
+
+ // The header should be prefixed by 4 bytes of length.
+ EXPECT_EQ(4, header->length());
+ EXPECT_TRUE(header->length() < header->computeChainDataLength());
+ EXPECT_TRUE(header->isChained());
+
+ // Now make sure the length is correct.
+ Cursor cursor(header.get());
+ auto prefixed_len = cursor.readBE<uint32_t>();
+ EXPECT_EQ(prefixed_len, header->next()->length());
+}
+
+TEST(ClientSerializerTest, TestHeaderDecode) {
+ ClientSerializer ser;
+ auto buf = ser.header("elliott");
+ auto header_buf = buf->next();
+ ConnectionHeader h;
+
+ EXPECT_TRUE(h.ParseFromArray(header_buf->data(), header_buf->length()));
+ EXPECT_EQ("elliott", h.user_info().effective_user());
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/serde/client-serializer.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/client-serializer.cc b/hbase-native-client/serde/client-serializer.cc
new file mode 100644
index 0000000..881b6e4
--- /dev/null
+++ b/hbase-native-client/serde/client-serializer.cc
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#include "serde/client-serializer.h"
+
+#include <folly/io/Cursor.h>
+#include <folly/Logging.h>
+#include <google/protobuf/io/coded_stream.h>
+#include <google/protobuf/io/zero_copy_stream_impl_lite.h>
+
+#include "if/HBase.pb.h"
+#include "if/RPC.pb.h"
+
+using namespace hbase;
+
+using folly::IOBuf;
+using folly::io::RWPrivateCursor;
+using google::protobuf::Message;
+using google::protobuf::io::ArrayOutputStream;
+using google::protobuf::io::CodedOutputStream;
+using google::protobuf::io::ZeroCopyOutputStream;
+using std::string;
+using std::unique_ptr;
+
+static const std::string PREAMBLE = "HBas";
+static const std::string INTERFACE = "ClientService";
+static const uint8_t RPC_VERSION = 0;
+static const uint8_t DEFAULT_AUTH_TYPE = 80;
+
+ClientSerializer::ClientSerializer() : auth_type_(DEFAULT_AUTH_TYPE) {}
+
+unique_ptr<IOBuf> ClientSerializer::preamble() {
+ auto magic = IOBuf::copyBuffer(PREAMBLE, 0, 2);
+ magic->append(2);
+ RWPrivateCursor c(magic.get());
+ c.skip(4);
+ // Version
+ c.write(RPC_VERSION);
+ // Standard security aka Please don't lie to me.
+ c.write(auth_type_);
+ return magic;
+}
+
+unique_ptr<IOBuf> ClientSerializer::header(const string &user) {
+ pb::ConnectionHeader h;
+
+ // TODO(eclark): Make this not a total lie.
+ h.mutable_user_info()->set_effective_user(user);
+ // The service name that we want to talk to.
+ //
+ // Right now we're completely ignoring the service interface.
+ // That may or may not be the correct thing to do.
+ // It worked for a while with the java client; until it
+ // didn't.
+ h.set_service_name(INTERFACE);
+ return prepend_length(serialize_message(h));
+}
+
+unique_ptr<IOBuf> ClientSerializer::request(const uint32_t call_id,
+ const string &method,
+ const Message *msg) {
+ pb::RequestHeader rq;
+ rq.set_method_name(method);
+ rq.set_call_id(call_id);
+ rq.set_request_param(msg != nullptr);
+ auto ser_header = serialize_delimited(rq);
+ if (msg != nullptr) {
+ auto ser_req = serialize_delimited(*msg);
+ ser_header->appendChain(std::move(ser_req));
+ }
+
+ return prepend_length(std::move(ser_header));
+}
+
+unique_ptr<IOBuf> ClientSerializer::prepend_length(unique_ptr<IOBuf> msg) {
+ // Java ints are 4 long. So create a buffer that large
+ auto len_buf = IOBuf::create(4);
+ // Then make those bytes visible.
+ len_buf->append(4);
+
+ RWPrivateCursor c(len_buf.get());
+ // Get the size of the data to be pushed out the network.
+ auto size = msg->computeChainDataLength();
+
+ // Write the length to this IOBuf.
+ c.writeBE(static_cast<uint32_t>(size));
+
+ // Then attach the origional to the back of len_buf
+ len_buf->appendChain(std::move(msg));
+ return len_buf;
+}
+
+unique_ptr<IOBuf> ClientSerializer::serialize_delimited(const Message &msg) {
+ // Get the buffer size needed for just the message.
+ int msg_size = msg.ByteSize();
+ int buf_size = CodedOutputStream::VarintSize32(msg_size) + msg_size;
+
+ // Create a buffer big enough to hold the varint and the object.
+ auto buf = IOBuf::create(buf_size);
+ buf->append(buf_size);
+
+ // Create the array output stream.
+ ArrayOutputStream aos{buf->writableData(), static_cast<int>(buf->length())};
+ // Wrap the ArrayOuputStream in the coded output stream to allow writing
+ // Varint32
+ CodedOutputStream cos{&aos};
+
+ // Write out the size.
+ cos.WriteVarint32(msg_size);
+
+ // Now write the rest out.
+ // We're using the protobuf output streams here to keep track
+ // of where in the output array we are rather than IOBuf.
+ msg.SerializeWithCachedSizesToArray(
+ cos.GetDirectBufferForNBytesAndAdvance(msg_size));
+
+ // Return the buffer.
+ return buf;
+}
+// TODO(eclark): Make this 1 copy.
+unique_ptr<IOBuf> ClientSerializer::serialize_message(const Message &msg) {
+ auto buf = IOBuf::copyBuffer(msg.SerializeAsString());
+ return buf;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dc0f0175/hbase-native-client/serde/client-serializer.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/client-serializer.h b/hbase-native-client/serde/client-serializer.h
new file mode 100644
index 0000000..685095d
--- /dev/null
+++ b/hbase-native-client/serde/client-serializer.h
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <folly/io/IOBuf.h>
+#include <string>
+#include <cstdint>
+
+// Forward
+namespace google {
+namespace protobuf {
+class Message;
+}
+}
+namespace hbase {
+class Request;
+}
+
+namespace hbase {
+class ClientSerializer {
+public:
+ ClientSerializer();
+ std::unique_ptr<folly::IOBuf> preamble();
+ std::unique_ptr<folly::IOBuf> header(const std::string &user);
+ std::unique_ptr<folly::IOBuf> request(const uint32_t call_id,
+ const std::string &method,
+ const google::protobuf::Message *msg);
+ std::unique_ptr<folly::IOBuf>
+ serialize_delimited(const google::protobuf::Message &msg);
+
+ std::unique_ptr<folly::IOBuf>
+ serialize_message(const google::protobuf::Message &msg);
+
+ std::unique_ptr<folly::IOBuf>
+ prepend_length(std::unique_ptr<folly::IOBuf> msg);
+
+ uint8_t auth_type_;
+};
+} // namespace hbase
[25/50] [abbrv] hbase git commit: HBASE-15045 Keep
hbase-native-client/if and hbase-protocol in sync.
Posted by ec...@apache.org.
HBASE-15045 Keep hbase-native-client/if and hbase-protocol in sync.
Summary: On every start of docker make sure that protos are up to date.
Test Plan: Added the script and it updated the protos, but didn't delete BUCK.
Differential Revision: https://reviews.facebook.net/D56763
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bbeaacb6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bbeaacb6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bbeaacb6
Branch: refs/heads/HBASE-14850
Commit: bbeaacb6d43b29746abc78f31c422c7706938f8a
Parents: 098028c
Author: Elliott Clark <ec...@apache.org>
Authored: Wed Apr 13 15:59:13 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/bin/copy-protobuf.sh | 8 ++
hbase-native-client/bin/start-docker.sh | 22 +++-
hbase-native-client/if/Admin.proto | 1 +
hbase-native-client/if/Client.proto | 8 +-
hbase-native-client/if/ClusterStatus.proto | 3 +
hbase-native-client/if/Filter.proto | 1 +
hbase-native-client/if/Master.proto | 47 +++++++-
hbase-native-client/if/MasterProcedure.proto | 40 +++++++
hbase-native-client/if/RSGroup.proto | 34 ++++++
hbase-native-client/if/RSGroupAdmin.proto | 136 ++++++++++++++++++++++
hbase-native-client/if/ZooKeeper.proto | 13 +++
11 files changed, 304 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/bbeaacb6/hbase-native-client/bin/copy-protobuf.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/copy-protobuf.sh b/hbase-native-client/bin/copy-protobuf.sh
new file mode 100755
index 0000000..c9e70f2
--- /dev/null
+++ b/hbase-native-client/bin/copy-protobuf.sh
@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+
+BIN_DIR=$(dirname "$0")
+PB_SOURCE_DIR="${BIN_DIR}/../../hbase-protocol/src/main/protobuf/"
+PB_DEST_DIR="${BIN_DIR}/../if/"
+rsync -r --delete --exclude BUCK ${PB_SOURCE_DIR} ${PB_DEST_DIR}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bbeaacb6/hbase-native-client/bin/start-docker.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/start-docker.sh b/hbase-native-client/bin/start-docker.sh
index 725ed6a..9cbd8b7 100755
--- a/hbase-native-client/bin/start-docker.sh
+++ b/hbase-native-client/bin/start-docker.sh
@@ -23,22 +23,38 @@ set -x
eval "$(docker-machine env docker-vm)"
eval "$(docker-machine env dinghy)"
-# Build the image
-docker build -t hbase_native .
+BIN_DIR=$(pushd `dirname "$0"` 2>&1 > /dev/null && pwd && popd 2>&1 > /dev/null)
+BASE_DIR=$(pushd "${BIN_DIR}/../" 2>&1 > /dev/null && pwd && popd 2>&1 > /dev/null)
+${BIN_DIR}/copy-protobuf.sh
+# Go into the base dir. This just makes things cleaner.
+pushd ${BASE_DIR}
+
+# Make sure that there is a thrid-party dir.
mkdir third-party || true
+
+# Get gtest
+# TODO(eclark): Remove this ( see HBASE-15427 )
if [[ ! -d third-party/googletest ]]; then
git clone https://github.com/google/googletest.git third-party/googletest
fi
+# We don't want to have to re-download all the jars in docker if we can help it
if [[ ! -d ~/.m2 ]]; then
echo "~/.m2 directory doesn't exist. Check Apache Maven is installed."
exit 1
fi;
+# Build the image
+#
+# This shouldn't be needed after the development environment is a little more stable.
+docker build -t hbase_native .
+
+# After the image is built run the thing
docker run -p 16010:16010/tcp \
-e "JAVA_HOME=/usr/lib/jvm/java-8-oracle" \
- -v ${PWD}/..:/usr/src/hbase \
+ -v ${BASE_DIR}/..:/usr/src/hbase \
-v ~/.m2:/root/.m2 \
-it hbase_native /bin/bash
+popd
http://git-wip-us.apache.org/repos/asf/hbase/blob/bbeaacb6/hbase-native-client/if/Admin.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/Admin.proto b/hbase-native-client/if/Admin.proto
index e905340..a1905a4 100644
--- a/hbase-native-client/if/Admin.proto
+++ b/hbase-native-client/if/Admin.proto
@@ -25,6 +25,7 @@ option java_generic_services = true;
option java_generate_equals_and_hash = true;
option optimize_for = SPEED;
+import "Client.proto";
import "HBase.proto";
import "WAL.proto";
http://git-wip-us.apache.org/repos/asf/hbase/blob/bbeaacb6/hbase-native-client/if/Client.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/Client.proto b/hbase-native-client/if/Client.proto
index ca9abdc..8a4d459 100644
--- a/hbase-native-client/if/Client.proto
+++ b/hbase-native-client/if/Client.proto
@@ -402,6 +402,11 @@ message RegionLoadStats {
optional int32 compactionPressure = 3 [default = 0];
}
+message MultiRegionLoadStats{
+ repeated RegionSpecifier region = 1;
+ repeated RegionLoadStats stat = 2;
+}
+
/**
* Either a Result or an Exception NameBytesPair (keyed by
* exception name whose value is the exception stringified)
@@ -416,7 +421,7 @@ message ResultOrException {
// result if this was a coprocessor service call
optional CoprocessorServiceResult service_result = 4;
// current load on the region
- optional RegionLoadStats loadStats = 5;
+ optional RegionLoadStats loadStats = 5 [deprecated=true];
}
/**
@@ -445,6 +450,7 @@ message MultiResponse {
repeated RegionActionResult regionActionResult = 1;
// used for mutate to indicate processed only
optional bool processed = 2;
+ optional MultiRegionLoadStats regionStatistics = 3;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bbeaacb6/hbase-native-client/if/ClusterStatus.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/ClusterStatus.proto b/hbase-native-client/if/ClusterStatus.proto
index 228be7e..54bc0c3 100644
--- a/hbase-native-client/if/ClusterStatus.proto
+++ b/hbase-native-client/if/ClusterStatus.proto
@@ -136,6 +136,9 @@ message RegionLoad {
/** the most recent sequence Id of store from cache flush */
repeated StoreSequenceId store_complete_sequence_id = 18;
+
+ /** the current total filtered read requests made to region */
+ optional uint64 filtered_read_requests_count = 19;
}
/* Server-level protobufs */
http://git-wip-us.apache.org/repos/asf/hbase/blob/bbeaacb6/hbase-native-client/if/Filter.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/Filter.proto b/hbase-native-client/if/Filter.proto
index 67d5717..1fa6697 100644
--- a/hbase-native-client/if/Filter.proto
+++ b/hbase-native-client/if/Filter.proto
@@ -146,6 +146,7 @@ message SkipFilter {
message TimestampsFilter {
repeated int64 timestamps = 1 [packed=true];
+ optional bool can_hint = 2;
}
message ValueFilter {
http://git-wip-us.apache.org/repos/asf/hbase/blob/bbeaacb6/hbase-native-client/if/Master.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/Master.proto b/hbase-native-client/if/Master.proto
index 4d3a2e1..1f7a3b7 100644
--- a/hbase-native-client/if/Master.proto
+++ b/hbase-native-client/if/Master.proto
@@ -186,6 +186,7 @@ message CreateNamespaceRequest {
}
message CreateNamespaceResponse {
+ optional uint64 proc_id = 1;
}
message DeleteNamespaceRequest {
@@ -195,6 +196,7 @@ message DeleteNamespaceRequest {
}
message DeleteNamespaceResponse {
+ optional uint64 proc_id = 1;
}
message ModifyNamespaceRequest {
@@ -204,6 +206,7 @@ message ModifyNamespaceRequest {
}
message ModifyNamespaceResponse {
+ optional uint64 proc_id = 1;
}
message GetNamespaceDescriptorRequest {
@@ -276,6 +279,29 @@ message IsBalancerEnabledResponse {
required bool enabled = 1;
}
+enum MasterSwitchType {
+ SPLIT = 0;
+ MERGE = 1;
+}
+
+message SetSplitOrMergeEnabledRequest {
+ required bool enabled = 1;
+ optional bool synchronous = 2;
+ repeated MasterSwitchType switch_types = 3;
+}
+
+message SetSplitOrMergeEnabledResponse {
+ repeated bool prev_value = 1;
+}
+
+message IsSplitOrMergeEnabledRequest {
+ required MasterSwitchType switch_type = 1;
+}
+
+message IsSplitOrMergeEnabledResponse {
+ required bool enabled = 1;
+}
+
message NormalizeRequest {
}
@@ -344,9 +370,12 @@ message DeleteSnapshotResponse {
message RestoreSnapshotRequest {
required SnapshotDescription snapshot = 1;
+ optional uint64 nonce_group = 2 [default = 0];
+ optional uint64 nonce = 3 [default = 0];
}
message RestoreSnapshotResponse {
+ required uint64 proc_id = 1;
}
/* if you don't send the snapshot, then you will get it back
@@ -630,6 +659,19 @@ service MasterService {
returns(IsBalancerEnabledResponse);
/**
+ * Turn the split or merge switch on or off.
+ * If synchronous is true, it waits until current operation call, if outstanding, to return.
+ */
+ rpc SetSplitOrMergeEnabled(SetSplitOrMergeEnabledRequest)
+ returns(SetSplitOrMergeEnabledResponse);
+
+ /**
+ * Query whether the split or merge switch is on/off.
+ */
+ rpc IsSplitOrMergeEnabled(IsSplitOrMergeEnabledRequest)
+ returns(IsSplitOrMergeEnabledResponse);
+
+ /**
* Run region normalizer. Can NOT run for various reasons. Check logs.
*/
rpc Normalize(NormalizeRequest)
@@ -696,11 +738,6 @@ service MasterService {
rpc RestoreSnapshot(RestoreSnapshotRequest) returns(RestoreSnapshotResponse);
/**
- * Determine if the snapshot restore is done yet.
- */
- rpc IsRestoreSnapshotDone(IsRestoreSnapshotDoneRequest) returns(IsRestoreSnapshotDoneResponse);
-
- /**
* Execute a distributed procedure.
*/
rpc ExecProcedure(ExecProcedureRequest) returns(ExecProcedureResponse);
http://git-wip-us.apache.org/repos/asf/hbase/blob/bbeaacb6/hbase-native-client/if/MasterProcedure.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/MasterProcedure.proto b/hbase-native-client/if/MasterProcedure.proto
index 2d2aff4..87aae6a 100644
--- a/hbase-native-client/if/MasterProcedure.proto
+++ b/hbase-native-client/if/MasterProcedure.proto
@@ -222,6 +222,46 @@ message DisableTableStateData {
required bool skip_table_state_check = 3;
}
+message RestoreParentToChildRegionsPair {
+ required string parent_region_name = 1;
+ required string child1_region_name = 2;
+ required string child2_region_name = 3;
+}
+
+enum CloneSnapshotState {
+ CLONE_SNAPSHOT_PRE_OPERATION = 1;
+ CLONE_SNAPSHOT_WRITE_FS_LAYOUT = 2;
+ CLONE_SNAPSHOT_ADD_TO_META = 3;
+ CLONE_SNAPSHOT_ASSIGN_REGIONS = 4;
+ CLONE_SNAPSHOT_UPDATE_DESC_CACHE = 5;
+ CLONE_SNAPSHOT_POST_OPERATION = 6;
+}
+
+message CloneSnapshotStateData {
+ required UserInformation user_info = 1;
+ required SnapshotDescription snapshot = 2;
+ required TableSchema table_schema = 3;
+ repeated RegionInfo region_info = 4;
+ repeated RestoreParentToChildRegionsPair parent_to_child_regions_pair_list = 5;
+}
+
+enum RestoreSnapshotState {
+ RESTORE_SNAPSHOT_PRE_OPERATION = 1;
+ RESTORE_SNAPSHOT_UPDATE_TABLE_DESCRIPTOR = 2;
+ RESTORE_SNAPSHOT_WRITE_FS_LAYOUT = 3;
+ RESTORE_SNAPSHOT_UPDATE_META = 4;
+}
+
+message RestoreSnapshotStateData {
+ required UserInformation user_info = 1;
+ required SnapshotDescription snapshot = 2;
+ required TableSchema modified_table_schema = 3;
+ repeated RegionInfo region_info_for_restore = 4;
+ repeated RegionInfo region_info_for_remove = 5;
+ repeated RegionInfo region_info_for_add = 6;
+ repeated RestoreParentToChildRegionsPair parent_to_child_regions_pair_list = 7;
+}
+
message ServerCrashStateData {
required ServerName server_name = 1;
optional bool distributed_log_replay = 2;
http://git-wip-us.apache.org/repos/asf/hbase/blob/bbeaacb6/hbase-native-client/if/RSGroup.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/RSGroup.proto b/hbase-native-client/if/RSGroup.proto
new file mode 100644
index 0000000..7358941
--- /dev/null
+++ b/hbase-native-client/if/RSGroup.proto
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "RSGroupProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+
+message RSGroupInfo {
+ required string name = 1;
+ repeated ServerName servers = 4;
+ repeated TableName tables = 3;
+}
+
http://git-wip-us.apache.org/repos/asf/hbase/blob/bbeaacb6/hbase-native-client/if/RSGroupAdmin.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/RSGroupAdmin.proto b/hbase-native-client/if/RSGroupAdmin.proto
new file mode 100644
index 0000000..fda9b09
--- /dev/null
+++ b/hbase-native-client/if/RSGroupAdmin.proto
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "RSGroupAdminProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "RSGroup.proto";
+
+/** Group level protobufs */
+
+message ListTablesOfRSGroupRequest {
+ required string r_s_group_name = 1;
+}
+
+message ListTablesOfRSGroupResponse {
+ repeated TableName table_name = 1;
+}
+
+message GetRSGroupInfoRequest {
+ required string r_s_group_name = 1;
+}
+
+message GetRSGroupInfoResponse {
+ optional RSGroupInfo r_s_group_info = 1;
+}
+
+message GetRSGroupInfoOfTableRequest {
+ required TableName table_name = 1;
+}
+
+message GetRSGroupInfoOfTableResponse {
+ optional RSGroupInfo r_s_group_info = 1;
+}
+
+message MoveServersRequest {
+ required string target_group = 1;
+ repeated ServerName servers = 3;
+}
+
+message MoveServersResponse {
+}
+
+message MoveTablesRequest {
+ required string target_group = 1;
+ repeated TableName table_name = 2;
+}
+
+message MoveTablesResponse {
+}
+
+message AddRSGroupRequest {
+ required string r_s_group_name = 1;
+}
+
+message AddRSGroupResponse {
+}
+
+message RemoveRSGroupRequest {
+ required string r_s_group_name = 1;
+}
+
+message RemoveRSGroupResponse {
+}
+
+message BalanceRSGroupRequest {
+ required string r_s_group_name = 1;
+}
+
+message BalanceRSGroupResponse {
+ required bool balanceRan = 1;
+}
+
+message ListRSGroupInfosRequest {
+}
+
+message ListRSGroupInfosResponse {
+ repeated RSGroupInfo r_s_group_info = 1;
+}
+
+message GetRSGroupInfoOfServerRequest {
+ required ServerName server = 2;
+}
+
+message GetRSGroupInfoOfServerResponse {
+ optional RSGroupInfo r_s_group_info = 1;
+}
+
+service RSGroupAdminService {
+ rpc GetRSGroupInfo(GetRSGroupInfoRequest)
+ returns (GetRSGroupInfoResponse);
+
+ rpc GetRSGroupInfoOfTable(GetRSGroupInfoOfTableRequest)
+ returns (GetRSGroupInfoOfTableResponse);
+
+ rpc GetRSGroupInfoOfServer(GetRSGroupInfoOfServerRequest)
+ returns (GetRSGroupInfoOfServerResponse);
+
+ rpc MoveServers(MoveServersRequest)
+ returns (MoveServersResponse);
+
+ rpc MoveTables(MoveTablesRequest)
+ returns (MoveTablesResponse);
+
+ rpc AddRSGroup(AddRSGroupRequest)
+ returns (AddRSGroupResponse);
+
+ rpc RemoveRSGroup(RemoveRSGroupRequest)
+ returns (RemoveRSGroupResponse);
+
+ rpc BalanceRSGroup(BalanceRSGroupRequest)
+ returns (BalanceRSGroupResponse);
+
+ rpc ListRSGroupInfos(ListRSGroupInfosRequest)
+ returns (ListRSGroupInfosResponse);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/bbeaacb6/hbase-native-client/if/ZooKeeper.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/ZooKeeper.proto b/hbase-native-client/if/ZooKeeper.proto
index 54652af..186d183 100644
--- a/hbase-native-client/if/ZooKeeper.proto
+++ b/hbase-native-client/if/ZooKeeper.proto
@@ -105,6 +105,11 @@ message DeprecatedTableState {
required State state = 1 [default = ENABLED];
}
+message TableCF {
+ optional TableName table_name = 1;
+ repeated bytes families = 2;
+}
+
/**
* Used by replication. Holds a replication peer key.
*/
@@ -115,6 +120,7 @@ message ReplicationPeer {
optional string replicationEndpointImpl = 2;
repeated BytesBytesPair data = 3;
repeated NameStringPair configuration = 4;
+ repeated TableCF table_cfs = 5;
}
/**
@@ -153,3 +159,10 @@ message TableLock {
optional string purpose = 5;
optional int64 create_time = 6;
}
+
+/**
+ * State of the switch.
+ */
+message SwitchState {
+ optional bool enabled = 1;
+}
\ No newline at end of file
[40/50] [abbrv] hbase git commit: HBASE-15655 Set up doxygen for
documentation
Posted by ec...@apache.org.
HBASE-15655 Set up doxygen for documentation
Summary:
Add on doxygen config.
It outputs to the docs folder.
Added on a makefile
Test Plan:
doxygen hbase.doxygen
Differential Revision: https://reviews.facebook.net/D57663
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f034294f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f034294f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f034294f
Branch: refs/heads/HBASE-14850
Commit: f034294f3a72bcb6c11330d6fa1f9b6c028f1f1c
Parents: 190fc7a
Author: Elliott Clark <ec...@apache.org>
Authored: Wed May 4 18:19:33 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
.gitignore | 1 +
hbase-native-client/Dockerfile | 2 +-
hbase-native-client/Makefile | 20 +++
hbase-native-client/hbase.doxygen | 319 +++++++++++++++++++++++++++++++++
4 files changed, 341 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f034294f/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index 626a4d0..1b1e9d5 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,3 +16,4 @@ hbase-*/test
*.ipr
patchprocess/
dependency-reduced-pom.xml
+docs/
http://git-wip-us.apache.org/repos/asf/hbase/blob/f034294f/hbase-native-client/Dockerfile
----------------------------------------------------------------------
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index 8e33e74..be0cbc6 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -24,7 +24,7 @@ ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O2 -p
ENV JAVA_HOME="/usr/lib/jvm/java-8-openjdk-amd64/"
-RUN apt-get install -y vim maven inetutils-ping python-pip && \
+RUN apt-get install -y vim maven inetutils-ping python-pip doxygen graphviz && \
pip install yapf && \
apt-get -qq clean && \
apt-get -y -qq autoremove && \
http://git-wip-us.apache.org/repos/asf/hbase/blob/f034294f/hbase-native-client/Makefile
----------------------------------------------------------------------
diff --git a/hbase-native-client/Makefile b/hbase-native-client/Makefile
new file mode 100644
index 0000000..90ec6ea
--- /dev/null
+++ b/hbase-native-client/Makefile
@@ -0,0 +1,20 @@
+build:
+ $(shell buck build core/... )
+
+check:
+ $(shell buck test --all --no-results-cache )
+
+doc:
+ $(shell doxygen hbase.doxygen > /dev/null )
+
+clean:
+ $(shell rm -rf docs )
+
+help:
+ @echo "Available targets:"
+ @echo ""
+ @echo " build : will build everything."
+ @echo " clean : will remove the docs folder"
+ @echo " check : will test everything."
+
+all: build doc check
http://git-wip-us.apache.org/repos/asf/hbase/blob/f034294f/hbase-native-client/hbase.doxygen
----------------------------------------------------------------------
diff --git a/hbase-native-client/hbase.doxygen b/hbase-native-client/hbase.doxygen
new file mode 100644
index 0000000..22e2d21
--- /dev/null
+++ b/hbase-native-client/hbase.doxygen
@@ -0,0 +1,319 @@
+# Doxyfile 1.8.11
+
+#---------------------------------------------------------------------------
+# Project related configuration options
+#---------------------------------------------------------------------------
+DOXYFILE_ENCODING = UTF-8
+PROJECT_NAME = "HBase Native Client"
+PROJECT_NUMBER =
+PROJECT_BRIEF =
+PROJECT_LOGO = ../hbase-server/src/main/resources/hbase-webapps/static/hbase_logo_small.png
+OUTPUT_DIRECTORY = docs
+CREATE_SUBDIRS = YES
+ALLOW_UNICODE_NAMES = NO
+OUTPUT_LANGUAGE = English
+BRIEF_MEMBER_DESC = YES
+REPEAT_BRIEF = YES
+ABBREVIATE_BRIEF =
+ALWAYS_DETAILED_SEC = NO
+INLINE_INHERITED_MEMB = NO
+FULL_PATH_NAMES = NO
+STRIP_FROM_PATH =
+STRIP_FROM_INC_PATH =
+SHORT_NAMES = NO
+JAVADOC_AUTOBRIEF = NO
+QT_AUTOBRIEF = NO
+MULTILINE_CPP_IS_BRIEF = NO
+INHERIT_DOCS = YES
+SEPARATE_MEMBER_PAGES = NO
+TAB_SIZE = 4
+ALIASES =
+TCL_SUBST =
+OPTIMIZE_OUTPUT_FOR_C = NO
+OPTIMIZE_OUTPUT_JAVA = NO
+OPTIMIZE_FOR_FORTRAN = NO
+OPTIMIZE_OUTPUT_VHDL = NO
+EXTENSION_MAPPING =
+MARKDOWN_SUPPORT = YES
+AUTOLINK_SUPPORT = YES
+BUILTIN_STL_SUPPORT = NO
+CPP_CLI_SUPPORT = NO
+SIP_SUPPORT = NO
+IDL_PROPERTY_SUPPORT = YES
+DISTRIBUTE_GROUP_DOC = NO
+GROUP_NESTED_COMPOUNDS = NO
+SUBGROUPING = YES
+INLINE_GROUPED_CLASSES = NO
+INLINE_SIMPLE_STRUCTS = NO
+TYPEDEF_HIDES_STRUCT = NO
+LOOKUP_CACHE_SIZE = 0
+#---------------------------------------------------------------------------
+# Build related configuration options
+#---------------------------------------------------------------------------
+EXTRACT_ALL = NO
+EXTRACT_PRIVATE = NO
+EXTRACT_PACKAGE = NO
+EXTRACT_STATIC = NO
+EXTRACT_LOCAL_CLASSES = YES
+EXTRACT_LOCAL_METHODS = NO
+EXTRACT_ANON_NSPACES = NO
+HIDE_UNDOC_MEMBERS = NO
+HIDE_UNDOC_CLASSES = NO
+HIDE_FRIEND_COMPOUNDS = NO
+HIDE_IN_BODY_DOCS = NO
+INTERNAL_DOCS = NO
+CASE_SENSE_NAMES = YES
+HIDE_SCOPE_NAMES = NO
+HIDE_COMPOUND_REFERENCE= NO
+SHOW_INCLUDE_FILES = YES
+SHOW_GROUPED_MEMB_INC = NO
+FORCE_LOCAL_INCLUDES = NO
+INLINE_INFO = YES
+SORT_MEMBER_DOCS = YES
+SORT_BRIEF_DOCS = NO
+SORT_MEMBERS_CTORS_1ST = NO
+SORT_GROUP_NAMES = NO
+SORT_BY_SCOPE_NAME = NO
+STRICT_PROTO_MATCHING = NO
+GENERATE_TODOLIST = YES
+GENERATE_TESTLIST = YES
+GENERATE_BUGLIST = YES
+GENERATE_DEPRECATEDLIST= YES
+ENABLED_SECTIONS =
+MAX_INITIALIZER_LINES = 30
+SHOW_USED_FILES = YES
+SHOW_FILES = YES
+SHOW_NAMESPACES = YES
+FILE_VERSION_FILTER =
+LAYOUT_FILE =
+CITE_BIB_FILES =
+#---------------------------------------------------------------------------
+# Configuration options related to warning and progress messages
+#---------------------------------------------------------------------------
+QUIET = NO
+WARNINGS = YES
+WARN_IF_UNDOCUMENTED = YES
+WARN_IF_DOC_ERROR = YES
+WARN_NO_PARAMDOC = NO
+WARN_AS_ERROR = NO
+WARN_FORMAT = "$file:$line: $text"
+WARN_LOGFILE =
+#---------------------------------------------------------------------------
+# Configuration options related to the input files
+#---------------------------------------------------------------------------
+INPUT =
+INPUT_ENCODING = UTF-8
+FILE_PATTERNS =
+RECURSIVE = YES
+EXCLUDE = third-party/ buck-out/
+EXCLUDE_SYMLINKS = NO
+EXCLUDE_PATTERNS = /*-test.cc
+EXCLUDE_SYMBOLS =
+EXAMPLE_PATH =
+EXAMPLE_PATTERNS =
+EXAMPLE_RECURSIVE = NO
+IMAGE_PATH =
+INPUT_FILTER =
+FILTER_PATTERNS =
+FILTER_SOURCE_FILES = NO
+FILTER_SOURCE_PATTERNS =
+USE_MDFILE_AS_MAINPAGE =
+#---------------------------------------------------------------------------
+# Configuration options related to source browsing
+#---------------------------------------------------------------------------
+SOURCE_BROWSER = NO
+INLINE_SOURCES = NO
+STRIP_CODE_COMMENTS = YES
+REFERENCED_BY_RELATION = NO
+REFERENCES_RELATION = NO
+REFERENCES_LINK_SOURCE = YES
+SOURCE_TOOLTIPS = YES
+USE_HTAGS = NO
+VERBATIM_HEADERS = YES
+CLANG_ASSISTED_PARSING = NO
+CLANG_OPTIONS =
+#---------------------------------------------------------------------------
+# Configuration options related to the alphabetical class index
+#---------------------------------------------------------------------------
+ALPHABETICAL_INDEX = YES
+COLS_IN_ALPHA_INDEX = 5
+IGNORE_PREFIX =
+#---------------------------------------------------------------------------
+# Configuration options related to the HTML output
+#---------------------------------------------------------------------------
+GENERATE_HTML = YES
+HTML_OUTPUT = html
+HTML_FILE_EXTENSION = .html
+HTML_HEADER =
+HTML_FOOTER =
+HTML_STYLESHEET =
+HTML_EXTRA_STYLESHEET =
+HTML_EXTRA_FILES =
+HTML_COLORSTYLE_HUE = 220
+HTML_COLORSTYLE_SAT = 100
+HTML_COLORSTYLE_GAMMA = 80
+HTML_TIMESTAMP = NO
+HTML_DYNAMIC_SECTIONS = NO
+HTML_INDEX_NUM_ENTRIES = 100
+GENERATE_DOCSET = NO
+DOCSET_FEEDNAME = "Doxygen generated docs"
+DOCSET_BUNDLE_ID = org.doxygen.Project
+DOCSET_PUBLISHER_ID = org.doxygen.Publisher
+DOCSET_PUBLISHER_NAME = Publisher
+GENERATE_HTMLHELP = NO
+CHM_FILE =
+HHC_LOCATION =
+GENERATE_CHI = NO
+CHM_INDEX_ENCODING =
+BINARY_TOC = NO
+TOC_EXPAND = NO
+GENERATE_QHP = NO
+QCH_FILE =
+QHP_NAMESPACE = org.doxygen.Project
+QHP_VIRTUAL_FOLDER = doc
+QHP_CUST_FILTER_NAME =
+QHP_CUST_FILTER_ATTRS =
+QHP_SECT_FILTER_ATTRS =
+QHG_LOCATION =
+GENERATE_ECLIPSEHELP = NO
+ECLIPSE_DOC_ID = org.doxygen.Project
+DISABLE_INDEX = NO
+GENERATE_TREEVIEW = NO
+ENUM_VALUES_PER_LINE = 4
+TREEVIEW_WIDTH = 250
+EXT_LINKS_IN_WINDOW = NO
+FORMULA_FONTSIZE = 10
+FORMULA_TRANSPARENT = YES
+USE_MATHJAX = NO
+MATHJAX_FORMAT = HTML-CSS
+MATHJAX_RELPATH = http://cdn.mathjax.org/mathjax/latest
+MATHJAX_EXTENSIONS =
+MATHJAX_CODEFILE =
+SEARCHENGINE = YES
+SERVER_BASED_SEARCH = NO
+EXTERNAL_SEARCH = NO
+SEARCHENGINE_URL =
+SEARCHDATA_FILE = searchdata.xml
+EXTERNAL_SEARCH_ID =
+EXTRA_SEARCH_MAPPINGS =
+#---------------------------------------------------------------------------
+# Configuration options related to the LaTeX output
+#---------------------------------------------------------------------------
+GENERATE_LATEX = YES
+LATEX_OUTPUT = latex
+LATEX_CMD_NAME = latex
+MAKEINDEX_CMD_NAME = makeindex
+COMPACT_LATEX = NO
+PAPER_TYPE = a4
+EXTRA_PACKAGES =
+LATEX_HEADER =
+LATEX_FOOTER =
+LATEX_EXTRA_STYLESHEET =
+LATEX_EXTRA_FILES =
+PDF_HYPERLINKS = YES
+USE_PDFLATEX = YES
+LATEX_BATCHMODE = NO
+LATEX_HIDE_INDICES = NO
+LATEX_SOURCE_CODE = NO
+LATEX_BIB_STYLE = plain
+LATEX_TIMESTAMP = NO
+#---------------------------------------------------------------------------
+# Configuration options related to the RTF output
+#---------------------------------------------------------------------------
+GENERATE_RTF = NO
+RTF_OUTPUT = rtf
+COMPACT_RTF = NO
+RTF_HYPERLINKS = NO
+RTF_STYLESHEET_FILE =
+RTF_EXTENSIONS_FILE =
+RTF_SOURCE_CODE = NO
+#---------------------------------------------------------------------------
+# Configuration options related to the man page output
+#---------------------------------------------------------------------------
+GENERATE_MAN = NO
+MAN_OUTPUT = man
+MAN_EXTENSION = .3
+MAN_SUBDIR =
+MAN_LINKS = NO
+#---------------------------------------------------------------------------
+# Configuration options related to the XML output
+#---------------------------------------------------------------------------
+GENERATE_XML = NO
+XML_OUTPUT = xml
+XML_PROGRAMLISTING = YES
+#---------------------------------------------------------------------------
+# Configuration options related to the DOCBOOK output
+#---------------------------------------------------------------------------
+GENERATE_DOCBOOK = NO
+DOCBOOK_OUTPUT = docbook
+DOCBOOK_PROGRAMLISTING = NO
+#---------------------------------------------------------------------------
+# Configuration options for the AutoGen Definitions output
+#---------------------------------------------------------------------------
+GENERATE_AUTOGEN_DEF = NO
+#---------------------------------------------------------------------------
+# Configuration options related to the Perl module output
+#---------------------------------------------------------------------------
+GENERATE_PERLMOD = NO
+PERLMOD_LATEX = NO
+PERLMOD_PRETTY = YES
+PERLMOD_MAKEVAR_PREFIX =
+#---------------------------------------------------------------------------
+# Configuration options related to the preprocessor
+#---------------------------------------------------------------------------
+ENABLE_PREPROCESSING = YES
+MACRO_EXPANSION = NO
+EXPAND_ONLY_PREDEF = NO
+SEARCH_INCLUDES = YES
+INCLUDE_PATH =
+INCLUDE_FILE_PATTERNS =
+PREDEFINED =
+EXPAND_AS_DEFINED =
+SKIP_FUNCTION_MACROS = YES
+#---------------------------------------------------------------------------
+# Configuration options related to external references
+#---------------------------------------------------------------------------
+TAGFILES =
+GENERATE_TAGFILE =
+ALLEXTERNALS = NO
+EXTERNAL_GROUPS = YES
+EXTERNAL_PAGES = YES
+PERL_PATH = /usr/bin/perl
+#---------------------------------------------------------------------------
+# Configuration options related to the dot tool
+#---------------------------------------------------------------------------
+CLASS_DIAGRAMS = YES
+MSCGEN_PATH =
+DIA_PATH =
+HIDE_UNDOC_RELATIONS = YES
+HAVE_DOT = YES
+DOT_NUM_THREADS = 0
+DOT_FONTNAME = Helvetica
+DOT_FONTSIZE = 10
+DOT_FONTPATH =
+CLASS_GRAPH = YES
+COLLABORATION_GRAPH = YES
+GROUP_GRAPHS = YES
+UML_LOOK = NO
+UML_LIMIT_NUM_FIELDS = 10
+TEMPLATE_RELATIONS = NO
+INCLUDE_GRAPH = YES
+INCLUDED_BY_GRAPH = YES
+CALL_GRAPH = NO
+CALLER_GRAPH = NO
+GRAPHICAL_HIERARCHY = YES
+DIRECTORY_GRAPH = YES
+DOT_IMAGE_FORMAT = png
+INTERACTIVE_SVG = NO
+DOT_PATH =
+DOTFILE_DIRS =
+MSCFILE_DIRS =
+DIAFILE_DIRS =
+PLANTUML_JAR_PATH =
+PLANTUML_INCLUDE_PATH =
+DOT_GRAPH_MAX_NODES = 50
+MAX_DOT_GRAPH_DEPTH = 0
+DOT_TRANSPARENT = NO
+DOT_MULTI_TARGETS = NO
+GENERATE_LEGEND = YES
+DOT_CLEANUP = YES
[43/50] [abbrv] hbase git commit: HBASE-15731 Add on a connection pool
Posted by ec...@apache.org.
HBASE-15731 Add on a connection pool
Summary:
Add on a connection pool protected by read write mutex.
Add on a service filter that will remove a connection from a connection pool when closed
Test Plan: Need to add on tests.
Differential Revision: https://reviews.facebook.net/D57411
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/baf863f6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/baf863f6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/baf863f6
Branch: refs/heads/HBASE-14850
Commit: baf863f6829d5d68f706e7b49b360eb1d2927708
Parents: c1140d7
Author: Elliott Clark <ec...@apache.org>
Authored: Thu Apr 28 19:53:42 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/connection/BUCK | 5 +
.../connection/connection-factory.cc | 4 +-
.../connection/connection-factory.h | 5 +-
.../connection/connection-pool-test.cc | 77 ++++++++++++++
.../connection/connection-pool.cc | 89 ++++++++++++++++
.../connection/connection-pool.h | 59 +++++++++++
hbase-native-client/connection/service.h | 4 +-
hbase-native-client/core/BUCK | 4 -
hbase-native-client/core/client.h | 3 -
hbase-native-client/core/get-request.cc | 19 ----
hbase-native-client/core/get-request.h | 35 -------
hbase-native-client/core/get-result.cc | 19 ----
hbase-native-client/core/get-result.h | 32 ------
hbase-native-client/core/location-cache.cc | 1 -
hbase-native-client/core/simple-client.cc | 105 +++++++++++--------
15 files changed, 297 insertions(+), 164 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/baf863f6/hbase-native-client/connection/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/BUCK b/hbase-native-client/connection/BUCK
index d393885..96f2136 100644
--- a/hbase-native-client/connection/BUCK
+++ b/hbase-native-client/connection/BUCK
@@ -22,6 +22,7 @@ cxx_library(name="connection",
"client-dispatcher.h",
"client-handler.h",
"connection-factory.h",
+ "connection-pool.h",
"pipeline.h",
"request.h",
"response.h",
@@ -31,6 +32,7 @@ cxx_library(name="connection",
"client-dispatcher.cc",
"client-handler.cc",
"connection-factory.cc",
+ "connection-pool.cc",
"pipeline.cc",
"request.cc",
],
@@ -42,3 +44,6 @@ cxx_library(name="connection",
"//third-party:wangle",
],
visibility=['//core/...', ], )
+cxx_test(name="connection-pool-test",
+ srcs=["connection-pool-test.cc", ],
+ deps=[":connection", ], )
http://git-wip-us.apache.org/repos/asf/hbase/blob/baf863f6/hbase-native-client/connection/connection-factory.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-factory.cc b/hbase-native-client/connection/connection-factory.cc
index 7073f9d..b546269 100644
--- a/hbase-native-client/connection/connection-factory.cc
+++ b/hbase-native-client/connection/connection-factory.cc
@@ -45,8 +45,8 @@ ConnectionFactory::ConnectionFactory() {
bootstrap_.pipelineFactory(std::make_shared<RpcPipelineFactory>());
}
-std::shared_ptr<Service<std::unique_ptr<Request>, Response>>
-ConnectionFactory::make_connection(std::string host, int port) {
+std::shared_ptr<HBaseService>
+ConnectionFactory::make_connection(const std::string &host, int port) {
// Connect to a given server
// Then when connected create a ClientDispactcher.
auto pipeline = bootstrap_.connect(SocketAddress(host, port, true)).get();
http://git-wip-us.apache.org/repos/asf/hbase/blob/baf863f6/hbase-native-client/connection/connection-factory.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-factory.h b/hbase-native-client/connection/connection-factory.h
index 8d1d2f0..5a45316 100644
--- a/hbase-native-client/connection/connection-factory.h
+++ b/hbase-native-client/connection/connection-factory.h
@@ -31,8 +31,9 @@ namespace hbase {
class ConnectionFactory {
public:
ConnectionFactory();
- std::shared_ptr<wangle::Service<std::unique_ptr<Request>, Response>>
- make_connection(std::string host, int port);
+
+ virtual std::shared_ptr<HBaseService> make_connection(const std::string &host,
+ int port);
private:
wangle::ClientBootstrap<SerializePipeline> bootstrap_;
http://git-wip-us.apache.org/repos/asf/hbase/blob/baf863f6/hbase-native-client/connection/connection-pool-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-pool-test.cc b/hbase-native-client/connection/connection-pool-test.cc
new file mode 100644
index 0000000..975bc5e
--- /dev/null
+++ b/hbase-native-client/connection/connection-pool-test.cc
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "connection/connection-pool.h"
+
+#include <folly/Logging.h>
+#include <gmock/gmock.h>
+
+#include "connection/connection-factory.h"
+#include "if/HBase.pb.h"
+
+using namespace hbase;
+
+using hbase::pb::ServerName;
+using ::testing::Return;
+using ::testing::_;
+
+class MockConnectionFactory : public ConnectionFactory {
+public:
+ MOCK_METHOD2(make_connection,
+ std::shared_ptr<HBaseService>(const std::string &hostname,
+ int port));
+};
+
+class MockServiceBase : public HBaseService {
+public:
+ folly::Future<Response> operator()(std::unique_ptr<Request> req) override {
+ return do_operation(req.get());
+ }
+ virtual folly::Future<Response> do_operation(Request *req) {
+ return folly::makeFuture<Response>(Response{});
+ }
+};
+
+class MockService : public MockServiceBase {
+public:
+ MOCK_METHOD1(do_operation, folly::Future<Response>(Request *));
+};
+
+TEST(TestConnectionPool, TestOnlyCreateOnce) {
+ std::string hostname{"hostname"};
+ auto mock_service = std::make_shared<MockService>();
+ uint32_t port{999};
+
+ LOG(ERROR) << "About to make a MockConnectionFactory";
+ auto mock_cf = std::make_shared<MockConnectionFactory>();
+ EXPECT_CALL((*mock_cf), make_connection(_, _))
+ .Times(1)
+ .WillRepeatedly(Return(mock_service));
+ ConnectionPool cp{mock_cf};
+
+ LOG(ERROR) << "Created ConnectionPool";
+
+ ServerName sn;
+ sn.set_host_name(hostname);
+ sn.set_port(port);
+
+ auto result = cp.get(sn);
+ ASSERT_TRUE(result != nullptr);
+ result = cp.get(sn);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/baf863f6/hbase-native-client/connection/connection-pool.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-pool.cc b/hbase-native-client/connection/connection-pool.cc
new file mode 100644
index 0000000..a967df2
--- /dev/null
+++ b/hbase-native-client/connection/connection-pool.cc
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "connection/connection-pool.h"
+
+#include <wangle/service/Service.h>
+
+using std::mutex;
+using std::unique_ptr;
+using std::shared_ptr;
+using hbase::pb::ServerName;
+using wangle::ServiceFilter;
+using folly::SharedMutexWritePriority;
+
+namespace hbase {
+
+class RemoveServiceFilter
+ : public ServiceFilter<unique_ptr<Request>, Response> {
+
+public:
+ RemoveServiceFilter(std::shared_ptr<HBaseService> service, ServerName sn,
+ ConnectionPool *cp)
+ : ServiceFilter<unique_ptr<Request>, Response>(service), sn_(sn),
+ cp_(cp) {}
+
+ folly::Future<folly::Unit> close() override {
+ if (!released.exchange(true)) {
+ return this->service_->close().then(
+ [this]() { this->cp_->close(this->sn_); });
+ } else {
+ return folly::makeFuture();
+ }
+ }
+
+ virtual bool isAvailable() override { return service_->isAvailable(); }
+
+ folly::Future<Response> operator()(unique_ptr<Request> req) override {
+ return (*this->service_)(std::move(req));
+ }
+
+private:
+ std::atomic<bool> released{false};
+ hbase::pb::ServerName sn_;
+ ConnectionPool *cp_;
+};
+
+ConnectionPool::ConnectionPool() : cf_(std::make_shared<ConnectionFactory>()) {}
+ConnectionPool::ConnectionPool(std::shared_ptr<ConnectionFactory> cf)
+ : cf_(cf) {}
+
+std::shared_ptr<HBaseService> ConnectionPool::get(const ServerName &sn) {
+ SharedMutexWritePriority::UpgradeHolder holder(map_mutex_);
+ auto found = connections_.find(sn);
+ if (found == connections_.end() || found->second == nullptr) {
+ SharedMutexWritePriority::WriteHolder holder(std::move(holder));
+ auto new_con = cf_->make_connection(sn.host_name(), sn.port());
+ auto wrapped = std::make_shared<RemoveServiceFilter>(new_con, sn, this);
+ connections_[sn] = wrapped;
+ return new_con;
+ }
+ return found->second;
+}
+void ConnectionPool::close(ServerName sn) {
+ SharedMutexWritePriority::WriteHolder holder(map_mutex_);
+
+ auto found = connections_.find(sn);
+ if (found == connections_.end() || found->second == nullptr) {
+ return;
+ }
+ auto service = found->second;
+ connections_.erase(found);
+}
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/baf863f6/hbase-native-client/connection/connection-pool.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-pool.h b/hbase-native-client/connection/connection-pool.h
new file mode 100644
index 0000000..394cd71
--- /dev/null
+++ b/hbase-native-client/connection/connection-pool.h
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <folly/SharedMutex.h>
+#include <mutex>
+#include <unordered_map>
+
+#include "connection/connection-factory.h"
+#include "connection/service.h"
+#include "if/HBase.pb.h"
+
+namespace hbase {
+struct MyServerNameEquals {
+ bool operator()(const hbase::pb::ServerName &lhs,
+ const hbase::pb::ServerName &rhs) const {
+ return lhs.host_name() == rhs.host_name() && lhs.port() == rhs.port();
+ }
+};
+struct MyServerNameHash {
+ std::size_t operator()(hbase::pb::ServerName const &s) const {
+ std::size_t h1 = std::hash<std::string>()(s.host_name());
+ std::size_t h2 = std::hash<uint32_t>()(s.port());
+ return h1 ^ (h2 << 1);
+ }
+};
+
+class ConnectionPool {
+public:
+ ConnectionPool();
+ explicit ConnectionPool(std::shared_ptr<ConnectionFactory> cf);
+ std::shared_ptr<HBaseService> get(const hbase::pb::ServerName &sn);
+ void close(hbase::pb::ServerName sn);
+
+private:
+ std::shared_ptr<ConnectionFactory> cf_;
+ std::unordered_map<hbase::pb::ServerName, std::shared_ptr<HBaseService>,
+ MyServerNameHash, MyServerNameEquals>
+ connections_;
+ folly::SharedMutexWritePriority map_mutex_;
+};
+
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/baf863f6/hbase-native-client/connection/service.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/service.h b/hbase-native-client/connection/service.h
index feb14ec..79f087d 100644
--- a/hbase-native-client/connection/service.h
+++ b/hbase-native-client/connection/service.h
@@ -18,9 +18,11 @@
*/
#pragma once
+#include <wangle/service/Service.h>
+
#include "connection/request.h"
#include "connection/response.h"
namespace hbase {
-using HBaseService = wangle::Service<Request, Response>;
+using HBaseService = wangle::Service<std::unique_ptr<Request>, Response>;
} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/baf863f6/hbase-native-client/core/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index 9db6fda..e555ba4 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -21,8 +21,6 @@ cxx_library(
exported_headers=[
"client.h",
"connection.h",
- "get-request.h",
- "get-result.h",
"hbase_macros.h",
"location-cache.h",
"table-name.h",
@@ -32,8 +30,6 @@ cxx_library(
],
srcs=[
"client.cc",
- "get-request.cc",
- "get-result.cc",
"location-cache.cc",
"meta-utils.cc",
"table-name.cc",
http://git-wip-us.apache.org/repos/asf/hbase/blob/baf863f6/hbase-native-client/core/client.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client.h b/hbase-native-client/core/client.h
index b583285..4bed751 100644
--- a/hbase-native-client/core/client.h
+++ b/hbase-native-client/core/client.h
@@ -24,8 +24,6 @@
#include <string>
-#include "core/get-request.h"
-#include "core/get-result.h"
#include "core/location-cache.h"
#include "if/Cell.pb.h"
@@ -33,7 +31,6 @@ namespace hbase {
class Client {
public:
explicit Client(std::string quorum_spec);
- folly::Future<GetResult> get(const GetRequest &get_request);
private:
LocationCache location_cache_;
http://git-wip-us.apache.org/repos/asf/hbase/blob/baf863f6/hbase-native-client/core/get-request.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/get-request.cc b/hbase-native-client/core/get-request.cc
deleted file mode 100644
index e927ccc..0000000
--- a/hbase-native-client/core/get-request.cc
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#include "core/get-request.h"
http://git-wip-us.apache.org/repos/asf/hbase/blob/baf863f6/hbase-native-client/core/get-request.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/get-request.h b/hbase-native-client/core/get-request.h
deleted file mode 100644
index bb755c5..0000000
--- a/hbase-native-client/core/get-request.h
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#pragma once
-
-#include <string>
-
-#include "core/table-name.h"
-
-namespace hbase {
-
-class GetRequest {
-public:
- GetRequest(TableName table_name, std::string key);
-
-private:
- TableName table_name_;
- std::string key_;
-};
-} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/baf863f6/hbase-native-client/core/get-result.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/get-result.cc b/hbase-native-client/core/get-result.cc
deleted file mode 100644
index 7eea483..0000000
--- a/hbase-native-client/core/get-result.cc
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#include "core/get-result.h"
http://git-wip-us.apache.org/repos/asf/hbase/blob/baf863f6/hbase-native-client/core/get-result.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/get-result.h b/hbase-native-client/core/get-result.h
deleted file mode 100644
index a49ad98..0000000
--- a/hbase-native-client/core/get-result.h
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#pragma once
-
-#include <string>
-
-namespace hbase {
-
-class GetResult {
-public:
- explicit GetResult(std::string key);
-
-private:
- std::string key_;
-};
-} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/baf863f6/hbase-native-client/core/location-cache.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
index 5925f4a..c81deba 100644
--- a/hbase-native-client/core/location-cache.cc
+++ b/hbase-native-client/core/location-cache.cc
@@ -77,7 +77,6 @@ ServerName LocationCache::ReadMetaLocation() {
int zk_result =
zoo_get(this->zk_, META_ZNODE_NAME, 0,
reinterpret_cast<char *>(buf->writableData()), &len, nullptr);
- LOG(ERROR) << "len = " << len;
if (zk_result != ZOK || len < 9) {
LOG(ERROR) << "Error getting meta location.";
throw runtime_error("Error getting meta location");
http://git-wip-us.apache.org/repos/asf/hbase/blob/baf863f6/hbase-native-client/core/simple-client.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/simple-client.cc b/hbase-native-client/core/simple-client.cc
index 2cb6200..11dcd68 100644
--- a/hbase-native-client/core/simple-client.cc
+++ b/hbase-native-client/core/simple-client.cc
@@ -25,7 +25,7 @@
#include <chrono>
#include <iostream>
-#include "connection/connection-factory.h"
+#include "connection/connection-pool.h"
#include "core/client.h"
#include "if/Client.pb.h"
#include "if/ZooKeeper.pb.h"
@@ -33,13 +33,20 @@
using namespace folly;
using namespace std;
using namespace std::chrono;
-using namespace hbase;
-using namespace hbase::pb;
-using namespace google::protobuf;
+using hbase::Response;
+using hbase::Request;
+using hbase::HBaseService;
+using hbase::LocationCache;
+using hbase::ConnectionPool;
+using hbase::pb::ServerName;
+using hbase::pb::RegionSpecifier_RegionSpecifierType;
+using hbase::pb::GetRequest;
+using hbase::pb::GetResponse;
// TODO(eclark): remove the need for this.
DEFINE_string(region, "1588230740", "What region to send a get to");
DEFINE_string(row, "test", "What row to get");
+DEFINE_string(zookeeper, "localhost:2181", "What zk quorum to talk to");
int main(int argc, char *argv[]) {
google::SetUsageMessage(
@@ -48,46 +55,52 @@ int main(int argc, char *argv[]) {
google::InitGoogleLogging(argv[0]);
// Create a connection factory
- ConnectionFactory cf;
-
- LocationCache cache{"localhost:2181", wangle::getCPUExecutor()};
-
- auto result = cache.LocateMeta().get();
-
- // Create a connection to the local host
- auto conn = cf.make_connection(result.host_name(), result.port());
-
- // Send the request
- auto r = Request::get();
-
- // This is a get request so make that
- auto req_msg = static_pointer_cast<hbase::pb::GetRequest>(r->req_msg());
-
- // Set what region
- req_msg->mutable_region()->set_value(FLAGS_region);
- // It's always this.
- req_msg->mutable_region()->set_type(
- RegionSpecifier_RegionSpecifierType::
- RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
-
- // What row.
- req_msg->mutable_get()->set_row(FLAGS_row);
-
- // Send it.
- auto resp = (*conn)(std::move(r)).get(milliseconds(5000));
-
- auto get_resp = std::static_pointer_cast<GetResponse>(resp.response());
- cout << "GetResponse has_result = " << get_resp->has_result() << '\n';
- if (get_resp->has_result()) {
- auto &r = get_resp->result();
- cout << "Result cell_size = " << r.cell_size() << endl;
- for (auto &cell : r.cell()) {
- cout << "\trow = " << cell.row() << " family = " << cell.family()
- << " qualifier = " << cell.qualifier()
- << " timestamp = " << cell.timestamp() << " value = " << cell.value()
- << endl;
- }
- }
-
- return 0;
+ ConnectionPool cp;
+ auto cpu_ex = wangle::getCPUExecutor();
+ LocationCache cache{FLAGS_zookeeper, cpu_ex};
+ auto result =
+ cache.LocateMeta()
+ .then([&cp = cp](ServerName sn) { return cp.get(sn); })
+ .then([](shared_ptr<HBaseService> con) {
+ // Send the request
+ auto r = Request::get();
+ // This is a get request so make that
+ auto req_msg = static_pointer_cast<GetRequest>(r->req_msg());
+ // Set what region
+ req_msg->mutable_region()->set_value(FLAGS_region);
+ // It's always this.
+ req_msg->mutable_region()->set_type(
+ RegionSpecifier_RegionSpecifierType::
+ RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
+
+ // What row.
+ req_msg->mutable_get()->set_row(FLAGS_row);
+
+ return (*con)(std::move(r));
+ })
+ .then([](Response resp) {
+ return static_pointer_cast<GetResponse>(resp.response());
+ })
+ .via(cpu_ex.get())
+ .then([](shared_ptr<GetResponse> get_resp) {
+ cout << "GetResponse has_result = " << get_resp->has_result()
+ << '\n';
+ if (get_resp->has_result()) {
+ auto &r = get_resp->result();
+ cout << "Result cell_size = " << r.cell_size() << endl;
+ for (auto &cell : r.cell()) {
+ cout << "\trow = " << cell.row()
+ << " family = " << cell.family()
+ << " qualifier = " << cell.qualifier()
+ << " timestamp = " << cell.timestamp()
+ << " value = " << cell.value() << endl;
+ }
+ return 0;
+ }
+
+ return 1;
+ })
+ .get(milliseconds(5000));
+
+ return result;
}
[11/50] [abbrv] hbase git commit: HBASE-15823 Use call once for user
util
Posted by ec...@apache.org.
HBASE-15823 Use call once for user util
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/dea1eb64
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/dea1eb64
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/dea1eb64
Branch: refs/heads/HBASE-14850
Commit: dea1eb64bd6ef85ef0ca9a4c1f5c7b30043ba953
Parents: e718c7d
Author: Elliott Clark <ec...@apache.org>
Authored: Fri May 13 13:07:03 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/utils/user-util.cc | 13 ++-----------
hbase-native-client/utils/user-util.h | 6 ++----
2 files changed, 4 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea1eb64/hbase-native-client/utils/user-util.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/utils/user-util.cc b/hbase-native-client/utils/user-util.cc
index 3d963b3..9e170e0 100644
--- a/hbase-native-client/utils/user-util.cc
+++ b/hbase-native-client/utils/user-util.cc
@@ -27,22 +27,14 @@
using namespace hbase;
using namespace std;
-UserUtil::UserUtil() : init_{false}, user_name_{"drwho"}, m_() {}
+UserUtil::UserUtil() : once_flag_{}, user_name_{"drwho"} {}
string UserUtil::user_name() {
- if (!init_) {
- compute_user_name();
- }
+ std::call_once(once_flag_, [this]() { compute_user_name(); });
return user_name_;
}
void UserUtil::compute_user_name() {
- lock_guard<mutex> lock(m_);
-
- if (init_) {
- return;
- }
-
// According to the man page of getpwuid
// this should never be free'd
//
@@ -52,6 +44,5 @@ void UserUtil::compute_user_name() {
// make sure that we got something.
if (passwd && passwd->pw_name) {
user_name_ = string{passwd->pw_name};
- init_ = true;
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/dea1eb64/hbase-native-client/utils/user-util.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/utils/user-util.h b/hbase-native-client/utils/user-util.h
index 0b4cc73..fdfc0c8 100644
--- a/hbase-native-client/utils/user-util.h
+++ b/hbase-native-client/utils/user-util.h
@@ -19,8 +19,7 @@
#pragma once
-#include <atomic>
-#include <memory>
+#include <string>
#include <mutex>
namespace hbase {
@@ -49,8 +48,7 @@ private:
* Compute the username. This will block.
*/
void compute_user_name();
- std::atomic<bool> init_;
+ std::once_flag once_flag_;
std::string user_name_;
- std::mutex m_;
};
} // namespace hbase
[22/50] [abbrv] hbase git commit: HBASE-14854 Read meta location from
zk
Posted by ec...@apache.org.
HBASE-14854 Read meta location from zk
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/26ba4ed4
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/26ba4ed4
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/26ba4ed4
Branch: refs/heads/HBASE-14850
Commit: 26ba4ed461600a0bc8581d2ba9ba4230b71e992b
Parents: 27ee8ba
Author: Elliott Clark <ec...@apache.org>
Authored: Sat Mar 5 00:09:08 2016 -0800
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/Dockerfile | 20 ++-
hbase-native-client/bin/start-docker.sh | 5 +-
hbase-native-client/core/BUCK | 106 +++++++-------
.../core/HBaseNativeClientTestEnv.cc | 42 ------
.../core/SampleNativeClientTest.cc | 28 ----
hbase-native-client/core/location-cache-test.cc | 14 ++
hbase-native-client/core/location-cache.cc | 67 +++++++++
hbase-native-client/core/location-cache.h | 35 +++++
.../core/native-client-test-env.cc | 42 ++++++
.../core/simple-native-client-test.cc | 28 ++++
hbase-native-client/core/test_env.h | 2 +
hbase-native-client/if/BUCK | 1 +
hbase-native-client/third-party/BUCK | 138 ++++++++++---------
13 files changed, 339 insertions(+), 189 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/26ba4ed4/hbase-native-client/Dockerfile
----------------------------------------------------------------------
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
index 1364d22..36959a5 100644
--- a/hbase-native-client/Dockerfile
+++ b/hbase-native-client/Dockerfile
@@ -15,7 +15,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-FROM cpp_update
+FROM pjameson/buck-folly-watchman
ARG CC=/usr/bin/gcc-5
ARG CXX=/usr/bin/g++-5
@@ -25,20 +25,26 @@ ARG CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=0 -fPIC -g -fno-omit-frame-pointer -O3 -p
RUN apt-get install -y clang-format-3.7 vim maven inetutils-ping
RUN git clone --depth 1 --branch v2.6.1 https://github.com/google/protobuf.git /usr/src/protobuf && \
cd /usr/src/protobuf/ && \
+ ldconfig && \
./autogen.sh && \
- ./configure --disable-shared && \
+ ./configure && \
make && \
- make check && \
- make install
+ make install && \
+ make clean && \
+ rm -rf .git
+
RUN cd /usr/src && \
wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && \
tar zxf zookeeper-3.4.8.tar.gz && \
rm -rf zookeeper-3.4.8.tar.gz && \
cd zookeeper-3.4.8 && \
cd src/c && \
- ./configure --disable-shared && \
+ ldconfig && \
+ ./configure && \
make && \
make install && \
- make clean
+ make clean
+
+RUN ldconfig
-WORKDIR /usr/local/src/hbase/hbase-native-client
+WORKDIR /usr/src/hbase/hbase-native-client
http://git-wip-us.apache.org/repos/asf/hbase/blob/26ba4ed4/hbase-native-client/bin/start-docker.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/start-docker.sh b/hbase-native-client/bin/start-docker.sh
index 4426705..725ed6a 100755
--- a/hbase-native-client/bin/start-docker.sh
+++ b/hbase-native-client/bin/start-docker.sh
@@ -19,8 +19,11 @@
set -e
set -x
+# Try out some standard docker machine names that could work
eval "$(docker-machine env docker-vm)"
eval "$(docker-machine env dinghy)"
+
+# Build the image
docker build -t hbase_native .
@@ -36,6 +39,6 @@ fi;
docker run -p 16010:16010/tcp \
-e "JAVA_HOME=/usr/lib/jvm/java-8-oracle" \
- -v ${PWD}/..:/usr/local/src/hbase \
+ -v ${PWD}/..:/usr/src/hbase \
-v ~/.m2:/root/.m2 \
-it hbase_native /bin/bash
http://git-wip-us.apache.org/repos/asf/hbase/blob/26ba4ed4/hbase-native-client/core/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index ef027a1..817b5a0 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -15,52 +15,62 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+cxx_library(name="core",
+ headers=[
+ "admin.h",
+ "client.h",
+ "connection.h",
+ "connection_attr.h",
+ "delete.h",
+ "get.h",
+ "hbase_macros.h",
+ "mutation.h",
+ "put.h",
+ "scanner.h",
+ "location-cache.h",
+ ],
+ srcs=[
+ "admin.cc",
+ "client.cc",
+ "connection.cc",
+ "get.cc",
+ "mutation.cc",
+ "put.cc",
+ "delete.cc",
+ "scanner.cc",
+ "location-cache.cc",
+ ],
+ deps=[
+ "//if:if",
+ "//third-party:zookeeper_mt",
+ "//third-party:folly",
+ "//third-party:wangle",
+ ],
+ visibility=[
+ 'PUBLIC',
+ ], )
-cxx_binary(
- name = "core",
- headers = [
- "admin.h",
- "client.h",
- "connection.h",
- "connection_attr.h",
- "delete.h",
- "get.h",
- "hbase_macros.h",
- "mutation.h",
- "put.h",
- "scanner.h",
- ],
- srcs = [
- "admin.cc",
- "client.cc",
- "connection.cc",
- "get.cc",
- "mutation.cc",
- "put.cc",
- "delete.cc",
- "scanner.cc",
- ],
- deps = [
- "//if:if",
- "//third-party:folly",
- "//third-party:wangle",
- ],
- visibility = [
- 'PUBLIC',
- ],
-)
-
-cxx_test(
- name = "core_test",
- headers = [
- "test_env.h",
- ],
- srcs = [
- "HBaseNativeClientTestEnv.cc",
- "SampleNativeClientTest.cc",
- ],
- deps = [
- ":core",
- ],
- run_test_separately = True,
-)
+cxx_test(name="simple-test",
+ headers=[
+ "test_env.h",
+ ],
+ srcs=[
+ "native-client-test-env.cc",
+ "simple-native-client-test.cc",
+ ],
+ deps=[
+ ":core",
+ ],
+ run_test_separately=True, )
+cxx_test(name="location-cache-test",
+ headers=[
+ "test_env.h",
+ ],
+ srcs=[
+ "native-client-test-env.cc",
+ "location-cache-test.cc",
+ ],
+ deps=[
+ ":core",
+ ],
+ run_test_separately=True, )
http://git-wip-us.apache.org/repos/asf/hbase/blob/26ba4ed4/hbase-native-client/core/HBaseNativeClientTestEnv.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/HBaseNativeClientTestEnv.cc b/hbase-native-client/core/HBaseNativeClientTestEnv.cc
deleted file mode 100644
index b8cb8db..0000000
--- a/hbase-native-client/core/HBaseNativeClientTestEnv.cc
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include <gtest/gtest.h>
-#include <core/test_env.h>
-
-namespace {
-
-class HBaseNativeClientTestEnv : public ::testing::Environment {
- public:
- void SetUp() override {
- init_test_env();
- }
-
- void TearDown() override {
- clean_test_env();
- }
-};
-
-} // anonymous
-
-int main(int argc, char** argv) {
- testing::InitGoogleTest(&argc, argv);
- ::testing::AddGlobalTestEnvironment(new HBaseNativeClientTestEnv());
- return RUN_ALL_TESTS();
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26ba4ed4/hbase-native-client/core/SampleNativeClientTest.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/SampleNativeClientTest.cc b/hbase-native-client/core/SampleNativeClientTest.cc
deleted file mode 100644
index ef564f7..0000000
--- a/hbase-native-client/core/SampleNativeClientTest.cc
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "gtest/gtest.h"
-
-/**
- * Sample test.
- */
-TEST(SampleTest, sample) {
- EXPECT_TRUE(true);
-}
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/26ba4ed4/hbase-native-client/core/location-cache-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache-test.cc b/hbase-native-client/core/location-cache-test.cc
new file mode 100644
index 0000000..3106e36
--- /dev/null
+++ b/hbase-native-client/core/location-cache-test.cc
@@ -0,0 +1,14 @@
+#include <gtest/gtest.h>
+#include <folly/Memory.h>
+#include <wangle/concurrent/GlobalExecutor.h>
+
+#include "location-cache.h"
+using namespace hbase;
+
+TEST(LocationCacheTest, TestGetMetaNodeContents) {
+ // TODO(elliott): need to make a test utility for this.
+ LocationCache cache{"localhost:2181", wangle::getCPUExecutor()};
+ auto result = cache.LocateMeta();
+ result.wait();
+ ASSERT_FALSE(result.hasException());
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26ba4ed4/hbase-native-client/core/location-cache.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
new file mode 100644
index 0000000..cf61e24
--- /dev/null
+++ b/hbase-native-client/core/location-cache.cc
@@ -0,0 +1,67 @@
+#include "location-cache.h"
+
+#include <folly/Logging.h>
+
+#include "if/ZooKeeper.pb.h"
+
+using namespace std;
+using namespace folly;
+using namespace hbase::pb;
+
+namespace hbase {
+
+// TODO(elliott): make this configurable on client creation
+const static string META_LOCATION = "/hbase/meta-region-server";
+
+LocationCache::LocationCache(string quorum_spec,
+ shared_ptr<folly::Executor> executor)
+ : quorum_spec_(quorum_spec), executor_(executor), meta_promise_(nullptr) {
+ zk_ = zookeeper_init(quorum_spec.c_str(), nullptr, 1000, 0, 0, 0);
+}
+
+LocationCache::~LocationCache() {
+ zookeeper_close(zk_);
+ zk_ = nullptr;
+ LOG(INFO) << "Closed connection to ZooKeeper.";
+}
+
+Future<ServerName> LocationCache::LocateMeta() {
+ lock_guard<mutex> g(meta_lock_);
+ if (meta_promise_ == nullptr) {
+ this->RefreshMetaLocation();
+ }
+ return meta_promise_->getFuture();
+}
+
+void LocationCache::InvalidateMeta() {
+ if (meta_promise_ != nullptr) {
+ lock_guard<mutex> g(meta_lock_);
+ meta_promise_ = nullptr;
+ }
+}
+
+/// MUST hold the meta_lock_
+void LocationCache::RefreshMetaLocation() {
+ meta_promise_ = make_unique<SharedPromise<ServerName>>();
+ executor_->add([&] {
+ meta_promise_->setWith([&] { return this->ReadMetaLocation(); });
+ });
+}
+
+ServerName LocationCache::ReadMetaLocation() {
+ char contents[4096];
+ int len = sizeof(contents);
+ // TODO(elliott): handle disconnects/reconntion as needed.
+ int zk_result =
+ zoo_get(this->zk_, META_LOCATION.c_str(), 0, contents, &len, nullptr);
+
+ if (zk_result != ZOK) {
+ LOG(ERROR) << "Error getting meta location.";
+ throw runtime_error("Error getting meta location");
+ }
+
+ MetaRegionServer mrs;
+ mrs.ParseFromArray(contents, len);
+ return mrs.server();
+}
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26ba4ed4/hbase-native-client/core/location-cache.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h
new file mode 100644
index 0000000..8dc2760
--- /dev/null
+++ b/hbase-native-client/core/location-cache.h
@@ -0,0 +1,35 @@
+#pragma once
+
+#include <memory>
+#include <mutex>
+
+#include <zookeeper/zookeeper.h>
+#include <folly/futures/Future.h>
+#include <folly/futures/SharedPromise.h>
+
+#include <folly/Executor.h>
+#include "if/HBase.pb.h"
+
+namespace hbase {
+class LocationCache {
+public:
+ explicit LocationCache(std::string quorum_spec,
+ std::shared_ptr<folly::Executor> executor);
+ ~LocationCache();
+ // Meta Related Methods.
+ // These are only public until testing is complete
+ folly::Future<hbase::pb::ServerName> LocateMeta();
+ void InvalidateMeta();
+
+private:
+ void RefreshMetaLocation();
+ hbase::pb::ServerName ReadMetaLocation();
+
+ std::string quorum_spec_;
+ std::shared_ptr<folly::Executor> executor_;
+ std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> meta_promise_;
+ std::mutex meta_lock_;
+
+ zhandle_t *zk_;
+};
+} // hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/26ba4ed4/hbase-native-client/core/native-client-test-env.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/native-client-test-env.cc b/hbase-native-client/core/native-client-test-env.cc
new file mode 100644
index 0000000..a86961f
--- /dev/null
+++ b/hbase-native-client/core/native-client-test-env.cc
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <gtest/gtest.h>
+#include <core/test_env.h>
+
+namespace {
+
+class NativeClientTestEnv : public ::testing::Environment {
+ public:
+ void SetUp() override {
+ init_test_env();
+ }
+
+ void TearDown() override {
+ clean_test_env();
+ }
+};
+
+} // anonymous
+
+int main(int argc, char** argv) {
+ testing::InitGoogleTest(&argc, argv);
+ ::testing::AddGlobalTestEnvironment(new NativeClientTestEnv());
+ return RUN_ALL_TESTS();
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/26ba4ed4/hbase-native-client/core/simple-native-client-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/simple-native-client-test.cc b/hbase-native-client/core/simple-native-client-test.cc
new file mode 100644
index 0000000..ef564f7
--- /dev/null
+++ b/hbase-native-client/core/simple-native-client-test.cc
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "gtest/gtest.h"
+
+/**
+ * Sample test.
+ */
+TEST(SampleTest, sample) {
+ EXPECT_TRUE(true);
+}
+
http://git-wip-us.apache.org/repos/asf/hbase/blob/26ba4ed4/hbase-native-client/core/test_env.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/test_env.h b/hbase-native-client/core/test_env.h
index 5796ae1..79bdbec 100644
--- a/hbase-native-client/core/test_env.h
+++ b/hbase-native-client/core/test_env.h
@@ -17,6 +17,8 @@
*
*/
+#pragma once
+
#include <cstdlib>
inline void init_test_env() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/26ba4ed4/hbase-native-client/if/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/BUCK b/hbase-native-client/if/BUCK
index 3490a05..9b989b5 100644
--- a/hbase-native-client/if/BUCK
+++ b/hbase-native-client/if/BUCK
@@ -33,4 +33,5 @@ cxx_library(
+ [':' + x for x in CC_FILENAMES]
+ [ ':' + x for x in HEADER_FILENAMES ],
visibility = [ 'PUBLIC', ],
+ exported_deps = ['//third-party:protobuf']
)
http://git-wip-us.apache.org/repos/asf/hbase/blob/26ba4ed4/hbase-native-client/third-party/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/third-party/BUCK b/hbase-native-client/third-party/BUCK
index e577a5f..6548695 100644
--- a/hbase-native-client/third-party/BUCK
+++ b/hbase-native-client/third-party/BUCK
@@ -15,85 +15,97 @@
# See the License for the specific language governing permissions and
# limitations under the License.
-def add_system_libs(names = [], lib_dir = "/usr/lib/x86_64-linux-gnu", deps = [], exported_linker_flags = []):
- rules = []
- for name in names:
- gen_rule_name = "gen_lib{}".format(name)
- genrule(
- name = gen_rule_name,
- out = gen_rule_name,
- bash = "mkdir -p $OUT && cp {}/lib{}.* $OUT".format(lib_dir,name),
- )
- prebuilt_cxx_library(
- name = name,
- lib_name = name,
- lib_dir = '$(location :{})'.format(gen_rule_name),
- force_static = True,
- deps = deps,
- visibility = [ 'PUBLIC' ],
- exported_linker_flags = exported_linker_flags,
- )
- rules.append(":" + name)
- return rules
-system_libs = [
- "unwind",
- "lzma",
-]
+def add_system_libs(names=[],
+ lib_dir="/usr/lib/x86_64-linux-gnu",
+ deps=[],
+ exported_deps=[],
+ exported_linker_flags=[]):
+ rules = []
+ for name in names:
+ gen_rule_name = "gen_lib{}".format(name)
+ genrule(name=gen_rule_name,
+ out=gen_rule_name,
+ bash="mkdir -p $OUT && cp {}/lib{}.a $OUT".format(lib_dir,
+ name), )
+ prebuilt_cxx_library(name=name,
+ lib_name=name,
+ lib_dir='$(location :{})'.format(gen_rule_name),
+ deps=deps,
+ force_static = True,
+ exported_deps=exported_deps,
+ visibility=['PUBLIC'],
+ exported_linker_flags=exported_linker_flags, )
+ rules.append(":" + name)
+ return rules
+
+
+system_libs = ["unwind", "lzma", "event", ]
local_libs = [
- "double-conversion",
- "glog",
- "gflags",
- "protobuf",
- "zookeeper_mt",
- "boost_regex",
+ "double-conversion",
+ "boost_regex",
+ "boost_context",
+ "boost_thread",
+ "boost_system",
+ "boost_filesystem",
+ "boost_program_options",
+ "boost_chrono",
+ "gflags",
+ "glog",
+ "protobuf",
]
+
+
tp_dep_rules = add_system_libs(system_libs) \
- + add_system_libs(local_libs, lib_dir = "/usr/local/lib")
-folly = add_system_libs(
- ['folly'],
- lib_dir = '/usr/local/lib',
- deps = tp_dep_rules,
- exported_linker_flags = [ "-pthread", "-lstdc++",]
- )
-folly_bench = add_system_libs(
- ['follybenchmark'],
- lib_dir = '/usr/local/lib',
- deps = tp_dep_rules + folly,
- exported_linker_flags = [ "-pthread", "-lstdc++",]
- )
-wangle = add_system_libs(
- ['wangle'],
- lib_dir = '/usr/local/lib',
- deps = tp_dep_rules + folly,
- exported_linker_flags = [ "-pthread", "-lstdc++",]
- )
+ + add_system_libs(local_libs, lib_dir = "/usr/local/lib")
+
+zookeeper = add_system_libs(["zookeeper_mt"], lib_dir = "/usr/local/lib")
+folly = add_system_libs(['folly'],
+ lib_dir='/usr/local/lib',
+ exported_deps=tp_dep_rules,
+ exported_linker_flags=["-pthread",
+ "-lstdc++", ])
+folly_bench = add_system_libs(['follybenchmark'],
+ lib_dir='/usr/local/lib',
+ exported_deps=tp_dep_rules + folly,
+ exported_linker_flags=["-pthread",
+ "-lstdc++", ])
+wangle = add_system_libs(['wangle'],
+ lib_dir='/usr/local/lib',
+ exported_deps=tp_dep_rules + folly,
+ exported_linker_flags=["-pthread",
+ "-lstdc++", ])
+
+
+genrule(
+name = "gen_zk",
+out = "gen_zk",
+bash = "mkdir -p $OUT && wget http://www-us.apache.org/dist/zookeeper/zookeeper-3.4.8/zookeeper-3.4.8.tar.gz && tar zxf zookeeper-3.4.8.tar.gz && rm -rf zookeeper-3.4.8.tar.gz && cd zookeeper-3.4.8 && cd src/c && ./configure --prefix=$OUT && make && make install && cd $OUT && rm -rf zookeeper-3.4.8*"
+)
cxx_library(
- name = 'google-test',
- srcs = [
+ name = 'google-test',
+ srcs = [
'googletest/googletest/src/gtest-all.cc',
'googletest/googlemock/src/gmock-all.cc',
'googletest/googlemock/src/gmock_main.cc',
- ],
- header_namespace = '',
- exported_headers = subdir_glob([
+ ],
+ header_namespace = '',
+ exported_headers = subdir_glob([
('googletest/googletest/include', '**/*.h'),
('googletest/googlemock/include', '**/*.h'),
- ]),
- headers = subdir_glob([
+ ]),
+ headers = subdir_glob([
('googletest/googletest', 'src/*.h'),
('googletest/googletest', 'src/*.cc'),
('googletest/googlemock', 'src/*.h'),
('googletest/googlemock', 'src/*.cc'),
- ]),
- exported_linker_flags = [
+ ]),
+ exported_linker_flags = [
"-pthread",
"-lstdc++",
- ],
- visibility = [
+ ],
+ visibility = [
'PUBLIC',
- ],
- deps = [
- ]
+ ],
)
[19/50] [abbrv] hbase git commit: HBASE-14853 Add on protobuf to c++
chain
Posted by ec...@apache.org.
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/Master.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/Master.proto b/hbase-native-client/if/Master.proto
new file mode 100644
index 0000000..4d3a2e1
--- /dev/null
+++ b/hbase-native-client/if/Master.proto
@@ -0,0 +1,778 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// All to do with the Master. Includes schema management since these
+// changes are run by the Master process.
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "MasterProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "Client.proto";
+import "ClusterStatus.proto";
+import "ErrorHandling.proto";
+import "Procedure.proto";
+import "Quota.proto";
+
+/* Column-level protobufs */
+
+message AddColumnRequest {
+ required TableName table_name = 1;
+ required ColumnFamilySchema column_families = 2;
+ optional uint64 nonce_group = 3 [default = 0];
+ optional uint64 nonce = 4 [default = 0];
+}
+
+message AddColumnResponse {
+ optional uint64 proc_id = 1;
+}
+
+message DeleteColumnRequest {
+ required TableName table_name = 1;
+ required bytes column_name = 2;
+ optional uint64 nonce_group = 3 [default = 0];
+ optional uint64 nonce = 4 [default = 0];
+}
+
+message DeleteColumnResponse {
+ optional uint64 proc_id = 1;
+}
+
+message ModifyColumnRequest {
+ required TableName table_name = 1;
+ required ColumnFamilySchema column_families = 2;
+ optional uint64 nonce_group = 3 [default = 0];
+ optional uint64 nonce = 4 [default = 0];
+}
+
+message ModifyColumnResponse {
+ optional uint64 proc_id = 1;
+}
+
+/* Region-level Protos */
+
+message MoveRegionRequest {
+ required RegionSpecifier region = 1;
+ optional ServerName dest_server_name = 2;
+}
+
+message MoveRegionResponse {
+}
+
+/**
+ * Dispatch merging the specified regions.
+ */
+message DispatchMergingRegionsRequest {
+ required RegionSpecifier region_a = 1;
+ required RegionSpecifier region_b = 2;
+ optional bool forcible = 3 [default = false];
+}
+
+message DispatchMergingRegionsResponse {
+}
+
+message AssignRegionRequest {
+ required RegionSpecifier region = 1;
+}
+
+message AssignRegionResponse {
+}
+
+message UnassignRegionRequest {
+ required RegionSpecifier region = 1;
+ optional bool force = 2 [default = false];
+}
+
+message UnassignRegionResponse {
+}
+
+message OfflineRegionRequest {
+ required RegionSpecifier region = 1;
+}
+
+message OfflineRegionResponse {
+}
+
+/* Table-level protobufs */
+
+message CreateTableRequest {
+ required TableSchema table_schema = 1;
+ repeated bytes split_keys = 2;
+ optional uint64 nonce_group = 3 [default = 0];
+ optional uint64 nonce = 4 [default = 0];
+}
+
+message CreateTableResponse {
+ optional uint64 proc_id = 1;
+}
+
+message DeleteTableRequest {
+ required TableName table_name = 1;
+ optional uint64 nonce_group = 2 [default = 0];
+ optional uint64 nonce = 3 [default = 0];
+}
+
+message DeleteTableResponse {
+ optional uint64 proc_id = 1;
+}
+
+message TruncateTableRequest {
+ required TableName tableName = 1;
+ optional bool preserveSplits = 2 [default = false];
+ optional uint64 nonce_group = 3 [default = 0];
+ optional uint64 nonce = 4 [default = 0];
+}
+
+message TruncateTableResponse {
+ optional uint64 proc_id = 1;
+}
+
+message EnableTableRequest {
+ required TableName table_name = 1;
+ optional uint64 nonce_group = 2 [default = 0];
+ optional uint64 nonce = 3 [default = 0];
+}
+
+message EnableTableResponse {
+ optional uint64 proc_id = 1;
+}
+
+message DisableTableRequest {
+ required TableName table_name = 1;
+ optional uint64 nonce_group = 2 [default = 0];
+ optional uint64 nonce = 3 [default = 0];
+}
+
+message DisableTableResponse {
+ optional uint64 proc_id = 1;
+}
+
+message ModifyTableRequest {
+ required TableName table_name = 1;
+ required TableSchema table_schema = 2;
+ optional uint64 nonce_group = 3 [default = 0];
+ optional uint64 nonce = 4 [default = 0];
+}
+
+message ModifyTableResponse {
+ optional uint64 proc_id = 1;
+}
+
+/* Namespace-level protobufs */
+
+message CreateNamespaceRequest {
+ required NamespaceDescriptor namespaceDescriptor = 1;
+ optional uint64 nonce_group = 2 [default = 0];
+ optional uint64 nonce = 3 [default = 0];
+}
+
+message CreateNamespaceResponse {
+}
+
+message DeleteNamespaceRequest {
+ required string namespaceName = 1;
+ optional uint64 nonce_group = 2 [default = 0];
+ optional uint64 nonce = 3 [default = 0];
+}
+
+message DeleteNamespaceResponse {
+}
+
+message ModifyNamespaceRequest {
+ required NamespaceDescriptor namespaceDescriptor = 1;
+ optional uint64 nonce_group = 2 [default = 0];
+ optional uint64 nonce = 3 [default = 0];
+}
+
+message ModifyNamespaceResponse {
+}
+
+message GetNamespaceDescriptorRequest {
+ required string namespaceName = 1;
+}
+
+message GetNamespaceDescriptorResponse {
+ required NamespaceDescriptor namespaceDescriptor = 1;
+}
+
+message ListNamespaceDescriptorsRequest {
+}
+
+message ListNamespaceDescriptorsResponse {
+ repeated NamespaceDescriptor namespaceDescriptor = 1;
+}
+
+message ListTableDescriptorsByNamespaceRequest {
+ required string namespaceName = 1;
+}
+
+message ListTableDescriptorsByNamespaceResponse {
+ repeated TableSchema tableSchema = 1;
+}
+
+message ListTableNamesByNamespaceRequest {
+ required string namespaceName = 1;
+}
+
+message ListTableNamesByNamespaceResponse {
+ repeated TableName tableName = 1;
+}
+
+/* Cluster-level protobufs */
+
+
+message ShutdownRequest {
+}
+
+message ShutdownResponse {
+}
+
+message StopMasterRequest {
+}
+
+message StopMasterResponse {
+}
+
+message BalanceRequest {
+ optional bool force = 1;
+}
+
+message BalanceResponse {
+ required bool balancer_ran = 1;
+}
+
+message SetBalancerRunningRequest {
+ required bool on = 1;
+ optional bool synchronous = 2;
+}
+
+message SetBalancerRunningResponse {
+ optional bool prev_balance_value = 1;
+}
+
+message IsBalancerEnabledRequest {
+}
+
+message IsBalancerEnabledResponse {
+ required bool enabled = 1;
+}
+
+message NormalizeRequest {
+}
+
+message NormalizeResponse {
+ required bool normalizer_ran = 1;
+}
+
+message SetNormalizerRunningRequest {
+ required bool on = 1;
+}
+
+message SetNormalizerRunningResponse {
+ optional bool prev_normalizer_value = 1;
+}
+
+message IsNormalizerEnabledRequest {
+}
+
+message IsNormalizerEnabledResponse {
+ required bool enabled = 1;
+}
+
+message RunCatalogScanRequest {
+}
+
+message RunCatalogScanResponse {
+ optional int32 scan_result = 1;
+}
+
+message EnableCatalogJanitorRequest {
+ required bool enable = 1;
+}
+
+message EnableCatalogJanitorResponse {
+ optional bool prev_value = 1;
+}
+
+message IsCatalogJanitorEnabledRequest {
+}
+
+message IsCatalogJanitorEnabledResponse {
+ required bool value = 1;
+}
+
+message SnapshotRequest {
+ required SnapshotDescription snapshot = 1;
+}
+
+message SnapshotResponse {
+ required int64 expected_timeout = 1;
+}
+
+message GetCompletedSnapshotsRequest {
+}
+
+message GetCompletedSnapshotsResponse {
+ repeated SnapshotDescription snapshots = 1;
+}
+
+message DeleteSnapshotRequest {
+ required SnapshotDescription snapshot = 1;
+}
+
+message DeleteSnapshotResponse {
+}
+
+message RestoreSnapshotRequest {
+ required SnapshotDescription snapshot = 1;
+}
+
+message RestoreSnapshotResponse {
+}
+
+/* if you don't send the snapshot, then you will get it back
+ * in the response (if the snapshot is done) so you can check the snapshot
+ */
+message IsSnapshotDoneRequest {
+ optional SnapshotDescription snapshot = 1;
+}
+
+message IsSnapshotDoneResponse {
+ optional bool done = 1 [default = false];
+ optional SnapshotDescription snapshot = 2;
+}
+
+message IsRestoreSnapshotDoneRequest {
+ optional SnapshotDescription snapshot = 1;
+}
+
+message IsRestoreSnapshotDoneResponse {
+ optional bool done = 1 [default = false];
+}
+
+message GetSchemaAlterStatusRequest {
+ required TableName table_name = 1;
+}
+
+message GetSchemaAlterStatusResponse {
+ optional uint32 yet_to_update_regions = 1;
+ optional uint32 total_regions = 2;
+}
+
+message GetTableDescriptorsRequest {
+ repeated TableName table_names = 1;
+ optional string regex = 2;
+ optional bool include_sys_tables = 3 [default=false];
+ optional string namespace = 4;
+}
+
+message GetTableDescriptorsResponse {
+ repeated TableSchema table_schema = 1;
+}
+
+message GetTableNamesRequest {
+ optional string regex = 1;
+ optional bool include_sys_tables = 2 [default=false];
+ optional string namespace = 3;
+}
+
+message GetTableNamesResponse {
+ repeated TableName table_names = 1;
+}
+
+message GetTableStateRequest {
+ required TableName table_name = 1;
+}
+
+message GetTableStateResponse {
+ required TableState table_state = 1;
+}
+
+message GetClusterStatusRequest {
+}
+
+message GetClusterStatusResponse {
+ required ClusterStatus cluster_status = 1;
+}
+
+message IsMasterRunningRequest {
+}
+
+message IsMasterRunningResponse {
+ required bool is_master_running = 1;
+}
+
+message ExecProcedureRequest {
+ required ProcedureDescription procedure = 1;
+}
+
+message ExecProcedureResponse {
+ optional int64 expected_timeout = 1;
+ optional bytes return_data = 2;
+}
+
+message IsProcedureDoneRequest {
+ optional ProcedureDescription procedure = 1;
+}
+
+message IsProcedureDoneResponse {
+ optional bool done = 1 [default = false];
+ optional ProcedureDescription snapshot = 2;
+}
+
+message GetProcedureResultRequest {
+ required uint64 proc_id = 1;
+}
+
+message GetProcedureResultResponse {
+ enum State {
+ NOT_FOUND = 0;
+ RUNNING = 1;
+ FINISHED = 2;
+ }
+
+ required State state = 1;
+ optional uint64 start_time = 2;
+ optional uint64 last_update = 3;
+ optional bytes result = 4;
+ optional ForeignExceptionMessage exception = 5;
+}
+
+message AbortProcedureRequest {
+ required uint64 proc_id = 1;
+ optional bool mayInterruptIfRunning = 2 [default = true];
+}
+
+message AbortProcedureResponse {
+ required bool is_procedure_aborted = 1;
+}
+
+message ListProceduresRequest {
+}
+
+message ListProceduresResponse {
+ repeated Procedure procedure = 1;
+}
+
+message SetQuotaRequest {
+ optional string user_name = 1;
+ optional string user_group = 2;
+ optional string namespace = 3;
+ optional TableName table_name = 4;
+
+ optional bool remove_all = 5;
+ optional bool bypass_globals = 6;
+ optional ThrottleRequest throttle = 7;
+}
+
+message SetQuotaResponse {
+}
+
+message MajorCompactionTimestampRequest {
+ required TableName table_name = 1;
+}
+
+message MajorCompactionTimestampForRegionRequest {
+ required RegionSpecifier region = 1;
+}
+
+message MajorCompactionTimestampResponse {
+ required int64 compaction_timestamp = 1;
+}
+
+message SecurityCapabilitiesRequest {
+}
+
+message SecurityCapabilitiesResponse {
+ enum Capability {
+ SIMPLE_AUTHENTICATION = 0;
+ SECURE_AUTHENTICATION = 1;
+ AUTHORIZATION = 2;
+ CELL_AUTHORIZATION = 3;
+ CELL_VISIBILITY = 4;
+ }
+
+ repeated Capability capabilities = 1;
+}
+
+service MasterService {
+ /** Used by the client to get the number of regions that have received the updated schema */
+ rpc GetSchemaAlterStatus(GetSchemaAlterStatusRequest)
+ returns(GetSchemaAlterStatusResponse);
+
+ /** Get list of TableDescriptors for requested tables. */
+ rpc GetTableDescriptors(GetTableDescriptorsRequest)
+ returns(GetTableDescriptorsResponse);
+
+ /** Get the list of table names. */
+ rpc GetTableNames(GetTableNamesRequest)
+ returns(GetTableNamesResponse);
+
+ /** Return cluster status. */
+ rpc GetClusterStatus(GetClusterStatusRequest)
+ returns(GetClusterStatusResponse);
+
+ /** return true if master is available */
+ rpc IsMasterRunning(IsMasterRunningRequest) returns(IsMasterRunningResponse);
+
+ /** Adds a column to the specified table. */
+ rpc AddColumn(AddColumnRequest)
+ returns(AddColumnResponse);
+
+ /** Deletes a column from the specified table. Table must be disabled. */
+ rpc DeleteColumn(DeleteColumnRequest)
+ returns(DeleteColumnResponse);
+
+ /** Modifies an existing column on the specified table. */
+ rpc ModifyColumn(ModifyColumnRequest)
+ returns(ModifyColumnResponse);
+
+ /** Move the region region to the destination server. */
+ rpc MoveRegion(MoveRegionRequest)
+ returns(MoveRegionResponse);
+
+ /** Master dispatch merging the regions */
+ rpc DispatchMergingRegions(DispatchMergingRegionsRequest)
+ returns(DispatchMergingRegionsResponse);
+
+ /** Assign a region to a server chosen at random. */
+ rpc AssignRegion(AssignRegionRequest)
+ returns(AssignRegionResponse);
+
+ /**
+ * Unassign a region from current hosting regionserver. Region will then be
+ * assigned to a regionserver chosen at random. Region could be reassigned
+ * back to the same server. Use MoveRegion if you want
+ * to control the region movement.
+ */
+ rpc UnassignRegion(UnassignRegionRequest)
+ returns(UnassignRegionResponse);
+
+ /**
+ * Offline a region from the assignment manager's in-memory state. The
+ * region should be in a closed state and there will be no attempt to
+ * automatically reassign the region as in unassign. This is a special
+ * method, and should only be used by experts or hbck.
+ */
+ rpc OfflineRegion(OfflineRegionRequest)
+ returns(OfflineRegionResponse);
+
+ /** Deletes a table */
+ rpc DeleteTable(DeleteTableRequest)
+ returns(DeleteTableResponse);
+
+ /** Truncate a table */
+ rpc truncateTable(TruncateTableRequest)
+ returns(TruncateTableResponse);
+
+ /** Puts the table on-line (only needed if table has been previously taken offline) */
+ rpc EnableTable(EnableTableRequest)
+ returns(EnableTableResponse);
+
+ /** Take table offline */
+ rpc DisableTable(DisableTableRequest)
+ returns(DisableTableResponse);
+
+ /** Modify a table's metadata */
+ rpc ModifyTable(ModifyTableRequest)
+ returns(ModifyTableResponse);
+
+ /** Creates a new table asynchronously */
+ rpc CreateTable(CreateTableRequest)
+ returns(CreateTableResponse);
+
+ /** Shutdown an HBase cluster. */
+ rpc Shutdown(ShutdownRequest)
+ returns(ShutdownResponse);
+
+ /** Stop HBase Master only. Does not shutdown the cluster. */
+ rpc StopMaster(StopMasterRequest)
+ returns(StopMasterResponse);
+
+ /**
+ * Run the balancer. Will run the balancer and if regions to move, it will
+ * go ahead and do the reassignments. Can NOT run for various reasons.
+ * Check logs.
+ */
+ rpc Balance(BalanceRequest)
+ returns(BalanceResponse);
+
+ /**
+ * Turn the load balancer on or off.
+ * If synchronous is true, it waits until current balance() call, if outstanding, to return.
+ */
+ rpc SetBalancerRunning(SetBalancerRunningRequest)
+ returns(SetBalancerRunningResponse);
+
+ /**
+ * Query whether the Region Balancer is running.
+ */
+ rpc IsBalancerEnabled(IsBalancerEnabledRequest)
+ returns(IsBalancerEnabledResponse);
+
+ /**
+ * Run region normalizer. Can NOT run for various reasons. Check logs.
+ */
+ rpc Normalize(NormalizeRequest)
+ returns(NormalizeResponse);
+
+ /**
+ * Turn region normalizer on or off.
+ */
+ rpc SetNormalizerRunning(SetNormalizerRunningRequest)
+ returns(SetNormalizerRunningResponse);
+
+ /**
+ * Query whether region normalizer is enabled.
+ */
+ rpc IsNormalizerEnabled(IsNormalizerEnabledRequest)
+ returns(IsNormalizerEnabledResponse);
+
+ /** Get a run of the catalog janitor */
+ rpc RunCatalogScan(RunCatalogScanRequest)
+ returns(RunCatalogScanResponse);
+
+ /**
+ * Enable the catalog janitor on or off.
+ */
+ rpc EnableCatalogJanitor(EnableCatalogJanitorRequest)
+ returns(EnableCatalogJanitorResponse);
+
+ /**
+ * Query whether the catalog janitor is enabled.
+ */
+ rpc IsCatalogJanitorEnabled(IsCatalogJanitorEnabledRequest)
+ returns(IsCatalogJanitorEnabledResponse);
+
+ /**
+ * Call a master coprocessor endpoint
+ */
+ rpc ExecMasterService(CoprocessorServiceRequest)
+ returns(CoprocessorServiceResponse);
+
+ /**
+ * Create a snapshot for the given table.
+ */
+ rpc Snapshot(SnapshotRequest) returns(SnapshotResponse);
+
+ /**
+ * Get completed snapshots.
+ * Returns a list of snapshot descriptors for completed snapshots
+ */
+ rpc GetCompletedSnapshots(GetCompletedSnapshotsRequest) returns(GetCompletedSnapshotsResponse);
+
+ /**
+ * Delete an existing snapshot. This method can also be used to clean up an aborted snapshot.
+ */
+ rpc DeleteSnapshot(DeleteSnapshotRequest) returns(DeleteSnapshotResponse);
+
+ /**
+ * Determine if the snapshot is done yet.
+ */
+ rpc IsSnapshotDone(IsSnapshotDoneRequest) returns(IsSnapshotDoneResponse);
+
+ /**
+ * Restore a snapshot
+ */
+ rpc RestoreSnapshot(RestoreSnapshotRequest) returns(RestoreSnapshotResponse);
+
+ /**
+ * Determine if the snapshot restore is done yet.
+ */
+ rpc IsRestoreSnapshotDone(IsRestoreSnapshotDoneRequest) returns(IsRestoreSnapshotDoneResponse);
+
+ /**
+ * Execute a distributed procedure.
+ */
+ rpc ExecProcedure(ExecProcedureRequest) returns(ExecProcedureResponse);
+
+ /**
+ * Execute a distributed procedure with return data.
+ */
+ rpc ExecProcedureWithRet(ExecProcedureRequest) returns(ExecProcedureResponse);
+
+ /**
+ * Determine if the procedure is done yet.
+ */
+ rpc IsProcedureDone(IsProcedureDoneRequest) returns(IsProcedureDoneResponse);
+
+ /** return true if master is available */
+ /** rpc IsMasterRunning(IsMasterRunningRequest) returns(IsMasterRunningResponse); */
+
+ /** Modify a namespace's metadata */
+ rpc ModifyNamespace(ModifyNamespaceRequest)
+ returns(ModifyNamespaceResponse);
+
+ /** Creates a new namespace synchronously */
+ rpc CreateNamespace(CreateNamespaceRequest)
+ returns(CreateNamespaceResponse);
+
+ /** Deletes namespace synchronously */
+ rpc DeleteNamespace(DeleteNamespaceRequest)
+ returns(DeleteNamespaceResponse);
+
+ /** Get a namespace descriptor by name */
+ rpc GetNamespaceDescriptor(GetNamespaceDescriptorRequest)
+ returns(GetNamespaceDescriptorResponse);
+
+ /** returns a list of namespaces */
+ rpc ListNamespaceDescriptors(ListNamespaceDescriptorsRequest)
+ returns(ListNamespaceDescriptorsResponse);
+
+ /** returns a list of tables for a given namespace*/
+ rpc ListTableDescriptorsByNamespace(ListTableDescriptorsByNamespaceRequest)
+ returns(ListTableDescriptorsByNamespaceResponse);
+
+ /** returns a list of tables for a given namespace*/
+ rpc ListTableNamesByNamespace(ListTableNamesByNamespaceRequest)
+ returns(ListTableNamesByNamespaceResponse);
+
+ /** returns table state */
+ rpc GetTableState(GetTableStateRequest)
+ returns(GetTableStateResponse);
+
+ /** Apply the new quota settings */
+ rpc SetQuota(SetQuotaRequest) returns(SetQuotaResponse);
+
+ /** Returns the timestamp of the last major compaction */
+ rpc getLastMajorCompactionTimestamp(MajorCompactionTimestampRequest)
+ returns(MajorCompactionTimestampResponse);
+
+ /** Returns the timestamp of the last major compaction */
+ rpc getLastMajorCompactionTimestampForRegion(MajorCompactionTimestampForRegionRequest)
+ returns(MajorCompactionTimestampResponse);
+
+ rpc getProcedureResult(GetProcedureResultRequest)
+ returns(GetProcedureResultResponse);
+
+ /** Returns the security capabilities in effect on the cluster */
+ rpc getSecurityCapabilities(SecurityCapabilitiesRequest)
+ returns(SecurityCapabilitiesResponse);
+
+ /** Abort a procedure */
+ rpc AbortProcedure(AbortProcedureRequest)
+ returns(AbortProcedureResponse);
+
+ /** returns a list of procedures */
+ rpc ListProcedures(ListProceduresRequest)
+ returns(ListProceduresResponse);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/MasterProcedure.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/MasterProcedure.proto b/hbase-native-client/if/MasterProcedure.proto
new file mode 100644
index 0000000..2d2aff4
--- /dev/null
+++ b/hbase-native-client/if/MasterProcedure.proto
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "MasterProcedureProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "RPC.proto";
+
+// ============================================================================
+// WARNING - Compatibility rules
+// ============================================================================
+// This .proto contains the data serialized by the master procedures.
+// Each procedure has some state stored to know, which step were executed
+// and what were the parameters or data created by the previous steps.
+// new code should be able to handle the old format or at least fail cleanly
+// triggering a rollback/cleanup.
+//
+// Procedures that are inheriting from a StateMachineProcedure have an enum:
+// - Do not change the number of the 'State' enums.
+// doing so, will cause executing the wrong 'step' on the pending
+// procedures when they will be replayed.
+// - Do not remove items from the enum, new code must be able to handle
+// all the previous 'steps'. There may be pending procedure ready to be
+// recovered replayed. alternative you can make sure that not-known state
+// will result in a failure that will rollback the already executed steps.
+// ============================================================================
+
+enum CreateTableState {
+ CREATE_TABLE_PRE_OPERATION = 1;
+ CREATE_TABLE_WRITE_FS_LAYOUT = 2;
+ CREATE_TABLE_ADD_TO_META = 3;
+ CREATE_TABLE_ASSIGN_REGIONS = 4;
+ CREATE_TABLE_UPDATE_DESC_CACHE = 5;
+ CREATE_TABLE_POST_OPERATION = 6;
+}
+
+message CreateTableStateData {
+ required UserInformation user_info = 1;
+ required TableSchema table_schema = 2;
+ repeated RegionInfo region_info = 3;
+}
+
+enum ModifyTableState {
+ MODIFY_TABLE_PREPARE = 1;
+ MODIFY_TABLE_PRE_OPERATION = 2;
+ MODIFY_TABLE_UPDATE_TABLE_DESCRIPTOR = 3;
+ MODIFY_TABLE_REMOVE_REPLICA_COLUMN = 4;
+ MODIFY_TABLE_DELETE_FS_LAYOUT = 5;
+ MODIFY_TABLE_POST_OPERATION = 6;
+ MODIFY_TABLE_REOPEN_ALL_REGIONS = 7;
+}
+
+message ModifyTableStateData {
+ required UserInformation user_info = 1;
+ optional TableSchema unmodified_table_schema = 2;
+ required TableSchema modified_table_schema = 3;
+ required bool delete_column_family_in_modify = 4;
+}
+
+enum TruncateTableState {
+ TRUNCATE_TABLE_PRE_OPERATION = 1;
+ TRUNCATE_TABLE_REMOVE_FROM_META = 2;
+ TRUNCATE_TABLE_CLEAR_FS_LAYOUT = 3;
+ TRUNCATE_TABLE_CREATE_FS_LAYOUT = 4;
+ TRUNCATE_TABLE_ADD_TO_META = 5;
+ TRUNCATE_TABLE_ASSIGN_REGIONS = 6;
+ TRUNCATE_TABLE_POST_OPERATION = 7;
+}
+
+message TruncateTableStateData {
+ required UserInformation user_info = 1;
+ required bool preserve_splits = 2;
+ optional TableName table_name = 3;
+ optional TableSchema table_schema = 4;
+ repeated RegionInfo region_info = 5;
+}
+
+enum DeleteTableState {
+ DELETE_TABLE_PRE_OPERATION = 1;
+ DELETE_TABLE_REMOVE_FROM_META = 2;
+ DELETE_TABLE_CLEAR_FS_LAYOUT = 3;
+ DELETE_TABLE_UPDATE_DESC_CACHE = 4;
+ DELETE_TABLE_UNASSIGN_REGIONS = 5;
+ DELETE_TABLE_POST_OPERATION = 6;
+}
+
+message DeleteTableStateData {
+ required UserInformation user_info = 1;
+ required TableName table_name = 2;
+ repeated RegionInfo region_info = 3;
+}
+
+enum CreateNamespaceState {
+ CREATE_NAMESPACE_PREPARE = 1;
+ CREATE_NAMESPACE_CREATE_DIRECTORY = 2;
+ CREATE_NAMESPACE_INSERT_INTO_NS_TABLE = 3;
+ CREATE_NAMESPACE_UPDATE_ZK = 4;
+ CREATE_NAMESPACE_SET_NAMESPACE_QUOTA = 5;
+}
+
+message CreateNamespaceStateData {
+ required NamespaceDescriptor namespace_descriptor = 1;
+}
+
+enum ModifyNamespaceState {
+ MODIFY_NAMESPACE_PREPARE = 1;
+ MODIFY_NAMESPACE_UPDATE_NS_TABLE = 2;
+ MODIFY_NAMESPACE_UPDATE_ZK = 3;
+}
+
+message ModifyNamespaceStateData {
+ required NamespaceDescriptor namespace_descriptor = 1;
+ optional NamespaceDescriptor unmodified_namespace_descriptor = 2;
+}
+
+enum DeleteNamespaceState {
+ DELETE_NAMESPACE_PREPARE = 1;
+ DELETE_NAMESPACE_DELETE_FROM_NS_TABLE = 2;
+ DELETE_NAMESPACE_REMOVE_FROM_ZK = 3;
+ DELETE_NAMESPACE_DELETE_DIRECTORIES = 4;
+ DELETE_NAMESPACE_REMOVE_NAMESPACE_QUOTA = 5;
+}
+
+message DeleteNamespaceStateData {
+ required string namespace_name = 1;
+ optional NamespaceDescriptor namespace_descriptor = 2;
+}
+
+enum AddColumnFamilyState {
+ ADD_COLUMN_FAMILY_PREPARE = 1;
+ ADD_COLUMN_FAMILY_PRE_OPERATION = 2;
+ ADD_COLUMN_FAMILY_UPDATE_TABLE_DESCRIPTOR = 3;
+ ADD_COLUMN_FAMILY_POST_OPERATION = 4;
+ ADD_COLUMN_FAMILY_REOPEN_ALL_REGIONS = 5;
+}
+
+message AddColumnFamilyStateData {
+ required UserInformation user_info = 1;
+ required TableName table_name = 2;
+ required ColumnFamilySchema columnfamily_schema = 3;
+ optional TableSchema unmodified_table_schema = 4;
+}
+
+enum ModifyColumnFamilyState {
+ MODIFY_COLUMN_FAMILY_PREPARE = 1;
+ MODIFY_COLUMN_FAMILY_PRE_OPERATION = 2;
+ MODIFY_COLUMN_FAMILY_UPDATE_TABLE_DESCRIPTOR = 3;
+ MODIFY_COLUMN_FAMILY_POST_OPERATION = 4;
+ MODIFY_COLUMN_FAMILY_REOPEN_ALL_REGIONS = 5;
+}
+
+message ModifyColumnFamilyStateData {
+ required UserInformation user_info = 1;
+ required TableName table_name = 2;
+ required ColumnFamilySchema columnfamily_schema = 3;
+ optional TableSchema unmodified_table_schema = 4;
+}
+
+enum DeleteColumnFamilyState {
+ DELETE_COLUMN_FAMILY_PREPARE = 1;
+ DELETE_COLUMN_FAMILY_PRE_OPERATION = 2;
+ DELETE_COLUMN_FAMILY_UPDATE_TABLE_DESCRIPTOR = 3;
+ DELETE_COLUMN_FAMILY_DELETE_FS_LAYOUT = 4;
+ DELETE_COLUMN_FAMILY_POST_OPERATION = 5;
+ DELETE_COLUMN_FAMILY_REOPEN_ALL_REGIONS = 6;
+}
+
+message DeleteColumnFamilyStateData {
+ required UserInformation user_info = 1;
+ required TableName table_name = 2;
+ required bytes columnfamily_name = 3;
+ optional TableSchema unmodified_table_schema = 4;
+}
+
+enum EnableTableState {
+ ENABLE_TABLE_PREPARE = 1;
+ ENABLE_TABLE_PRE_OPERATION = 2;
+ ENABLE_TABLE_SET_ENABLING_TABLE_STATE = 3;
+ ENABLE_TABLE_MARK_REGIONS_ONLINE = 4;
+ ENABLE_TABLE_SET_ENABLED_TABLE_STATE = 5;
+ ENABLE_TABLE_POST_OPERATION = 6;
+}
+
+message EnableTableStateData {
+ required UserInformation user_info = 1;
+ required TableName table_name = 2;
+ required bool skip_table_state_check = 3;
+}
+
+enum DisableTableState {
+ DISABLE_TABLE_PREPARE = 1;
+ DISABLE_TABLE_PRE_OPERATION = 2;
+ DISABLE_TABLE_SET_DISABLING_TABLE_STATE = 3;
+ DISABLE_TABLE_MARK_REGIONS_OFFLINE = 4;
+ DISABLE_TABLE_SET_DISABLED_TABLE_STATE = 5;
+ DISABLE_TABLE_POST_OPERATION = 6;
+}
+
+message DisableTableStateData {
+ required UserInformation user_info = 1;
+ required TableName table_name = 2;
+ required bool skip_table_state_check = 3;
+}
+
+message ServerCrashStateData {
+ required ServerName server_name = 1;
+ optional bool distributed_log_replay = 2;
+ repeated RegionInfo regions_on_crashed_server = 3;
+ repeated RegionInfo regions_assigned = 4;
+ optional bool carrying_meta = 5;
+ optional bool should_split_wal = 6 [default = true];
+}
+
+enum ServerCrashState {
+ SERVER_CRASH_START = 1;
+ SERVER_CRASH_PROCESS_META = 2;
+ SERVER_CRASH_GET_REGIONS = 3;
+ SERVER_CRASH_NO_SPLIT_LOGS = 4;
+ SERVER_CRASH_SPLIT_LOGS = 5;
+ SERVER_CRASH_PREPARE_LOG_REPLAY = 6;
+ // Removed SERVER_CRASH_CALC_REGIONS_TO_ASSIGN = 7;
+ SERVER_CRASH_ASSIGN = 8;
+ SERVER_CRASH_WAIT_ON_ASSIGN = 9;
+ SERVER_CRASH_FINISH = 100;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/MultiRowMutation.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/MultiRowMutation.proto b/hbase-native-client/if/MultiRowMutation.proto
new file mode 100644
index 0000000..747afac
--- /dev/null
+++ b/hbase-native-client/if/MultiRowMutation.proto
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+import "Client.proto";
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "MultiRowMutationProtos";
+option java_generate_equals_and_hash = true;
+option java_generic_services = true;
+option optimize_for = SPEED;
+
+message MultiRowMutationProcessorRequest{
+}
+
+message MultiRowMutationProcessorResponse{
+}
+
+message MutateRowsRequest {
+ repeated MutationProto mutation_request = 1;
+ optional uint64 nonce_group = 2;
+ optional uint64 nonce = 3;
+}
+
+message MutateRowsResponse {
+}
+
+service MultiRowMutationService {
+ rpc MutateRows(MutateRowsRequest)
+ returns(MutateRowsResponse);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/Procedure.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/Procedure.proto b/hbase-native-client/if/Procedure.proto
new file mode 100644
index 0000000..55e44a4
--- /dev/null
+++ b/hbase-native-client/if/Procedure.proto
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "ProcedureProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "ErrorHandling.proto";
+
+enum ProcedureState {
+ INITIALIZING = 1; // Procedure in construction, not yet added to the executor
+ RUNNABLE = 2; // Procedure added to the executor, and ready to be executed
+ WAITING = 3; // The procedure is waiting on children to be completed
+ WAITING_TIMEOUT = 4; // The procedure is waiting a timout or an external event
+ ROLLEDBACK = 5; // The procedure failed and was rolledback
+ FINISHED = 6; // The procedure execution is completed. may need a rollback if failed.
+}
+
+/**
+ * Procedure metadata, serialized by the ProcedureStore to be able to recover the old state.
+ */
+message Procedure {
+ // internal "static" state
+ required string class_name = 1; // full classname to be able to instantiate the procedure
+ optional uint64 parent_id = 2; // parent if not a root-procedure otherwise not set
+ required uint64 proc_id = 3;
+ required uint64 start_time = 4;
+ optional string owner = 5;
+
+ // internal "runtime" state
+ required ProcedureState state = 6;
+ repeated uint32 stack_id = 7; // stack indices in case the procedure was running
+ required uint64 last_update = 8;
+ optional uint32 timeout = 9;
+
+ // user state/results
+ optional ForeignExceptionMessage exception = 10;
+ optional bytes result = 11; // opaque (user) result structure
+ optional bytes state_data = 12; // opaque (user) procedure internal-state
+
+ // Nonce to prevent same procedure submit by multiple times
+ optional uint64 nonce_group = 13 [default = 0];
+ optional uint64 nonce = 14 [default = 0];
+}
+
+/**
+ * SequentialProcedure data
+ */
+message SequentialProcedureData {
+ required bool executed = 1;
+}
+
+/**
+ * StateMachineProcedure data
+ */
+message StateMachineProcedureData {
+ repeated uint32 state = 1;
+}
+
+/**
+ * Procedure WAL header
+ */
+message ProcedureWALHeader {
+ required uint32 version = 1;
+ required uint32 type = 2;
+ required uint64 log_id = 3;
+ required uint64 min_proc_id = 4;
+}
+
+/**
+ * Procedure WAL trailer
+ */
+message ProcedureWALTrailer {
+ required uint32 version = 1;
+ required uint64 tracker_pos = 2;
+}
+
+message ProcedureStoreTracker {
+ message TrackerNode {
+ required uint64 start_id = 1;
+ repeated uint64 updated = 2;
+ repeated uint64 deleted = 3;
+ }
+
+ repeated TrackerNode node = 1;
+}
+
+message ProcedureWALEntry {
+ enum Type {
+ PROCEDURE_WAL_EOF = 1;
+ PROCEDURE_WAL_INIT = 2;
+ PROCEDURE_WAL_INSERT = 3;
+ PROCEDURE_WAL_UPDATE = 4;
+ PROCEDURE_WAL_DELETE = 5;
+ PROCEDURE_WAL_COMPACT = 6;
+ }
+
+ required Type type = 1;
+ repeated Procedure procedure = 2;
+ optional uint64 proc_id = 3;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/Quota.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/Quota.proto b/hbase-native-client/if/Quota.proto
new file mode 100644
index 0000000..a8303b1
--- /dev/null
+++ b/hbase-native-client/if/Quota.proto
@@ -0,0 +1,76 @@
+ /**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "QuotaProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+
+enum QuotaScope {
+ CLUSTER = 1;
+ MACHINE = 2;
+}
+
+message TimedQuota {
+ required TimeUnit time_unit = 1;
+ optional uint64 soft_limit = 2;
+ optional float share = 3;
+ optional QuotaScope scope = 4 [default = MACHINE];
+}
+
+enum ThrottleType {
+ REQUEST_NUMBER = 1;
+ REQUEST_SIZE = 2;
+ WRITE_NUMBER = 3;
+ WRITE_SIZE = 4;
+ READ_NUMBER = 5;
+ READ_SIZE = 6;
+}
+
+message Throttle {
+ optional TimedQuota req_num = 1;
+ optional TimedQuota req_size = 2;
+
+ optional TimedQuota write_num = 3;
+ optional TimedQuota write_size = 4;
+
+ optional TimedQuota read_num = 5;
+ optional TimedQuota read_size = 6;
+}
+
+message ThrottleRequest {
+ optional ThrottleType type = 1;
+ optional TimedQuota timed_quota = 2;
+}
+
+enum QuotaType {
+ THROTTLE = 1;
+}
+
+message Quotas {
+ optional bool bypass_globals = 1 [default = false];
+ optional Throttle throttle = 2;
+}
+
+message QuotaUsage {
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/RPC.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/RPC.proto b/hbase-native-client/if/RPC.proto
new file mode 100644
index 0000000..59bb03d
--- /dev/null
+++ b/hbase-native-client/if/RPC.proto
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+import "Tracing.proto";
+import "HBase.proto";
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "RPCProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+// See https://issues.apache.org/jira/browse/HBASE-7898 for high-level
+// description of RPC specification.
+//
+// On connection setup, the client sends six bytes of preamble -- a four
+// byte magic, a byte of version, and a byte of authentication type.
+//
+// We then send a "ConnectionHeader" protobuf of user information and the
+// 'protocol' or 'service' that is to be run over this connection as well as
+// info such as codecs and compression to use when we send cell blocks(see below).
+// This connection header protobuf is prefaced by an int that holds the length
+// of this connection header (this is NOT a varint). The pb connection header
+// is sent with Message#writeTo. The server throws an exception if it doesn't
+// like what it was sent noting what it is objecting too. Otherwise, the server
+// says nothing and is open for business.
+//
+// Hereafter the client makes requests and the server returns responses.
+//
+// Requests look like this:
+//
+// <An int with the total length of the request>
+// <RequestHeader Message written out using Message#writeDelimitedTo>
+// <Optionally a Request Parameter Message written out using Message#writeDelimitedTo>
+// <Optionally a Cell block>
+//
+// ...where the Request Parameter Message is whatever the method name stipulated
+// in the RequestHeader expects; e.g. if the method is a scan, then the pb
+// Request Message is a GetRequest, or a ScanRequest. A block of Cells
+// optionally follows. The presence of a Request param Message and/or a
+// block of Cells will be noted in the RequestHeader.
+//
+// Response is the mirror of the request:
+//
+// <An int with the total length of the response>
+// <ResponseHeader Message written out using Message#writeDelimitedTo>
+// <Optionally a Response Result Message written out using Message#writeDelimitedTo>
+// <Optionally a Cell block>
+//
+// ...where the Response Message is the response type that goes with the
+// method specified when making the request and the follow on Cell blocks may
+// or may not be there -- read the response header to find out if one following.
+// If an exception, it will be included inside the Response Header.
+//
+// Any time we write a pb, we do it with Message#writeDelimitedTo EXCEPT when
+// the connection header is sent; this is prefaced by an int with its length
+// and the pb connection header is then written with Message#writeTo.
+//
+
+// User Information proto. Included in ConnectionHeader on connection setup
+message UserInformation {
+ required string effective_user = 1;
+ optional string real_user = 2;
+}
+
+// This is sent on connection setup after the connection preamble is sent.
+message ConnectionHeader {
+ optional UserInformation user_info = 1;
+ optional string service_name = 2;
+ // Cell block codec we will use sending over optional cell blocks. Server throws exception
+ // if cannot deal. Null means no codec'ing going on so we are pb all the time (SLOW!!!)
+ optional string cell_block_codec_class = 3;
+ // Compressor we will use if cell block is compressed. Server will throw exception if not supported.
+ // Class must implement hadoop's CompressionCodec Interface. Can't compress if no codec.
+ optional string cell_block_compressor_class = 4;
+ optional VersionInfo version_info = 5;
+}
+
+// Optional Cell block Message. Included in client RequestHeader
+message CellBlockMeta {
+ // Length of the following cell block. Could calculate it but convenient having it too hand.
+ optional uint32 length = 1;
+}
+
+// At the RPC layer, this message is used to carry
+// the server side exception to the RPC client.
+message ExceptionResponse {
+ // Class name of the exception thrown from the server
+ optional string exception_class_name = 1;
+ // Exception stack trace from the server side
+ optional string stack_trace = 2;
+ // Optional hostname. Filled in for some exceptions such as region moved
+ // where exception gives clue on where the region may have moved.
+ optional string hostname = 3;
+ optional int32 port = 4;
+ // Set if we are NOT to retry on receipt of this exception
+ optional bool do_not_retry = 5;
+}
+
+// Header sent making a request.
+message RequestHeader {
+ // Monotonically increasing call_id to keep track of RPC requests and their response
+ optional uint32 call_id = 1;
+ optional RPCTInfo trace_info = 2;
+ optional string method_name = 3;
+ // If true, then a pb Message param follows.
+ optional bool request_param = 4;
+ // If present, then an encoded data block follows.
+ optional CellBlockMeta cell_block_meta = 5;
+ // 0 is NORMAL priority. 200 is HIGH. If no priority, treat it as NORMAL.
+ // See HConstants.
+ optional uint32 priority = 6;
+}
+
+message ResponseHeader {
+ optional uint32 call_id = 1;
+ // If present, then request threw an exception and no response message (else we presume one)
+ optional ExceptionResponse exception = 2;
+ // If present, then an encoded data block follows.
+ optional CellBlockMeta cell_block_meta = 3;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/RegionNormalizer.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/RegionNormalizer.proto b/hbase-native-client/if/RegionNormalizer.proto
new file mode 100644
index 0000000..e5305d6
--- /dev/null
+++ b/hbase-native-client/if/RegionNormalizer.proto
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers to represent the state of the load balancer.
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "RegionNormalizerProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+message RegionNormalizerState {
+ optional bool normalizer_on = 1;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/RegionServerStatus.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/RegionServerStatus.proto b/hbase-native-client/if/RegionServerStatus.proto
new file mode 100644
index 0000000..fda9de2
--- /dev/null
+++ b/hbase-native-client/if/RegionServerStatus.proto
@@ -0,0 +1,158 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// This file contains protocol buffers that are used for RegionServerStatusProtocol.
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "RegionServerStatusProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "ClusterStatus.proto";
+
+message RegionServerStartupRequest {
+ /** Port number this regionserver is up on */
+ required uint32 port = 1;
+
+ /** This servers' startcode */
+ required uint64 server_start_code = 2;
+
+ /** Current time of the region server in ms */
+ required uint64 server_current_time = 3;
+
+ /** hostname for region server, optional */
+ optional string use_this_hostname_instead = 4;
+}
+
+message RegionServerStartupResponse {
+ /**
+ * Configuration for the regionserver to use: e.g. filesystem,
+ * hbase rootdir, the hostname to use creating the RegionServer ServerName,
+ * etc
+ */
+ repeated NameStringPair map_entries = 1;
+}
+
+message RegionServerReportRequest {
+ required ServerName server = 1;
+
+ /** load the server is under */
+ optional ServerLoad load = 2;
+}
+
+message RegionServerReportResponse {
+}
+
+message ReportRSFatalErrorRequest {
+ /** name of the server experiencing the error */
+ required ServerName server = 1;
+
+ /** informative text to expose in the master logs and UI */
+ required string error_message = 2;
+}
+
+message ReportRSFatalErrorResponse {
+}
+
+message GetLastFlushedSequenceIdRequest {
+ /** region name */
+ required bytes region_name = 1;
+}
+
+message GetLastFlushedSequenceIdResponse {
+ /** the last WAL sequence id flushed from MemStore to HFile for the region */
+ required uint64 last_flushed_sequence_id = 1;
+
+ /** the last WAL sequence id flushed from MemStore to HFile for stores of the region */
+ repeated StoreSequenceId store_last_flushed_sequence_id = 2;
+}
+
+message RegionStateTransition {
+ required TransitionCode transition_code = 1;
+
+ /** Mutliple regions are involved during merging/splitting */
+ repeated RegionInfo region_info = 2;
+
+ /** For newly opened region, the open seq num is needed */
+ optional uint64 open_seq_num = 3;
+
+ enum TransitionCode {
+ OPENED = 0;
+ FAILED_OPEN = 1;
+ /** No failed_close, in which case region server will abort */
+ CLOSED = 2;
+
+ /** Ask master for ok to split/merge region(s) */
+ READY_TO_SPLIT = 3;
+ READY_TO_MERGE = 4;
+
+ SPLIT_PONR = 5;
+ MERGE_PONR = 6;
+
+ SPLIT = 7;
+ MERGED = 8;
+ SPLIT_REVERTED = 9;
+ MERGE_REVERTED = 10;
+ }
+}
+
+message ReportRegionStateTransitionRequest {
+ /** This region server's server name */
+ required ServerName server = 1;
+
+ repeated RegionStateTransition transition = 2;
+}
+
+message ReportRegionStateTransitionResponse {
+ /** Error message if failed to update the region state */
+ optional string error_message = 1;
+}
+
+service RegionServerStatusService {
+ /** Called when a region server first starts. */
+ rpc RegionServerStartup(RegionServerStartupRequest)
+ returns(RegionServerStartupResponse);
+
+ /** Called to report the load the RegionServer is under. */
+ rpc RegionServerReport(RegionServerReportRequest)
+ returns(RegionServerReportResponse);
+
+ /**
+ * Called by a region server to report a fatal error that is causing it to
+ * abort.
+ */
+ rpc ReportRSFatalError(ReportRSFatalErrorRequest)
+ returns(ReportRSFatalErrorResponse);
+
+ /** Called to get the sequence id of the last MemStore entry flushed to an
+ * HFile for a specified region. Used by the region server to speed up
+ * log splitting. */
+ rpc GetLastFlushedSequenceId(GetLastFlushedSequenceIdRequest)
+ returns(GetLastFlushedSequenceIdResponse);
+
+ /**
+ * Called by a region server to report the progress of a region
+ * transition. If the request fails, the transition should
+ * be aborted.
+ */
+ rpc ReportRegionStateTransition(ReportRegionStateTransitionRequest)
+ returns(ReportRegionStateTransitionResponse);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/RowProcessor.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/RowProcessor.proto b/hbase-native-client/if/RowProcessor.proto
new file mode 100644
index 0000000..cf2f30f
--- /dev/null
+++ b/hbase-native-client/if/RowProcessor.proto
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Defines a protocol to perform multi row transactions.
+ * See BaseRowProcessorEndpoint for the implementation.
+ * See HRegion#processRowsWithLocks() for details.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "RowProcessorProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+message ProcessRequest {
+ required string row_processor_class_name = 1;
+ optional string row_processor_initializer_message_name = 2;
+ optional bytes row_processor_initializer_message = 3;
+ optional uint64 nonce_group = 4;
+ optional uint64 nonce = 5;
+}
+
+message ProcessResponse {
+ required bytes row_processor_result = 1;
+}
+
+service RowProcessorService {
+ rpc Process(ProcessRequest) returns (ProcessResponse);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/SecureBulkLoad.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/SecureBulkLoad.proto b/hbase-native-client/if/SecureBulkLoad.proto
new file mode 100644
index 0000000..814735b
--- /dev/null
+++ b/hbase-native-client/if/SecureBulkLoad.proto
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "SecureBulkLoadProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import 'HBase.proto';
+import 'Client.proto';
+
+message SecureBulkLoadHFilesRequest {
+ repeated BulkLoadHFileRequest.FamilyPath family_path = 1;
+ optional bool assign_seq_num = 2;
+ required DelegationToken fs_token = 3;
+ required string bulk_token = 4;
+}
+
+message SecureBulkLoadHFilesResponse {
+ required bool loaded = 1;
+}
+
+message DelegationToken {
+ optional bytes identifier = 1;
+ optional bytes password = 2;
+ optional string kind = 3;
+ optional string service = 4;
+}
+
+message PrepareBulkLoadRequest {
+ required TableName table_name = 1;
+}
+
+message PrepareBulkLoadResponse {
+ required string bulk_token = 1;
+}
+
+message CleanupBulkLoadRequest {
+ required string bulk_token = 1;
+
+}
+
+message CleanupBulkLoadResponse {
+}
+
+service SecureBulkLoadService {
+ rpc PrepareBulkLoad(PrepareBulkLoadRequest)
+ returns (PrepareBulkLoadResponse);
+
+ rpc SecureBulkLoadHFiles(SecureBulkLoadHFilesRequest)
+ returns (SecureBulkLoadHFilesResponse);
+
+ rpc CleanupBulkLoad(CleanupBulkLoadRequest)
+ returns (CleanupBulkLoadResponse);
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/Snapshot.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/Snapshot.proto b/hbase-native-client/if/Snapshot.proto
new file mode 100644
index 0000000..ae1a1e6
--- /dev/null
+++ b/hbase-native-client/if/Snapshot.proto
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "SnapshotProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "FS.proto";
+import "HBase.proto";
+
+message SnapshotFileInfo {
+ enum Type {
+ HFILE = 1;
+ WAL = 2;
+ }
+
+ required Type type = 1;
+
+ optional string hfile = 3;
+
+ optional string wal_server = 4;
+ optional string wal_name = 5;
+}
+
+message SnapshotRegionManifest {
+ optional int32 version = 1;
+
+ required RegionInfo region_info = 2;
+ repeated FamilyFiles family_files = 3;
+
+ message StoreFile {
+ required string name = 1;
+ optional Reference reference = 2;
+
+ // TODO: Add checksums or other fields to verify the file
+ optional uint64 file_size = 3;
+ }
+
+ message FamilyFiles {
+ required bytes family_name = 1;
+ repeated StoreFile store_files = 2;
+ }
+}
+
+message SnapshotDataManifest {
+ required TableSchema table_schema = 1;
+ repeated SnapshotRegionManifest region_manifests = 2;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/Tracing.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/Tracing.proto b/hbase-native-client/if/Tracing.proto
new file mode 100644
index 0000000..5a64cfc
--- /dev/null
+++ b/hbase-native-client/if/Tracing.proto
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "TracingProtos";
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+//Used to pass through the information necessary to continue
+//a trace after an RPC is made. All we need is the traceid
+//(so we know the overarching trace this message is a part of), and
+//the id of the current span when this message was sent, so we know
+//what span caused the new span we will create when this message is received.
+message RPCTInfo {
+ optional int64 trace_id = 1;
+ optional int64 parent_id = 2;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/VisibilityLabels.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/VisibilityLabels.proto b/hbase-native-client/if/VisibilityLabels.proto
new file mode 100644
index 0000000..d2dc44d
--- /dev/null
+++ b/hbase-native-client/if/VisibilityLabels.proto
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "VisibilityLabelsProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "Client.proto";
+
+message VisibilityLabelsRequest {
+ repeated VisibilityLabel visLabel = 1;
+}
+
+message VisibilityLabel {
+ required bytes label = 1;
+ optional uint32 ordinal = 2;
+}
+
+message VisibilityLabelsResponse {
+ repeated RegionActionResult result = 1;
+}
+
+message SetAuthsRequest {
+ required bytes user = 1;
+ repeated bytes auth = 2;
+}
+
+message UserAuthorizations {
+ required bytes user = 1;
+ repeated uint32 auth = 2;
+}
+
+message MultiUserAuthorizations {
+ repeated UserAuthorizations userAuths = 1;
+}
+
+message GetAuthsRequest {
+ required bytes user = 1;
+}
+
+message GetAuthsResponse {
+ required bytes user = 1;
+ repeated bytes auth = 2;
+}
+
+message ListLabelsRequest {
+ optional string regex = 1;
+}
+
+message ListLabelsResponse {
+ repeated bytes label = 1;
+}
+
+service VisibilityLabelsService {
+ rpc addLabels(VisibilityLabelsRequest)
+ returns (VisibilityLabelsResponse);
+ rpc setAuths(SetAuthsRequest)
+ returns (VisibilityLabelsResponse);
+ rpc clearAuths(SetAuthsRequest)
+ returns (VisibilityLabelsResponse);
+ rpc getAuths(GetAuthsRequest)
+ returns (GetAuthsResponse);
+ rpc listLabels(ListLabelsRequest)
+ returns (ListLabelsResponse);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/WAL.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/WAL.proto b/hbase-native-client/if/WAL.proto
new file mode 100644
index 0000000..cb9bd8f
--- /dev/null
+++ b/hbase-native-client/if/WAL.proto
@@ -0,0 +1,172 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "WALProtos";
+option java_generic_services = false;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "Client.proto";
+
+message WALHeader {
+ optional bool has_compression = 1;
+ optional bytes encryption_key = 2;
+ optional bool has_tag_compression = 3;
+ optional string writer_cls_name = 4;
+ optional string cell_codec_cls_name = 5;
+}
+
+/*
+ * Protocol buffer version of WALKey; see WALKey comment, not really a key but WALEdit header
+ * for some KVs
+ */
+message WALKey {
+ required bytes encoded_region_name = 1;
+ required bytes table_name = 2;
+ required uint64 log_sequence_number = 3;
+ required uint64 write_time = 4;
+ /*
+ This parameter is deprecated in favor of clusters which
+ contains the list of clusters that have consumed the change.
+ It is retained so that the log created by earlier releases (0.94)
+ can be read by the newer releases.
+ */
+ optional UUID cluster_id = 5 [deprecated=true];
+
+ repeated FamilyScope scopes = 6;
+ optional uint32 following_kv_count = 7;
+
+ /*
+ This field contains the list of clusters that have
+ consumed the change
+ */
+ repeated UUID cluster_ids = 8;
+
+ optional uint64 nonceGroup = 9;
+ optional uint64 nonce = 10;
+ optional uint64 orig_sequence_number = 11;
+
+/*
+ optional CustomEntryType custom_entry_type = 9;
+
+ enum CustomEntryType {
+ COMPACTION = 0;
+ }
+*/
+}
+
+enum ScopeType {
+ REPLICATION_SCOPE_LOCAL = 0;
+ REPLICATION_SCOPE_GLOBAL = 1;
+}
+
+message FamilyScope {
+ required bytes family = 1;
+ required ScopeType scope_type = 2;
+}
+
+/**
+ * Custom WAL entries
+ */
+
+/**
+ * Special WAL entry to hold all related to a compaction.
+ * Written to WAL before completing compaction. There is
+ * sufficient info in the below message to complete later
+ * the * compaction should we fail the WAL write.
+ */
+message CompactionDescriptor {
+ required bytes table_name = 1; // TODO: WALKey already stores these, might remove
+ required bytes encoded_region_name = 2;
+ required bytes family_name = 3;
+ repeated string compaction_input = 4; // relative to store dir
+ repeated string compaction_output = 5;
+ required string store_home_dir = 6; // relative to region dir
+ optional bytes region_name = 7; // full region name
+}
+
+/**
+ * Special WAL entry to hold all related to a flush.
+ */
+message FlushDescriptor {
+ enum FlushAction {
+ START_FLUSH = 0;
+ COMMIT_FLUSH = 1;
+ ABORT_FLUSH = 2;
+ CANNOT_FLUSH = 3; // marker for indicating that a flush has been requested but cannot complete
+ }
+
+ message StoreFlushDescriptor {
+ required bytes family_name = 1;
+ required string store_home_dir = 2; //relative to region dir
+ repeated string flush_output = 3; // relative to store dir (if this is a COMMIT_FLUSH)
+ }
+
+ required FlushAction action = 1;
+ required bytes table_name = 2;
+ required bytes encoded_region_name = 3;
+ optional uint64 flush_sequence_number = 4;
+ repeated StoreFlushDescriptor store_flushes = 5;
+ optional bytes region_name = 6; // full region name
+}
+
+message StoreDescriptor {
+ required bytes family_name = 1;
+ required string store_home_dir = 2; //relative to region dir
+ repeated string store_file = 3; // relative to store dir
+}
+
+/**
+ * Special WAL entry used for writing bulk load events to WAL
+ */
+message BulkLoadDescriptor {
+ required TableName table_name = 1;
+ required bytes encoded_region_name = 2;
+ repeated StoreDescriptor stores = 3;
+ required int64 bulkload_seq_num = 4;
+}
+
+/**
+ * Special WAL entry to hold all related to a region event (open/close).
+ */
+message RegionEventDescriptor {
+ enum EventType {
+ REGION_OPEN = 0;
+ REGION_CLOSE = 1;
+ }
+
+ required EventType event_type = 1;
+ required bytes table_name = 2;
+ required bytes encoded_region_name = 3;
+ optional uint64 log_sequence_number = 4;
+ repeated StoreDescriptor stores = 5;
+ optional ServerName server = 6; // Server who opened the region
+ optional bytes region_name = 7; // full region name
+}
+
+/**
+ * A trailer that is appended to the end of a properly closed WAL file.
+ * If missing, this is either a legacy or a corrupted WAL file.
+ * N.B. This trailer currently doesn't contain any information and we
+ * purposefully don't expose it in the WAL APIs. It's for future growth.
+ */
+message WALTrailer {
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/if/ZooKeeper.proto
----------------------------------------------------------------------
diff --git a/hbase-native-client/if/ZooKeeper.proto b/hbase-native-client/if/ZooKeeper.proto
new file mode 100644
index 0000000..54652af
--- /dev/null
+++ b/hbase-native-client/if/ZooKeeper.proto
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// ZNode data in hbase are serialized protobufs with a four byte
+// 'magic' 'PBUF' prefix.
+package hbase.pb;
+
+option java_package = "org.apache.hadoop.hbase.protobuf.generated";
+option java_outer_classname = "ZooKeeperProtos";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+option optimize_for = SPEED;
+
+import "HBase.proto";
+import "ClusterStatus.proto";
+
+/**
+ * Content of the meta-region-server znode.
+ */
+message MetaRegionServer {
+ // The ServerName hosting the meta region currently, or destination server,
+ // if meta region is in transition.
+ required ServerName server = 1;
+ // The major version of the rpc the server speaks. This is used so that
+ // clients connecting to the cluster can have prior knowledge of what version
+ // to send to a RegionServer. AsyncHBase will use this to detect versions.
+ optional uint32 rpc_version = 2;
+
+ // State of the region transition. OPEN means fully operational 'hbase:meta'
+ optional RegionState.State state = 3;
+}
+
+/**
+ * Content of the master znode.
+ */
+message Master {
+ // The ServerName of the current Master
+ required ServerName master = 1;
+ // Major RPC version so that clients can know what version the master can accept.
+ optional uint32 rpc_version = 2;
+ optional uint32 info_port = 3;
+}
+
+/**
+ * Content of the '/hbase/running', cluster state, znode.
+ */
+message ClusterUp {
+ // If this znode is present, cluster is up. Currently
+ // the data is cluster start_date.
+ required string start_date = 1;
+}
+
+/**
+ * WAL SplitLog directory znodes have this for content. Used doing distributed
+ * WAL splitting. Holds current state and name of server that originated split.
+ */
+message SplitLogTask {
+ enum State {
+ UNASSIGNED = 0;
+ OWNED = 1;
+ RESIGNED = 2;
+ DONE = 3;
+ ERR = 4;
+ }
+ enum RecoveryMode {
+ UNKNOWN = 0;
+ LOG_SPLITTING = 1;
+ LOG_REPLAY = 2;
+ }
+ required State state = 1;
+ required ServerName server_name = 2;
+ optional RecoveryMode mode = 3 [default = UNKNOWN];
+}
+
+/**
+ * The znode that holds state of table.
+ * Deprected, table state is stored in table descriptor on HDFS.
+ */
+message DeprecatedTableState {
+ // Table's current state
+ enum State {
+ ENABLED = 0;
+ DISABLED = 1;
+ DISABLING = 2;
+ ENABLING = 3;
+ }
+ // This is the table's state. If no znode for a table,
+ // its state is presumed enabled. See o.a.h.h.zookeeper.ZKTable class
+ // for more.
+ required State state = 1 [default = ENABLED];
+}
+
+/**
+ * Used by replication. Holds a replication peer key.
+ */
+message ReplicationPeer {
+ // clusterkey is the concatenation of the slave cluster's
+ // hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent
+ required string clusterkey = 1;
+ optional string replicationEndpointImpl = 2;
+ repeated BytesBytesPair data = 3;
+ repeated NameStringPair configuration = 4;
+}
+
+/**
+ * Used by replication. Holds whether enabled or disabled
+ */
+message ReplicationState {
+ enum State {
+ ENABLED = 0;
+ DISABLED = 1;
+ }
+ required State state = 1;
+}
+
+/**
+ * Used by replication. Holds the current position in an WAL file.
+ */
+message ReplicationHLogPosition {
+ required int64 position = 1;
+}
+
+/**
+ * Used by replication. Used to lock a region server during failover.
+ */
+message ReplicationLock {
+ required string lock_owner = 1;
+}
+
+/**
+ * Metadata associated with a table lock in zookeeper
+ */
+message TableLock {
+ optional TableName table_name = 1;
+ optional ServerName lock_owner = 2;
+ optional int64 thread_id = 3;
+ optional bool is_shared = 4;
+ optional string purpose = 5;
+ optional int64 create_time = 6;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/rpc/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hbase-native-client/rpc/CMakeLists.txt b/hbase-native-client/rpc/CMakeLists.txt
deleted file mode 100644
index 2456923..0000000
--- a/hbase-native-client/rpc/CMakeLists.txt
+++ /dev/null
@@ -1,17 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/e00e3a04/hbase-native-client/third-party/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/third-party/BUCK b/hbase-native-client/third-party/BUCK
index 89d2c93..b7baa86 100644
--- a/hbase-native-client/third-party/BUCK
+++ b/hbase-native-client/third-party/BUCK
@@ -21,7 +21,8 @@ def add_system_libs(names = []):
prebuilt_cxx_library(
name = name,
lib_name = name,
- lib_dir = "/usr/lib/x86_64-linux-gnu"
+ lib_dir = "/usr/lib/x86_64-linux-gnu",
+ visibility = [ 'PUBLIC', ],
)
rules.append(":" + name)
@@ -30,6 +31,7 @@ def add_system_libs(names = []):
system_libs = [
"double-conversion",
"glog",
+ "protobuf",
"gflags",
"unwind",
"lzma",
[15/50] [abbrv] hbase git commit: HBASE-15739 Add region location
lookup from meta
Posted by ec...@apache.org.
HBASE-15739 Add region location lookup from meta
Summary:
Add the ability for location-cache to read from meta.
There's still no cache. Or any parsing.
Test Plan: Needs tests.
Differential Revision: https://reviews.facebook.net/D57489
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/f74379e6
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/f74379e6
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/f74379e6
Branch: refs/heads/HBASE-14850
Commit: f74379e6875a3f64c185eaefd0cb4594f9ca8022
Parents: baf863f
Author: Elliott Clark <ec...@apache.org>
Authored: Sat Apr 30 17:12:11 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/connection/BUCK | 1 +
.../connection/client-dispatcher.cc | 2 +
.../connection/client-dispatcher.h | 3 +-
.../connection/client-handler.cc | 5 +-
hbase-native-client/connection/client-handler.h | 2 +-
.../connection/connection-factory.cc | 2 +-
.../connection/connection-factory.h | 1 +
.../connection/connection-pool.cc | 14 ++---
hbase-native-client/connection/pipeline.cc | 2 +
hbase-native-client/connection/pipeline.h | 1 +
hbase-native-client/connection/response.h | 6 ++-
hbase-native-client/core/BUCK | 3 +-
hbase-native-client/core/location-cache.cc | 37 ++++++++++++--
hbase-native-client/core/location-cache.h | 19 ++++++-
hbase-native-client/core/meta-utils.cc | 54 +++++++++++++++++++-
hbase-native-client/core/meta-utils.h | 9 +++-
hbase-native-client/core/region-location.h | 45 ++++++++++++++++
hbase-native-client/core/simple-client.cc | 48 ++---------------
hbase-native-client/core/table-name-test.cc | 28 +++++-----
hbase-native-client/core/table-name.cc | 34 ------------
hbase-native-client/core/table-name.h | 35 ++++++-------
hbase-native-client/serde/BUCK | 1 +
hbase-native-client/utils/BUCK | 7 +--
hbase-native-client/utils/user-util.cc | 2 +
hbase-native-client/utils/user-util.h | 3 +-
25 files changed, 231 insertions(+), 133 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/connection/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/BUCK b/hbase-native-client/connection/BUCK
index 96f2136..39b8be5 100644
--- a/hbase-native-client/connection/BUCK
+++ b/hbase-native-client/connection/BUCK
@@ -43,6 +43,7 @@ cxx_library(name="connection",
"//third-party:folly",
"//third-party:wangle",
],
+ compiler_flags=['-Weffc++'],
visibility=['//core/...', ], )
cxx_test(name="connection-pool-test",
srcs=["connection-pool-test.cc", ],
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/connection/client-dispatcher.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-dispatcher.cc b/hbase-native-client/connection/client-dispatcher.cc
index eea0a17..817adc1 100644
--- a/hbase-native-client/connection/client-dispatcher.cc
+++ b/hbase-native-client/connection/client-dispatcher.cc
@@ -22,6 +22,8 @@ using namespace folly;
using namespace hbase;
using namespace wangle;
+ClientDispatcher::ClientDispatcher() : requests_(), current_call_id_(9) {}
+
void ClientDispatcher::read(Context *ctx, Response in) {
auto call_id = in.call_id();
auto search = requests_.find(call_id);
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/connection/client-dispatcher.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-dispatcher.h b/hbase-native-client/connection/client-dispatcher.h
index 877e877..826fc6a 100644
--- a/hbase-native-client/connection/client-dispatcher.h
+++ b/hbase-native-client/connection/client-dispatcher.h
@@ -30,6 +30,7 @@ class ClientDispatcher
: public wangle::ClientDispatcherBase<SerializePipeline,
std::unique_ptr<Request>, Response> {
public:
+ ClientDispatcher();
void read(Context *ctx, Response in) override;
folly::Future<Response> operator()(std::unique_ptr<Request> arg) override;
folly::Future<folly::Unit> close(Context *ctx) override;
@@ -45,6 +46,6 @@ private:
//
// uint32_t has a max of 4Billion so 10 more or less is
// not a big deal.
- uint32_t current_call_id_ = 10;
+ uint32_t current_call_id_;
};
} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/connection/client-handler.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-handler.cc b/hbase-native-client/connection/client-handler.cc
index abcf5c1..3180f4e 100644
--- a/hbase-native-client/connection/client-handler.cc
+++ b/hbase-native-client/connection/client-handler.cc
@@ -34,8 +34,11 @@ using namespace folly;
using namespace wangle;
using hbase::pb::ResponseHeader;
using hbase::pb::GetResponse;
+using google::protobuf::Message;
-ClientHandler::ClientHandler(std::string user_name) : user_name_(user_name) {}
+ClientHandler::ClientHandler(std::string user_name)
+ : user_name_(user_name), need_send_header_(true), ser_(), deser_(),
+ resp_msgs_() {}
void ClientHandler::read(Context *ctx, std::unique_ptr<IOBuf> buf) {
if (LIKELY(buf != nullptr)) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/connection/client-handler.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/client-handler.h b/hbase-native-client/connection/client-handler.h
index 41bb883..68513de 100644
--- a/hbase-native-client/connection/client-handler.h
+++ b/hbase-native-client/connection/client-handler.h
@@ -47,7 +47,7 @@ public:
std::unique_ptr<Request> r) override;
private:
- bool need_send_header_ = true;
+ bool need_send_header_;
std::string user_name_;
ClientSerializer ser_;
ClientDeserializer deser_;
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/connection/connection-factory.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-factory.cc b/hbase-native-client/connection/connection-factory.cc
index b546269..9102d60 100644
--- a/hbase-native-client/connection/connection-factory.cc
+++ b/hbase-native-client/connection/connection-factory.cc
@@ -40,7 +40,7 @@ using namespace folly;
using namespace hbase;
using namespace wangle;
-ConnectionFactory::ConnectionFactory() {
+ConnectionFactory::ConnectionFactory() : bootstrap_() {
bootstrap_.group(std::make_shared<wangle::IOThreadPoolExecutor>(1));
bootstrap_.pipelineFactory(std::make_shared<RpcPipelineFactory>());
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/connection/connection-factory.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-factory.h b/hbase-native-client/connection/connection-factory.h
index 5a45316..fc4e161 100644
--- a/hbase-native-client/connection/connection-factory.h
+++ b/hbase-native-client/connection/connection-factory.h
@@ -31,6 +31,7 @@ namespace hbase {
class ConnectionFactory {
public:
ConnectionFactory();
+ virtual ~ConnectionFactory() = default;
virtual std::shared_ptr<HBaseService> make_connection(const std::string &host,
int port);
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/connection/connection-pool.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/connection-pool.cc b/hbase-native-client/connection/connection-pool.cc
index a967df2..72c1306 100644
--- a/hbase-native-client/connection/connection-pool.cc
+++ b/hbase-native-client/connection/connection-pool.cc
@@ -35,14 +35,14 @@ class RemoveServiceFilter
public:
RemoveServiceFilter(std::shared_ptr<HBaseService> service, ServerName sn,
- ConnectionPool *cp)
+ ConnectionPool &cp)
: ServiceFilter<unique_ptr<Request>, Response>(service), sn_(sn),
cp_(cp) {}
folly::Future<folly::Unit> close() override {
if (!released.exchange(true)) {
return this->service_->close().then(
- [this]() { this->cp_->close(this->sn_); });
+ [this]() { this->cp_.close(this->sn_); });
} else {
return folly::makeFuture();
}
@@ -57,12 +57,14 @@ public:
private:
std::atomic<bool> released{false};
hbase::pb::ServerName sn_;
- ConnectionPool *cp_;
+ ConnectionPool &cp_;
};
-ConnectionPool::ConnectionPool() : cf_(std::make_shared<ConnectionFactory>()) {}
+ConnectionPool::ConnectionPool()
+ : cf_(std::make_shared<ConnectionFactory>()), connections_(), map_mutex_() {
+}
ConnectionPool::ConnectionPool(std::shared_ptr<ConnectionFactory> cf)
- : cf_(cf) {}
+ : cf_(cf), connections_(), map_mutex_() {}
std::shared_ptr<HBaseService> ConnectionPool::get(const ServerName &sn) {
SharedMutexWritePriority::UpgradeHolder holder(map_mutex_);
@@ -70,7 +72,7 @@ std::shared_ptr<HBaseService> ConnectionPool::get(const ServerName &sn) {
if (found == connections_.end() || found->second == nullptr) {
SharedMutexWritePriority::WriteHolder holder(std::move(holder));
auto new_con = cf_->make_connection(sn.host_name(), sn.port());
- auto wrapped = std::make_shared<RemoveServiceFilter>(new_con, sn, this);
+ auto wrapped = std::make_shared<RemoveServiceFilter>(new_con, sn, *this);
connections_[sn] = wrapped;
return new_con;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/connection/pipeline.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/pipeline.cc b/hbase-native-client/connection/pipeline.cc
index b9f5e0b..39cce8b 100644
--- a/hbase-native-client/connection/pipeline.cc
+++ b/hbase-native-client/connection/pipeline.cc
@@ -30,6 +30,8 @@ using namespace folly;
using namespace hbase;
using namespace wangle;
+RpcPipelineFactory::RpcPipelineFactory() : user_util_() {}
+
SerializePipeline::Ptr
RpcPipelineFactory::newPipeline(std::shared_ptr<AsyncTransportWrapper> sock) {
auto pipeline = SerializePipeline::create();
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/connection/pipeline.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/pipeline.h b/hbase-native-client/connection/pipeline.h
index 6c4f4ff..e90cb93 100644
--- a/hbase-native-client/connection/pipeline.h
+++ b/hbase-native-client/connection/pipeline.h
@@ -31,6 +31,7 @@ using SerializePipeline =
class RpcPipelineFactory : public wangle::PipelineFactory<SerializePipeline> {
public:
+ RpcPipelineFactory();
SerializePipeline::Ptr
newPipeline(std::shared_ptr<folly::AsyncTransportWrapper> sock) override;
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/connection/response.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/connection/response.h b/hbase-native-client/connection/response.h
index a7f7939..d2eb19a 100644
--- a/hbase-native-client/connection/response.h
+++ b/hbase-native-client/connection/response.h
@@ -31,10 +31,12 @@ namespace hbase {
class Response {
public:
- Response() : call_id_(0) {}
+ Response() : call_id_(0), response_(nullptr) {}
uint32_t call_id() { return call_id_; }
void set_call_id(uint32_t call_id) { call_id_ = call_id; }
- std::shared_ptr<google::protobuf::Message> response() { return response_; }
+ std::shared_ptr<google::protobuf::Message> response() const {
+ return response_;
+ }
void set_response(std::shared_ptr<google::protobuf::Message> response) {
response_ = std::move(response);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/core/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
index e555ba4..447248b 100644
--- a/hbase-native-client/core/BUCK
+++ b/hbase-native-client/core/BUCK
@@ -22,6 +22,7 @@ cxx_library(
"client.h",
"connection.h",
"hbase_macros.h",
+ "region-location.h",
"location-cache.h",
"table-name.h",
# TODO: move this out of exported
@@ -32,7 +33,6 @@ cxx_library(
"client.cc",
"location-cache.cc",
"meta-utils.cc",
- "table-name.cc",
],
deps=[
"//connection:connection",
@@ -42,6 +42,7 @@ cxx_library(
"//third-party:wangle",
"//third-party:zookeeper_mt",
],
+ compiler_flags=['-Weffc++'],
visibility=[
'PUBLIC',
], )
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/core/location-cache.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.cc b/hbase-native-client/core/location-cache.cc
index c81deba..539051a 100644
--- a/hbase-native-client/core/location-cache.cc
+++ b/hbase-native-client/core/location-cache.cc
@@ -21,20 +21,31 @@
#include <folly/Logging.h>
#include <folly/io/IOBuf.h>
+#include "connection/response.h"
+#include "if/Client.pb.h"
#include "if/ZooKeeper.pb.h"
#include "serde/zk-deserializer.h"
using namespace std;
using namespace folly;
-using namespace hbase::pb;
-using namespace hbase;
+
+using hbase::Response;
+using hbase::LocationCache;
+using hbase::RegionLocation;
+using hbase::HBaseService;
+using hbase::pb::ScanResponse;
+using hbase::pb::TableName;
+using hbase::pb::ServerName;
+using hbase::pb::MetaRegionServer;
+using hbase::pb::RegionInfo;
// TODO(eclark): make this configurable on client creation
static const char META_ZNODE_NAME[] = "/hbase/meta-region-server";
LocationCache::LocationCache(string quorum_spec,
shared_ptr<folly::Executor> executor)
- : quorum_spec_(quorum_spec), executor_(executor), meta_promise_(nullptr) {
+ : quorum_spec_(quorum_spec), executor_(executor), meta_promise_(nullptr),
+ meta_lock_(), cp_(), meta_util_() {
zk_ = zookeeper_init(quorum_spec.c_str(), nullptr, 1000, 0, 0, 0);
}
@@ -89,3 +100,23 @@ ServerName LocationCache::ReadMetaLocation() {
}
return mrs.server();
}
+
+Future<RegionLocation> LocationCache::locateFromMeta(const TableName &tn,
+ const string &row) {
+ return this->LocateMeta()
+ .then([&](ServerName sn) { return this->cp_.get(sn); })
+ .then([&](std::shared_ptr<HBaseService> service) {
+ return (*service)(std::move(meta_util_.make_meta_request(tn, row)));
+ })
+ .then([&](Response resp) {
+ // take the protobuf response and make it into
+ // a region location.
+ return this->parse_response(std::move(resp));
+ });
+}
+
+RegionLocation LocationCache::parse_response(const Response &resp) {
+ auto resp_msg = static_pointer_cast<ScanResponse>(resp.response());
+ LOG(ERROR) << "resp_msg = " << resp_msg->DebugString();
+ return RegionLocation{RegionInfo{}, ServerName{}, nullptr};
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/core/location-cache.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/location-cache.h b/hbase-native-client/core/location-cache.h
index ce97c43..cfd6838 100644
--- a/hbase-native-client/core/location-cache.h
+++ b/hbase-native-client/core/location-cache.h
@@ -27,9 +27,19 @@
#include <mutex>
#include <string>
-#include "if/HBase.pb.h"
+#include "connection/connection-pool.h"
+#include "core/meta-utils.h"
+#include "core/table-name.h"
+#include "core/region-location.h"
namespace hbase {
+
+class Request;
+class Response;
+namespace pb {
+class ServerName;
+}
+
class LocationCache {
public:
explicit LocationCache(std::string quorum_spec,
@@ -38,6 +48,9 @@ public:
// Meta Related Methods.
// These are only public until testing is complete
folly::Future<hbase::pb::ServerName> LocateMeta();
+ folly::Future<RegionLocation> locateFromMeta(const hbase::pb::TableName &tn,
+ const std::string &row);
+ RegionLocation parse_response(const Response &resp);
void InvalidateMeta();
private:
@@ -48,7 +61,11 @@ private:
std::shared_ptr<folly::Executor> executor_;
std::unique_ptr<folly::SharedPromise<hbase::pb::ServerName>> meta_promise_;
std::mutex meta_lock_;
+ ConnectionPool cp_;
+ MetaUtil meta_util_;
+
+ // TODO: migrate this to a smart pointer with a deleter.
zhandle_t *zk_;
};
} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/core/meta-utils.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/meta-utils.cc b/hbase-native-client/core/meta-utils.cc
index 7c7d45f..d2fdd88 100644
--- a/hbase-native-client/core/meta-utils.cc
+++ b/hbase-native-client/core/meta-utils.cc
@@ -21,11 +21,61 @@
#include <folly/Conv.h>
+#include "connection/request.h"
+#include "connection/response.h"
#include "core/table-name.h"
+#include "if/Client.pb.h"
-using namespace hbase;
+using hbase::pb::TableName;
+using hbase::MetaUtil;
+using hbase::Request;
+using hbase::Response;
+using hbase::pb::ScanRequest;
+using hbase::pb::RegionSpecifier_RegionSpecifierType;
+
+static const std::string META_REGION = "1588230740";
std::string MetaUtil::region_lookup_rowkey(const TableName &tn,
- const std::string &row) {
+ const std::string &row) const {
return folly::to<std::string>(tn, ",", row, ",", "999999999999999999");
}
+
+std::unique_ptr<Request>
+MetaUtil::make_meta_request(const TableName tn, const std::string &row) const {
+ auto request = Request::scan();
+ auto msg = std::static_pointer_cast<ScanRequest>(request->req_msg());
+
+ msg->set_number_of_rows(1);
+ msg->set_close_scanner(true);
+
+ // Set the region this scan goes to
+ auto region = msg->mutable_region();
+ region->set_value(META_REGION);
+ region->set_type(RegionSpecifier_RegionSpecifierType::
+ RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
+
+ auto scan = msg->mutable_scan();
+ // We don't care about before, just now.
+ scan->set_max_versions(1);
+ // Meta should be cached at all times.
+ scan->set_cache_blocks(true);
+ // We only want one row right now.
+ //
+ // TODO(eclark): Figure out if we should get more.
+ scan->set_caching(1);
+ // Close the scan after we have data.
+ scan->set_small(true);
+ // We know where to start but not where to end.
+ scan->set_reversed(true);
+ // Give me everything or nothing.
+ scan->set_allow_partial_results(false);
+
+ // Set the columns that we need
+ auto info_col = scan->add_column();
+ info_col->set_family("info");
+ info_col->add_qualifier("server");
+ info_col->add_qualifier("regioninfo");
+
+ scan->set_start_row(region_lookup_rowkey(tn, row));
+ return request;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/core/meta-utils.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/meta-utils.h b/hbase-native-client/core/meta-utils.h
index 391d0ea..e007d02 100644
--- a/hbase-native-client/core/meta-utils.h
+++ b/hbase-native-client/core/meta-utils.h
@@ -20,11 +20,16 @@
#include <string>
+#include "connection/Request.h"
+#include "core/table-name.h"
+
namespace hbase {
-class TableName;
class MetaUtil {
public:
- std::string region_lookup_rowkey(const TableName &tn, const std::string &row);
+ std::string region_lookup_rowkey(const hbase::pb::TableName &tn,
+ const std::string &row) const;
+ std::unique_ptr<Request> make_meta_request(const hbase::pb::TableName tn,
+ const std::string &row) const;
};
} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/core/region-location.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/region-location.h b/hbase-native-client/core/region-location.h
new file mode 100644
index 0000000..a46b8e2
--- /dev/null
+++ b/hbase-native-client/core/region-location.h
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+#pragma once
+
+#include <memory>
+
+#include "connection/service.h"
+#include "if/HBase.pb.h"
+
+
+namespace hbase {
+
+class RegionLocation {
+public:
+ RegionLocation(hbase::pb::RegionInfo ri, hbase::pb::ServerName sn,
+ std::shared_ptr<HBaseService> service)
+ : ri_(ri), sn_(sn), service_(service) {}
+
+ const hbase::pb::RegionInfo& region_info() { return ri_; }
+ const hbase::pb::ServerName& server_name() { return sn_; }
+ std::shared_ptr<HBaseService> service() { return service_; }
+
+private:
+ hbase::pb::RegionInfo ri_;
+ hbase::pb::ServerName sn_;
+ std::shared_ptr<HBaseService> service_;
+};
+
+} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/core/simple-client.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/simple-client.cc b/hbase-native-client/core/simple-client.cc
index 11dcd68..ab614e4 100644
--- a/hbase-native-client/core/simple-client.cc
+++ b/hbase-native-client/core/simple-client.cc
@@ -27,6 +27,7 @@
#include "connection/connection-pool.h"
#include "core/client.h"
+#include "core/table-name.h"
#include "if/Client.pb.h"
#include "if/ZooKeeper.pb.h"
@@ -38,13 +39,14 @@ using hbase::Request;
using hbase::HBaseService;
using hbase::LocationCache;
using hbase::ConnectionPool;
+using hbase::TableNameUtil;
using hbase::pb::ServerName;
using hbase::pb::RegionSpecifier_RegionSpecifierType;
using hbase::pb::GetRequest;
using hbase::pb::GetResponse;
// TODO(eclark): remove the need for this.
-DEFINE_string(region, "1588230740", "What region to send a get to");
+DEFINE_string(table, "t", "What region to send a get");
DEFINE_string(row, "test", "What row to get");
DEFINE_string(zookeeper, "localhost:2181", "What zk quorum to talk to");
@@ -59,48 +61,8 @@ int main(int argc, char *argv[]) {
auto cpu_ex = wangle::getCPUExecutor();
LocationCache cache{FLAGS_zookeeper, cpu_ex};
auto result =
- cache.LocateMeta()
- .then([&cp = cp](ServerName sn) { return cp.get(sn); })
- .then([](shared_ptr<HBaseService> con) {
- // Send the request
- auto r = Request::get();
- // This is a get request so make that
- auto req_msg = static_pointer_cast<GetRequest>(r->req_msg());
- // Set what region
- req_msg->mutable_region()->set_value(FLAGS_region);
- // It's always this.
- req_msg->mutable_region()->set_type(
- RegionSpecifier_RegionSpecifierType::
- RegionSpecifier_RegionSpecifierType_ENCODED_REGION_NAME);
-
- // What row.
- req_msg->mutable_get()->set_row(FLAGS_row);
-
- return (*con)(std::move(r));
- })
- .then([](Response resp) {
- return static_pointer_cast<GetResponse>(resp.response());
- })
- .via(cpu_ex.get())
- .then([](shared_ptr<GetResponse> get_resp) {
- cout << "GetResponse has_result = " << get_resp->has_result()
- << '\n';
- if (get_resp->has_result()) {
- auto &r = get_resp->result();
- cout << "Result cell_size = " << r.cell_size() << endl;
- for (auto &cell : r.cell()) {
- cout << "\trow = " << cell.row()
- << " family = " << cell.family()
- << " qualifier = " << cell.qualifier()
- << " timestamp = " << cell.timestamp()
- << " value = " << cell.value() << endl;
- }
- return 0;
- }
-
- return 1;
- })
+ cache.locateFromMeta(TableNameUtil::create(FLAGS_table), FLAGS_row)
.get(milliseconds(5000));
- return result;
+ return 0;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/core/table-name-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/table-name-test.cc b/hbase-native-client/core/table-name-test.cc
index 8aff375..7bad3f1 100644
--- a/hbase-native-client/core/table-name-test.cc
+++ b/hbase-native-client/core/table-name-test.cc
@@ -25,26 +25,30 @@
#include "core/table-name.h"
using namespace hbase;
+using hbase::pb::TableName;
TEST(TestTableName, TestToStringNoDefault) {
- TableName tn{"TestTableName"};
+ TableName tn;
+ tn.set_qualifier("TestTableName");
std::string result = folly::to<std::string>(tn);
ASSERT_EQ(result.find("default"), std::string::npos);
ASSERT_EQ("TestTableName", result);
}
+
+TEST(TestTableName, TestToStringNoDefaltWhenSet) {
+ TableName tn;
+ tn.set_namespace_("default");
+ tn.set_qualifier("TestTableName");
+ std::string result = folly::to<std::string>(tn);
+ ASSERT_EQ(result.find("default"), std::string::npos);
+ ASSERT_EQ("TestTableName", result);
+}
+
TEST(TestTableName, TestToStringIncludeNS) {
- TableName tn{"hbase", "acl"};
+ TableName tn;
+ tn.set_namespace_("hbase");
+ tn.set_qualifier("acl");
std::string result = folly::to<std::string>(tn);
ASSERT_EQ(result.find("hbase"), 0);
ASSERT_EQ("hbase:acl", result);
}
-TEST(TestTableName, TestIsDefault) {
- TableName default_t1{"in_default"};
- ASSERT_TRUE(default_t1.is_default_name_space());
-
- TableName default_t2{"default", "in_also"};
- ASSERT_TRUE(default_t2.is_default_name_space());
-
- TableName non_default{"testing", "hmm"};
- ASSERT_FALSE(non_default.is_default_name_space());
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/core/table-name.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/table-name.cc b/hbase-native-client/core/table-name.cc
deleted file mode 100644
index 33087fd..0000000
--- a/hbase-native-client/core/table-name.cc
+++ /dev/null
@@ -1,34 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-#include "core/table-name.h"
-
-#include <folly/Conv.h>
-
-using namespace hbase;
-
-TableName::TableName(std::string table_name)
- : name_space_("default"), table_(table_name) {}
-TableName::TableName(std::string name_space, std::string table_name)
- : name_space_(name_space), table_(table_name) {}
-bool TableName::is_default_name_space() const {
- return name_space_.length() == 0 || name_space_ == "default";
-}
-bool TableName::operator==(const TableName &other) const {
- return name_space_ == other.name_space_ && table_ == other.table_;
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/core/table-name.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/table-name.h b/hbase-native-client/core/table-name.h
index 816f7da..1612667 100644
--- a/hbase-native-client/core/table-name.h
+++ b/hbase-native-client/core/table-name.h
@@ -21,33 +21,30 @@
#include <memory>
#include <string>
+#include "if/HBase.pb.h"
#include <folly/Conv.h>
namespace hbase {
-
-// This is the core class of a HBase client.
-class TableName {
-public:
- explicit TableName(std::string table_name);
- explicit TableName(std::string name_space, std::string table_name);
-
- std::string name_space() const { return name_space_; };
- std::string table() const { return table_; };
- bool is_default_name_space() const;
- bool operator==(const TableName &other) const;
-
-private:
- std::string name_space_;
- std::string table_;
-};
+namespace pb {
// Provide folly::to<std::string>(TableName);
template <class String> void toAppend(const TableName &in, String *result) {
- if (in.is_default_name_space()) {
- folly::toAppend(in.table(), result);
+ if (!in.has_namespace_() || in.namespace_() == "default") {
+ folly::toAppend(in.qualifier(), result);
} else {
- folly::toAppend(in.name_space(), ':', in.table(), result);
+ folly::toAppend(in.namespace_(), ':', in.qualifier(), result);
}
}
+} // namespace pb
+
+class TableNameUtil {
+public:
+ static ::hbase::pb::TableName create(std::string table_name) {
+ ::hbase::pb::TableName tn;
+ tn.set_namespace_("default");
+ tn.set_qualifier(table_name);
+ return tn;
+ }
+};
} // namespace hbase
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/serde/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/BUCK b/hbase-native-client/serde/BUCK
index 0014c0b..539a221 100644
--- a/hbase-native-client/serde/BUCK
+++ b/hbase-native-client/serde/BUCK
@@ -34,6 +34,7 @@ cxx_library(name="serde",
":client-serializer-test",
":client-deserializer-test",
],
+ compiler_flags=['-Weffc++'],
visibility=[
'PUBLIC',
], )
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/utils/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/utils/BUCK b/hbase-native-client/utils/BUCK
index ad12f7d..2ab6a89 100644
--- a/hbase-native-client/utils/BUCK
+++ b/hbase-native-client/utils/BUCK
@@ -25,12 +25,13 @@ cxx_library(name="utils",
deps=[
'//third-party:folly',
],
+ tests=[
+ ":user-util-test"
+ ],
visibility=[
'PUBLIC',
],
- tests=[
- ":user-util-test"
- ], )
+ compiler_flags=['-Weffc++'], )
cxx_test(name="user-util-test",
srcs=[
"user-util-test.cc",
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/utils/user-util.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/utils/user-util.cc b/hbase-native-client/utils/user-util.cc
index b801537..3d963b3 100644
--- a/hbase-native-client/utils/user-util.cc
+++ b/hbase-native-client/utils/user-util.cc
@@ -27,6 +27,8 @@
using namespace hbase;
using namespace std;
+UserUtil::UserUtil() : init_{false}, user_name_{"drwho"}, m_() {}
+
string UserUtil::user_name() {
if (!init_) {
compute_user_name();
http://git-wip-us.apache.org/repos/asf/hbase/blob/f74379e6/hbase-native-client/utils/user-util.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/utils/user-util.h b/hbase-native-client/utils/user-util.h
index 9ca28e3..da95d78 100644
--- a/hbase-native-client/utils/user-util.h
+++ b/hbase-native-client/utils/user-util.h
@@ -26,11 +26,12 @@
namespace hbase {
class UserUtil {
public:
+ UserUtil();
std::string user_name();
private:
void compute_user_name();
- std::atomic<bool> init_{false};
+ std::atomic<bool> init_;
std::string user_name_;
std::mutex m_;
};
[10/50] [abbrv] hbase git commit: HBASE-16044 Fix 'hbase shell'
output parsing in graceful_stop.sh
Posted by ec...@apache.org.
HBASE-16044 Fix 'hbase shell' output parsing in graceful_stop.sh
Change-Id: I94b2f665105837d31281be66d620b6152c3bc9d1
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a396ae77
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a396ae77
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a396ae77
Branch: refs/heads/HBASE-14850
Commit: a396ae773a0f7dbcd6f54fc8133b7a9dc30d8431
Parents: 3b5fbf8
Author: Apekshit Sharma <ap...@apache.org>
Authored: Tue Jun 28 16:55:09 2016 -0700
Committer: Apekshit Sharma <ap...@apache.org>
Committed: Mon Jul 11 11:28:33 2016 -0700
----------------------------------------------------------------------
bin/graceful_stop.sh | 2 +-
bin/rolling-restart.sh | 2 +-
hbase-shell/src/main/ruby/shell/commands/balance_switch.rb | 4 +++-
hbase-shell/src/main/ruby/shell/commands/balancer_enabled.rb | 4 +++-
4 files changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/a396ae77/bin/graceful_stop.sh
----------------------------------------------------------------------
diff --git a/bin/graceful_stop.sh b/bin/graceful_stop.sh
index 8d08cc9..89e3dd9 100755
--- a/bin/graceful_stop.sh
+++ b/bin/graceful_stop.sh
@@ -106,7 +106,7 @@ if [ "$nob" == "true" ]; then
HBASE_BALANCER_STATE=false
else
log "Disabling load balancer"
- HBASE_BALANCER_STATE=$(echo 'balance_switch false' | "$bin"/hbase --config "${HBASE_CONF_DIR}" shell | tail -3 | head -1)
+ HBASE_BALANCER_STATE=$(echo 'balance_switch false' | "$bin"/hbase --config "${HBASE_CONF_DIR}" shell -n | tail -1)
log "Previous balancer state was $HBASE_BALANCER_STATE"
fi
http://git-wip-us.apache.org/repos/asf/hbase/blob/a396ae77/bin/rolling-restart.sh
----------------------------------------------------------------------
diff --git a/bin/rolling-restart.sh b/bin/rolling-restart.sh
index 530b76b..9ac4d64 100755
--- a/bin/rolling-restart.sh
+++ b/bin/rolling-restart.sh
@@ -189,7 +189,7 @@ else
zkrs="$zparent/$zkrs"
online_regionservers=`$bin/hbase zkcli ls $zkrs 2>&1 | tail -1 | sed "s/\[//" | sed "s/\]//"`
echo "Disabling load balancer"
- HBASE_BALANCER_STATE=$(echo 'balance_switch false' | "$bin"/hbase --config "${HBASE_CONF_DIR}" shell | tail -3 | head -1)
+ HBASE_BALANCER_STATE=$(echo 'balance_switch false' | "$bin"/hbase --config "${HBASE_CONF_DIR}" shell -n | tail -1)
echo "Previous balancer state was $HBASE_BALANCER_STATE"
for rs in $online_regionservers
http://git-wip-us.apache.org/repos/asf/hbase/blob/a396ae77/hbase-shell/src/main/ruby/shell/commands/balance_switch.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell/commands/balance_switch.rb b/hbase-shell/src/main/ruby/shell/commands/balance_switch.rb
index 107d267..4076cf6 100644
--- a/hbase-shell/src/main/ruby/shell/commands/balance_switch.rb
+++ b/hbase-shell/src/main/ruby/shell/commands/balance_switch.rb
@@ -31,7 +31,9 @@ EOF
end
def command(enableDisable)
- formatter.row([admin.balance_switch(enableDisable)? "true" : "false"])
+ prev_state = admin.balance_switch(enableDisable)? "true" : "false"
+ formatter.row(["Previous balancer state : #{prev_state}"])
+ prev_state
end
end
end
http://git-wip-us.apache.org/repos/asf/hbase/blob/a396ae77/hbase-shell/src/main/ruby/shell/commands/balancer_enabled.rb
----------------------------------------------------------------------
diff --git a/hbase-shell/src/main/ruby/shell/commands/balancer_enabled.rb b/hbase-shell/src/main/ruby/shell/commands/balancer_enabled.rb
index 6a75ab5..c3d7a2b 100644
--- a/hbase-shell/src/main/ruby/shell/commands/balancer_enabled.rb
+++ b/hbase-shell/src/main/ruby/shell/commands/balancer_enabled.rb
@@ -30,7 +30,9 @@ EOF
end
def command()
- formatter.row([admin.balancer_enabled?.to_s])
+ state = admin.balancer_enabled?
+ formatter.row([state.to_s])
+ state
end
end
end
[17/50] [abbrv] hbase git commit: HBASE-14852 Update build env
Posted by ec...@apache.org.
HBASE-14852 Update build env
Also includes HBASE-14858 Clean up so core is ready for development on a recent version of c++
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/8556e3a3
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/8556e3a3
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/8556e3a3
Branch: refs/heads/HBASE-14850
Commit: 8556e3a348fb4e4d1610ab44084b70631331a98a
Parents: a396ae7
Author: Elliott Clark <ec...@apache.org>
Authored: Thu Nov 19 16:28:11 2015 -0800
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/.buckconfig | 2 +
hbase-native-client/.gitignore | 23 +--
hbase-native-client/CMakeLists.txt | 157 -------------------
hbase-native-client/Dockerfile | 22 +++
hbase-native-client/README.md | 15 +-
hbase-native-client/bin/build-all.sh | 41 -----
hbase-native-client/bin/build-thirdparty.sh | 64 --------
hbase-native-client/bin/download-thirdparty.sh | 70 ---------
hbase-native-client/bin/hbase-client-env.sh | 47 ------
hbase-native-client/bin/start-docker.sh | 32 ++++
.../cmake_modules/FindGTest.cmake | 53 -------
.../cmake_modules/FindLibEv.cmake | 47 ------
hbase-native-client/core/BUCK | 50 ++++++
hbase-native-client/core/admin.cc | 20 +++
hbase-native-client/core/admin.h | 22 +++
hbase-native-client/core/client.cc | 38 +++++
hbase-native-client/core/client.h | 24 +++
hbase-native-client/core/connection.cc | 20 +++
hbase-native-client/core/connection.h | 26 +++
hbase-native-client/core/connection_attr.h | 24 +++
hbase-native-client/core/delete.cc | 21 +++
hbase-native-client/core/delete.h | 27 ++++
hbase-native-client/core/get.cc | 20 +++
hbase-native-client/core/get.h | 22 +++
hbase-native-client/core/hbase_macros.h | 56 +++++++
hbase-native-client/core/mutation.cc | 41 +++++
hbase-native-client/core/mutation.h | 58 +++++++
hbase-native-client/core/put.cc | 21 +++
hbase-native-client/core/put.h | 27 ++++
hbase-native-client/core/scanner.cc | 20 +++
hbase-native-client/core/scanner.h | 22 +++
hbase-native-client/rpc/CMakeLists.txt | 17 ++
hbase-native-client/src/async/CMakeLists.txt | 32 ----
hbase-native-client/src/async/get-test.cc | 59 -------
hbase-native-client/src/async/hbase_admin.cc | 57 -------
hbase-native-client/src/async/hbase_admin.h | 69 --------
hbase-native-client/src/async/hbase_client.cc | 47 ------
hbase-native-client/src/async/hbase_client.h | 60 -------
.../src/async/hbase_connection.cc | 37 -----
.../src/async/hbase_connection.h | 52 ------
hbase-native-client/src/async/hbase_errno.h | 23 ---
hbase-native-client/src/async/hbase_get.cc | 61 -------
hbase-native-client/src/async/hbase_get.h | 73 ---------
.../src/async/hbase_mutations.cc | 111 -------------
hbase-native-client/src/async/hbase_mutations.h | 119 --------------
hbase-native-client/src/async/hbase_result.cc | 37 -----
hbase-native-client/src/async/hbase_result.h | 44 ------
hbase-native-client/src/async/hbase_scanner.cc | 59 -------
hbase-native-client/src/async/hbase_scanner.h | 80 ----------
hbase-native-client/src/async/mutations-test.cc | 102 ------------
hbase-native-client/src/core/CMakeLists.txt | 31 ----
hbase-native-client/src/core/admin.cc | 20 ---
hbase-native-client/src/core/admin.h | 25 ---
hbase-native-client/src/core/client.cc | 20 ---
hbase-native-client/src/core/client.h | 25 ---
hbase-native-client/src/core/connection.cc | 22 ---
hbase-native-client/src/core/connection.h | 26 ---
hbase-native-client/src/core/connection_attr.h | 30 ----
hbase-native-client/src/core/delete.cc | 22 ---
hbase-native-client/src/core/delete.h | 29 ----
hbase-native-client/src/core/get.cc | 20 ---
hbase-native-client/src/core/get.h | 26 ---
.../src/core/hbase_connection_attr.cc | 41 -----
.../src/core/hbase_connection_attr.h | 51 ------
hbase-native-client/src/core/hbase_macros.h | 60 -------
hbase-native-client/src/core/hbase_types.h | 83 ----------
hbase-native-client/src/core/mutation.cc | 42 -----
hbase-native-client/src/core/mutation.h | 48 ------
hbase-native-client/src/core/put.cc | 22 ---
hbase-native-client/src/core/put.h | 29 ----
hbase-native-client/src/core/scanner.cc | 20 ---
hbase-native-client/src/core/scanner.h | 25 ---
hbase-native-client/src/rpc/CMakeLists.txt | 17 --
hbase-native-client/src/sync/CMakeLists.txt | 24 ---
hbase-native-client/src/sync/hbase_admin.cc | 51 ------
hbase-native-client/src/sync/hbase_admin.h | 61 -------
.../src/sync/hbase_connection.cc | 37 -----
hbase-native-client/src/sync/hbase_connection.h | 52 ------
hbase-native-client/third-party/BUCK | 105 +++++++++++++
pom.xml | 4 +
80 files changed, 757 insertions(+), 2582 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/.buckconfig
----------------------------------------------------------------------
diff --git a/hbase-native-client/.buckconfig b/hbase-native-client/.buckconfig
new file mode 100644
index 0000000..3227a2a
--- /dev/null
+++ b/hbase-native-client/.buckconfig
@@ -0,0 +1,2 @@
+[cxx]
+ gtest_dep = //third-party/googletest/googletest:google-test
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/.gitignore
----------------------------------------------------------------------
diff --git a/hbase-native-client/.gitignore b/hbase-native-client/.gitignore
index f14b3be..b172fe9 100644
--- a/hbase-native-client/.gitignore
+++ b/hbase-native-client/.gitignore
@@ -14,25 +14,12 @@
#python
*.pyc
-# CMake Generated Files
-CMakeCache.txt
-CMakeFiles
-Makefile
-cmake_install.CMakeCache
-cmake_install.cmake
-install_manifest.txt
-CTestTestfile.cmake
-build
-Testing
+# Buck
+buck-out
+.buckd
+*.swp
# Thirdparty dirs
-
-thirdparty/glog-*
-thirdparty/gtest-*
-thirdparty/gtest
-thirdparty/protobuf-*
-thirdparty/libevent-*
-thirdparty/libev-*
-thirdparty/installed
+third-party/googletest*
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hbase-native-client/CMakeLists.txt b/hbase-native-client/CMakeLists.txt
deleted file mode 100644
index 819f178..0000000
--- a/hbase-native-client/CMakeLists.txt
+++ /dev/null
@@ -1,157 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-cmake_minimum_required(VERSION 2.6)
-
-# generate CTest input files
-enable_testing()
-
-# where to find cmake modules
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_SOURCE_DIR}/cmake_modules")
-
-# if no build build type is specified, default to debug builds
-if (NOT CMAKE_BUILD_TYPE)
- set(CMAKE_BUILD_TYPE Debug)
-endif(NOT CMAKE_BUILD_TYPE)
-
-STRING(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE)
-
-set(CXX_COMMON_FLAGS "-Wall -Wextra -std=c++11")
-set(C_COMMON_FLAGS "-Wall -Wextra -std=c11")
-
-set(CXX_FLAGS_DEBUG "-ggdb -g")
-set(C_FLAGS_DEBUG "-ggdb")
-
-set(CXX_FLAGS_FASTDEBUG "-ggdb -O1")
-set(C_FLAGS_FASTDEBUG "-ggdb -O1")
-
-set(CXX_FLAGS_RELEASE "-O3 -g -DNDEBUG -Wno-strict-aliasing")
-set(C_FLAGS_RELEASE "-O3 -g -DNDEBUG -Wno-strict-aliasing")
-
-# if no build build type is specified, default to debug builds
-if (NOT CMAKE_BUILD_TYPE)
- set(CMAKE_BUILD_TYPE Debug)
-endif(NOT CMAKE_BUILD_TYPE)
-
-string(TOUPPER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE)
-
-# Set compile flags based on the build type.
-message("Configured for ${CMAKE_BUILD_TYPE} build (set with cmake -DCMAKE_BUILD_TYPE={release,debug,...})")
-if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG")
- set(CMAKE_CXX_FLAGS ${CXX_FLAGS_DEBUG})
- set(CMAKE_C_FLAGS ${C_FLAGS_DEBUG})
-elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "FASTDEBUG")
- set(CMAKE_CXX_FLAGS ${CXX_FLAGS_FASTDEBUG})
- set(CMAKE_C_FLAGS ${C_FLAGS_FASTDEBUG})
-elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")
- set(CMAKE_CXX_FLAGS ${CXX_FLAGS_RELEASE})
- set(CMAKE_C_FLAGS ${C_FLAGS_RELEASE})
-else()
- message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}")
-endif ()
-
-set(CMAKE_CXX_FLAGS "${CXX_COMMON_FLAGS} ${CMAKE_CXX_FLAGS}")
-set(CMAKE_C_FLAGS "${C_COMMON_FLAGS} ${CMAKE_C_FLAGS}")
-
-
-# set compile output directory
-string(TOLOWER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_LOWERCASE)
-set(BUILD_OUTPUT_ROOT_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/build/${CMAKE_BUILD_TYPE_LOWERCASE}/")
-
-# Link build/latest to the current build directory, to avoid developers
-# accidentally running the latest debug build when in fact they're building
-# release builds.
-file(MAKE_DIRECTORY ${BUILD_OUTPUT_ROOT_DIRECTORY})
-execute_process(COMMAND rm -f ${CMAKE_CURRENT_SOURCE_DIR}/build/latest)
-execute_process(COMMAND ln -sf ${BUILD_OUTPUT_ROOT_DIRECTORY}
- ${CMAKE_CURRENT_SOURCE_DIR}/build/latest)
-
-# where to put generated libraries
-set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
-set(ARCHIVE_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
-set(CMAKE_LIBRARY_OUTPUT_DIRECTORY "${BUILD_OUTPUT_ROOT_DIRECTORY}")
-
-# where to put generated binaries
-set(EXECUTABLE_OUTPUT_PATH "${BUILD_OUTPUT_ROOT_DIRECTORY}")
-
-include_directories(src)
-
-#####################
-# Third Party
-#####################
-
-# find boost headers and libs
-set(Boost_DEBUG TRUE)
-set(Boost_USE_MULTITHREADED ON)
-find_package(Boost REQUIRED COMPONENTS thread system-mt)
-include_directories(${Boost_INCLUDE_DIRS})
-
-find_package(Protobuf REQUIRED)
-include_directories(${PROTOBUF_INCLUDE_DIRS})
-add_library(protobuf STATIC IMPORTED)
-set_target_properties(protobuf PROPERTIES IMPORTED_LOCATION "${PROTOBUF_STATIC_LIBRARY}")
-
-find_package(LibEv REQUIRED)
-include_directories(${LIBEV_INCLUDE_DIR})
-add_library(libev STATIC IMPORTED)
-
-
-set(HBASE_LIBS
- ${PROTOBUF}
- ${LIBEV}
- ${Boost_LIBRARIES}
-)
-#########
-# Testing Config
-#########
-
-# find GTest headers and libs
-find_package(GTest REQUIRED)
-include_directories(${GTEST_INCLUDE_DIR})
-add_library(gtest STATIC IMPORTED)
-set_target_properties(gtest PROPERTIES IMPORTED_LOCATION "${GTEST_LIBS}")
-
-set(HBASE_TEST_LIBS ${HBASE_LIBS} ${GTEST_LIBS})
-set(HBASE_ASYNC_TEST_LIBS ${HBASE_LIBS} ${GTEST_LIBS} hbase-async)
-set(HBASE_SYNC_TEST_LIBS ${HBASE_LIBS} ${GTEST_LIBS} hbase-sync)
-
-function(ADD_HBASE_ASYNC_TEST TEST_NAME)
- add_executable(${TEST_NAME} ${TEST_NAME}.cc)
- target_link_libraries(${TEST_NAME} ${HBASE_ASYNC_TEST_LIBS})
- add_test(${TEST_NAME} "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
-endfunction()
-
-
-function(ADD_HBASE_SYNC_TEST TEST_NAME)
- add_executable(${TEST_NAME} ${TEST_NAME}.cc)
- target_link_libraries(${TEST_NAME} ${HBASE_ASYNC_TEST_LIBS})
- add_test(${TEST_NAME} "${EXECUTABLE_OUTPUT_PATH}/${TEST_NAME}")
-endfunction()
-
-
-#####
-# Actual project definition
-#####
-add_subdirectory(src/async)
-add_subdirectory(src/sync)
-
-add_subdirectory(src/core)
-add_subdirectory(src/rpc)
-
-
-add_library(hbase-async SHARED $<TARGET_OBJECTS:hcore> $<TARGET_OBJECTS:hasync>)
-add_library(hbase-sync SHARED $<TARGET_OBJECTS:hcore> $<TARGET_OBJECTS:hsync>)
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/Dockerfile
----------------------------------------------------------------------
diff --git a/hbase-native-client/Dockerfile b/hbase-native-client/Dockerfile
new file mode 100644
index 0000000..70e823b
--- /dev/null
+++ b/hbase-native-client/Dockerfile
@@ -0,0 +1,22 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM pjameson/buck-folly-watchman
+
+RUN apt-get install -y libprotobuf-dev protobuf-compiler clang-format-3.7 vim
+
+WORKDIR /usr/local/src/hbase/hbase-native-client
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/README.md
----------------------------------------------------------------------
diff --git a/hbase-native-client/README.md b/hbase-native-client/README.md
index 0bb4d48..76df596 100644
--- a/hbase-native-client/README.md
+++ b/hbase-native-client/README.md
@@ -19,11 +19,10 @@ under the License.
# hbase-native-client
-Native client for HBase 0.96
+Native client for HBase
-This is a C library that implements a
-HBase client. It's thread safe and libEv
-based.
+This is a C/C++ library that implements a
+HBase client.
## Design Philosphy
@@ -51,3 +50,11 @@ the hbase_ prefix is assumed to be implementation private.
All C apis and typedefs will be prefixed with hb_.
All typedefs end with _t.
+
+
+## Docker
+
+The build environment is docker. This should keep a consistent
+build environment for everyone. Buck the build system works
+best with mmap'd files. On OSX this means that vmwarefusion
+works the best. However it should work with just the defaults.
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/bin/build-all.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/build-all.sh b/hbase-native-client/bin/build-all.sh
deleted file mode 100755
index a3e45de..0000000
--- a/hbase-native-client/bin/build-all.sh
+++ /dev/null
@@ -1,41 +0,0 @@
-#! /usr/bin/env bash
-#
-#/**
-# * Copyright The Apache Software Foundation
-# *
-# * Licensed to the Apache Software Foundation (ASF) under one
-# * or more contributor license agreements. See the NOTICE file
-# * distributed with this work for additional information
-# * regarding copyright ownership. The ASF licenses this file
-# * to you under the Apache License, Version 2.0 (the
-# * "License"); you may not use this file except in compliance
-# * with the License. You may obtain a copy of the License at
-# *
-# * http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# */
-
-set -e
-set -x
-
-SOURCE="${BASH_SOURCE[0]}"
-while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
- DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
- SOURCE="$(readlink "$SOURCE")"
- [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
-done
-DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-
-source ${DIR}/hbase-client-env.sh
-
-${DIR}/download-thirdparty.sh
-${DIR}/build-thirdparty.sh
-cd ${HBASE_DIR}
-cmake .
-make clean all
-make test
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/bin/build-thirdparty.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/build-thirdparty.sh b/hbase-native-client/bin/build-thirdparty.sh
deleted file mode 100755
index 097c1a4..0000000
--- a/hbase-native-client/bin/build-thirdparty.sh
+++ /dev/null
@@ -1,64 +0,0 @@
-#! /usr/bin/env bash
-#
-#/**
-# * Copyright The Apache Software Foundation
-# *
-# * Licensed to the Apache Software Foundation (ASF) under one
-# * or more contributor license agreements. See the NOTICE file
-# * distributed with this work for additional information
-# * regarding copyright ownership. The ASF licenses this file
-# * to you under the Apache License, Version 2.0 (the
-# * "License"); you may not use this file except in compliance
-# * with the License. You may obtain a copy of the License at
-# *
-# * http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# */
-
-set -x
-set -e
-
-SOURCE="${BASH_SOURCE[0]}"
-while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
- DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
- SOURCE="$(readlink "$SOURCE")"
- [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
-done
-DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-
-source ${DIR}/../bin/hbase-client-env.sh
-
-# On some systems, autotools installs libraries to lib64 rather than lib. Fix
-# this by setting up lib64 as a symlink to lib. We have to do this step first
-# to handle cases where one third-party library depends on another.
-mkdir -p "${HBASE_PREFIX}/lib"
-cd ${HBASE_PREFIX}
-ln -sf lib "${HBASE_PREFIX}/lib64"
-
-if [ ! -f gtest ]; then
- cd ${HBASE_GTEST_DIR}
- cmake .
- make -j4
- cd ..
- ln -sf ${HBASE_GTEST_DIR} gtest
-fi
-
-if [ ! -f ${HBASE_PREFIX}/lib/libprotobuf.a ]; then
- cd ${HBASE_PROTOBUF_DIR}
- ./configure --with-pic --disable-shared --prefix=${HBASE_PREFIX}
- make -j4 install
-fi
-
-if [ ! -f ${HBASE_PREFIX}/lib/libev.a ]; then
- cd ${HBASE_LIBEV_DIR}
- ./configure --with-pic --disable-shared --prefix=${HBASE_PREFIX}
- make -j4 install
-fi
-
-echo "---------------------"
-echo "Thirdparty dependencies built and installed into $HBASE_PREFIX successfully"
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/bin/download-thirdparty.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/download-thirdparty.sh b/hbase-native-client/bin/download-thirdparty.sh
deleted file mode 100755
index db98aae..0000000
--- a/hbase-native-client/bin/download-thirdparty.sh
+++ /dev/null
@@ -1,70 +0,0 @@
-#! /usr/bin/env bash
-#
-#/**
-# * Copyright The Apache Software Foundation
-# *
-# * Licensed to the Apache Software Foundation (ASF) under one
-# * or more contributor license agreements. See the NOTICE file
-# * distributed with this work for additional information
-# * regarding copyright ownership. The ASF licenses this file
-# * to you under the Apache License, Version 2.0 (the
-# * "License"); you may not use this file except in compliance
-# * with the License. You may obtain a copy of the License at
-# *
-# * http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# */
-
-set -x
-set -e
-
-SOURCE="${BASH_SOURCE[0]}"
-while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
- DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
- SOURCE="$(readlink "$SOURCE")"
- [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
-done
-DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-
-source ${DIR}/../bin/hbase-client-env.sh
-
-mkdir -p ${HBASE_TP_DIR}
-cd ${HBASE_TP_DIR}
-
-
-if [ ! -d ${HBASE_GTEST_DIR} ]; then
- echo "Fetching gtest"
- wget -c http://googletest.googlecode.com/files/gtest-${HBASE_GTEST_VERSION}.zip
- unzip gtest-${HBASE_GTEST_VERSION}.zip
- rm gtest-${HBASE_GTEST_VERSION}.zip
-fi
-
-if [ ! -d ${HBASE_PROTOBUF_DIR} ]; then
- echo "Fetching protobuf"
- wget -c http://protobuf.googlecode.com/files/protobuf-${HBASE_PROTOBUF_VERSION}.tar.gz
- tar xzf protobuf-${HBASE_PROTOBUF_VERSION}.tar.gz
- rm protobuf-${HBASE_PROTOBUF_VERSION}.tar.gz
-fi
-
-if [ ! -d $HBASE_LIBEV_DIR ]; then
- echo "Fetching libev"
- wget -c http://dist.schmorp.de/libev/libev-${HBASE_LIBEV_VERSION}.tar.gz
- tar zxf libev-${HBASE_LIBEV_VERSION}.tar.gz
- rm -rf libev-${HBASE_LIBEV_VERSION}.tar.gz
-fi
-
-if [ ! -d $HBASE_PREFIX/bin/cpplint.py ]; then
- echo "Fetching cpplint"
- mkdir -p $HBASE_PREFIX/bin/
- wget -O $HBASE_PREFIX/bin/cpplint.py http://google-styleguide.googlecode.com/svn/trunk/cpplint/cpplint.py
- chmod +x $HBASE_PREFIX/bin/cpplint.py
-fi
-
-
-echo "---------------"
-echo "Thirdparty dependencies downloaded successfully"
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/bin/hbase-client-env.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/hbase-client-env.sh b/hbase-native-client/bin/hbase-client-env.sh
deleted file mode 100644
index 96763da..0000000
--- a/hbase-native-client/bin/hbase-client-env.sh
+++ /dev/null
@@ -1,47 +0,0 @@
-#! /usr/bin/env bash
-#
-#/**
-# * Copyright The Apache Software Foundation
-# *
-# * Licensed to the Apache Software Foundation (ASF) under one
-# * or more contributor license agreements. See the NOTICE file
-# * distributed with this work for additional information
-# * regarding copyright ownership. The ASF licenses this file
-# * to you under the Apache License, Version 2.0 (the
-# * "License"); you may not use this file except in compliance
-# * with the License. You may obtain a copy of the License at
-# *
-# * http://www.apache.org/licenses/LICENSE-2.0
-# *
-# * Unless required by applicable law or agreed to in writing, software
-# * distributed under the License is distributed on an "AS IS" BASIS,
-# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# * See the License for the specific language governing permissions and
-# * limitations under the License.
-# */
-
-SOURCE="${BASH_SOURCE[0]}"
-while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
- DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
- SOURCE="$(readlink "$SOURCE")"
- [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to
- # resolve it relative to the path where
- # the symlink file was located
-done
-DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-HBASE_DIR="${DIR}/../"
-HBASE_TP_DIR="${HBASE_DIR}/thirdparty/"
-
-HBASE_PREFIX=$HBASE_TP_DIR/installed
-
-HBASE_GTEST_VERSION=1.7.0
-HBASE_GTEST_DIR=$HBASE_TP_DIR/gtest-$HBASE_GTEST_VERSION
-
-HBASE_PROTOBUF_VERSION=2.5.0
-HBASE_PROTOBUF_DIR=$HBASE_TP_DIR/protobuf-$HBASE_PROTOBUF_VERSION
-
-HBASE_LIBEV_VERSION=4.15
-HBASE_LIBEV_DIR=$HBASE_TP_DIR/libev-$HBASE_LIBEV_VERSION
-
-# use the compiled tools
-export PATH=$HBASE_PREFIX/bin:$PATH
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/bin/start-docker.sh
----------------------------------------------------------------------
diff --git a/hbase-native-client/bin/start-docker.sh b/hbase-native-client/bin/start-docker.sh
new file mode 100755
index 0000000..1c9b02e
--- /dev/null
+++ b/hbase-native-client/bin/start-docker.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set -x
+
+eval "$(docker-machine env docker-vm)"
+docker build -t hbase_native .
+
+
+mkdir third-party || true
+if [[ ! -d third-party/googletest ]]; then
+ git clone https://github.com/google/googletest.git third-party/googletest
+fi
+
+
+docker run -v ${PWD}/..:/usr/local/src/hbase -it hbase_native /bin/bash
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/cmake_modules/FindGTest.cmake
----------------------------------------------------------------------
diff --git a/hbase-native-client/cmake_modules/FindGTest.cmake b/hbase-native-client/cmake_modules/FindGTest.cmake
deleted file mode 100644
index 8d27876..0000000
--- a/hbase-native-client/cmake_modules/FindGTest.cmake
+++ /dev/null
@@ -1,53 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set( GTEST_INCLUDE_SEARCH
- ${CMAKE_SOURCE_DIR}/thirdparty/gtest/include
-)
-
-set( GTEST_LIB_SEARCH
- ${CMAKE_SOURCE_DIR}/thirdparty/gtest
-)
-
-find_path(GTEST_INCLUDE
- NAMES gtest/gtest.h
- PATHS ${GTEST_INCLUDE_SEARCH}
- NO_DEFAULT_PATH
-)
-
-find_library(GTEST_LIBRARY
- NAMES gtest
- PATHS ${GTEST_LIB_SEARCH}
- NO_DEFAULT_PATH
-)
-
-find_library(GTEST_LIBRARY_MAIN
- NAMES gtest_main
- PATHS ${GTEST_LIB_SEARCH}
- NO_DEFAULT_PATH
-)
-
-if (GTEST_INCLUDE AND GTEST_LIBRARY AND GTEST_LIBRARY_MAIN)
- set(GTEST_LIBS ${GTEST_LIBRARY} ${GTEST_LIBRARY_MAIN})
- set(GTEST_INCLUDE_DIR ${GTEST_INCLUDE})
- set(GTEST_FOUND TRUE)
-endif()
-
-mark_as_advanced(
- GTEST_INCLUDE_DIR
- GTEST_LIBS
-)
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/cmake_modules/FindLibEv.cmake
----------------------------------------------------------------------
diff --git a/hbase-native-client/cmake_modules/FindLibEv.cmake b/hbase-native-client/cmake_modules/FindLibEv.cmake
deleted file mode 100644
index 5fd3e86..0000000
--- a/hbase-native-client/cmake_modules/FindLibEv.cmake
+++ /dev/null
@@ -1,47 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set( LIBEV_INCLUDE_SEARCH
- ${CMAKE_SOURCE_DIR}/thirdparty/installed/include
-)
-
-set( LIBEV_LIB_SEARCH
- ${CMAKE_SOURCE_DIR}/thirdparty/installed/include
-)
-
-find_path(LIBEV_INCLUDE
- NAMES ev++.h
- PATHS ${LIBEV_INCLUDE_SEARCH}
- NO_DEFAULT_PATH
-)
-
-find_library(LIBEV_LIBRARY
- NAMES ev
- PATHS ${LIBEV_LIB_SEARCH}
- NO_DEFAULT_PATH
-)
-
-if (LIBEV_INCLUDE_PATH AND LIBEV_LIBRARY)
- set(LIBEV_LIBS ${LIBEV_LIBRARY})
- set(LIBEV_INCLUDE_DIR ${LIBEV_INCLUDE})
- set(LIBEV_FOUND TRUE)
-endif()
-
-mark_as_advanced(
- LIBEV_INCLUDE_DIR
- LIBEV_LIBS
-)
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/core/BUCK
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/BUCK b/hbase-native-client/core/BUCK
new file mode 100644
index 0000000..a7eaa9e
--- /dev/null
+++ b/hbase-native-client/core/BUCK
@@ -0,0 +1,50 @@
+##
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+cxx_binary(
+ name = "core",
+ headers = [
+ "admin.h",
+ "client.h",
+ "connection.h",
+ "connection_attr.h",
+ "delete.h",
+ "get.h",
+ "hbase_macros.h",
+ "mutation.h",
+ "put.h",
+ "scanner.h",
+ ],
+ srcs = [
+ "admin.cc",
+ "client.cc",
+ "connection.cc",
+ "get.cc",
+ "mutation.cc",
+ "put.cc",
+ "delete.cc",
+ "scanner.cc",
+ ],
+ deps = [
+ "//third-party:folly",
+ "//third-party:wangle",
+ ],
+ visibility = [
+ 'PUBLIC',
+ ],
+)
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/core/admin.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/admin.cc b/hbase-native-client/core/admin.cc
new file mode 100644
index 0000000..897e6bf
--- /dev/null
+++ b/hbase-native-client/core/admin.cc
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/admin.h"
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/core/admin.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/admin.h b/hbase-native-client/core/admin.h
new file mode 100644
index 0000000..775181c
--- /dev/null
+++ b/hbase-native-client/core/admin.h
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+class Admin {};
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/core/client.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client.cc b/hbase-native-client/core/client.cc
new file mode 100644
index 0000000..98cf38a
--- /dev/null
+++ b/hbase-native-client/core/client.cc
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/client.h"
+
+#include <folly/Logging.h>
+#include <folly/Random.h>
+#include <glog/logging.h>
+#include <gflags/gflags.h>
+
+using namespace folly;
+
+int main(int argc, char *argv[]) {
+ google::ParseCommandLineFlags(&argc, &argv, true);
+ google::InitGoogleLogging(argv[0]);
+
+ FB_LOG_EVERY_MS(INFO, 10000) << "Hello";
+ for (long i = 0; i < 10000000; i++) {
+ FB_LOG_EVERY_MS(INFO, 1) << Random::rand32();
+ }
+ return 0;
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/core/client.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/client.h b/hbase-native-client/core/client.h
new file mode 100644
index 0000000..921cecc
--- /dev/null
+++ b/hbase-native-client/core/client.h
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <folly/io/IOBuf.h>
+
+class Client {};
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/core/connection.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/connection.cc b/hbase-native-client/core/connection.cc
new file mode 100644
index 0000000..e9a28eb
--- /dev/null
+++ b/hbase-native-client/core/connection.cc
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "core/connection.h"
+
+void Connection::set_zk_quorum(char *zk_q) { this->zk_quorum = zk_q; }
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/core/connection.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/connection.h b/hbase-native-client/core/connection.h
new file mode 100644
index 0000000..5c9d6b8
--- /dev/null
+++ b/hbase-native-client/core/connection.h
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+class Connection {
+ char *zk_quorum;
+
+public:
+ void set_zk_quorum(char *zk_q);
+};
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/core/connection_attr.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/connection_attr.h b/hbase-native-client/core/connection_attr.h
new file mode 100644
index 0000000..a312005
--- /dev/null
+++ b/hbase-native-client/core/connection_attr.h
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include "core/hbase_macros.h"
+
+class ConnectionAttr {};
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/core/delete.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/delete.cc b/hbase-native-client/core/delete.cc
new file mode 100644
index 0000000..57030be
--- /dev/null
+++ b/hbase-native-client/core/delete.cc
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "core/delete.h"
+
+Delete::~Delete() {}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/core/delete.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/delete.h b/hbase-native-client/core/delete.h
new file mode 100644
index 0000000..34f6a6c
--- /dev/null
+++ b/hbase-native-client/core/delete.h
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include "core/mutation.h"
+
+class Delete : public Mutation {
+public:
+ ~Delete();
+};
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/core/get.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/get.cc b/hbase-native-client/core/get.cc
new file mode 100644
index 0000000..9e11332
--- /dev/null
+++ b/hbase-native-client/core/get.cc
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/get.h"
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/core/get.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/get.h b/hbase-native-client/core/get.h
new file mode 100644
index 0000000..b4b5912
--- /dev/null
+++ b/hbase-native-client/core/get.h
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+class Get {};
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/core/hbase_macros.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/hbase_macros.h b/hbase-native-client/core/hbase_macros.h
new file mode 100644
index 0000000..48304f0
--- /dev/null
+++ b/hbase-native-client/core/hbase_macros.h
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/**
+ * The following code block define API as the tag for exported
+ * functions. The library should be compiled with symbols visibility
+ * set to hidden by default and only the exported functions should be
+ * tagged as HBASE_API.
+ *
+ * When building the library on Windows, compile with compiler flag
+ * "-D_LIBHBASE_IMPLEMENTATION_", whereas when linking application with
+ * this library, this compiler flag should not be used.
+ */
+#if defined _WIN32 || defined __CYGWIN__
+#ifdef _LIBHBASE_IMPLEMENTATION_
+#define API __declspec(dllexport)
+#else
+#ifdef _LIBHBASE_TEST_
+#define HBASE_API
+#else
+#define HBASE_API __declspec(dllimport)
+#endif
+#endif
+#else
+#if __GNUC__ >= 4
+#define HBASE_API __attribute__((visibility("default")))
+#else
+#define HBASE_API
+#endif
+#endif
+
+#ifdef __cplusplus
+} // extern "C"
+#endif // __cplusplus
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/core/mutation.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/mutation.cc b/hbase-native-client/core/mutation.cc
new file mode 100644
index 0000000..52910d5
--- /dev/null
+++ b/hbase-native-client/core/mutation.cc
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/mutation.h"
+
+void Mutation::set_namespace(char *name_space, size_t name_space_length) {
+ this->name_space = name_space;
+ this->name_space_length = name_space_length;
+}
+
+void Mutation::set_table(char *table, size_t table_length) {
+ this->table = table;
+ this->table_length = table_length;
+}
+
+void Mutation::set_row(unsigned char *row, size_t row_length) {
+ this->row = row;
+ this->row_length = row_length;
+}
+
+void Mutation::set_durability(durability_type durability) {
+ this->durability = durability;
+}
+
+Mutation::~Mutation() {}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/core/mutation.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/mutation.h b/hbase-native-client/core/mutation.h
new file mode 100644
index 0000000..1880571
--- /dev/null
+++ b/hbase-native-client/core/mutation.h
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include <stdlib.h>
+
+typedef enum {
+ DELETE_ONE_VERSION,
+ DELETE_MULTIPLE_VERSIONS,
+ DELETE_FAMILY,
+ DELETE_FAMILY_VERSION
+} delete_type;
+
+typedef enum {
+ USE_DEFAULT,
+ SKIP_WAL,
+ ASYNC_WAL,
+ SYNC_WAL,
+ HSYNC_WAL
+} durability_type;
+
+class Mutation {
+ char *name_space;
+ size_t name_space_length;
+
+ char *table;
+ size_t table_length;
+
+ unsigned char *row;
+ size_t row_length;
+
+ durability_type durability;
+
+public:
+ void set_namespace(char *name_space, size_t name_space_length);
+ void set_table(char *table, size_t table_length);
+ void set_row(unsigned char *row, size_t row_length);
+ void set_durability(durability_type durability);
+
+ virtual ~Mutation();
+};
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/core/put.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/put.cc b/hbase-native-client/core/put.cc
new file mode 100644
index 0000000..806a478
--- /dev/null
+++ b/hbase-native-client/core/put.cc
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "core/put.h"
+
+Put::~Put() {}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/core/put.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/put.h b/hbase-native-client/core/put.h
new file mode 100644
index 0000000..d34aa0d
--- /dev/null
+++ b/hbase-native-client/core/put.h
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+#include "core/mutation.h"
+
+class Put : public Mutation {
+public:
+ ~Put();
+};
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/core/scanner.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/scanner.cc b/hbase-native-client/core/scanner.cc
new file mode 100644
index 0000000..a10e444
--- /dev/null
+++ b/hbase-native-client/core/scanner.cc
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include "core/scanner.h"
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/core/scanner.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/core/scanner.h b/hbase-native-client/core/scanner.h
new file mode 100644
index 0000000..180865a
--- /dev/null
+++ b/hbase-native-client/core/scanner.h
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#pragma once
+
+class Scanner {};
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/rpc/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hbase-native-client/rpc/CMakeLists.txt b/hbase-native-client/rpc/CMakeLists.txt
new file mode 100644
index 0000000..2456923
--- /dev/null
+++ b/hbase-native-client/rpc/CMakeLists.txt
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/async/CMakeLists.txt b/hbase-native-client/src/async/CMakeLists.txt
deleted file mode 100644
index b2ed461..0000000
--- a/hbase-native-client/src/async/CMakeLists.txt
+++ /dev/null
@@ -1,32 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set( ASYNC_SRC
- hbase_admin.cc
- hbase_client.cc
- hbase_connection.cc
- hbase_get.cc
- hbase_mutations.cc
- hbase_result.cc
- hbase_scanner.cc
-)
-
-
-add_library(hasync OBJECT ${ASYNC_SRC})
-
-ADD_HBASE_ASYNC_TEST(mutations-test)
-ADD_HBASE_ASYNC_TEST(get-test)
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/get-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/async/get-test.cc b/hbase-native-client/src/async/get-test.cc
deleted file mode 100644
index df94ce2..0000000
--- a/hbase-native-client/src/async/get-test.cc
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include <pthread.h>
-
-#include "gtest/gtest.h"
-#include "async/hbase_get.h"
-#include "async/hbase_client.h"
-
-
-pthread_cond_t cv;
-pthread_mutex_t mutex;
-
-TEST(GetTest, TestPut) {
- char tn[] = "T1";
- hb_byte_t row[] = "ROW";
-
- hb_client_t client = NULL;
- hb_get_t get = NULL;
- int32_t s1 = -1;
-
- pthread_cond_init(&cv, NULL);
- pthread_mutex_init(&mutex, NULL);
-
- s1 = hb_client_create(&client, NULL);
- EXPECT_EQ(0, s1);
-
- s1 = hb_get_create(&get);
- EXPECT_EQ(0, s1);
-
- hb_get_set_table(get, tn, 2);
- hb_get_set_row(get, row, 3);
-
- /*
- * TODO:
- * This is currently a NO-OP as there is no CB.
- */
- hb_get_send(client, get, NULL, NULL);
-
- hb_client_destroy(client, NULL, NULL);
-
- EXPECT_EQ(0, s1);
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/hbase_admin.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/async/hbase_admin.cc b/hbase-native-client/src/async/hbase_admin.cc
deleted file mode 100644
index 17ff2a7..0000000
--- a/hbase-native-client/src/async/hbase_admin.cc
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "async/hbase_admin.h"
-
-#include <stdlib.h>
-#include <stdbool.h>
-
-#include "core/admin.h"
-#include "async/hbase_connection.h"
-
-int32_t hb_admin_create(hb_admin_t* admin_ptr,
- hb_connection_t connection) {
- (*admin_ptr) = reinterpret_cast<hb_admin_t>(new Admin());
- return 0;
-}
-
-/*
- * Disconnect the admin releasing any internal objects
- * or connections created in the background.
- */
-int32_t hb_admin_destroy(hb_admin_t admin,
- hb_admin_disconnection_cb cb, void * extra) {
- if (cb)
- cb(0, admin, extra);
- free(admin);
- return 0;
-}
-
-/*
- * See if a table exists.
- */
-int32_t hb_admin_table_exists(hb_admin_t admin,
- char * name_space, size_t name_space_length,
- char * table, size_t table_length,
- hb_admin_table_exists_cb cb, void * extra) {
- if (cb)
- cb(0, admin, name_space, name_space_length,
- table, table_length, true, extra);
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/hbase_admin.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/async/hbase_admin.h b/hbase-native-client/src/async/hbase_admin.h
deleted file mode 100644
index f03a83f..0000000
--- a/hbase-native-client/src/async/hbase_admin.h
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef ASYNC_HBASE_ADMIN_H_
-#define ASYNC_HBASE_ADMIN_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdlib.h>
-#include <stdbool.h>
-
-#include "core/hbase_macros.h"
-#include "async/hbase_connection.h"
-
-
-typedef void (* hb_admin_disconnection_cb)( int32_t status,
- hb_admin_t admin, void * extra);
-typedef void (* hb_admin_table_exists_cb)(int32_t status, hb_admin_t admin,
- char * name_space, size_t name_space_length,
- char * table, size_t table_length, bool exsists, void * extra);
-
-/**
- * Create a new hb_admin.
- * All fields are initialized to the defaults. If you want to set
- * connection or other properties, set those before calling any
- * RPC functions.
- */
-HBASE_API int32_t hb_admin_create(hb_admin_t* admin_ptr,
- hb_connection_t connection);
-
-/*
- * Disconnect the admin releasing any internal objects
- * or connections created in the background.
- */
-HBASE_API int32_t hb_admin_destroy(hb_admin_t admin,
- hb_admin_disconnection_cb cb, void * extra);
-
-/*
- * See if a table exists.
- */
-HBASE_API int32_t hb_admin_table_exists(hb_admin_t admin,
- char * name_space, size_t name_space_length,
- char * table, size_t table_length,
- hb_admin_table_exists_cb cb, void * extra);
-
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // ASYNC_HBASE_ADMIN_H_
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/hbase_client.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/async/hbase_client.cc b/hbase-native-client/src/async/hbase_client.cc
deleted file mode 100644
index 98e3dbf..0000000
--- a/hbase-native-client/src/async/hbase_client.cc
+++ /dev/null
@@ -1,47 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "async/hbase_client.h"
-
-#include <pthread.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "core/client.h"
-#include "async/hbase_connection.h"
-
-int32_t hb_client_create(hb_client_t* client_ptr,
- hb_connection_t connection) {
- (*client_ptr) = reinterpret_cast<hb_client_t>(new Client());
- if (client_ptr == NULL)
- return -1; // TODO(eclark): setup the errno file.
- return 0;
-}
-
-int32_t hb_client_destroy(hb_client_t client,
- hb_client_disconnection_cb cb, void * extra) {
- if (client == NULL)
- return -2;
- if (cb) {
- cb(0, client, extra);
- }
- free(client);
- return 0;
-}
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/hbase_client.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/async/hbase_client.h b/hbase-native-client/src/async/hbase_client.h
deleted file mode 100644
index 34c3d98..0000000
--- a/hbase-native-client/src/async/hbase_client.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef ASYNC_HBASE_CLIENT_H_
-#define ASYNC_HBASE_CLIENT_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "core/hbase_macros.h"
-#include "core/hbase_types.h"
-
-/*
- * Client disconnection callback typedef
- *
- * This is called after the connections are closed, but just
- * before the client is freed.
- */
-typedef void (* hb_client_disconnection_cb)( int32_t status,
- hb_client_t client, void * extra);
-
-/**
- * Create an hb_client_t.
- *
- * If connection is null then all defaults will be used.
- */
-HBASE_API int32_t hb_client_create(hb_client_t * client_ptr,
- hb_connection_t connection);
-
-/*
- * Disconnect the client releasing any internal objects
- * or connections created in the background.
- */
-HBASE_API int32_t hb_client_destroy(hb_client_t client,
- hb_client_disconnection_cb cb, void * extra);
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // ASYNC_HBASE_CLIENT_H_
-
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/hbase_connection.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/async/hbase_connection.cc b/hbase-native-client/src/async/hbase_connection.cc
deleted file mode 100644
index 57fa4f5..0000000
--- a/hbase-native-client/src/async/hbase_connection.cc
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "async/hbase_connection.h"
-
-#include "core/connection.h"
-#include "core/hbase_types.h"
-
-extern "C" {
-int32_t hb_connection_create(hb_connection_t * connection_ptr,
- hb_connection_attr_t connection_attr) {
- (*connection_ptr) = reinterpret_cast<hb_connection_t>(new Connection());
- if ((*connection_ptr) == NULL)
- return -1;
- return 0;
-}
-int32_t hb_connection_destroy(hb_connection_t connection) {
- free(connection);
- return 0;
-}
-} // extern "C"
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/hbase_connection.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/async/hbase_connection.h b/hbase-native-client/src/async/hbase_connection.h
deleted file mode 100644
index 343f36c..0000000
--- a/hbase-native-client/src/async/hbase_connection.h
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef ASYNC_HBASE_CONNECTION_H_
-#define ASYNC_HBASE_CONNECTION_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "core/hbase_macros.h"
-#include "core/hbase_types.h"
-#include "core/hbase_connection_attr.h"
-
-#include <stdlib.h>
-
-/**
- * Create an hb_connection.
- *
- * if connection_attr is null everything will be left as default
- */
-HBASE_API int32_t hb_connection_create(hb_connection_t * connection_ptr,
- hb_connection_attr_t connection_attr);
-
-/**
- * Destroy the connection and free all resources allocated at creation
- * time.
- */
-HBASE_API int32_t hb_connection_destroy(hb_connection_t connection);
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // ASYNC_HBASE_CONNECTION_H_
-
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/hbase_errno.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/async/hbase_errno.h b/hbase-native-client/src/async/hbase_errno.h
deleted file mode 100644
index 1698bbd..0000000
--- a/hbase-native-client/src/async/hbase_errno.h
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef ASYNC_HBASE_ERRNO_H_
-#define ASYNC_HBASE_ERRNO_H_
-
-#endif // ASYNC_HBASE_ERRNO_H_
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/hbase_get.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/async/hbase_get.cc b/hbase-native-client/src/async/hbase_get.cc
deleted file mode 100644
index 32048d8..0000000
--- a/hbase-native-client/src/async/hbase_get.cc
+++ /dev/null
@@ -1,61 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "async/hbase_get.h"
-
-#include <stdlib.h>
-#include <string.h>
-
-#include "core/get.h"
-
-int32_t hb_get_create(hb_get_t * get_ptr) {
- (*get_ptr) = reinterpret_cast<hb_get_t>(new Get());
- if ((*get_ptr) == NULL) {
- return -1;
- }
- return 0;
-}
-
-int32_t hb_get_destroy(hb_get_t get) {
- free(get);
- return 0;
-}
-
-int32_t hb_get_set_row(hb_get_t get, unsigned char * row,
- size_t row_length) {
- return 0;
-}
-
-int32_t hb_get_set_table(hb_get_t get,
- char * table, size_t table_length) {
- return 0;
-}
-
-int32_t hb_get_set_namespace(hb_get_t get,
- char * name_space, size_t name_space_length) {
- return 0;
-}
-
-int32_t hb_get_send(hb_client_t client,
- hb_get_t get, hb_get_cb cb, void * extra) {
- if (cb) {
- cb(0, client, get, NULL, extra);
- }
- return 0;
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/hbase_get.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/async/hbase_get.h b/hbase-native-client/src/async/hbase_get.h
deleted file mode 100644
index 7d91c08..0000000
--- a/hbase-native-client/src/async/hbase_get.h
+++ /dev/null
@@ -1,73 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef ASYNC_HBASE_GET_H_
-#define ASYNC_HBASE_GET_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "core/hbase_macros.h"
-#include "core/hbase_types.h"
-
-/**
- * Allocate a new get structure.
- * Ownership passes to the caller.
- */
-HBASE_API int32_t hb_get_create(hb_get_t * get_ptr);
-
-/**
- * Destroy and free a get structure.
- */
-HBASE_API int32_t hb_get_destroy(hb_get_t get);
-
-/**
- * set the row of this get.
- */
-HBASE_API int32_t hb_get_set_row(hb_get_t get, hb_byte_t * row,
- size_t row_length);
-
-/**
- * Set the table.
- */
-HBASE_API int32_t hb_get_set_table(hb_get_t get,
- char * table, size_t table_length);
-
-/**
- * Set the namespace this get is targeting.
- */
-HBASE_API int32_t hb_get_set_namespace(hb_get_t get,
- char * name_space, size_t name_space_length);
-
-/*
- * get call back typedef.
- */
-typedef void (* hb_get_cb)(int32_t status, hb_client_t client,
- hb_get_t get, hb_result_t results, void * extra);
-
-HBASE_API int32_t hb_get_send(hb_client_t client,
- hb_get_t get, hb_get_cb cb, void * extra);
-
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // ASYNC_HBASE_GET_H_
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/hbase_mutations.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/async/hbase_mutations.cc b/hbase-native-client/src/async/hbase_mutations.cc
deleted file mode 100644
index 2456dc0..0000000
--- a/hbase-native-client/src/async/hbase_mutations.cc
+++ /dev/null
@@ -1,111 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "async/hbase_mutations.h"
-
-#include <stdlib.h>
-
-#include "core/hbase_types.h"
-#include "core/mutation.h"
-#include "core/put.h"
-#include "core/delete.h"
-
-#include "async/hbase_result.h"
-
-extern "C" {
-int32_t hb_put_create(hb_put_t* put_ptr) {
- (*put_ptr) = reinterpret_cast<hb_put_t>(new Put());
- return 0;
-}
-
-int32_t hb_delete_create(hb_delete_t * delete_ptr) {
- (*delete_ptr) = reinterpret_cast<hb_delete_t>(new Delete());
- return 0;
-}
-
-int32_t hb_increment_create(hb_increment_t * increment_ptr) {
- return 0;
-}
-
-int32_t hb_append_create(hb_append_t * append_ptr) {
- return 0;
-}
-
-int32_t hb_mutation_destroy(hb_mutation_t mutation) {
- return 0;
-}
-
-HBASE_API int32_t hb_mutation_set_namespace(hb_mutation_t mutation,
- char * name_space, size_t name_space_length) {
- Mutation * m = reinterpret_cast<Mutation *>(mutation);
- m->set_namespace(name_space, name_space_length);
- return 0;
-}
-
-HBASE_API int32_t hb_mutation_set_table(hb_mutation_t mutation,
- char * table, size_t table_length) {
- Mutation * m = reinterpret_cast<Mutation *>(mutation);
- m->set_namespace(table, table_length);
- return 0;
-}
-
-int32_t hb_mutation_set_row(hb_mutation_t mutation,
- unsigned char * rk, size_t row_length) {
- Mutation * m = reinterpret_cast<Mutation *>(mutation);
- m->set_row(rk, row_length);
- return 0;
-}
-
-int32_t hb_mutation_set_durability(hb_mutation_t mutation,
- hb_durability_type durability) {
- Mutation * m = reinterpret_cast<Mutation *>(mutation);
- m->set_durability(durability);
- return 0;
-}
-
-int32_t hb_put_add_cell(hb_put_t put, hb_cell_t * cell) {
- return 0;
-}
-
-int32_t hb_delete_add_col(hb_increment_t incr,
- unsigned char * family, size_t family_length,
- unsigned char * qual, size_t qual_length) {
- return 0;
-}
-
-int32_t hb_increment_add_value(hb_increment_t incr,
- unsigned char * family, size_t family_length,
- unsigned char * qual, size_t qual_length,
- int64_t ammount) {
- return 0;
-}
-
-int32_t hb_append_add_cell(hb_append_t put, hb_cell_t * cell) {
- return 0;
-}
-
-int32_t hb_mutation_send(hb_client_t client,
- hb_mutation_t mutation, hb_mutation_cb cb,
- void * extra) {
- if (cb) {
- cb(0, client, mutation, NULL, extra);
- }
- return 0;
-}
-} // extern "C"
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/hbase_mutations.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/async/hbase_mutations.h b/hbase-native-client/src/async/hbase_mutations.h
deleted file mode 100644
index 4000390..0000000
--- a/hbase-native-client/src/async/hbase_mutations.h
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#ifndef ASYNC_HBASE_MUTATIONS_H_
-#define ASYNC_HBASE_MUTATIONS_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "core/hbase_types.h"
-#include "async/hbase_result.h"
-
-// Creation methods
-
-/**
- * Create a put.
- * Ownership passes to the caller.
- */
-HBASE_API int32_t hb_put_create(hb_put_t * put_ptr);
-
-/**
- * Create a delete
- * Ownership passes to the caller.
- */
-HBASE_API int32_t hb_delete_create(hb_delete_t * delete_ptr);
-
-/**
- * Create an increment
- * Ownership passes to the caller.
- */
-HBASE_API int32_t hb_increment_create(hb_increment_t * increment_ptr);
-
-/**
- * Create an append
- * Ownership passes to the caller.
- */
-HBASE_API int32_t hb_append_create(hb_append_t * append_ptr);
-
-/**
- * Destroy the mutation.
- * All internal structures are cleaned up. However any backing
- * data structures passed in by the user are not cleaned up.
- */
-HBASE_API int32_t hb_mutation_destroy(hb_mutation_t mutation);
-
-// Shared setters.
-HBASE_API int32_t hb_mutation_set_namespace(hb_mutation_t mutation,
- char * name_space, size_t name_space_length);
-HBASE_API int32_t hb_mutation_set_table(hb_mutation_t mutation,
- char * table, size_t table_length);
-HBASE_API int32_t hb_mutation_set_row(hb_mutation_t mutation,
- unsigned char * rk, size_t row_length);
-HBASE_API int32_t hb_mutation_set_durability(hb_mutation_t mutation,
- hb_durability_type durability);
-
-// Put Setters etc.
-HBASE_API int32_t hb_put_add_cell(hb_put_t put, hb_cell_t * cell);
-
-// Delete
-HBASE_API int32_t hb_delete_add_col(hb_increment_t incr,
- unsigned char * family, size_t family_length,
- unsigned char * qual, size_t qual_length);
-
-// Increment
-HBASE_API int32_t hb_increment_add_value(hb_increment_t incr,
- unsigned char * family, size_t family_length,
- unsigned char * qual, size_t qual_length,
- int64_t ammount);
-
-// Append
-HBASE_API int32_t hb_append_add_cell(hb_append_t put, hb_cell_t * cell);
-
-// Now that the mutations are created and populated
-// The real meat of the client is below.
-
-/*
- * mutation call back typedef
- */
-typedef void (* hb_mutation_cb)(int32_t status,
- hb_client_t client, hb_mutation_t mutation,
- hb_result_t result, void * extra);
-
-/*
- * Queue a single mutation. This mutation will be
- * sent out in the background and can be batched with
- * other requests destined for the same server.
- *
- * The call back will be executed after the response
- * is received from the RegionServer. Even if the
- * mutation was batched with other requests,
- * the call back will be invoked for every mutation
- * individually.
- */
-HBASE_API int32_t hb_mutation_send(hb_client_t client,
- hb_mutation_t mutation, hb_mutation_cb cb,
- void * extra);
-
-#ifdef __cplusplus
-} // extern "C"
-#endif // __cplusplus
-
-#endif // ASYNC_HBASE_MUTATIONS_H_
http://git-wip-us.apache.org/repos/asf/hbase/blob/8556e3a3/hbase-native-client/src/async/hbase_result.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/src/async/hbase_result.cc b/hbase-native-client/src/async/hbase_result.cc
deleted file mode 100644
index 9351270..0000000
--- a/hbase-native-client/src/async/hbase_result.cc
+++ /dev/null
@@ -1,37 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *
- */
-
-#include "async/hbase_result.h"
-
-#include "core/hbase_types.h"
-
-int32_t hb_result_get_cells(hb_result_t result,
- hb_cell_t ** cell_ptr, size_t * num_cells) {
- return 0;
-}
-
-int32_t hb_result_get_table(hb_result_t result,
- char ** table, size_t * table_length) {
- return 0;
-}
-
-int32_t hb_result_get_namespace(hb_result_t result,
- char ** name_space, size_t * name_space_length) {
- return 0;
-}
[27/50] [abbrv] hbase git commit: HBASE-15761 Add on more server name
tests
Posted by ec...@apache.org.
HBASE-15761 Add on more server name tests
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/297a4dff
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/297a4dff
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/297a4dff
Branch: refs/heads/HBASE-14850
Commit: 297a4dff638649f7777346e83c8cc685f0009fb3
Parents: fdf0023
Author: Elliott Clark <ec...@apache.org>
Authored: Wed May 4 12:04:18 2016 -0700
Committer: Elliott Clark <ec...@apache.org>
Committed: Mon Jul 11 16:47:26 2016 -0700
----------------------------------------------------------------------
hbase-native-client/serde/server-name-test.cc | 18 ++++++++++++++++++
hbase-native-client/serde/server-name.h | 4 +++-
2 files changed, 21 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/297a4dff/hbase-native-client/serde/server-name-test.cc
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/server-name-test.cc b/hbase-native-client/serde/server-name-test.cc
index 35dcbc1..2281fa2 100644
--- a/hbase-native-client/serde/server-name-test.cc
+++ b/hbase-native-client/serde/server-name-test.cc
@@ -30,3 +30,21 @@ TEST(TestServerName, TestMakeServerName) {
ASSERT_EQ("test", sn.host_name());
ASSERT_EQ(123, sn.port());
}
+
+TEST(TestServerName, TestIps) {
+ auto sn = folly::to<ServerName>("127.0.0.1:999");
+ ASSERT_EQ("127.0.0.1", sn.host_name());
+ ASSERT_EQ(999, sn.port());
+}
+
+TEST(TestServerName, TestThrow) {
+ ASSERT_ANY_THROW(folly::to<ServerName>("Ther's no colon here"));
+}
+
+TEST(TestServerName, TestIPV6) {
+ auto sn = folly::to<ServerName>("[::::1]:123");
+
+ ASSERT_EQ("[::::1]", sn.host_name());
+ ASSERT_EQ(123, sn.port());
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/297a4dff/hbase-native-client/serde/server-name.h
----------------------------------------------------------------------
diff --git a/hbase-native-client/serde/server-name.h b/hbase-native-client/serde/server-name.h
index bdba087..9844465 100644
--- a/hbase-native-client/serde/server-name.h
+++ b/hbase-native-client/serde/server-name.h
@@ -12,7 +12,9 @@ template <class String> void parseTo(String in, ServerName &out) {
std::string s = folly::to<std::string>(in);
auto delim = s.rfind(":");
- DCHECK(delim != std::string::npos);
+ if (delim == std::string::npos) {
+ throw std::runtime_error("Couldn't parse server name");
+ }
out.set_host_name(s.substr(0, delim));
// Now keep everything after the : (delim + 1) to the end.
out.set_port(folly::to<int>(s.substr(delim + 1)));