You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/07/13 06:03:46 UTC
[34/51] [abbrv] impala git commit: IMPALA-7006: Add KRPC folders from
kudu@334ecafd
http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/cert.h
----------------------------------------------------------------------
diff --git a/be/src/kudu/security/cert.h b/be/src/kudu/security/cert.h
new file mode 100644
index 0000000..4629883
--- /dev/null
+++ b/be/src/kudu/security/cert.h
@@ -0,0 +1,119 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include <openssl/asn1.h>
+
+#include "kudu/gutil/port.h"
+#include "kudu/security/openssl_util.h"
+
+typedef struct X509_name_st X509_NAME;
+
+namespace boost {
+template <class T>
+class optional;
+}
+
+namespace kudu {
+
+class Status;
+
+namespace security {
+
+class PrivateKey;
+class PublicKey;
+
+// Convert an X509_NAME object to a human-readable string.
+std::string X509NameToString(X509_NAME* name);
+
+// Return the OpenSSL NID for the custom X509 extension where we store
+// our Kerberos principal in IPKI certs.
+int GetKuduKerberosPrincipalOidNid();
+
+// A wrapper class around the STACK_OF(X509) object. This can either hold one certificate or
+// a chain of certificates.
+// TODO(unknown): Currently, there isn't a mechanism to add to the chain. Implement it when needed.
+class Cert : public RawDataWrapper<STACK_OF(X509)> {
+ public:
+ Status FromString(const std::string& data, DataFormat format) WARN_UNUSED_RESULT;
+ Status ToString(std::string* data, DataFormat format) const WARN_UNUSED_RESULT;
+ Status FromFile(const std::string& fpath, DataFormat format) WARN_UNUSED_RESULT;
+
+ int chain_len() const { return sk_X509_num(data_.get()); }
+
+ std::string SubjectName() const;
+ std::string IssuerName() const;
+
+ // Return DNS names from the SAN extension field of the end-user cert.
+ std::vector<std::string> Hostnames() const;
+
+ // Return the 'userId' extension of the end-user cert, if set.
+ boost::optional<std::string> UserId() const;
+
+ // Return the Kerberos principal encoded in the end-user certificate, if set.
+ boost::optional<std::string> KuduKerberosPrincipal() const;
+
+ // Check whether the specified private key matches the end-user certificate.
+ // Return Status::OK() if key match the end-user certificate.
+ Status CheckKeyMatch(const PrivateKey& key) const WARN_UNUSED_RESULT;
+
+ // Returns the 'tls-server-end-point' channel bindings for the end-user certificate as
+ // specified in RFC 5929.
+ Status GetServerEndPointChannelBindings(std::string* channel_bindings) const WARN_UNUSED_RESULT;
+
+ // Adopts the provided STACK_OF(X509), and increments the reference count of the X509 cert
+ // contained within it. Currently, only one certificate should be contained in the stack.
+ void AdoptAndAddRefRawData(RawDataType* data);
+
+ // Adopts the provided X509 certificate, and replaces the current underlying STACK_OF(X509).
+ void AdoptX509(X509* cert);
+
+ // Adopts the provided X509 certificate, increments its reference count and replaces the current
+ // underlying STACK_OF(X509).
+ void AdoptAndAddRefX509(X509* cert);
+
+ // Returns the end-user certificate's public key.
+ Status GetPublicKey(PublicKey* key) const WARN_UNUSED_RESULT;
+
+ // Get the first certificate in the chain, otherwise known as the 'end-user' certificate.
+ X509* GetTopOfChainX509() const;
+};
+
+class CertSignRequest : public RawDataWrapper<X509_REQ> {
+ public:
+ Status FromString(const std::string& data, DataFormat format) WARN_UNUSED_RESULT;
+ Status ToString(std::string* data, DataFormat format) const WARN_UNUSED_RESULT;
+ Status FromFile(const std::string& fpath, DataFormat format) WARN_UNUSED_RESULT;
+
+ // Returns a clone of the CSR.
+ //
+ // Whether this clone is deep or shallow (i.e. only a reference count is
+ // incremented) depends on the version of OpenSSL. Either way, the right
+ // thing happens when the clone goes out of scope.
+ CertSignRequest Clone() const;
+
+ // Returns the CSR's public key.
+ Status GetPublicKey(PublicKey* key) const WARN_UNUSED_RESULT;
+};
+
+} // namespace security
+} // namespace kudu
http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/crypto-test.cc
----------------------------------------------------------------------
diff --git a/be/src/kudu/security/crypto-test.cc b/be/src/kudu/security/crypto-test.cc
new file mode 100644
index 0000000..c1e32df
--- /dev/null
+++ b/be/src/kudu/security/crypto-test.cc
@@ -0,0 +1,257 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <cstring>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include <gtest/gtest.h>
+
+#include "kudu/gutil/strings/strip.h"
+#include "kudu/gutil/strings/substitute.h"
+#include "kudu/security/crypto.h"
+#include "kudu/security/openssl_util.h"
+#include "kudu/security/test/test_certs.h"
+#include "kudu/util/env.h"
+#include "kudu/util/path_util.h"
+#include "kudu/util/slice.h"
+#include "kudu/util/status.h"
+#include "kudu/util/test_macros.h"
+#include "kudu/util/test_util.h"
+#include "kudu/util/url-coding.h"
+
+using std::pair;
+using std::string;
+using std::vector;
+using strings::Substitute;
+
+namespace kudu {
+namespace security {
+
+// Test for various crypto-related functionality in the security library.
+class CryptoTest : public KuduTest {
+ public:
+ CryptoTest() :
+ pem_dir_(GetTestPath("pem")),
+ private_key_file_(JoinPathSegments(pem_dir_, "private_key.pem")),
+ public_key_file_(JoinPathSegments(pem_dir_, "public_key.pem")),
+ corrupted_private_key_file_(JoinPathSegments(pem_dir_,
+ "corrupted.private_key.pem")),
+ corrupted_public_key_file_(JoinPathSegments(pem_dir_,
+ "corrupted.public_key.pem")) {
+ }
+
+ void SetUp() override {
+ ASSERT_OK(env_->CreateDir(pem_dir_));
+ ASSERT_OK(WriteStringToFile(env_, kCaPrivateKey, private_key_file_));
+ ASSERT_OK(WriteStringToFile(env_, kCaPublicKey, public_key_file_));
+ ASSERT_OK(WriteStringToFile(env_,
+ string(kCaPrivateKey, strlen(kCaPrivateKey) / 2),
+ corrupted_private_key_file_));
+ ASSERT_OK(WriteStringToFile(env_,
+ string(kCaPublicKey, strlen(kCaPublicKey) / 2),
+ corrupted_public_key_file_));
+ }
+
+ protected:
+ template<typename Key>
+ void CheckToAndFromString(const Key& key_ref, DataFormat format) {
+ SCOPED_TRACE(Substitute("ToAndFromString for $0 format",
+ DataFormatToString(format)));
+ string key_ref_str;
+ ASSERT_OK(key_ref.ToString(&key_ref_str, format));
+ Key key;
+ ASSERT_OK(key.FromString(key_ref_str, format));
+ string key_str;
+ ASSERT_OK(key.ToString(&key_str, format));
+ ASSERT_EQ(key_ref_str, key_str);
+ }
+
+ const string pem_dir_;
+
+ const string private_key_file_;
+ const string public_key_file_;
+ const string corrupted_private_key_file_;
+ const string corrupted_public_key_file_;
+};
+
+// Check input/output of RSA private keys in PEM format.
+TEST_F(CryptoTest, RsaPrivateKeyInputOutputPEM) {
+ PrivateKey key;
+ ASSERT_OK(key.FromFile(private_key_file_, DataFormat::PEM));
+ string key_str;
+ ASSERT_OK(key.ToString(&key_str, DataFormat::PEM));
+ RemoveExtraWhitespace(&key_str);
+
+ string ref_key_str(kCaPrivateKey);
+ RemoveExtraWhitespace(&ref_key_str);
+ EXPECT_EQ(ref_key_str, key_str);
+}
+
+// Check input of corrupted RSA private keys in PEM format.
+TEST_F(CryptoTest, CorruptedRsaPrivateKeyInputPEM) {
+ static const string kFiles[] = {
+ corrupted_private_key_file_,
+ public_key_file_,
+ corrupted_public_key_file_,
+ "/bin/sh"
+ };
+ for (const auto& file : kFiles) {
+ PrivateKey key;
+ const Status s = key.FromFile(file, DataFormat::PEM);
+ EXPECT_TRUE(s.IsRuntimeError()) << s.ToString();
+ }
+}
+
+// Check input/output of RSA public keys in PEM format.
+TEST_F(CryptoTest, RsaPublicKeyInputOutputPEM) {
+ PublicKey key;
+ ASSERT_OK(key.FromFile(public_key_file_, DataFormat::PEM));
+ string key_str;
+ ASSERT_OK(key.ToString(&key_str, DataFormat::PEM));
+ RemoveExtraWhitespace(&key_str);
+
+ string ref_key_str(kCaPublicKey);
+ RemoveExtraWhitespace(&ref_key_str);
+ EXPECT_EQ(ref_key_str, key_str);
+}
+
+// Check input of corrupted RSA public keys in PEM format.
+TEST_F(CryptoTest, CorruptedRsaPublicKeyInputPEM) {
+ static const string kFiles[] = {
+ corrupted_public_key_file_,
+ private_key_file_,
+ corrupted_private_key_file_,
+ "/bin/sh"
+ };
+ for (const auto& file : kFiles) {
+ PublicKey key;
+ const Status s = key.FromFile(file, DataFormat::PEM);
+ EXPECT_TRUE(s.IsRuntimeError()) << s.ToString();
+ }
+}
+
+// Check extraction of the public part from RSA private keys par.
+TEST_F(CryptoTest, RsaExtractPublicPartFromPrivateKey) {
+ // Load the reference RSA private key.
+ PrivateKey private_key;
+ ASSERT_OK(private_key.FromString(kCaPrivateKey, DataFormat::PEM));
+
+ PublicKey public_key;
+ ASSERT_OK(private_key.GetPublicKey(&public_key));
+ string str_public_key;
+ ASSERT_OK(public_key.ToString(&str_public_key, DataFormat::PEM));
+ RemoveExtraWhitespace(&str_public_key);
+
+ string ref_str_public_key(kCaPublicKey);
+ RemoveExtraWhitespace(&ref_str_public_key);
+ EXPECT_EQ(ref_str_public_key, str_public_key);
+}
+
+class CryptoKeySerDesTest :
+ public CryptoTest,
+ public ::testing::WithParamInterface<DataFormat> {
+};
+
+// Check the transformation chains for RSA public/private keys:
+// internal -> PEM -> internal -> PEM
+// internal -> DER -> internal -> DER
+TEST_P(CryptoKeySerDesTest, ToAndFromString) {
+ const auto format = GetParam();
+
+ // Generate private RSA key.
+ PrivateKey private_key;
+ ASSERT_OK(GeneratePrivateKey(2048, &private_key));
+ NO_FATALS(CheckToAndFromString(private_key, format));
+
+ // Extract public part of the key.
+ PublicKey public_key;
+ ASSERT_OK(private_key.GetPublicKey(&public_key));
+ NO_FATALS(CheckToAndFromString(public_key, format));
+}
+
+INSTANTIATE_TEST_CASE_P(
+ DataFormats, CryptoKeySerDesTest,
+ ::testing::Values(DataFormat::DER, DataFormat::PEM));
+
+// Check making crypto signatures against the reference data.
+TEST_F(CryptoTest, MakeVerifySignatureRef) {
+ static const vector<pair<string, string>> kRefElements = {
+ { kDataTiny, kSignatureTinySHA512 },
+ { kDataShort, kSignatureShortSHA512 },
+ { kDataLong, kSignatureLongSHA512 },
+ };
+
+ // Load the reference RSA private key.
+ PrivateKey private_key;
+ ASSERT_OK(private_key.FromString(kCaPrivateKey, DataFormat::PEM));
+
+ // Load the reference RSA public key.
+ PublicKey public_key;
+ ASSERT_OK(public_key.FromString(kCaPublicKey, DataFormat::PEM));
+
+ for (const auto& e : kRefElements) {
+ string sig;
+ ASSERT_OK(private_key.MakeSignature(DigestType::SHA512, e.first, &sig));
+
+ // Ad-hoc verification: check the produced signature matches the reference.
+ string sig_base64;
+ Base64Encode(sig, &sig_base64);
+ EXPECT_EQ(e.second, sig_base64);
+
+ // Verify the signature cryptographically.
+ EXPECT_OK(public_key.VerifySignature(DigestType::SHA512, e.first, sig));
+ }
+}
+
+TEST_F(CryptoTest, VerifySignatureWrongData) {
+ static const vector<string> kRefSignatures = {
+ kSignatureTinySHA512,
+ kSignatureShortSHA512,
+ kSignatureLongSHA512,
+ };
+
+ // Load the reference RSA public key.
+ PublicKey key;
+ ASSERT_OK(key.FromString(kCaPublicKey, DataFormat::PEM));
+
+ for (const auto& e : kRefSignatures) {
+ string signature;
+ ASSERT_TRUE(Base64Decode(e, &signature));
+ Status s = key.VerifySignature(DigestType::SHA512,
+ "non-expected-data", signature);
+ EXPECT_TRUE(s.IsCorruption()) << s.ToString();
+ }
+}
+
+TEST_F(CryptoTest, TestGenerateNonce) {
+ string nonce;
+ ASSERT_OK(GenerateNonce(&nonce));
+
+ // Do some basic validation on the returned nonce.
+ ASSERT_EQ(kNonceSize, nonce.size());
+ ASSERT_NE(string(kNonceSize, '\0'), nonce);
+
+ // Nonces should be unique, by definition.
+ string another_nonce;
+ ASSERT_OK(GenerateNonce(&another_nonce));
+ ASSERT_NE(nonce, another_nonce);
+}
+
+} // namespace security
+} // namespace kudu
http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/crypto.cc
----------------------------------------------------------------------
diff --git a/be/src/kudu/security/crypto.cc b/be/src/kudu/security/crypto.cc
new file mode 100644
index 0000000..234d193
--- /dev/null
+++ b/be/src/kudu/security/crypto.cc
@@ -0,0 +1,276 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "kudu/security/crypto.h"
+
+#include <memory>
+#include <ostream>
+#include <string>
+
+#include <glog/logging.h>
+#include <openssl/bio.h>
+#include <openssl/bn.h>
+#include <openssl/err.h>
+#include <openssl/evp.h>
+#include <openssl/opensslv.h>
+#include <openssl/ossl_typ.h>
+#include <openssl/pem.h>
+#include <openssl/rand.h>
+#include <openssl/rsa.h>
+#include <openssl/x509.h>
+
+#include "kudu/gutil/strings/substitute.h"
+#include "kudu/security/openssl_util.h"
+#include "kudu/security/openssl_util_bio.h"
+#include "kudu/util/status.h"
+
+using std::string;
+using strings::Substitute;
+
+namespace kudu {
+namespace security {
+
+const size_t kNonceSize = 16;
+
+namespace {
+
+// Writing the private key from an EVP_PKEY has a different
+// signature than the rest of the write functions, so we
+// have to provide this wrapper.
+int PemWritePrivateKey(BIO* bio, EVP_PKEY* key) {
+ auto rsa = ssl_make_unique(EVP_PKEY_get1_RSA(key));
+ return PEM_write_bio_RSAPrivateKey(
+ bio, rsa.get(), nullptr, nullptr, 0, nullptr, nullptr);
+}
+
+int PemWritePublicKey(BIO* bio, EVP_PKEY* key) {
+ auto rsa = ssl_make_unique(EVP_PKEY_get1_RSA(key));
+ return PEM_write_bio_RSA_PUBKEY(bio, rsa.get());
+}
+
+int DerWritePublicKey(BIO* bio, EVP_PKEY* key) {
+ auto rsa = ssl_make_unique(EVP_PKEY_get1_RSA(key));
+ return i2d_RSA_PUBKEY_bio(bio, rsa.get());
+}
+
+} // anonymous namespace
+
+template<> struct SslTypeTraits<BIGNUM> {
+ static constexpr auto kFreeFunc = &BN_free;
+};
+struct RsaPrivateKeyTraits : public SslTypeTraits<EVP_PKEY> {
+ static constexpr auto kReadPemFunc = &PEM_read_bio_PrivateKey;
+ static constexpr auto kReadDerFunc = &d2i_PrivateKey_bio;
+ static constexpr auto kWritePemFunc = &PemWritePrivateKey;
+ static constexpr auto kWriteDerFunc = &i2d_PrivateKey_bio;
+};
+struct RsaPublicKeyTraits : public SslTypeTraits<EVP_PKEY> {
+ static constexpr auto kReadPemFunc = &PEM_read_bio_PUBKEY;
+ static constexpr auto kReadDerFunc = &d2i_PUBKEY_bio;
+ static constexpr auto kWritePemFunc = &PemWritePublicKey;
+ static constexpr auto kWriteDerFunc = &DerWritePublicKey;
+};
+template<> struct SslTypeTraits<RSA> {
+ static constexpr auto kFreeFunc = &RSA_free;
+};
+template<> struct SslTypeTraits<EVP_MD_CTX> {
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+ static constexpr auto kFreeFunc = &EVP_MD_CTX_destroy;
+#else
+ static constexpr auto kFreeFunc = &EVP_MD_CTX_free;
+#endif
+};
+
+namespace {
+
+const EVP_MD* GetMessageDigest(DigestType digest_type) {
+ switch (digest_type) {
+ case DigestType::SHA256: return EVP_sha256();
+ case DigestType::SHA512: return EVP_sha512();
+ }
+ LOG(FATAL) << "unknown digest type";
+}
+
+} // anonymous namespace
+
+
+Status PublicKey::FromString(const std::string& data, DataFormat format) {
+ return ::kudu::security::FromString<RawDataType, RsaPublicKeyTraits>(
+ data, format, &data_);
+}
+
+Status PublicKey::ToString(std::string* data, DataFormat format) const {
+ return ::kudu::security::ToString<RawDataType, RsaPublicKeyTraits>(
+ data, format, data_.get());
+}
+
+Status PublicKey::FromFile(const std::string& fpath, DataFormat format) {
+ return ::kudu::security::FromFile<RawDataType, RsaPublicKeyTraits>(
+ fpath, format, &data_);
+}
+
+Status PublicKey::FromBIO(BIO* bio, DataFormat format) {
+ return ::kudu::security::FromBIO<RawDataType, RsaPublicKeyTraits>(
+ bio, format, &data_);
+}
+
+// Modeled after code in $OPENSSL_ROOT/apps/dgst.c
+Status PublicKey::VerifySignature(DigestType digest,
+ const std::string& data,
+ const std::string& signature) const {
+ SCOPED_OPENSSL_NO_PENDING_ERRORS;
+ const EVP_MD* md = GetMessageDigest(digest);
+ auto md_ctx = ssl_make_unique(EVP_MD_CTX_create());
+
+ OPENSSL_RET_NOT_OK(EVP_DigestVerifyInit(md_ctx.get(), nullptr, md, nullptr, GetRawData()),
+ "error initializing verification digest");
+ OPENSSL_RET_NOT_OK(EVP_DigestVerifyUpdate(md_ctx.get(), data.data(), data.size()),
+ "error verifying data signature");
+#if OPENSSL_VERSION_NUMBER < 0x10002000L
+ unsigned char* sig_data = reinterpret_cast<unsigned char*>(
+ const_cast<char*>(signature.data()));
+#else
+ const unsigned char* sig_data = reinterpret_cast<const unsigned char*>(
+ signature.data());
+#endif
+ // The success is indicated by return code 1. All other values means
+ // either wrong signature or error while performing signature verification.
+ const int rc = EVP_DigestVerifyFinal(md_ctx.get(), sig_data, signature.size());
+ if (rc < 0 || rc > 1) {
+ return Status::RuntimeError(
+ Substitute("error verifying data signature: $0", GetOpenSSLErrors()));
+ }
+ if (rc == 0) {
+ // No sense stringifying the internal OpenSSL error, since a bad verification
+ // is self-explanatory.
+ ERR_clear_error();
+ return Status::Corruption("data signature verification failed");
+ }
+
+ return Status::OK();
+}
+
+Status PublicKey::Equals(const PublicKey& other, bool* equals) const {
+ SCOPED_OPENSSL_NO_PENDING_ERRORS;
+ int cmp = EVP_PKEY_cmp(data_.get(), other.data_.get());
+ switch (cmp) {
+ case -2:
+ return Status::NotSupported("failed to compare public keys");
+ case -1: // Key types are different; treat this as not equal
+ case 0: // Keys are not equal
+ *equals = false;
+ return Status::OK();
+ case 1:
+ *equals = true;
+ return Status::OK();
+ default:
+ return Status::RuntimeError("unexpected public key comparison result", std::to_string(cmp));
+ }
+}
+
+Status PrivateKey::FromString(const std::string& data, DataFormat format) {
+ return ::kudu::security::FromString<RawDataType, RsaPrivateKeyTraits>(
+ data, format, &data_);
+}
+
+Status PrivateKey::ToString(std::string* data, DataFormat format) const {
+ return ::kudu::security::ToString<RawDataType, RsaPrivateKeyTraits>(
+ data, format, data_.get());
+}
+
+Status PrivateKey::FromFile(const std::string& fpath, DataFormat format,
+ const PasswordCallback& password_cb) {
+ return ::kudu::security::FromFile<RawDataType, RsaPrivateKeyTraits>(
+ fpath, format, &data_, password_cb);
+}
+
+// The code is modeled after $OPENSSL_ROOT/apps/rsa.c code: there is
+// corresponding functionality to read public part from RSA private/public
+// keypair.
+Status PrivateKey::GetPublicKey(PublicKey* public_key) const {
+ SCOPED_OPENSSL_NO_PENDING_ERRORS;
+ CHECK(public_key);
+ auto rsa = ssl_make_unique(EVP_PKEY_get1_RSA(CHECK_NOTNULL(data_.get())));
+ if (PREDICT_FALSE(!rsa)) {
+ return Status::RuntimeError(GetOpenSSLErrors());
+ }
+ auto tmp = ssl_make_unique(BIO_new(BIO_s_mem()));
+ CHECK(tmp);
+ // Export public key in DER format into the temporary buffer.
+ OPENSSL_RET_NOT_OK(i2d_RSA_PUBKEY_bio(tmp.get(), rsa.get()),
+ "error extracting public RSA key");
+ // Read the public key into the result placeholder.
+ RETURN_NOT_OK(public_key->FromBIO(tmp.get(), DataFormat::DER));
+
+ return Status::OK();
+}
+
+// Modeled after code in $OPENSSL_ROOT/apps/dgst.c
+Status PrivateKey::MakeSignature(DigestType digest,
+ const std::string& data,
+ std::string* signature) const {
+ SCOPED_OPENSSL_NO_PENDING_ERRORS;
+ CHECK(signature);
+ const EVP_MD* md = GetMessageDigest(digest);
+ auto md_ctx = ssl_make_unique(EVP_MD_CTX_create());
+
+ OPENSSL_RET_NOT_OK(EVP_DigestSignInit(md_ctx.get(), nullptr, md, nullptr, GetRawData()),
+ "error initializing signing digest");
+ OPENSSL_RET_NOT_OK(EVP_DigestSignUpdate(md_ctx.get(), data.data(), data.size()),
+ "error signing data");
+ size_t sig_len = EVP_PKEY_size(GetRawData());
+ static const size_t kSigBufSize = 4 * 1024;
+ CHECK(sig_len <= kSigBufSize);
+ unsigned char buf[kSigBufSize];
+ OPENSSL_RET_NOT_OK(EVP_DigestSignFinal(md_ctx.get(), buf, &sig_len),
+ "error finalizing data signature");
+ *signature = string(reinterpret_cast<char*>(buf), sig_len);
+
+ return Status::OK();
+}
+
+Status GeneratePrivateKey(int num_bits, PrivateKey* ret) {
+ SCOPED_OPENSSL_NO_PENDING_ERRORS;
+ CHECK(ret);
+ InitializeOpenSSL();
+ auto key = ssl_make_unique(EVP_PKEY_new());
+ {
+ auto bn = ssl_make_unique(BN_new());
+ OPENSSL_CHECK_OK(BN_set_word(bn.get(), RSA_F4));
+ auto rsa = ssl_make_unique(RSA_new());
+ OPENSSL_RET_NOT_OK(
+ RSA_generate_key_ex(rsa.get(), num_bits, bn.get(), nullptr),
+ "error generating RSA key");
+ OPENSSL_RET_NOT_OK(
+ EVP_PKEY_set1_RSA(key.get(), rsa.get()), "error assigning RSA key");
+ }
+ ret->AdoptRawData(key.release());
+
+ return Status::OK();
+}
+
+Status GenerateNonce(string* s) {
+ SCOPED_OPENSSL_NO_PENDING_ERRORS;
+ CHECK_NOTNULL(s);
+ unsigned char buf[kNonceSize];
+ OPENSSL_RET_NOT_OK(RAND_bytes(buf, sizeof(buf)), "failed to generate nonce");
+ s->assign(reinterpret_cast<char*>(buf), kNonceSize);
+ return Status::OK();
+}
+
+} // namespace security
+} // namespace kudu
http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/crypto.h
----------------------------------------------------------------------
diff --git a/be/src/kudu/security/crypto.h b/be/src/kudu/security/crypto.h
new file mode 100644
index 0000000..145c405
--- /dev/null
+++ b/be/src/kudu/security/crypto.h
@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <cstddef>
+#include <string>
+
+#include <openssl/bio.h>
+#include <openssl/rsa.h>
+
+#include "kudu/gutil/port.h"
+#include "kudu/security/openssl_util.h"
+
+// Forward declarations for the OpenSSL typedefs.
+typedef struct rsa_st RSA;
+typedef struct bio_st BIO;
+
+namespace kudu {
+
+class Status;
+
+namespace security {
+
+extern const size_t kNonceSize;
+
+// Supported message digests for data signing and signature verification.
+enum class DigestType {
+ SHA256,
+ SHA512,
+};
+
+// A class with generic public key interface, but actually it represents
+// an RSA key.
+class PublicKey : public RawDataWrapper<EVP_PKEY> {
+ public:
+ ~PublicKey() {}
+
+ Status FromString(const std::string& data, DataFormat format) WARN_UNUSED_RESULT;
+ Status ToString(std::string* data, DataFormat format) const WARN_UNUSED_RESULT;
+ Status FromFile(const std::string& fpath, DataFormat format) WARN_UNUSED_RESULT;
+
+ Status FromBIO(BIO* bio, DataFormat format) WARN_UNUSED_RESULT;
+
+ // Using the key, verify data signature using the specified message
+ // digest algorithm for signature verification.
+ // The input signature should be in in raw format (i.e. no base64 encoding).
+ Status VerifySignature(DigestType digest,
+ const std::string& data,
+ const std::string& signature) const WARN_UNUSED_RESULT;
+
+ // Sets 'equals' to true if the other public key equals this.
+ Status Equals(const PublicKey& other, bool* equals) const WARN_UNUSED_RESULT;
+};
+
+// A class with generic private key interface, but actually it represents
+// an RSA private key. It's important to have PrivateKey and PublicKey
+// be different types to avoid accidental leakage of private keys.
+class PrivateKey : public RawDataWrapper<EVP_PKEY> {
+ public:
+ ~PrivateKey() {}
+
+ Status FromString(const std::string& data, DataFormat format) WARN_UNUSED_RESULT;
+ Status ToString(std::string* data, DataFormat format) const WARN_UNUSED_RESULT;
+
+ // If 'cb' is set, it will be called to obtain the password necessary to decrypt
+ // the private key file in 'fpath'.
+ Status FromFile(const std::string& fpath, DataFormat format,
+ const PasswordCallback& password_cb = PasswordCallback()) WARN_UNUSED_RESULT;
+
+ // Output the public part of the keypair into the specified placeholder.
+ Status GetPublicKey(PublicKey* public_key) const WARN_UNUSED_RESULT;
+
+ // Using the key, generate data signature using the specified
+ // message digest algorithm. The result signature is in raw format
+ // (i.e. no base64 encoding).
+ Status MakeSignature(DigestType digest,
+ const std::string& data,
+ std::string* signature) const WARN_UNUSED_RESULT;
+};
+
+// Utility method to generate private keys.
+Status GeneratePrivateKey(int num_bits, PrivateKey* ret) WARN_UNUSED_RESULT;
+
+// Generates a nonce of size kNonceSize, and writes it to the provided string.
+Status GenerateNonce(std::string* s) WARN_UNUSED_RESULT;
+
+} // namespace security
+} // namespace kudu
http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/init.cc
----------------------------------------------------------------------
diff --git a/be/src/kudu/security/init.cc b/be/src/kudu/security/init.cc
new file mode 100644
index 0000000..ee754eb
--- /dev/null
+++ b/be/src/kudu/security/init.cc
@@ -0,0 +1,465 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "kudu/security/init.h"
+
+#include <algorithm>
+#include <cctype>
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <ctime>
+#include <functional>
+#include <memory>
+#include <mutex>
+#include <ostream>
+#include <random>
+#include <string>
+#include <type_traits>
+
+#include <boost/optional/optional.hpp>
+#include <gflags/gflags.h>
+#include <glog/logging.h>
+#include <krb5/krb5.h>
+
+#include "kudu/gutil/macros.h"
+#include "kudu/gutil/ref_counted.h"
+#include "kudu/gutil/strings/substitute.h"
+#include "kudu/gutil/strings/util.h"
+#include "kudu/util/flag_tags.h"
+#include "kudu/util/monotime.h"
+#include "kudu/util/net/net_util.h"
+#include "kudu/util/rw_mutex.h"
+#include "kudu/util/scoped_cleanup.h"
+#include "kudu/util/status.h"
+#include "kudu/util/thread.h"
+
+#ifndef __APPLE__
+static constexpr bool kDefaultSystemAuthToLocal = true;
+#else
+// macOS's Heimdal library has a no-op implementation of
+// krb5_aname_to_localname, so instead we just use the simple
+// implementation.
+static constexpr bool kDefaultSystemAuthToLocal = false;
+#endif
+DEFINE_bool(use_system_auth_to_local, kDefaultSystemAuthToLocal,
+ "When enabled, use the system krb5 library to map Kerberos principal "
+ "names to local (short) usernames. If not enabled, the first component "
+ "of the principal will be used as the short name. For example, "
+ "'kudu/foo.example.com@EXAMPLE' will map to 'kudu'.");
+TAG_FLAG(use_system_auth_to_local, advanced);
+
+
+using std::mt19937;
+using std::random_device;
+using std::string;
+using std::uniform_int_distribution;
+using std::uniform_real_distribution;
+using strings::Substitute;
+
+namespace kudu {
+namespace security {
+
+namespace {
+
+class KinitContext;
+
+// Global context for usage of the Krb5 library.
+krb5_context g_krb5_ctx;
+
+// Global instance of the context used by the kinit/reacquire thread.
+KinitContext* g_kinit_ctx;
+
+// This lock is used to avoid a race while reacquiring the kerberos ticket.
+// The race can occur between the time we reinitialize the cache and the
+// time when we actually store the new credentials back in the cache.
+RWMutex* g_kerberos_reinit_lock;
+
+class KinitContext {
+ public:
+ KinitContext();
+
+ // Equivalent implementation of 'kinit -kt <keytab path> <principal>'.
+ //
+ // This logs in from the given keytab as the given principal, returning
+ // RuntimeError if any part of this process fails.
+ //
+ // If the log-in is successful, then the default ticket cache is overwritten
+ // with the credentials of the newly logged-in principal.
+ Status Kinit(const string& keytab_path, const string& principal);
+
+ // Acquires a new Ticket Granting Ticket (TGT).
+ //
+ // Renews the existing ticket if possible, or acquires a new Ticket Granting
+ // Ticket (TGT).
+ Status DoRenewal();
+
+ // Calculates the next sleep interval based on the 'ticket_end_timestamp_' and
+ // adds some jitter so that all the nodes do not hit the KDC at the same time.
+ //
+ // If 'num_retries' > 0, it calls GetBackedOffRenewInterval() to return a backed
+ // off interval.
+ int32_t GetNextRenewInterval(uint32_t num_retries);
+
+ // Returns a value based on 'time_remaining' that increases exponentially with
+ // 'num_retries', with a random jitter of +/- 0%-50% of that value.
+ int32_t GetBackedOffRenewInterval(int32_t time_remaining, uint32_t num_retries);
+
+ const string& principal_str() const { return principal_str_; }
+ const string& username_str() const { return username_str_; }
+
+ private:
+ krb5_principal principal_;
+ krb5_keytab keytab_;
+ krb5_ccache ccache_;
+ krb5_get_init_creds_opt* opts_;
+
+ // The stringified principal and username that we are logged in as.
+ string principal_str_, username_str_;
+
+ // This is the time that the current TGT in use expires.
+ int32_t ticket_end_timestamp_;
+};
+
+Status Krb5CallToStatus(krb5_context ctx, krb5_error_code code) {
+ if (code == 0) return Status::OK();
+
+ std::unique_ptr<const char, std::function<void(const char*)>> err_msg(
+ krb5_get_error_message(ctx, code),
+ std::bind(krb5_free_error_message, ctx, std::placeholders::_1));
+ return Status::RuntimeError(err_msg.get());
+}
+#define KRB5_RETURN_NOT_OK_PREPEND(call, prepend) \
+ RETURN_NOT_OK_PREPEND(Krb5CallToStatus(g_krb5_ctx, (call)), (prepend))
+
+
+void InitKrb5Ctx() {
+ static std::once_flag once;
+ std::call_once(once, [&]() {
+ CHECK_EQ(krb5_init_context(&g_krb5_ctx), 0);
+ });
+}
+
+KinitContext::KinitContext() {}
+
+// Port of the data_eq() implementation from krb5/k5-int.h
+inline int data_eq(krb5_data d1, krb5_data d2) {
+ return (d1.length == d2.length && !memcmp(d1.data, d2.data, d1.length));
+}
+
+// Port of the data_eq_string() implementation from krb5/k5-int.h
+inline int data_eq_string(krb5_data d, const char *s) {
+ return (d.length == strlen(s) && !memcmp(d.data, s, d.length));
+}
+
+Status Krb5UnparseName(krb5_principal princ, string* name) {
+ char* c_name;
+ KRB5_RETURN_NOT_OK_PREPEND(krb5_unparse_name(g_krb5_ctx, princ, &c_name),
+ "krb5_unparse_name");
+ SCOPED_CLEANUP({
+ krb5_free_unparsed_name(g_krb5_ctx, c_name);
+ });
+ *name = c_name;
+ return Status::OK();
+}
+
+// Periodically calls DoRenewal().
+void RenewThread() {
+ uint32_t failure_retries = 0;
+ while (true) {
+ // This thread is run immediately after the first Kinit, so sleep first.
+ SleepFor(MonoDelta::FromSeconds(g_kinit_ctx->GetNextRenewInterval(failure_retries)));
+
+ Status s = g_kinit_ctx->DoRenewal();
+ WARN_NOT_OK(s, "Kerberos reacquire error: ");
+ if (!s.ok()) {
+ ++failure_retries;
+ } else {
+ failure_retries = 0;
+ }
+ }
+}
+
+int32_t KinitContext::GetNextRenewInterval(uint32_t num_retries) {
+ int32_t time_remaining = ticket_end_timestamp_ - time(nullptr);
+
+ // If the last ticket reacqusition was a failure, we back off our retry attempts exponentially.
+ if (num_retries > 0) return GetBackedOffRenewInterval(time_remaining, num_retries);
+
+ // If the time remaining between now and ticket expiry is:
+ // * > 10 minutes: We attempt to reacquire the ticket between 5 seconds and 5 minutes before the
+ // ticket expires.
+ // * 5 - 10 minutes: We attempt to reacquire the ticket betwen 5 seconds and 1 minute before the
+ // ticket expires.
+ // * < 5 minutes: Attempt to reacquire the ticket every 'time_remaining'.
+ // The jitter is added to make sure that every server doesn't flood the KDC at the same time.
+ random_device rd;
+ mt19937 generator(rd());
+ if (time_remaining > 600) {
+ uniform_int_distribution<> dist(5, 300);
+ return time_remaining - dist(generator);
+ } else if (time_remaining > 300) {
+ uniform_int_distribution<> dist(5, 60);
+ return time_remaining - dist(generator);
+ }
+ return time_remaining;
+}
+
+int32_t KinitContext::GetBackedOffRenewInterval(int32_t time_remaining, uint32_t num_retries) {
+ // The minimum sleep interval after a failure will be 60 seconds.
+ int32_t next_interval = std::max(time_remaining, 60);
+ int32_t base_time = std::min(next_interval * (1 << num_retries), INT32_MAX);
+ random_device rd;
+ mt19937 generator(rd());
+ uniform_real_distribution<> dist(0.5, 1.5);
+ return static_cast<int32_t>(base_time * dist(generator));
+}
+
+Status KinitContext::DoRenewal() {
+
+ krb5_cc_cursor cursor;
+ // Setup a cursor to iterate through the credential cache.
+ KRB5_RETURN_NOT_OK_PREPEND(krb5_cc_start_seq_get(g_krb5_ctx, ccache_, &cursor),
+ "Failed to peek into ccache");
+ SCOPED_CLEANUP({
+ krb5_cc_end_seq_get(g_krb5_ctx, ccache_, &cursor); });
+
+ krb5_creds creds;
+ memset(&creds, 0, sizeof(krb5_creds));
+
+ krb5_error_code rc;
+ // Iterate through the credential cache.
+ while (!(rc = krb5_cc_next_cred(g_krb5_ctx, ccache_, &cursor, &creds))) {
+ SCOPED_CLEANUP({
+ krb5_free_cred_contents(g_krb5_ctx, &creds); });
+ if (krb5_is_config_principal(g_krb5_ctx, creds.server)) continue;
+
+ // We only want to reacquire the TGT (Ticket Granting Ticket). Ignore all other tickets.
+ // This follows the same format as is_local_tgt() from krb5:src/clients/klist/klist.c
+ if (creds.server->length != 2 ||
+ data_eq(creds.server->data[1], principal_->realm) == 0 ||
+ data_eq_string(creds.server->data[0], KRB5_TGS_NAME) == 0 ||
+ data_eq(creds.server->realm, principal_->realm) == 0) {
+ continue;
+ }
+
+ krb5_creds new_creds;
+ memset(&new_creds, 0, sizeof(krb5_creds));
+ SCOPED_CLEANUP({
+ krb5_free_cred_contents(g_krb5_ctx, &new_creds); });
+ // Acquire a new ticket using the keytab. This ticket will automatically be put into the
+ // credential cache.
+ {
+ std::lock_guard<RWMutex> l(*g_kerberos_reinit_lock);
+ KRB5_RETURN_NOT_OK_PREPEND(krb5_get_init_creds_keytab(g_krb5_ctx, &new_creds, principal_,
+ keytab_, 0 /* valid from now */,
+ nullptr /* TKT service name */,
+ opts_),
+ "Reacquire error: unable to login from keytab");
+#if !defined(HAVE_KRB5_GET_INIT_CREDS_OPT_SET_OUT_CCACHE)
+ // Heimdal krb5 doesn't have the 'krb5_get_init_creds_opt_set_out_ccache' option,
+ // so use this alternate route.
+ KRB5_RETURN_NOT_OK_PREPEND(krb5_cc_initialize(g_krb5_ctx, ccache_, principal_),
+ "Reacquire error: could not init ccache");
+
+ KRB5_RETURN_NOT_OK_PREPEND(krb5_cc_store_cred(g_krb5_ctx, ccache_, &new_creds),
+ "Reacquire error: could not store creds in cache");
+#endif
+ }
+ LOG(INFO) << "Successfully reacquired a new kerberos TGT";
+ ticket_end_timestamp_ = new_creds.times.endtime;
+ break;
+ }
+ return Status::OK();
+}
+
+Status KinitContext::Kinit(const string& keytab_path, const string& principal) {
+ InitKrb5Ctx();
+
+ // Parse the principal
+ KRB5_RETURN_NOT_OK_PREPEND(krb5_parse_name(g_krb5_ctx, principal.c_str(), &principal_),
+ "could not parse principal");
+
+ KRB5_RETURN_NOT_OK_PREPEND(krb5_kt_resolve(g_krb5_ctx, keytab_path.c_str(), &keytab_),
+ "unable to resolve keytab");
+
+ KRB5_RETURN_NOT_OK_PREPEND(krb5_cc_default(g_krb5_ctx, &ccache_),
+ "unable to get default credentials cache");
+
+ KRB5_RETURN_NOT_OK_PREPEND(krb5_get_init_creds_opt_alloc(g_krb5_ctx, &opts_),
+ "unable to allocate get_init_creds_opt struct");
+
+#if defined(HAVE_KRB5_GET_INIT_CREDS_OPT_SET_OUT_CCACHE)
+ KRB5_RETURN_NOT_OK_PREPEND(krb5_get_init_creds_opt_set_out_ccache(g_krb5_ctx, opts_, ccache_),
+ "unable to set init_creds options");
+#endif
+
+ krb5_creds creds;
+ KRB5_RETURN_NOT_OK_PREPEND(krb5_get_init_creds_keytab(g_krb5_ctx, &creds, principal_, keytab_,
+ 0 /* valid from now */,
+ nullptr /* TKT service name */, opts_),
+ "unable to login from keytab");
+ SCOPED_CLEANUP({
+ krb5_free_cred_contents(g_krb5_ctx, &creds); });
+
+ ticket_end_timestamp_ = creds.times.endtime;
+
+#if !defined(HAVE_KRB5_GET_INIT_CREDS_OPT_SET_OUT_CCACHE)
+ // Heimdal krb5 doesn't have the 'krb5_get_init_creds_opt_set_out_ccache' option,
+ // so use this alternate route.
+ KRB5_RETURN_NOT_OK_PREPEND(krb5_cc_initialize(g_krb5_ctx, ccache_, principal_),
+ "could not init ccache");
+
+ KRB5_RETURN_NOT_OK_PREPEND(krb5_cc_store_cred(g_krb5_ctx, ccache_, &creds),
+ "could not store creds in cache");
+#endif
+
+ // Convert the logged-in principal back to a string. This may be different than
+ // 'principal', since the default realm will be filled in based on the Kerberos
+ // configuration if not originally specified.
+ RETURN_NOT_OK_PREPEND(Krb5UnparseName(principal_, &principal_str_),
+ "could not stringify the logged-in principal");
+ RETURN_NOT_OK_PREPEND(MapPrincipalToLocalName(principal_str_, &username_str_),
+ "could not map own logged-in principal to a short username");
+
+ LOG(INFO) << "Logged in from keytab as " << principal_str_
+ << " (short username " << username_str_ << ")";
+
+ return Status::OK();
+}
+
+// 'in_principal' is the user specified principal to use with Kerberos. It may have a token
+// in the string of the form '_HOST', which if present, needs to be replaced with the FQDN of the
+// current host.
+// 'out_principal' has the final principal with which one may Kinit.
+Status GetConfiguredPrincipal(const std::string& in_principal, string* out_principal) {
+ *out_principal = in_principal;
+ const auto& kHostToken = "_HOST";
+ if (in_principal.find(kHostToken) != string::npos) {
+ string hostname;
+ // Try to fill in either the FQDN or hostname.
+ if (!GetFQDN(&hostname).ok()) {
+ RETURN_NOT_OK(GetHostname(&hostname));
+ }
+ // Hosts in principal names are canonicalized to lower-case.
+ std::transform(hostname.begin(), hostname.end(), hostname.begin(), tolower);
+ GlobalReplaceSubstring(kHostToken, hostname, out_principal);
+ }
+ return Status::OK();
+}
+} // anonymous namespace
+
+
+RWMutex* KerberosReinitLock() {
+ return g_kerberos_reinit_lock;
+}
+
+Status CanonicalizeKrb5Principal(std::string* principal) {
+ InitKrb5Ctx();
+ krb5_principal princ;
+ KRB5_RETURN_NOT_OK_PREPEND(krb5_parse_name(g_krb5_ctx, principal->c_str(), &princ),
+ "could not parse principal");
+ SCOPED_CLEANUP({
+ krb5_free_principal(g_krb5_ctx, princ);
+ });
+ RETURN_NOT_OK_PREPEND(Krb5UnparseName(princ, principal),
+ "failed to convert principal back to string");
+ return Status::OK();
+}
+
+Status MapPrincipalToLocalName(const std::string& principal, std::string* local_name) {
+ InitKrb5Ctx();
+ krb5_principal princ;
+ KRB5_RETURN_NOT_OK_PREPEND(krb5_parse_name(g_krb5_ctx, principal.c_str(), &princ),
+ "could not parse principal");
+ SCOPED_CLEANUP({
+ krb5_free_principal(g_krb5_ctx, princ);
+ });
+ char buf[1024];
+ krb5_error_code rc = KRB5_LNAME_NOTRANS;
+ if (FLAGS_use_system_auth_to_local) {
+ rc = krb5_aname_to_localname(g_krb5_ctx, princ, arraysize(buf), buf);
+ }
+ if (rc == KRB5_LNAME_NOTRANS || rc == KRB5_PLUGIN_NO_HANDLE) {
+ // No name mapping specified, or krb5-based name mapping is disabled.
+ //
+ // We fall back to simply taking the first component of the principal, for
+ // compatibility with the default behavior of Hadoop.
+ //
+ // NOTE: KRB5_PLUGIN_NO_HANDLE isn't typically expected here, but works around
+ // a bug in SSSD's auth_to_local implementation: https://pagure.io/SSSD/sssd/issue/3459
+ //
+ // TODO(todd): we should support custom configured auth-to-local mapping, since
+ // most Hadoop ecosystem components do not load them from krb5.conf.
+ if (princ->length > 0) {
+ local_name->assign(princ->data[0].data, princ->data[0].length);
+ return Status::OK();
+ }
+ return Status::NotFound("unable to find first component of principal");
+ }
+ if (rc == KRB5_CONFIG_NOTENUFSPACE) {
+ return Status::InvalidArgument("mapped username too large");
+ }
+ KRB5_RETURN_NOT_OK_PREPEND(rc, "krb5_aname_to_localname");
+ if (strlen(buf) == 0) {
+ return Status::InvalidArgument("principal mapped to empty username");
+ }
+ local_name->assign(buf);
+ return Status::OK();
+}
+
+boost::optional<string> GetLoggedInPrincipalFromKeytab() {
+ if (!g_kinit_ctx) return boost::none;
+ return g_kinit_ctx->principal_str();
+}
+
+boost::optional<string> GetLoggedInUsernameFromKeytab() {
+ if (!g_kinit_ctx) return boost::none;
+ return g_kinit_ctx->username_str();
+}
+
+Status InitKerberosForServer(const std::string& raw_principal, const std::string& keytab_file,
+ const std::string& krb5ccname, bool disable_krb5_replay_cache) {
+ if (keytab_file.empty()) return Status::OK();
+
+ setenv("KRB5CCNAME", krb5ccname.c_str(), 1);
+ setenv("KRB5_KTNAME", keytab_file.c_str(), 1);
+
+ if (disable_krb5_replay_cache) {
+ // KUDU-1897: disable the Kerberos replay cache. The KRPC protocol includes a
+ // per-connection server-generated nonce to protect against replay attacks
+ // when authenticating via Kerberos. The replay cache has many performance and
+ // implementation issues.
+ setenv("KRB5RCACHETYPE", "none", 1);
+ }
+
+ g_kinit_ctx = new KinitContext();
+ string configured_principal;
+ RETURN_NOT_OK(GetConfiguredPrincipal(raw_principal, &configured_principal));
+ RETURN_NOT_OK_PREPEND(g_kinit_ctx->Kinit(
+ keytab_file, configured_principal), "unable to kinit");
+
+ g_kerberos_reinit_lock = new RWMutex(RWMutex::Priority::PREFER_WRITING);
+ scoped_refptr<Thread> reacquire_thread;
+ // Start the reacquire thread.
+ RETURN_NOT_OK(Thread::Create("kerberos", "reacquire thread", &RenewThread, &reacquire_thread));
+
+ return Status::OK();
+}
+
+} // namespace security
+} // namespace kudu
http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/init.h
----------------------------------------------------------------------
diff --git a/be/src/kudu/security/init.h b/be/src/kudu/security/init.h
new file mode 100644
index 0000000..8b1519a
--- /dev/null
+++ b/be/src/kudu/security/init.h
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include <string>
+
+namespace boost {
+template <class T>
+class optional;
+}
+
+namespace kudu {
+
+class RWMutex;
+class Status;
+
+namespace security {
+
+// The default kerberos credential cache name.
+// Have the daemons use an in-memory ticket cache, so they don't accidentally
+// pick up credentials from test cases or any other daemon.
+static const std::string kKrb5CCName = "MEMORY:kudu";
+
+// Initializes Kerberos for a server. In particular, this processes
+// the '--keytab_file' command line flag.
+// 'raw_principal' is the principal to Kinit with after calling GetConfiguredPrincipal()
+// on it.
+// 'keytab_file' is the path to the kerberos keytab file. If it's an empty string, kerberos
+// will not be initialized.
+// 'krb5ccname' is passed into the KRB5CCNAME env var.
+// 'disable_krb5_replay_cache' if set to true, disables the kerberos replay cache by setting
+// the KRB5RCACHETYPE env var to "none".
+Status InitKerberosForServer(const std::string& raw_principal,
+ const std::string& keytab_file,
+ const std::string& krb5ccname = kKrb5CCName,
+ bool disable_krb5_replay_cache = true);
+
+// Returns the process lock 'kerberos_reinit_lock'
+// This lock is taken in write mode while the ticket is being reacquired, and
+// taken in read mode before using the SASL library which might require a ticket.
+RWMutex* KerberosReinitLock();
+
+// Return the full principal (user/host@REALM) that the server has used to
+// log in from the keytab.
+//
+// If the server has not logged in from a keytab, returns boost::none.
+boost::optional<std::string> GetLoggedInPrincipalFromKeytab();
+
+// Same, but returns the mapped short username.
+boost::optional<std::string> GetLoggedInUsernameFromKeytab();
+
+// Canonicalize the given principal name by adding '@DEFAULT_REALM' in the case that
+// the principal has no realm.
+//
+// TODO(todd): move to kerberos_util.h in the later patch in this series (the file doesn't
+// exist yet, and trying to avoid rebase pain).
+Status CanonicalizeKrb5Principal(std::string* principal);
+
+// Map the given Kerberos principal 'principal' to a short username (i.e. with no realm or
+// host component).
+//
+// This respects the "auth-to-local" mappings from the system krb5.conf. However, if no such
+// mapping can be found, we fall back to simply taking the first component of the principal.
+//
+// TODO(todd): move to kerberos_util.h in the later patch in this series (the file doesn't
+// exist yet, and trying to avoid rebase pain).
+Status MapPrincipalToLocalName(const std::string& principal, std::string* local_name);
+
+} // namespace security
+} // namespace kudu
http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/kerberos_util.cc
----------------------------------------------------------------------
diff --git a/be/src/kudu/security/kerberos_util.cc b/be/src/kudu/security/kerberos_util.cc
new file mode 100644
index 0000000..5e6d9b8
--- /dev/null
+++ b/be/src/kudu/security/kerberos_util.cc
@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "kudu/security/kerberos_util.h"
+#include "kudu/gutil/strings/split.h"
+#include "kudu/gutil/strings/stringpiece.h"
+
+#include <array>
+#include <utility>
+
+namespace kudu {
+namespace security {
+
+std::array<StringPiece, 3> SplitKerberosPrincipal(StringPiece principal) {
+
+ std::pair<StringPiece, StringPiece> user_realm = strings::Split(principal, "@");
+ std::pair<StringPiece, StringPiece> princ_host = strings::Split(user_realm.first, "/");
+ return {{princ_host.first, princ_host.second, user_realm.second}};
+}
+
+
+} // namespace security
+} // namespace kudu
http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/kerberos_util.h
----------------------------------------------------------------------
diff --git a/be/src/kudu/security/kerberos_util.h b/be/src/kudu/security/kerberos_util.h
new file mode 100644
index 0000000..4c27a86
--- /dev/null
+++ b/be/src/kudu/security/kerberos_util.h
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+#pragma once
+
+#include <array>
+
+class StringPiece;
+
+namespace kudu {
+namespace security {
+
+std::array<StringPiece, 3> SplitKerberosPrincipal(StringPiece principal);
+
+} // namespace security
+} // namespace kudu
http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/krb5_realm_override.cc
----------------------------------------------------------------------
diff --git a/be/src/kudu/security/krb5_realm_override.cc b/be/src/kudu/security/krb5_realm_override.cc
new file mode 100644
index 0000000..05e8af8
--- /dev/null
+++ b/be/src/kudu/security/krb5_realm_override.cc
@@ -0,0 +1,105 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// This file provides a workaround for tests running with Kerberos 1.11 or earlier.
+// These versions of Kerberos are missing a fix which allows service principals
+// to use IP addresses in their host component:
+//
+// http://krbdev.mit.edu/rt/Ticket/Display.html?id=7603
+//
+// We use such principals in external minicluster tests, where servers have IP addresses
+// like 127.x.y.z that have no corresponding reverse DNS.
+//
+// The file contains an implementation of krb5_get_host_realm which wraps the one
+// in the Kerberos library. It detects the return code that indicates the
+// above problem and falls back to the default realm/
+//
+// The wrapper is injected via linking it into tests as well as the
+// "security" library. The linkage invocation uses the '-Wl,--undefined'
+// linker flag to force linking even though no symbol here is explicitly
+// referenced.
+
+#include <dlfcn.h>
+
+#include <cerrno>
+#include <cstdlib>
+#include <cstring>
+
+#include <krb5/krb5.h>
+#include <glog/logging.h>
+
+extern "C" {
+
+// This symbol is exported from the static library so that other static-linked binaries
+// can reference it and force this compilation unit to be linked. Otherwise the linker
+// thinks it's unused and doesn't link it.
+int krb5_realm_override_loaded = 1;
+
+// Save the original function from the Kerberos library itself.
+// We use dlsym() to load all of them, since this file gets linked into
+// some test binaries that themselves may not link against libkrb5.so at all.
+static void* g_orig_krb5_get_host_realm;
+static void* g_orig_krb5_get_default_realm;
+static void* g_orig_krb5_free_default_realm;
+
+// We only enable our workaround if this environment variable is set.
+constexpr static const char* kEnvVar = "KUDU_ENABLE_KRB5_REALM_FIX";
+
+#define CALL_ORIG(func_name, ...) \
+ ((decltype(&func_name))g_orig_ ## func_name)(__VA_ARGS__)
+
+__attribute__((constructor))
+static void init_orig_func() {
+ g_orig_krb5_get_host_realm = dlsym(RTLD_NEXT, "krb5_get_host_realm");
+ g_orig_krb5_get_default_realm = dlsym(RTLD_NEXT, "krb5_get_default_realm");
+ g_orig_krb5_free_default_realm = dlsym(RTLD_NEXT, "krb5_free_default_realm");
+}
+
+krb5_error_code krb5_get_host_realm(krb5_context context, const char* host, char*** realmsp) {
+ CHECK(g_orig_krb5_get_host_realm);
+ CHECK(g_orig_krb5_get_default_realm);
+ CHECK(g_orig_krb5_free_default_realm);
+
+ krb5_error_code rc = CALL_ORIG(krb5_get_host_realm, context, host, realmsp);
+ if (rc != KRB5_ERR_NUMERIC_REALM || getenv(kEnvVar) == nullptr) {
+ return rc;
+ }
+ // If we get KRB5_ERR_NUMERIC_REALM, this is indicative of a Kerberos version
+ // which has not provided support for numeric addresses as service host names
+ // So, we fill in the default realm instead.
+ char* default_realm;
+ rc = CALL_ORIG(krb5_get_default_realm, context, &default_realm);
+ if (rc != 0) {
+ return rc;
+ }
+
+ char** ret_realms;
+ ret_realms = static_cast<char**>(malloc(2 * sizeof(*ret_realms)));
+ if (ret_realms == nullptr) return ENOMEM;
+ ret_realms[0] = strdup(default_realm);
+ if (ret_realms[0] == nullptr) {
+ free(ret_realms);
+ return ENOMEM;
+ }
+ ret_realms[1] = 0;
+ *realmsp = ret_realms;
+
+ CALL_ORIG(krb5_free_default_realm, context, default_realm);
+ return 0;
+}
+
+} // extern "C"
http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/openssl_util.cc
----------------------------------------------------------------------
diff --git a/be/src/kudu/security/openssl_util.cc b/be/src/kudu/security/openssl_util.cc
new file mode 100644
index 0000000..a32140f
--- /dev/null
+++ b/be/src/kudu/security/openssl_util.cc
@@ -0,0 +1,322 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "kudu/security/openssl_util.h"
+
+#include <cerrno>
+#include <cstdint>
+#include <cstdio>
+#include <mutex>
+#include <string>
+#include <vector>
+
+#include <glog/logging.h>
+#include <openssl/crypto.h>
+#include <openssl/err.h>
+#include <openssl/rand.h>
+
+#include "kudu/gutil/strings/split.h"
+#include "kudu/gutil/strings/strip.h"
+#include "kudu/gutil/strings/substitute.h"
+#include "kudu/util/debug/leakcheck_disabler.h"
+#include "kudu/util/errno.h"
+#include "kudu/util/mutex.h"
+#include "kudu/util/scoped_cleanup.h"
+#include "kudu/util/status.h"
+#include "kudu/util/subprocess.h"
+
+using std::ostringstream;
+using std::string;
+using std::vector;
+
+namespace kudu {
+namespace security {
+
+namespace {
+
+// Determine whether initialization was ever called.
+//
+// Thread safety:
+// - written by DoInitializeOpenSSL (single-threaded, due to std::call_once)
+// - read by DisableOpenSSLInitialization (must not be concurrent with above)
+bool g_ssl_is_initialized = false;
+
+// If true, then we expect someone else has initialized SSL.
+//
+// Thread safety:
+// - read by DoInitializeOpenSSL (single-threaded, due to std::call_once)
+// - written by DisableOpenSSLInitialization (must not be concurrent with above)
+bool g_disable_ssl_init = false;
+
+// Array of locks used by OpenSSL.
+// We use an intentionally-leaked C-style array here to avoid non-POD static data.
+//
+// As of OpenSSL 1.1, locking callbacks are no longer used.
+#if OPENSSL_VERSION_NUMBER < 0x10100000L
+Mutex* kCryptoLocks = nullptr;
+
+// Lock/Unlock the nth lock. Only to be used by OpenSSL.
+void LockingCB(int mode, int type, const char* /*file*/, int /*line*/) {
+ DCHECK(kCryptoLocks);
+ Mutex* m = &kCryptoLocks[type];
+ if (mode & CRYPTO_LOCK) {
+ m->lock();
+ } else {
+ m->unlock();
+ }
+}
+#endif
+
+Status CheckOpenSSLInitialized() {
+ if (!CRYPTO_get_locking_callback()) {
+ return Status::RuntimeError("Locking callback not initialized");
+ }
+ auto ctx = ssl_make_unique(SSL_CTX_new(SSLv23_method()));
+ if (!ctx) {
+ ERR_clear_error();
+ return Status::RuntimeError("SSL library appears uninitialized (cannot create SSL_CTX)");
+ }
+ return Status::OK();
+}
+
+void DoInitializeOpenSSL() {
+#if OPENSSL_VERSION_NUMBER > 0x10100000L
+ // The OPENSSL_init_ssl manpage [1] says "As of version 1.1.0 OpenSSL will
+ // automatically allocate all resources it needs so no explicit initialisation
+ // is required." However, eliding library initialization leads to a memory
+ // leak in some versions of OpenSSL 1.1 when the first OpenSSL is
+ // ERR_peek_error [2]. In Kudu this is often the
+ // case due to prolific application of SCOPED_OPENSSL_NO_PENDING_ERRORS.
+ //
+ // Rather than determine whether this particular OpenSSL instance is
+ // leak-free, we'll initialize the library explicitly.
+ //
+ // 1. https://www.openssl.org/docs/man1.1.0/ssl/OPENSSL_init_ssl.html
+ // 2. https://github.com/openssl/openssl/issues/5899
+ if (g_disable_ssl_init) {
+ VLOG(2) << "Not initializing OpenSSL (disabled by application)";
+ return;
+ }
+ CHECK_EQ(1, OPENSSL_init_ssl(0, nullptr));
+ SCOPED_OPENSSL_NO_PENDING_ERRORS;
+#else
+ // In case the user's thread has left some error around, clear it.
+ ERR_clear_error();
+ SCOPED_OPENSSL_NO_PENDING_ERRORS;
+ if (g_disable_ssl_init) {
+ VLOG(2) << "Not initializing OpenSSL (disabled by application)";
+ return;
+ }
+
+ // Check that OpenSSL isn't already initialized. If it is, it's likely
+ // we are embedded in (or embedding) another application/library which
+ // initializes OpenSSL, and we risk installing conflicting callbacks
+ // or crashing due to concurrent initialization attempts. In that case,
+ // log a warning.
+ auto ctx = ssl_make_unique(SSL_CTX_new(SSLv23_method()));
+ if (ctx) {
+ LOG(WARNING) << "It appears that OpenSSL has been previously initialized by "
+ << "code outside of Kudu. Please use kudu::client::DisableOpenSSLInitialization() "
+ << "to avoid potential crashes due to conflicting initialization.";
+ // Continue anyway; all of the below is idempotent, except for the locking callback,
+ // which we check before overriding. They aren't thread-safe, however -- that's why
+ // we try to get embedding applications to do the right thing here rather than risk a
+ // potential initialization race.
+ } else {
+ // As expected, SSL is not initialized, so SSL_CTX_new() failed. Make sure
+ // it didn't leave anything in our error queue.
+ ERR_clear_error();
+ }
+
+ SSL_load_error_strings();
+ SSL_library_init();
+ OpenSSL_add_all_algorithms();
+ RAND_poll();
+
+ if (!CRYPTO_get_locking_callback()) {
+ // Initialize the OpenSSL mutexes. We intentionally leak these, so ignore
+ // LSAN warnings.
+ debug::ScopedLeakCheckDisabler d;
+ int num_locks = CRYPTO_num_locks();
+ CHECK(!kCryptoLocks);
+ kCryptoLocks = new Mutex[num_locks];
+
+ // Callbacks used by OpenSSL required in a multi-threaded setting.
+ CRYPTO_set_locking_callback(LockingCB);
+ }
+#endif
+
+ g_ssl_is_initialized = true;
+}
+
+} // anonymous namespace
+
+// Reads a STACK_OF(X509) from the BIO and returns it.
+STACK_OF(X509)* PEM_read_STACK_OF_X509(BIO* bio, void* /* unused */, pem_password_cb* /* unused */,
+ void* /* unused */) {
+ // Extract information from the chain certificate.
+ STACK_OF(X509_INFO)* info = PEM_X509_INFO_read_bio(bio, nullptr, nullptr, nullptr);
+ if (!info) return nullptr;
+ SCOPED_CLEANUP({
+ sk_X509_INFO_pop_free(info, X509_INFO_free);
+ });
+
+ // Initialize the Stack.
+ STACK_OF(X509)* sk = sk_X509_new_null();
+
+ // Iterate through the chain certificate and add each one to the stack.
+ for (int i = 0; i < sk_X509_INFO_num(info); ++i) {
+ X509_INFO *stack_item = sk_X509_INFO_value(info, i);
+ sk_X509_push(sk, stack_item->x509);
+ // We don't want the ScopedCleanup to free the x509 certificates as well since we will
+ // use it as a part of the STACK_OF(X509) object to be returned, so we set it to nullptr.
+ // We will take the responsibility of freeing it when we are done with the STACK_OF(X509).
+ stack_item->x509 = nullptr;
+ }
+ return sk;
+}
+
+// Writes a STACK_OF(X509) to the BIO.
+int PEM_write_STACK_OF_X509(BIO* bio, STACK_OF(X509)* obj) {
+ int chain_len = sk_X509_num(obj);
+ // Iterate through the stack and add each one to the BIO.
+ for (int i = 0; i < chain_len; ++i) {
+ X509* cert_item = sk_X509_value(obj, i);
+ int ret = PEM_write_bio_X509(bio, cert_item);
+ if (ret <= 0) return ret;
+ }
+ return 1;
+}
+
+// Reads a single X509 certificate and returns a STACK_OF(X509) with the single certificate.
+STACK_OF(X509)* DER_read_STACK_OF_X509(BIO* bio, void* /* unused */) {
+ // We don't support chain certificates written in DER format.
+ auto x = ssl_make_unique(d2i_X509_bio(bio, nullptr));
+ if (!x) return nullptr;
+ STACK_OF(X509)* sk = sk_X509_new_null();
+ if (sk_X509_push(sk, x.get()) == 0) {
+ return nullptr;
+ }
+ x.release();
+ return sk;
+}
+
+// Writes a single X509 certificate that it gets from the STACK_OF(X509) 'obj'.
+int DER_write_STACK_OF_X509(BIO* bio, STACK_OF(X509)* obj) {
+ int chain_len = sk_X509_num(obj);
+ // We don't support chain certificates written in DER format.
+ DCHECK_EQ(chain_len, 1);
+ X509* cert_item = sk_X509_value(obj, 0);
+ if (cert_item == nullptr) return 0;
+ return i2d_X509_bio(bio, cert_item);
+}
+
+void free_STACK_OF_X509(STACK_OF(X509)* sk) {
+ sk_X509_pop_free(sk, X509_free);
+}
+
+Status DisableOpenSSLInitialization() {
+ if (g_disable_ssl_init) return Status::OK();
+ if (g_ssl_is_initialized) {
+ return Status::IllegalState("SSL already initialized. Initialization can only be disabled "
+ "before first usage.");
+ }
+ RETURN_NOT_OK(CheckOpenSSLInitialized());
+ g_disable_ssl_init = true;
+ return Status::OK();
+}
+
+void InitializeOpenSSL() {
+ static std::once_flag ssl_once;
+ std::call_once(ssl_once, DoInitializeOpenSSL);
+}
+
+string GetOpenSSLErrors() {
+ ostringstream serr;
+ uint32_t l;
+ int line, flags;
+ const char *file, *data;
+ bool is_first = true;
+ while ((l = ERR_get_error_line_data(&file, &line, &data, &flags)) != 0) {
+ if (is_first) {
+ is_first = false;
+ } else {
+ serr << " ";
+ }
+
+ char buf[256];
+ ERR_error_string_n(l, buf, sizeof(buf));
+ serr << buf << ":" << file << ":" << line;
+ if (flags & ERR_TXT_STRING) {
+ serr << ":" << data;
+ }
+ }
+ return serr.str();
+}
+
+string GetSSLErrorDescription(int error_code) {
+ switch (error_code) {
+ case SSL_ERROR_NONE: return "";
+ case SSL_ERROR_ZERO_RETURN: return "SSL_ERROR_ZERO_RETURN";
+ case SSL_ERROR_WANT_READ: return "SSL_ERROR_WANT_READ";
+ case SSL_ERROR_WANT_WRITE: return "SSL_ERROR_WANT_WRITE";
+ case SSL_ERROR_WANT_CONNECT: return "SSL_ERROR_WANT_CONNECT";
+ case SSL_ERROR_WANT_ACCEPT: return "SSL_ERROR_WANT_ACCEPT";
+ case SSL_ERROR_WANT_X509_LOOKUP: return "SSL_ERROR_WANT_X509_LOOKUP";
+ case SSL_ERROR_SYSCALL: {
+ string queued_error = GetOpenSSLErrors();
+ if (!queued_error.empty()) {
+ return queued_error;
+ }
+ return kudu::ErrnoToString(errno);
+ };
+ default: return GetOpenSSLErrors();
+ }
+}
+
+const string& DataFormatToString(DataFormat fmt) {
+ static const string kStrFormatUnknown = "UNKNOWN";
+ static const string kStrFormatDer = "DER";
+ static const string kStrFormatPem = "PEM";
+ switch (fmt) {
+ case DataFormat::DER:
+ return kStrFormatDer;
+ case DataFormat::PEM:
+ return kStrFormatPem;
+ default:
+ return kStrFormatUnknown;
+ }
+}
+
+Status GetPasswordFromShellCommand(const string& cmd, string* password) {
+ vector<string> argv = strings::Split(cmd, " ", strings::SkipEmpty());
+ if (argv.empty()) {
+ return Status::RuntimeError("invalid empty private key password command");
+ }
+ string stderr, stdout;
+ Status s = Subprocess::Call(argv, "" /* stdin */, &stdout, &stderr);
+ if (!s.ok()) {
+ return Status::RuntimeError(strings::Substitute(
+ "failed to run private key password command: $0", s.ToString()), stderr);
+ }
+ StripTrailingWhitespace(&stdout);
+ *password = stdout;
+ return Status::OK();
+}
+
+} // namespace security
+} // namespace kudu
http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/openssl_util.h
----------------------------------------------------------------------
diff --git a/be/src/kudu/security/openssl_util.h b/be/src/kudu/security/openssl_util.h
new file mode 100644
index 0000000..00c4ebf
--- /dev/null
+++ b/be/src/kudu/security/openssl_util.h
@@ -0,0 +1,217 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <functional>
+#include <memory>
+#include <ostream>
+#include <string>
+
+#include <glog/logging.h>
+#include <openssl/err.h>
+#include <openssl/pem.h>
+#include <openssl/ssl.h>
+#include <openssl/x509.h>
+
+#include "kudu/gutil/port.h"
+#include "kudu/util/status.h"
+
+// Forward declarations for the OpenSSL typedefs.
+typedef struct X509_req_st X509_REQ;
+typedef struct bio_st BIO;
+typedef struct evp_pkey_st EVP_PKEY;
+typedef struct ssl_ctx_st SSL_CTX;
+typedef struct ssl_st SSL;
+typedef struct x509_st X509;
+
+#define OPENSSL_CHECK_OK(call) \
+ CHECK_GT((call), 0)
+
+#define OPENSSL_RET_NOT_OK(call, msg) \
+ if ((call) <= 0) { \
+ return Status::RuntimeError((msg), GetOpenSSLErrors()); \
+ }
+
+#define OPENSSL_RET_IF_NULL(call, msg) \
+ if ((call) == nullptr) { \
+ return Status::RuntimeError((msg), GetOpenSSLErrors()); \
+ }
+
+// Scoped helper which DCHECKs that on both scope entry and exit, there are no
+// pending OpenSSL errors for the current thread.
+//
+// This allows us to avoid calling ERR_clear_error() defensively before every
+// OpenSSL call, but rather call it only when we get an error code indicating
+// there may be some pending error.
+//
+// Example usage:
+//
+// void MyFunc() {
+// SCOPED_OPENSSL_NO_PENDING_ERRORS;
+// ... use OpenSSL APIs ...
+// }
+#define SCOPED_OPENSSL_NO_PENDING_ERRORS \
+ kudu::security::internal::ScopedCheckNoPendingSSLErrors _no_ssl_errors(__PRETTY_FUNCTION__)
+
+namespace kudu {
+namespace security {
+
+using PasswordCallback = std::function<std::string(void)>;
+
+// Disable initialization of OpenSSL. Must be called before
+// any call to InitializeOpenSSL().
+Status DisableOpenSSLInitialization() WARN_UNUSED_RESULT;
+
+// Initializes static state required by the OpenSSL library.
+// This is a no-op if DisableOpenSSLInitialization() has been called.
+//
+// Safe to call multiple times.
+void InitializeOpenSSL();
+
+// Fetches errors from the OpenSSL error error queue, and stringifies them.
+//
+// The error queue will be empty after this method returns.
+//
+// See man(3) ERR_get_err for more discussion.
+std::string GetOpenSSLErrors();
+
+// Returns a string representation of the provided error code, which must be
+// from a prior call to the SSL_get_error function.
+//
+// If necessary, the OpenSSL error queue may be inspected and emptied as part of
+// this call, and/or 'errno' may be inspected. As a result, this method should
+// only be used directly after the error occurs, and from the same thread.
+//
+// See man(3) SSL_get_error for more discussion.
+std::string GetSSLErrorDescription(int error_code);
+
+// Runs the shell command 'cmd' which should give a password to a private key file
+// as the output.
+//
+// 'password' is populated with the password string if the command was a success.
+// An error Status object is returned otherwise.
+Status GetPasswordFromShellCommand(const std::string& cmd, std::string* password);
+
+// A generic wrapper for OpenSSL structures.
+template <typename T>
+using c_unique_ptr = std::unique_ptr<T, std::function<void(T*)>>;
+
+// For each SSL type, the Traits class provides the important OpenSSL
+// API functions.
+template<typename SSL_TYPE>
+struct SslTypeTraits {};
+
+template<> struct SslTypeTraits<X509> {
+ static constexpr auto kFreeFunc = &X509_free;
+ static constexpr auto kReadPemFunc = &PEM_read_bio_X509;
+ static constexpr auto kReadDerFunc = &d2i_X509_bio;
+ static constexpr auto kWritePemFunc = &PEM_write_bio_X509;
+ static constexpr auto kWriteDerFunc = &i2d_X509_bio;
+};
+
+// SslTypeTraits functions for Type STACK_OF(X509)
+STACK_OF(X509)* PEM_read_STACK_OF_X509(BIO* bio, void* /* unused */,
+ pem_password_cb* /* unused */, void* /* unused */);
+int PEM_write_STACK_OF_X509(BIO* bio, STACK_OF(X509)* obj);
+STACK_OF(X509)* DER_read_STACK_OF_X509(BIO* bio, void* /* unused */);
+int DER_write_STACK_OF_X509(BIO* bio, STACK_OF(X509)* obj);
+void free_STACK_OF_X509(STACK_OF(X509)* sk);
+
+template<> struct SslTypeTraits<STACK_OF(X509)> {
+ static constexpr auto kFreeFunc = &free_STACK_OF_X509;
+ static constexpr auto kReadPemFunc = &PEM_read_STACK_OF_X509;
+ static constexpr auto kReadDerFunc = &DER_read_STACK_OF_X509;
+ static constexpr auto kWritePemFunc = &PEM_write_STACK_OF_X509;
+ static constexpr auto kWriteDerFunc = &DER_write_STACK_OF_X509;
+};
+template<> struct SslTypeTraits<X509_EXTENSION> {
+ static constexpr auto kFreeFunc = &X509_EXTENSION_free;
+};
+template<> struct SslTypeTraits<X509_REQ> {
+ static constexpr auto kFreeFunc = &X509_REQ_free;
+ static constexpr auto kReadPemFunc = &PEM_read_bio_X509_REQ;
+ static constexpr auto kReadDerFunc = &d2i_X509_REQ_bio;
+ static constexpr auto kWritePemFunc = &PEM_write_bio_X509_REQ;
+ static constexpr auto kWriteDerFunc = &i2d_X509_REQ_bio;
+};
+template<> struct SslTypeTraits<EVP_PKEY> {
+ static constexpr auto kFreeFunc = &EVP_PKEY_free;
+};
+template<> struct SslTypeTraits<SSL_CTX> {
+ static constexpr auto kFreeFunc = &SSL_CTX_free;
+};
+
+template<typename SSL_TYPE, typename Traits = SslTypeTraits<SSL_TYPE>>
+c_unique_ptr<SSL_TYPE> ssl_make_unique(SSL_TYPE* d) {
+ return {d, Traits::kFreeFunc};
+}
+
+// Acceptable formats for keys, X509 certificates and X509 CSRs.
+enum class DataFormat {
+ DER = 0, // DER/ASN1 format (binary): for representing object on the wire
+ PEM = 1, // PEM format (ASCII): for storing on filesystem, printing, etc.
+};
+
+// Data format representation as a string.
+const std::string& DataFormatToString(DataFormat fmt);
+
+// Template wrapper for dynamically allocated entities with custom deleter.
+// Mostly, using it for xxx_st types from the OpenSSL crypto library.
+template<typename Type>
+class RawDataWrapper {
+ public:
+ typedef Type RawDataType;
+
+ RawDataType* GetRawData() const {
+ return data_.get();
+ }
+
+ void AdoptRawData(RawDataType* d) {
+ data_ = ssl_make_unique(d);
+ }
+
+ protected:
+ c_unique_ptr<RawDataType> data_;
+};
+
+
+namespace internal {
+
+// Implementation of SCOPED_OPENSSL_NO_PENDING_ERRORS. Use the macro form
+// instead of directly instantiating the implementation class.
+struct ScopedCheckNoPendingSSLErrors {
+ public:
+ explicit ScopedCheckNoPendingSSLErrors(const char* func)
+ : func_(func) {
+ DCHECK_EQ(ERR_peek_error(), 0)
+ << "Expected no pending OpenSSL errors on " << func_
+ << " entry, but had: " << GetOpenSSLErrors();
+ }
+ ~ScopedCheckNoPendingSSLErrors() {
+ DCHECK_EQ(ERR_peek_error(), 0)
+ << "Expected no pending OpenSSL errors on " << func_
+ << " exit, but had: " << GetOpenSSLErrors();
+ }
+
+ private:
+ const char* const func_;
+};
+
+} // namespace internal
+} // namespace security
+} // namespace kudu
http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/openssl_util_bio.h
----------------------------------------------------------------------
diff --git a/be/src/kudu/security/openssl_util_bio.h b/be/src/kudu/security/openssl_util_bio.h
new file mode 100644
index 0000000..c935b0b
--- /dev/null
+++ b/be/src/kudu/security/openssl_util_bio.h
@@ -0,0 +1,129 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include "kudu/security/openssl_util.h"
+
+#include <string>
+
+#include <glog/logging.h>
+#include <openssl/bio.h>
+#include <openssl/buffer.h>
+
+#include "kudu/gutil/strings/substitute.h"
+#include "kudu/util/status.h"
+
+namespace kudu {
+namespace security {
+
+template<> struct SslTypeTraits<BIO> {
+ static constexpr auto kFreeFunc = &BIO_free;
+};
+
+template<typename TYPE, typename Traits = SslTypeTraits<TYPE>>
+Status ToBIO(BIO* bio, DataFormat format, TYPE* obj) {
+ CHECK(bio);
+ CHECK(obj);
+ switch (format) {
+ case DataFormat::DER:
+ OPENSSL_RET_NOT_OK(Traits::kWriteDerFunc(bio, obj),
+ "error exporting data in DER format");
+ break;
+ case DataFormat::PEM:
+ OPENSSL_RET_NOT_OK(Traits::kWritePemFunc(bio, obj),
+ "error exporting data in PEM format");
+ break;
+ }
+ OPENSSL_RET_NOT_OK(BIO_flush(bio), "error flushing BIO");
+ return Status::OK();
+}
+
+// The callback which is called by the OpenSSL library when trying to decrypt
+// a password protected private key.
+inline int TLSPasswordCB(char* buf, int size, int /* rwflag */, void* userdata) {
+ const auto* cb = reinterpret_cast<const PasswordCallback*>(userdata);
+ std::string pw = (*cb)();
+ if (pw.size() >= size) {
+ LOG(ERROR) << "Provided key password is longer than maximum length "
+ << size;
+ return -1;
+ }
+ strncpy(buf, pw.c_str(), size);
+ return pw.size();
+}
+
+template<typename TYPE, typename Traits = SslTypeTraits<TYPE>>
+Status FromBIO(BIO* bio, DataFormat format, c_unique_ptr<TYPE>* ret,
+ const PasswordCallback& cb = PasswordCallback()) {
+ CHECK(bio);
+ switch (format) {
+ case DataFormat::DER:
+ *ret = ssl_make_unique(Traits::kReadDerFunc(bio, nullptr));
+ break;
+ case DataFormat::PEM:
+ *ret = ssl_make_unique(Traits::kReadPemFunc(bio, nullptr, &TLSPasswordCB,
+ const_cast<PasswordCallback*>(&cb)));
+ break;
+ }
+ if (PREDICT_FALSE(!*ret)) {
+ return Status::RuntimeError(GetOpenSSLErrors());
+ }
+ return Status::OK();
+}
+
+template<typename Type, typename Traits = SslTypeTraits<Type>>
+Status FromString(const std::string& data, DataFormat format,
+ c_unique_ptr<Type>* ret) {
+ const void* mdata = reinterpret_cast<const void*>(data.data());
+ auto bio = ssl_make_unique(BIO_new_mem_buf(
+#if OPENSSL_VERSION_NUMBER < 0x10002000L
+ const_cast<void*>(mdata),
+#else
+ mdata,
+#endif
+ data.size()));
+ RETURN_NOT_OK_PREPEND((FromBIO<Type, Traits>(bio.get(), format, ret)),
+ "unable to load data from memory");
+ return Status::OK();
+}
+
+template<typename Type, typename Traits = SslTypeTraits<Type>>
+Status ToString(std::string* data, DataFormat format, Type* obj) {
+ CHECK(data);
+ auto bio = ssl_make_unique(BIO_new(BIO_s_mem()));
+ RETURN_NOT_OK_PREPEND((ToBIO<Type, Traits>(bio.get(), format, obj)),
+ "error serializing data");
+ BUF_MEM* membuf;
+ OPENSSL_CHECK_OK(BIO_get_mem_ptr(bio.get(), &membuf));
+ data->assign(membuf->data, membuf->length);
+ return Status::OK();
+}
+
+template<typename Type, typename Traits = SslTypeTraits<Type>>
+Status FromFile(const std::string& fpath, DataFormat format,
+ c_unique_ptr<Type>* ret, const PasswordCallback& cb = PasswordCallback()) {
+ auto bio = ssl_make_unique(BIO_new(BIO_s_file()));
+ OPENSSL_RET_NOT_OK(BIO_read_filename(bio.get(), fpath.c_str()),
+ strings::Substitute("could not read data from file '$0'", fpath));
+ RETURN_NOT_OK_PREPEND((FromBIO<Type, Traits>(bio.get(), format, ret, cb)),
+ strings::Substitute("unable to load data from file '$0'", fpath));
+ return Status::OK();
+}
+
+} // namespace security
+} // namespace kudu
http://git-wip-us.apache.org/repos/asf/impala/blob/fcf190c4/be/src/kudu/security/security-test-util.cc
----------------------------------------------------------------------
diff --git a/be/src/kudu/security/security-test-util.cc b/be/src/kudu/security/security-test-util.cc
new file mode 100644
index 0000000..40b0938
--- /dev/null
+++ b/be/src/kudu/security/security-test-util.cc
@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "kudu/security/security-test-util.h"
+
+#include <cstdint>
+#include <string>
+
+#include <boost/optional/optional.hpp>
+
+#include "kudu/security/ca/cert_management.h"
+#include "kudu/security/cert.h"
+#include "kudu/security/crypto.h"
+#include "kudu/security/test/test_certs.h"
+#include "kudu/security/tls_context.h"
+#include "kudu/util/test_util.h"
+
+namespace kudu {
+namespace security {
+
+using ca::CaCertRequestGenerator;
+using ca::CertSigner;
+
+Status GenerateSelfSignedCAForTests(PrivateKey* ca_key, Cert* ca_cert) {
+ static const int64_t kRootCaCertExpirationSeconds = 24 * 60 * 60;
+ // Create a key for the self-signed CA.
+ //
+ // OpenSSL has a concept of "security levels" which, amongst other things,
+ // place certain restrictions on key strength. OpenSSL 1.0 defaults to level
+ // 0 (no restrictions) while 1.1 defaults to level 1, which requires RSA keys
+ // to have at least 1024 bits. For simplicity, we'll just use 1024 bits here,
+ // even though shorter keys would decrease test running time.
+ //
+ // See https://www.openssl.org/docs/man1.1.0/ssl/SSL_CTX_get_security_level.html
+ // for more details.
+ RETURN_NOT_OK(GeneratePrivateKey(1024, ca_key));
+
+ CaCertRequestGenerator::Config config = { "test-ca-cn" };
+ RETURN_NOT_OK(CertSigner::SelfSignCA(*ca_key,
+ config,
+ kRootCaCertExpirationSeconds,
+ ca_cert));
+ return Status::OK();
+}
+
+std::ostream& operator<<(std::ostream& o, PkiConfig c) {
+ switch (c) {
+ case PkiConfig::NONE: o << "NONE"; break;
+ case PkiConfig::SELF_SIGNED: o << "SELF_SIGNED"; break;
+ case PkiConfig::TRUSTED: o << "TRUSTED"; break;
+ case PkiConfig::SIGNED: o << "SIGNED"; break;
+ case PkiConfig::EXTERNALLY_SIGNED: o << "EXTERNALLY_SIGNED"; break;
+ }
+ return o;
+}
+
+Status ConfigureTlsContext(PkiConfig config,
+ const Cert& ca_cert,
+ const PrivateKey& ca_key,
+ TlsContext* tls_context) {
+ switch (config) {
+ case PkiConfig::NONE: break;
+ case PkiConfig::SELF_SIGNED:
+ RETURN_NOT_OK(tls_context->GenerateSelfSignedCertAndKey());
+ break;
+ case PkiConfig::TRUSTED:
+ RETURN_NOT_OK(tls_context->AddTrustedCertificate(ca_cert));
+ break;
+ case PkiConfig::SIGNED: {
+ RETURN_NOT_OK(tls_context->AddTrustedCertificate(ca_cert));
+ RETURN_NOT_OK(tls_context->GenerateSelfSignedCertAndKey());
+ Cert cert;
+ RETURN_NOT_OK(CertSigner(&ca_cert, &ca_key).Sign(*tls_context->GetCsrIfNecessary(), &cert));
+ RETURN_NOT_OK(tls_context->AdoptSignedCert(cert));
+ break;
+ };
+ case PkiConfig::EXTERNALLY_SIGNED: {
+ std::string cert_path, key_path;
+ // Write certificate and private key to file.
+ RETURN_NOT_OK(CreateTestSSLCertWithPlainKey(GetTestDataDirectory(), &cert_path, &key_path));
+ RETURN_NOT_OK(tls_context->LoadCertificateAndKey(cert_path, key_path));
+ RETURN_NOT_OK(tls_context->LoadCertificateAuthority(cert_path));
+ };
+ }
+ return Status::OK();
+}
+
+} // namespace security
+} // namespace kudu