You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@teaclave.apache.org by ms...@apache.org on 2019/12/18 21:29:15 UTC
[incubator-teaclave] branch master updated: [proto] Introduce
proto_gen (#180)
This is an automated email from the ASF dual-hosted git repository.
mssun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-teaclave.git
The following commit(s) were added to refs/heads/master by this push:
new f49dd15 [proto] Introduce proto_gen (#180)
f49dd15 is described below
commit f49dd151c55b2b8663b349f614a27e169ef1cb3e
Author: Mingshen Sun <bo...@mssun.me>
AuthorDate: Wed Dec 18 13:29:06 2019 -0800
[proto] Introduce proto_gen (#180)
---
cmake/UtilTargets.cmake | 2 -
cmake/tomls/Cargo.sgx_trusted_lib.toml | 1 -
mesatee_services/kms/proto/Cargo.toml | 3 -
mesatee_services/kms/proto/build.rs | 51 +++----
mesatee_services/kms/proto/src/lib.rs | 34 +----
.../kms/proto/src/prost_generated/kms_proto.rs | 146 ---------------------
mesatee_services/proto_gen/Cargo.toml | 17 +++
.../prost_build_generator.rs => proto_gen/main.rs} | 34 ++++-
mesatee_services/tdfs/external/proto/src/key.rs | 27 ----
teaclave_config/build.rs | 2 +
tests/integration_test/src/test.rs | 2 +
tests/integration_test/test_data/test.toml | 4 +-
12 files changed, 87 insertions(+), 236 deletions(-)
diff --git a/cmake/UtilTargets.cmake b/cmake/UtilTargets.cmake
index 8d1351c..2539719 100644
--- a/cmake/UtilTargets.cmake
+++ b/cmake/UtilTargets.cmake
@@ -7,7 +7,6 @@ add_custom_target(format
COMMAND RUSTUP_TOOLCHAIN=${RUSTUP_TOOLCHAIN} find ${MESATEE_PROJECT_ROOT}
-path ${MESATEE_PROJECT_ROOT}/third_party -prune -o
-path ${MESATEE_PROJECT_ROOT}/.git -prune -o
- -path *prost_generated -prune -o
-name "*.rs" -exec rustfmt {} +
COMMENT "Formating every .rs file"
DEPENDS prep
@@ -18,7 +17,6 @@ add_custom_target(check
COMMAND RUSTUP_TOOLCHAIN=${RUSTUP_TOOLCHAIN} find ${MESATEE_PROJECT_ROOT}
-path ${MESATEE_PROJECT_ROOT}/third_party -prune -o
-path ${MESATEE_PROJECT_ROOT}/.git -prune -o
- -path *prost_generated -prune -o
-name "*.rs" -exec rustfmt --check {} +
COMMENT "Checking the format of every .rs file"
DEPENDS prep
diff --git a/cmake/tomls/Cargo.sgx_trusted_lib.toml b/cmake/tomls/Cargo.sgx_trusted_lib.toml
index d86b1ea..f587d1f 100644
--- a/cmake/tomls/Cargo.sgx_trusted_lib.toml
+++ b/cmake/tomls/Cargo.sgx_trusted_lib.toml
@@ -87,7 +87,6 @@ webpki = { git = "https://github.com/mesalock-linux/webpki", branch =
webpki-roots = { git = "https://github.com/mesalock-linux/webpki-roots", branch = "mesalock_sgx" }
yasna = { git = "https://github.com/mesalock-linux/yasna.rs-sgx" }
rusty-machine = { git = "https://github.com/mesalock-linux/rusty-machine-sgx" }
-prost-build = { git = "https://github.com/mesalock-linux/prost-build-sgx-fake"}
bytes = { git = "https://github.com/mesalock-linux/bytes-sgx" }
prost = { git = "https://github.com/mesalock-linux/prost-sgx" }
prost-types = { git = "https://github.com/mesalock-linux/prost-sgx" }
diff --git a/mesatee_services/kms/proto/Cargo.toml b/mesatee_services/kms/proto/Cargo.toml
index f1b076b..16b68a5 100644
--- a/mesatee_services/kms/proto/Cargo.toml
+++ b/mesatee_services/kms/proto/Cargo.toml
@@ -26,6 +26,3 @@ mesatee_core = { version = "0.1.0" }
sgx_cov = { version = "0.1.0", optional = true }
sgx_tstd = { version = "1.0.9", features = ["net", "backtrace"], optional = true }
sgx_types = { version = "1.0.9" }
-
-[build-dependencies]
-prost-build = { version = "0.5.0" }
\ No newline at end of file
diff --git a/mesatee_services/kms/proto/build.rs b/mesatee_services/kms/proto/build.rs
index a646f8b..4efbbd7 100644
--- a/mesatee_services/kms/proto/build.rs
+++ b/mesatee_services/kms/proto/build.rs
@@ -15,33 +15,34 @@
// specific language governing permissions and limitations
// under the License.
-#[cfg(not(feature = "mesalock_sgx"))]
-include!("../../common/prost_build_generator.rs");
+use std::env;
+use std::process::Command;
+use std::str;
-#[cfg(not(feature = "mesalock_sgx"))]
fn main() {
+ let out_dir = env::var("OUT_DIR").expect("$OUT_DIR not set. Please build with cargo");
println!("cargo:rerun-if-changed=src/kms.proto");
- let src = PathBuf::from("src");
- let output = src.join("prost_generated");
- if !output.exists() {
- std::fs::create_dir(&output).expect("failed to create prost_generated dir");
+ println!("cargo:rerun-if-changed=build.rs");
+ let c = Command::new("cargo")
+ .args(&[
+ "run",
+ "--manifest-path",
+ "../../proto_gen/Cargo.toml",
+ "--",
+ "-p",
+ "src/kms.proto",
+ "-i",
+ ".",
+ "-d",
+ &out_dir,
+ ])
+ .output()
+ .expect("Cannot generate kms_proto.rs");
+ if !c.status.success() {
+ panic!(
+ "stdout: {:?}, stderr: {:?}",
+ str::from_utf8(&c.stderr).unwrap(),
+ str::from_utf8(&c.stderr).unwrap()
+ );
}
- let includes = &[src.clone()];
- let mut config = get_default_config();
- config.out_dir(output);
- let base64_field = [
- "AeadConfig.key",
- "AeadConfig.nonce",
- "AeadConfig.ad",
- "ProtectedFsConfig.key",
- ];
- for field_name in base64_field.iter() {
- config.field_attribute(field_name, "#[serde(with = \"crate::base64_coder\")]");
- }
- config
- .compile_protos(&[src.join("kms.proto")], includes)
- .unwrap();
}
-
-#[cfg(feature = "mesalock_sgx")]
-fn main() {}
diff --git a/mesatee_services/kms/proto/src/lib.rs b/mesatee_services/kms/proto/src/lib.rs
index 40427d8..602d5af 100644
--- a/mesatee_services/kms/proto/src/lib.rs
+++ b/mesatee_services/kms/proto/src/lib.rs
@@ -19,34 +19,14 @@
#[macro_use]
extern crate sgx_tstd as std;
+#[cfg(feature = "mesalock_sgx")]
+use std::prelude::v1::*;
+
+mod kms;
pub use crate::kms::*;
pub use crate::proto::KMSClient;
-mod kms;
-pub mod proto {
- #![allow(warnings)]
- #![allow(clippy)]
- #![allow(unknown_lints)]
- include!("prost_generated/kms_proto.rs");
-}
-
-mod base64_coder {
- use base64;
- use serde::{de, Deserialize, Deserializer, Serializer};
- #[cfg(feature = "mesalock_sgx")]
- use std::prelude::v1::*;
- pub fn serialize<S>(bytes: &[u8], serializer: S) -> Result<S::Ok, S::Error>
- where
- S: Serializer,
- {
- serializer.serialize_str(&base64::encode(bytes))
- }
-
- pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
- where
- D: Deserializer<'de>,
- {
- let s = <&str>::deserialize(deserializer)?;
- base64::decode(s).map_err(de::Error::custom)
- }
+pub mod proto {
+ #![allow(clippy::all)]
+ include!(concat!(env!("OUT_DIR"), "/kms_proto.rs"));
}
diff --git a/mesatee_services/kms/proto/src/prost_generated/kms_proto.rs b/mesatee_services/kms/proto/src/prost_generated/kms_proto.rs
deleted file mode 100644
index b78054a..0000000
--- a/mesatee_services/kms/proto/src/prost_generated/kms_proto.rs
+++ /dev/null
@@ -1,146 +0,0 @@
-#[derive(Clone, PartialEq, ::prost::Message)]
-#[derive(serde_derive::Serialize, serde_derive::Deserialize)]
-pub struct CreateKeyRequest {
- #[prost(enumeration="EncType", required, tag="1")]
- pub enc_type: i32,
-}
-#[derive(Clone, PartialEq, ::prost::Message)]
-#[derive(serde_derive::Serialize, serde_derive::Deserialize)]
-pub struct AeadConfig {
- #[prost(bytes, required, tag="1")]
- #[serde(with = "crate::base64_coder")]
- pub key: std::vec::Vec<u8>,
- #[prost(bytes, required, tag="2")]
- #[serde(with = "crate::base64_coder")]
- pub nonce: std::vec::Vec<u8>,
- #[prost(bytes, required, tag="3")]
- #[serde(with = "crate::base64_coder")]
- pub ad: std::vec::Vec<u8>,
-}
-#[derive(Clone, PartialEq, ::prost::Message)]
-#[derive(serde_derive::Serialize, serde_derive::Deserialize)]
-pub struct ProtectedFsConfig {
- #[prost(bytes, required, tag="1")]
- #[serde(with = "crate::base64_coder")]
- pub key: std::vec::Vec<u8>,
-}
-#[derive(Clone, PartialEq, ::prost::Message)]
-#[derive(serde_derive::Serialize, serde_derive::Deserialize)]
-pub struct KeyConfig {
- #[prost(oneof="key_config::Config", tags="1, 2")]
- pub config: ::std::option::Option<key_config::Config>,
-}
-pub mod key_config {
- #[derive(Clone, PartialEq, ::prost::Oneof)]
- #[derive(serde_derive::Serialize, serde_derive::Deserialize)]
- pub enum Config {
- #[prost(message, tag="1")]
- Aead(super::AeadConfig),
- #[prost(message, tag="2")]
- ProtectedFs(super::ProtectedFsConfig),
- }
-}
-#[derive(Clone, PartialEq, ::prost::Message)]
-#[derive(serde_derive::Serialize, serde_derive::Deserialize)]
-pub struct CreateKeyResponse {
- #[prost(string, required, tag="1")]
- pub key_id: std::string::String,
- #[prost(message, required, tag="2")]
- pub config: KeyConfig,
-}
-#[derive(Clone, PartialEq, ::prost::Message)]
-#[derive(serde_derive::Serialize, serde_derive::Deserialize)]
-pub struct GetKeyRequest {
- #[prost(string, required, tag="1")]
- pub key_id: std::string::String,
-}
-#[derive(Clone, PartialEq, ::prost::Message)]
-#[derive(serde_derive::Serialize, serde_derive::Deserialize)]
-pub struct GetKeyResponse {
- #[prost(message, required, tag="1")]
- pub config: KeyConfig,
-}
-#[derive(Clone, PartialEq, ::prost::Message)]
-#[derive(serde_derive::Serialize, serde_derive::Deserialize)]
-pub struct DeleteKeyRequest {
- #[prost(string, required, tag="1")]
- pub key_id: std::string::String,
-}
-#[derive(Clone, PartialEq, ::prost::Message)]
-#[derive(serde_derive::Serialize, serde_derive::Deserialize)]
-pub struct DeleteKeyResponse {
- #[prost(message, required, tag="1")]
- pub config: KeyConfig,
-}
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)]
-#[repr(i32)]
-#[derive(serde_derive::Serialize, serde_derive::Deserialize)]
-pub enum EncType {
- Aead = 0,
- ProtectedFs = 1,
-}
-#[derive(Clone, serde_derive::Serialize, serde_derive::Deserialize, Debug)]
-#[serde(tag = "type")]pub enum KMSRequest {
- GetKey(GetKeyRequest),
- DelKey(DeleteKeyRequest),
- CreateKey(CreateKeyRequest),
-}
-#[derive(Clone, serde_derive::Serialize, serde_derive::Deserialize, Debug)]
-#[serde(tag = "type")]
-pub enum KMSResponse {
- GetKey(GetKeyResponse),
- DelKey(DeleteKeyResponse),
- CreateKey(CreateKeyResponse),
-}
-pub trait KMSService {
- fn get_key(req: GetKeyRequest) -> mesatee_core::Result<GetKeyResponse>;
- fn del_key(req: DeleteKeyRequest) -> mesatee_core::Result<DeleteKeyResponse>;
- fn create_key(req: CreateKeyRequest) -> mesatee_core::Result<CreateKeyResponse>;
- fn dispatch(&self, req: KMSRequest) -> mesatee_core::Result<KMSResponse> {
- match req {
- KMSRequest::GetKey(req) => Self::get_key(req).map(KMSResponse::GetKey), KMSRequest::DelKey(req) => Self::del_key(req).map(KMSResponse::DelKey), KMSRequest::CreateKey(req) => Self::create_key(req).map(KMSResponse::CreateKey), }
- }
-}
-pub struct KMSClient {
- channel: mesatee_core::rpc::channel::SgxTrustedChannel<KMSRequest, KMSResponse>,
-}
-
-impl KMSClient {
- pub fn new(target: mesatee_core::config::TargetDesc) -> mesatee_core::Result<Self> {
- let addr = target.addr;
- let channel = match target.desc {
- mesatee_core::config::OutboundDesc::Sgx(enclave_addr) => {
- mesatee_core::rpc::channel::SgxTrustedChannel::<KMSRequest, KMSResponse>::new(addr, enclave_addr)?
- }
- };
- Ok(KMSClient { channel })
- }
-}
-impl KMSClient {
- pub fn get_key(&mut self, req: GetKeyRequest) -> mesatee_core::Result<GetKeyResponse> {
- let req = KMSRequest::GetKey(req);
- let resp = self.channel.invoke(req)?;
- match resp {
- KMSResponse::GetKey(resp) => Ok(resp),
- _ => Err(mesatee_core::Error::from(mesatee_core::ErrorKind::RPCResponseError)),
- }
- }
-
- pub fn del_key(&mut self, req: DeleteKeyRequest) -> mesatee_core::Result<DeleteKeyResponse> {
- let req = KMSRequest::DelKey(req);
- let resp = self.channel.invoke(req)?;
- match resp {
- KMSResponse::DelKey(resp) => Ok(resp),
- _ => Err(mesatee_core::Error::from(mesatee_core::ErrorKind::RPCResponseError)),
- }
- }
-
- pub fn create_key(&mut self, req: CreateKeyRequest) -> mesatee_core::Result<CreateKeyResponse> {
- let req = KMSRequest::CreateKey(req);
- let resp = self.channel.invoke(req)?;
- match resp {
- KMSResponse::CreateKey(resp) => Ok(resp),
- _ => Err(mesatee_core::Error::from(mesatee_core::ErrorKind::RPCResponseError)),
- }
- }
-}
diff --git a/mesatee_services/proto_gen/Cargo.toml b/mesatee_services/proto_gen/Cargo.toml
new file mode 100644
index 0000000..06fd52b
--- /dev/null
+++ b/mesatee_services/proto_gen/Cargo.toml
@@ -0,0 +1,17 @@
+[package]
+name = "proto_gen"
+version = "0.1.0"
+authors = ["MesaTEE Authors <de...@mesatee.org>"]
+description = "Generating Rust protocols from protobuf."
+license = "Apache-2.0"
+edition = "2018"
+
+[[bin]]
+name = "proto_gen"
+path = "main.rs"
+
+[dependencies]
+structopt = { version = "0.3" }
+prost-build = { version = "0.5.0" }
+prost = { version = "0.5.0" }
+prost-types = { version = "0.5.0" }
diff --git a/mesatee_services/common/prost_build_generator.rs b/mesatee_services/proto_gen/main.rs
similarity index 87%
rename from mesatee_services/common/prost_build_generator.rs
rename to mesatee_services/proto_gen/main.rs
index 4bbcb8f..68fd4a4 100644
--- a/mesatee_services/common/prost_build_generator.rs
+++ b/mesatee_services/proto_gen/main.rs
@@ -16,15 +16,18 @@
// under the License.
use prost_build;
-use std::path::PathBuf;
+use std::path;
+use structopt::StructOpt;
#[derive(Debug)]
pub struct MesaTEEServiceGenerator;
/// How to use prost. See kms as an example.
-/// 1. Define rpc messages with protobuf 2/3 syntax. protobuf 2 is recommended because we can avoid unneccessary option.
+/// 1. Define rpc messages with protobuf 2/3 syntax. protobuf 2 is recommended
+/// because we can avoid unneccessary option.
/// 2. Define services. Prost will generate corresponding sevices and clients.
-/// 3. Include ```prost_build_generator.rs``` and modify ```main function``` in the ```build.rs``` of the target library.
+/// 3. Include ```${OUT_DIR}/kms_proto.rs``` and provide serializer and
+/// deserializer if needed..
/// 4. Todo: add support for automatic authentication
const LINE_ENDING: &'static str = "\n";
impl MesaTEEServiceGenerator {
@@ -208,3 +211,28 @@ pub fn get_default_config() -> prost_build::Config {
);
config
}
+
+#[derive(Debug, StructOpt)]
+struct Cli {
+ #[structopt(short = "p", required = true)]
+ /// Paths to .proto files to compile. Any transitively imported .proto files
+ /// are automatically be included.
+ protos: Vec<path::PathBuf>,
+
+ #[structopt(short = "i", required = true)]
+ /// Paths to directories in which to search for imports. Directories are
+ /// searched in order. The .proto files passed in protos must be found in
+ /// one of the provided include directories.
+ includes: Vec<path::PathBuf>,
+
+ #[structopt(short = "d", required = true)]
+ /// Configures the output directory where generated Rust files will be written.
+ out_dir: path::PathBuf,
+}
+
+fn main() {
+ let args = Cli::from_args();
+ let mut config = get_default_config();
+ config.out_dir(args.out_dir);
+ config.compile_protos(&args.protos, &args.includes).unwrap();
+}
diff --git a/mesatee_services/tdfs/external/proto/src/key.rs b/mesatee_services/tdfs/external/proto/src/key.rs
index 3b11b2e..170e0f0 100644
--- a/mesatee_services/tdfs/external/proto/src/key.rs
+++ b/mesatee_services/tdfs/external/proto/src/key.rs
@@ -19,34 +19,7 @@ use serde_derive::*;
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq)]
pub struct AeadConfig {
- #[serde(with = "base64_coder")]
pub key: Vec<u8>,
- #[serde(with = "base64_coder")]
pub nonce: Vec<u8>,
- #[serde(with = "base64_coder")]
pub ad: Vec<u8>,
}
-
-mod base64_coder {
- // Insert std prelude in the top for the sgx feature
- #[cfg(feature = "mesalock_sgx")]
- use std::prelude::v1::*;
-
- extern crate base64;
- use serde::{de, Deserialize, Deserializer, Serializer};
-
- pub fn serialize<S>(bytes: &[u8], serializer: S) -> Result<S::Ok, S::Error>
- where
- S: Serializer,
- {
- serializer.serialize_str(&base64::encode(bytes))
- }
-
- pub fn deserialize<'de, D>(deserializer: D) -> Result<Vec<u8>, D::Error>
- where
- D: Deserializer<'de>,
- {
- let s = <&str>::deserialize(deserializer)?;
- base64::decode(s).map_err(de::Error::custom)
- }
-}
diff --git a/teaclave_config/build.rs b/teaclave_config/build.rs
index d7ea82c..3692645 100644
--- a/teaclave_config/build.rs
+++ b/teaclave_config/build.rs
@@ -13,11 +13,13 @@ fn main() {
let out_dir = env::var("OUT_DIR").expect("$OUT_DIR not set. Please build with cargo");
let dest_file = Path::new(&out_dir).join("build_config.rs");
println!("cargo:rerun-if-changed=build.config.toml");
+ println!("cargo:rerun-if-changed=build.rs");
let c = Command::new("cargo")
.args(&[
"run",
"--manifest-path",
"config_gen/Cargo.toml",
+ "--",
"build.config.toml",
&dest_file.to_string_lossy(),
])
diff --git a/tests/integration_test/src/test.rs b/tests/integration_test/src/test.rs
index e111e9e..530765e 100644
--- a/tests/integration_test/src/test.rs
+++ b/tests/integration_test/src/test.rs
@@ -117,6 +117,8 @@ impl Runner {
if pattern.is_match(&response) {
Ok(())
} else {
+ println!("response: {}", response);
+ println!("expected: {}", expected_response);
Err(failure::err_msg("Mismatched response").into())
}
}
diff --git a/tests/integration_test/test_data/test.toml b/tests/integration_test/test_data/test.toml
index a5750f3..13ebca2 100644
--- a/tests/integration_test/test_data/test.toml
+++ b/tests/integration_test/test_data/test.toml
@@ -53,7 +53,7 @@ payload = """
{"type":"Create","file_name":"file1","sha256":"this_is_a_hash","file_size":7,"user_id":"user1","user_token":"token1"}
"""
expected = """
-{"type":"Create","file_id":"*","access_path":"*","key_config":{"key":"*","nonce":"*","ad":"*"}}
+{"type":"Create","file_id":"*","access_path":"*","key_config":{"key":*,"nonce":*,"ad":*}}
"""
[[step]]
@@ -63,7 +63,7 @@ payload = """
{"type":"Get","file_id":"fake_file_record","user_id":"fake_file_owner","user_token":"token"}
"""
expected = """
-{"type":"Get","file_info":{"user_id":"fake_file_owner","file_name":"fake_file","sha256":"aaa","file_size":100,"access_path":"fake_file","task_id":null,"collaborator_list":[],"key_config":{"key":"QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUE=","nonce":"QUFBQUFBQUFBQUFB","ad":"QUFBQUE="}}}
+{"type":"Get","file_info":{"user_id":"fake_file_owner","file_name":"fake_file","sha256":"aaa","file_size":100,"access_path":"fake_file","task_id":null,"collaborator_list":[],"key_config":{"key":*,"nonce":*,"ad":*}}}
"""
[[step]]
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@teaclave.apache.org
For additional commands, e-mail: commits-help@teaclave.apache.org