You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@opendal.apache.org by xu...@apache.org on 2025/02/18 04:07:21 UTC
(opendal) branch main updated: feat: Implement github actions cache service v2 support (#5633)
This is an automated email from the ASF dual-hosted git repository.
xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/opendal.git
The following commit(s) were added to refs/heads/main by this push:
new 30c0c4fe9 feat: Implement github actions cache service v2 support (#5633)
30c0c4fe9 is described below
commit 30c0c4fe92b9be8e5db4a267b4cea0e4bdc33e20
Author: Xuanwo <gi...@xuanwo.io>
AuthorDate: Tue Feb 18 12:07:16 2025 +0800
feat: Implement github actions cache service v2 support (#5633)
---
.github/workflows/service_test_ghac.yml | 31 ++-
core/Cargo.lock | 10 +
core/Cargo.toml | 3 +-
core/src/raw/http_util/mod.rs | 1 +
core/src/raw/http_util/uri.rs | 6 +
core/src/services/azblob/mod.rs | 4 +-
core/src/services/ghac/backend.rs | 273 +++++-----------------
core/src/services/ghac/core.rs | 385 ++++++++++++++++++++++++++++++++
core/src/services/ghac/error.rs | 5 +-
core/src/services/ghac/mod.rs | 3 +
core/src/services/ghac/writer.rs | 159 ++++++++++---
11 files changed, 629 insertions(+), 251 deletions(-)
diff --git a/.github/workflows/service_test_ghac.yml b/.github/workflows/service_test_ghac.yml
index 0fcd067bf..e3c2221dc 100644
--- a/.github/workflows/service_test_ghac.yml
+++ b/.github/workflows/service_test_ghac.yml
@@ -37,7 +37,7 @@ concurrency:
cancel-in-progress: true
jobs:
- ghac:
+ ghac_v1:
runs-on: ubuntu-latest
if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork
permissions:
@@ -55,6 +55,35 @@ jobs:
script: |
core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+ core.exportVariable('ACTIONS_CACHE_SERVICE_V2', '');
+
+ - name: Test
+ shell: bash
+ working-directory: core
+ run: cargo test behavior --features tests,services-ghac
+ env:
+ OPENDAL_TEST: ghac
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+ ghac_v2:
+ runs-on: ubuntu-latest
+ if: github.event_name == 'push' || !github.event.pull_request.head.repo.fork
+ permissions:
+ actions: write
+ steps:
+ - uses: actions/checkout@v4
+ - name: Setup Rust toolchain
+ uses: ./.github/actions/setup
+ with:
+ need-nextest: true
+
+ - name: Configure Cache Env
+ uses: actions/github-script@v7
+ with:
+ script: |
+ core.exportVariable('ACTIONS_RESULTS_URL', process.env.ACTIONS_RESULTS_URL || '');
+ core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');
+ core.exportVariable('ACTIONS_CACHE_SERVICE_V2', 'on');
- name: Test
shell: bash
diff --git a/core/Cargo.lock b/core/Cargo.lock
index aa15df52a..70c6fd59e 100644
--- a/core/Cargo.lock
+++ b/core/Cargo.lock
@@ -3240,6 +3240,15 @@ dependencies = [
"wasm-bindgen",
]
+[[package]]
+name = "ghac"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a10bd5b898cac1a4de4a882a754b2ccaafead449348cfb420b48cd5c00ffd08b"
+dependencies = [
+ "prost 0.13.4",
+]
+
[[package]]
name = "ghost"
version = "0.1.18"
@@ -5143,6 +5152,7 @@ dependencies = [
"foundationdb",
"futures",
"getrandom 0.2.15",
+ "ghac",
"governor",
"hdfs-native",
"hdrs",
diff --git a/core/Cargo.toml b/core/Cargo.toml
index 552029864..c40a2432c 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -148,7 +148,7 @@ services-gcs = [
"reqsign?/reqwest_request",
]
services-gdrive = ["internal-path-cache"]
-services-ghac = []
+services-ghac = ["dep:ghac", "dep:prost", "services-azblob"]
services-github = []
services-gridfs = ["dep:mongodb", "dep:mongodb-internal-macros"]
services-hdfs = ["dep:hdrs"]
@@ -247,6 +247,7 @@ http = "1.1"
log = "0.4"
md-5 = "0.10"
# TODO: remove once_cell when lazy_lock is stable: https://doc.rust-lang.org/std/cell/struct.LazyCell.html
+ghac = { version = "0.2.0", optional = true }
once_cell = "1"
percent-encoding = "2"
quick-xml = { version = "0.36", features = ["serialize", "overlapped-lists"] }
diff --git a/core/src/raw/http_util/mod.rs b/core/src/raw/http_util/mod.rs
index c90b1e485..4b3619b66 100644
--- a/core/src/raw/http_util/mod.rs
+++ b/core/src/raw/http_util/mod.rs
@@ -53,6 +53,7 @@ pub use header::parse_multipart_boundary;
pub use header::parse_prefixed_headers;
mod uri;
+pub use uri::new_http_uri_invalid_error;
pub use uri::percent_decode_path;
pub use uri::percent_encode_path;
diff --git a/core/src/raw/http_util/uri.rs b/core/src/raw/http_util/uri.rs
index 1f3b893e0..aeaaee5e9 100644
--- a/core/src/raw/http_util/uri.rs
+++ b/core/src/raw/http_util/uri.rs
@@ -15,11 +15,17 @@
// specific language governing permissions and limitations
// under the License.
+use crate::*;
use percent_encoding::percent_decode_str;
use percent_encoding::utf8_percent_encode;
use percent_encoding::AsciiSet;
use percent_encoding::NON_ALPHANUMERIC;
+/// Parse http uri invalid error in to opendal::Error.
+pub fn new_http_uri_invalid_error(err: http::uri::InvalidUri) -> Error {
+ Error::new(ErrorKind::Unexpected, "parse http uri").set_source(err)
+}
+
/// PATH_ENCODE_SET is the encode set for http url path.
///
/// This set follows [encodeURIComponent](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/encodeURIComponent) which will encode all non-ASCII characters except `A-Z a-z 0-9 - _ . ! ~ * ' ( )`
diff --git a/core/src/services/azblob/mod.rs b/core/src/services/azblob/mod.rs
index 4b571a40f..c95e8b90d 100644
--- a/core/src/services/azblob/mod.rs
+++ b/core/src/services/azblob/mod.rs
@@ -16,7 +16,7 @@
// under the License.
#[cfg(feature = "services-azblob")]
-mod core;
+pub(crate) mod core;
#[cfg(feature = "services-azblob")]
mod delete;
#[cfg(feature = "services-azblob")]
@@ -24,7 +24,7 @@ mod error;
#[cfg(feature = "services-azblob")]
mod lister;
#[cfg(feature = "services-azblob")]
-mod writer;
+pub(crate) mod writer;
#[cfg(feature = "services-azblob")]
mod backend;
diff --git a/core/src/services/ghac/backend.rs b/core/src/services/ghac/backend.rs
index 423bde57e..165931ef5 100644
--- a/core/src/services/ghac/backend.rs
+++ b/core/src/services/ghac/backend.rs
@@ -18,40 +18,19 @@
use std::env;
use std::sync::Arc;
-use bytes::Buf;
-use bytes::Bytes;
-use http::header;
-use http::header::ACCEPT;
-use http::header::AUTHORIZATION;
-use http::header::CONTENT_LENGTH;
-use http::header::CONTENT_RANGE;
-use http::header::CONTENT_TYPE;
-use http::Request;
-use http::Response;
-use http::StatusCode;
-use log::debug;
-use serde::Deserialize;
-use serde::Serialize;
-
+use super::core::*;
use super::error::parse_error;
use super::writer::GhacWriter;
use crate::raw::*;
+use crate::services::ghac::core::GhacCore;
use crate::services::GhacConfig;
use crate::*;
-
-/// The base url for cache url.
-const CACHE_URL_BASE: &str = "_apis/artifactcache";
-/// Cache API requires to provide an accept header.
-const CACHE_HEADER_ACCEPT: &str = "application/json;api-version=6.0-preview.1";
-/// The cache url env for ghac.
-///
-/// The url will be like `https://artifactcache.actions.githubusercontent.com/<id>/`
-const ACTIONS_CACHE_URL: &str = "ACTIONS_CACHE_URL";
-/// The runtime token env for ghac.
-///
-/// This token will be valid for 6h and github action will running for 6
-/// hours at most. So we don't need to refetch it again.
-const ACTIONS_RUNTIME_TOKEN: &str = "ACTIONS_RUNTIME_TOKEN";
+use http::header;
+use http::Request;
+use http::Response;
+use http::StatusCode;
+use log::debug;
+use sha2::Digest;
fn value_or_env(
explicit_value: Option<String>,
@@ -164,7 +143,33 @@ impl Builder for GhacBuilder {
let root = normalize_root(&self.config.root.unwrap_or_default());
debug!("backend use root {}", root);
- let client = if let Some(client) = self.http_client {
+ let service_version = get_cache_service_version();
+ debug!("backend use service version {:?}", service_version);
+
+ let mut version = self
+ .config
+ .version
+ .clone()
+ .unwrap_or_else(|| "opendal".to_string());
+ debug!("backend use version {version}");
+ // ghac requires to use hex digest of Sha256 as version.
+ if matches!(service_version, GhacVersion::V2) {
+ let hash = sha2::Sha256::digest(&version);
+ version = format!("{:x}", hash);
+ }
+
+ let cache_url = self
+ .config
+ .endpoint
+ .unwrap_or_else(|| get_cache_service_url(service_version));
+ if cache_url.is_empty() {
+ return Err(Error::new(
+ ErrorKind::ConfigInvalid,
+ "cache url for ghac not found, maybe not in github action environment?".to_string(),
+ ));
+ }
+
+ let http_client = if let Some(client) = self.http_client {
client
} else {
HttpClient::new().map_err(|err| {
@@ -173,39 +178,31 @@ impl Builder for GhacBuilder {
})?
};
- let backend = GhacBackend {
+ let core = GhacCore {
root,
- cache_url: value_or_env(self.config.endpoint, ACTIONS_CACHE_URL, "Builder::build")?,
+ cache_url,
catch_token: value_or_env(
self.config.runtime_token,
ACTIONS_RUNTIME_TOKEN,
"Builder::build",
)?,
- version: self
- .config
- .version
- .clone()
- .unwrap_or_else(|| "opendal".to_string()),
+ version,
- client,
+ service_version,
+ http_client,
};
- Ok(backend)
+ Ok(GhacBackend {
+ core: Arc::new(core),
+ })
}
}
/// Backend for github action cache services.
#[derive(Debug, Clone)]
pub struct GhacBackend {
- // root should end with "/"
- root: String,
-
- cache_url: String,
- catch_token: String,
- version: String,
-
- pub client: HttpClient,
+ core: Arc<GhacCore>,
}
impl Access for GhacBackend {
@@ -221,8 +218,8 @@ impl Access for GhacBackend {
fn info(&self) -> Arc<AccessorInfo> {
let mut am = AccessorInfo::default();
am.set_scheme(Scheme::Ghac)
- .set_root(&self.root)
- .set_name(&self.version)
+ .set_root(&self.core.root)
+ .set_name(&self.core.version)
.set_native_capability(Capability {
stat: true,
stat_has_cache_control: true,
@@ -253,24 +250,13 @@ impl Access for GhacBackend {
///
/// In this way, we can support both self-hosted GHES and `github.com`.
async fn stat(&self, path: &str, _: OpStat) -> Result<RpStat> {
- let req = self.ghac_query(path)?;
-
- let resp = self.client.send(req).await?;
-
- let location = if resp.status() == StatusCode::OK {
- let slc = resp.into_body();
- let query_resp: GhacQueryResponse =
- serde_json::from_reader(slc.reader()).map_err(new_json_deserialize_error)?;
- query_resp.archive_location
- } else {
- return Err(parse_error(resp));
- };
+ let location = self.core.ghac_get_download_url(path).await?;
let req = Request::get(location)
.header(header::RANGE, "bytes=0-0")
.body(Buffer::new())
.map_err(new_request_build_error)?;
- let resp = self.client.send(req).await?;
+ let resp = self.core.http_client.send(req).await?;
let status = resp.status();
match status {
@@ -291,21 +277,16 @@ impl Access for GhacBackend {
}
async fn read(&self, path: &str, args: OpRead) -> Result<(RpRead, Self::Reader)> {
- let req = self.ghac_query(path)?;
+ let location = self.core.ghac_get_download_url(path).await?;
- let resp = self.client.send(req).await?;
+ let mut req = Request::get(location);
- let location = if resp.status() == StatusCode::OK {
- let slc = resp.into_body();
- let query_resp: GhacQueryResponse =
- serde_json::from_reader(slc.reader()).map_err(new_json_deserialize_error)?;
- query_resp.archive_location
- } else {
- return Err(parse_error(resp));
- };
+ if !args.range().is_full() {
+ req = req.header(header::RANGE, args.range().to_header());
+ }
+ let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
- let req = self.ghac_get_location(&location, args.range())?;
- let resp = self.client.fetch(req).await?;
+ let resp = self.core.http_client.fetch(req).await?;
let status = resp.status();
match status {
@@ -321,145 +302,11 @@ impl Access for GhacBackend {
}
async fn write(&self, path: &str, _: OpWrite) -> Result<(RpWrite, Self::Writer)> {
- let req = self.ghac_reserve(path)?;
-
- let resp = self.client.send(req).await?;
-
- let cache_id = if resp.status().is_success() {
- let slc = resp.into_body();
- let reserve_resp: GhacReserveResponse =
- serde_json::from_reader(slc.reader()).map_err(new_json_deserialize_error)?;
- reserve_resp.cache_id
- } else {
- return Err(parse_error(resp).map(|err| err.with_operation("Backend::ghac_reserve")));
- };
-
- Ok((RpWrite::default(), GhacWriter::new(self.clone(), cache_id)))
- }
-}
-
-impl GhacBackend {
- fn ghac_query(&self, path: &str) -> Result<Request<Buffer>> {
- let p = build_abs_path(&self.root, path);
-
- let url = format!(
- "{}{CACHE_URL_BASE}/cache?keys={}&version={}",
- self.cache_url,
- percent_encode_path(&p),
- self.version
- );
+ let url = self.core.ghac_get_upload_url(path).await?;
- let mut req = Request::get(&url);
- req = req.header(AUTHORIZATION, format!("Bearer {}", self.catch_token));
- req = req.header(ACCEPT, CACHE_HEADER_ACCEPT);
-
- let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
-
- Ok(req)
- }
-
- pub fn ghac_get_location(&self, location: &str, range: BytesRange) -> Result<Request<Buffer>> {
- let mut req = Request::get(location);
-
- if !range.is_full() {
- req = req.header(header::RANGE, range.to_header());
- }
-
- req.body(Buffer::new()).map_err(new_request_build_error)
- }
-
- fn ghac_reserve(&self, path: &str) -> Result<Request<Buffer>> {
- let p = build_abs_path(&self.root, path);
-
- let url = format!("{}{CACHE_URL_BASE}/caches", self.cache_url);
-
- let bs = serde_json::to_vec(&GhacReserveRequest {
- key: p,
- version: self.version.to_string(),
- })
- .map_err(new_json_serialize_error)?;
-
- let mut req = Request::post(&url);
- req = req.header(AUTHORIZATION, format!("Bearer {}", self.catch_token));
- req = req.header(ACCEPT, CACHE_HEADER_ACCEPT);
- req = req.header(CONTENT_LENGTH, bs.len());
- req = req.header(CONTENT_TYPE, "application/json");
-
- let req = req
- .body(Buffer::from(Bytes::from(bs)))
- .map_err(new_request_build_error)?;
-
- Ok(req)
- }
-
- pub fn ghac_upload(
- &self,
- cache_id: i64,
- offset: u64,
- size: u64,
- body: Buffer,
- ) -> Result<Request<Buffer>> {
- let url = format!("{}{CACHE_URL_BASE}/caches/{cache_id}", self.cache_url);
-
- let mut req = Request::patch(&url);
- req = req.header(AUTHORIZATION, format!("Bearer {}", self.catch_token));
- req = req.header(ACCEPT, CACHE_HEADER_ACCEPT);
- req = req.header(CONTENT_LENGTH, size);
- req = req.header(CONTENT_TYPE, "application/octet-stream");
- req = req.header(
- CONTENT_RANGE,
- BytesContentRange::default()
- .with_range(offset, offset + size - 1)
- .to_header(),
- );
-
- let req = req.body(body).map_err(new_request_build_error)?;
-
- Ok(req)
+ Ok((
+ RpWrite::default(),
+ GhacWriter::new(self.core.clone(), path.to_string(), url)?,
+ ))
}
-
- pub fn ghac_commit(&self, cache_id: i64, size: u64) -> Result<Request<Buffer>> {
- let url = format!("{}{CACHE_URL_BASE}/caches/{cache_id}", self.cache_url);
-
- let bs =
- serde_json::to_vec(&GhacCommitRequest { size }).map_err(new_json_serialize_error)?;
-
- let mut req = Request::post(&url);
- req = req.header(AUTHORIZATION, format!("Bearer {}", self.catch_token));
- req = req.header(ACCEPT, CACHE_HEADER_ACCEPT);
- req = req.header(CONTENT_TYPE, "application/json");
- req = req.header(CONTENT_LENGTH, bs.len());
-
- let req = req
- .body(Buffer::from(Bytes::from(bs)))
- .map_err(new_request_build_error)?;
-
- Ok(req)
- }
-}
-
-#[derive(Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct GhacQueryResponse {
- // Not used fields.
- // cache_key: String,
- // scope: String,
- archive_location: String,
-}
-
-#[derive(Serialize)]
-struct GhacReserveRequest {
- key: String,
- version: String,
-}
-
-#[derive(Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct GhacReserveResponse {
- cache_id: i64,
-}
-
-#[derive(Serialize)]
-struct GhacCommitRequest {
- size: u64,
}
diff --git a/core/src/services/ghac/core.rs b/core/src/services/ghac/core.rs
new file mode 100644
index 000000000..27f7c4f9f
--- /dev/null
+++ b/core/src/services/ghac/core.rs
@@ -0,0 +1,385 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use super::error::parse_error;
+use crate::raw::{
+ build_abs_path, new_json_deserialize_error, new_json_serialize_error, new_request_build_error,
+ percent_encode_path, HttpClient,
+};
+use crate::*;
+use ::ghac::v1 as ghac_types;
+use bytes::{Buf, Bytes};
+use http::header::{ACCEPT, AUTHORIZATION, CONTENT_LENGTH, CONTENT_TYPE};
+use http::{Request, StatusCode, Uri};
+use prost::Message;
+use serde::{Deserialize, Serialize};
+use std::env;
+use std::fmt::{Debug, Formatter};
+use std::str::FromStr;
+
+/// The base url for cache url.
+pub const CACHE_URL_BASE: &str = "_apis/artifactcache";
+/// The base url for cache service v2.
+pub const CACHE_URL_BASE_V2: &str = "twirp/github.actions.results.api.v1.CacheService";
+/// Cache API requires to provide an accept header.
+pub const CACHE_HEADER_ACCEPT: &str = "application/json;api-version=6.0-preview.1";
+/// The cache url env for ghac.
+///
+/// The url will be like `https://artifactcache.actions.githubusercontent.com/<id>/`
+pub const ACTIONS_CACHE_URL: &str = "ACTIONS_CACHE_URL";
+/// The runtime token env for ghac.
+///
+/// This token will be valid for 6h and github action will running for 6
+/// hours at most. So we don't need to refetch it again.
+pub const ACTIONS_RUNTIME_TOKEN: &str = "ACTIONS_RUNTIME_TOKEN";
+/// The cache service version env for ghac.
+pub const ACTIONS_CACHE_SERVICE_V2: &str = "ACTIONS_CACHE_SERVICE_V2";
+/// The results url env for ghac.
+pub const ACTIONS_RESULTS_URL: &str = "ACTIONS_RESULTS_URL";
+/// The content type for protobuf.
+pub const CONTENT_TYPE_JSON: &str = "application/json";
+/// The content type for protobuf.
+pub const CONTENT_TYPE_PROTOBUF: &str = "application/protobuf";
+
+/// The version of github action cache.
+#[derive(Clone, Copy, Debug)]
+pub enum GhacVersion {
+ V1,
+ V2,
+}
+
+/// Core for github action cache services.
+#[derive(Clone)]
+pub struct GhacCore {
+ // root should end with "/"
+ pub root: String,
+
+ pub cache_url: String,
+ pub catch_token: String,
+ pub version: String,
+
+ pub service_version: GhacVersion,
+ pub http_client: HttpClient,
+}
+
+impl Debug for GhacCore {
+ fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+ f.debug_struct("GhacCore")
+ .field("root", &self.root)
+ .field("cache_url", &self.cache_url)
+ .field("version", &self.version)
+ .field("service_version", &self.service_version)
+ .finish_non_exhaustive()
+ }
+}
+
+impl GhacCore {
+ pub async fn ghac_get_download_url(&self, path: &str) -> Result<String> {
+ let p = build_abs_path(&self.root, path);
+
+ match self.service_version {
+ GhacVersion::V1 => {
+ let url = format!(
+ "{}{CACHE_URL_BASE}/cache?keys={}&version={}",
+ self.cache_url,
+ percent_encode_path(&p),
+ self.version
+ );
+
+ let mut req = Request::get(&url);
+ req = req.header(AUTHORIZATION, format!("Bearer {}", self.catch_token));
+ req = req.header(ACCEPT, CACHE_HEADER_ACCEPT);
+
+ let req = req.body(Buffer::new()).map_err(new_request_build_error)?;
+ let resp = self.http_client.send(req).await?;
+ let location = if resp.status() == StatusCode::OK {
+ let slc = resp.into_body();
+ let query_resp: GhacQueryResponse = serde_json::from_reader(slc.reader())
+ .map_err(new_json_deserialize_error)?;
+ query_resp.archive_location
+ } else {
+ return Err(parse_error(resp));
+ };
+ Ok(location)
+ }
+ GhacVersion::V2 => {
+ let url = format!(
+ "{}{CACHE_URL_BASE_V2}/GetCacheEntryDownloadURL",
+ self.cache_url,
+ );
+
+ let req = ghac_types::GetCacheEntryDownloadUrlRequest {
+ key: p,
+ version: self.version.clone(),
+
+ metadata: None,
+ restore_keys: vec![],
+ };
+
+ let body = Buffer::from(req.encode_to_vec());
+
+ let req = Request::post(&url)
+ .header(AUTHORIZATION, format!("Bearer {}", self.catch_token))
+ .header(CONTENT_TYPE, CONTENT_TYPE_PROTOBUF)
+ .header(CONTENT_LENGTH, body.len())
+ .body(body)
+ .map_err(new_request_build_error)?;
+ let resp = self.http_client.send(req).await?;
+ let location = if resp.status() == StatusCode::OK {
+ let slc = resp.into_body();
+ let query_resp = ghac_types::GetCacheEntryDownloadUrlResponse::decode(slc)
+ .map_err(new_prost_decode_error)?;
+ if !query_resp.ok {
+ let mut err = Error::new(
+ ErrorKind::NotFound,
+ "GetCacheEntryDownloadURL returns non-ok, the key doesn't exist",
+ );
+
+ // GHAC is a cache service, so it's acceptable for it to occasionally not contain
+ // data that users have just written. However, we don't want users to always have
+ // to retry reading it, nor do we want our CI to fail constantly.
+ //
+ // Here's the trick: we check if the environment variable `OPENDAL_TEST` is set to `ghac`.
+ // If it is, we mark the error as temporary to allow retries in the test CI.
+ if env::var("OPENDAL_TEST") == Ok("ghac".to_string()) {
+ err = err.set_temporary();
+ }
+ return Err(err);
+ }
+ query_resp.signed_download_url
+ } else {
+ return Err(parse_error(resp));
+ };
+
+ Ok(location)
+ }
+ }
+ }
+
+ pub async fn ghac_get_upload_url(&self, path: &str) -> Result<String> {
+ let p = build_abs_path(&self.root, path);
+
+ match self.service_version {
+ GhacVersion::V1 => {
+ let url = format!("{}{CACHE_URL_BASE}/caches", self.cache_url);
+
+ let bs = serde_json::to_vec(&GhacReserveRequest {
+ key: p,
+ version: self.version.to_string(),
+ })
+ .map_err(new_json_serialize_error)?;
+
+ let mut req = Request::post(&url);
+ req = req.header(AUTHORIZATION, format!("Bearer {}", self.catch_token));
+ req = req.header(ACCEPT, CACHE_HEADER_ACCEPT);
+ req = req.header(CONTENT_TYPE, CONTENT_TYPE_JSON);
+ req = req.header(CONTENT_LENGTH, bs.len());
+
+ let req = req
+ .body(Buffer::from(Bytes::from(bs)))
+ .map_err(new_request_build_error)?;
+ let resp = self.http_client.send(req).await?;
+ let cache_id = if resp.status().is_success() {
+ let slc = resp.into_body();
+ let reserve_resp: GhacReserveResponse = serde_json::from_reader(slc.reader())
+ .map_err(new_json_deserialize_error)?;
+ reserve_resp.cache_id
+ } else {
+ return Err(
+ parse_error(resp).map(|err| err.with_operation("Backend::ghac_reserve"))
+ );
+ };
+
+ let url = format!("{}{CACHE_URL_BASE}/caches/{cache_id}", self.cache_url);
+ Ok(url)
+ }
+ GhacVersion::V2 => {
+ let url = format!("{}{CACHE_URL_BASE_V2}/CreateCacheEntry", self.cache_url,);
+
+ let req = ghac_types::CreateCacheEntryRequest {
+ key: p,
+ version: self.version.clone(),
+ metadata: None,
+ };
+
+ let body = Buffer::from(req.encode_to_vec());
+
+ let req = Request::post(&url)
+ .header(AUTHORIZATION, format!("Bearer {}", self.catch_token))
+ .header(CONTENT_TYPE, CONTENT_TYPE_PROTOBUF)
+ .header(CONTENT_LENGTH, body.len())
+ .body(body)
+ .map_err(new_request_build_error)?;
+ let resp = self.http_client.send(req).await?;
+ let location = if resp.status() == StatusCode::OK {
+ let (parts, slc) = resp.into_parts();
+ let query_resp = ghac_types::CreateCacheEntryResponse::decode(slc)
+ .map_err(new_prost_decode_error)?;
+ if !query_resp.ok {
+ return Err(Error::new(
+ ErrorKind::Unexpected,
+ "create cache entry returns non-ok",
+ )
+ .with_context("parts", format!("{:?}", parts)));
+ }
+ query_resp.signed_upload_url
+ } else {
+ return Err(parse_error(resp));
+ };
+ Ok(location)
+ }
+ }
+ }
+
+ pub async fn ghac_finalize_upload(&self, path: &str, url: &str, size: u64) -> Result<()> {
+ let p = build_abs_path(&self.root, path);
+
+ match self.service_version {
+ GhacVersion::V1 => {
+ let bs = serde_json::to_vec(&GhacCommitRequest { size })
+ .map_err(new_json_serialize_error)?;
+
+ let req = Request::post(url)
+ .header(AUTHORIZATION, format!("Bearer {}", self.catch_token))
+ .header(ACCEPT, CACHE_HEADER_ACCEPT)
+ .header(CONTENT_TYPE, CONTENT_TYPE_JSON)
+ .header(CONTENT_LENGTH, bs.len())
+ .body(Buffer::from(bs))
+ .map_err(new_request_build_error)?;
+ let resp = self.http_client.send(req).await?;
+ if resp.status().is_success() {
+ Ok(())
+ } else {
+ Err(parse_error(resp))
+ }
+ }
+ GhacVersion::V2 => {
+ let url = format!(
+ "{}{CACHE_URL_BASE_V2}/FinalizeCacheEntryUpload",
+ self.cache_url,
+ );
+
+ let req = ghac_types::FinalizeCacheEntryUploadRequest {
+ key: p,
+ version: self.version.clone(),
+ size_bytes: size as i64,
+
+ metadata: None,
+ };
+ let body = Buffer::from(req.encode_to_vec());
+
+ let req = Request::post(&url)
+ .header(AUTHORIZATION, format!("Bearer {}", self.catch_token))
+ .header(CONTENT_TYPE, CONTENT_TYPE_PROTOBUF)
+ .header(CONTENT_LENGTH, body.len())
+ .body(body)
+ .map_err(new_request_build_error)?;
+ let resp = self.http_client.send(req).await?;
+ if resp.status() != StatusCode::OK {
+ return Err(parse_error(resp));
+ };
+ Ok(())
+ }
+ }
+ }
+}
+
+/// Determines if the current environment is GitHub Enterprise Server (GHES)
+///
+/// We need to know this since GHES doesn't support ghac v2 yet.
+pub fn is_ghes() -> bool {
+ // Fetch GitHub Server URL with fallback to "https://github.com"
+ let server_url =
+ env::var("GITHUB_SERVER_URL").unwrap_or_else(|_| "https://github.com".to_string());
+
+ let Ok(url) = Uri::from_str(&server_url) else {
+ // We just return false if the URL is invalid
+ return false;
+ };
+
+ // Check against known non-GHES host patterns
+ let hostname = url.host().unwrap_or("").trim_end().to_lowercase();
+
+ let is_github_host = hostname == "github.com";
+ let is_ghe_host = hostname.ends_with(".ghe.com");
+ let is_localhost = hostname.ends_with(".localhost");
+
+ !is_github_host && !is_ghe_host && !is_localhost
+}
+
+/// Determines the cache service version based on environment
+pub fn get_cache_service_version() -> GhacVersion {
+ if is_ghes() {
+ // GHES only supports v1 regardless of feature flags
+ GhacVersion::V1
+ } else {
+ // Check for presence of non-empty ACTIONS_CACHE_SERVICE_V2
+ let value = env::var(ACTIONS_CACHE_SERVICE_V2).unwrap_or_default();
+ if value.is_empty() {
+ GhacVersion::V1
+ } else {
+ GhacVersion::V2
+ }
+ }
+}
+
+/// Returns the appropriate cache service URL based on version
+pub fn get_cache_service_url(version: GhacVersion) -> String {
+ match version {
+ GhacVersion::V1 => {
+ // Priority order for v1: CACHE_URL -> RESULTS_URL
+ env::var(ACTIONS_CACHE_URL)
+ .or_else(|_| env::var(ACTIONS_RESULTS_URL))
+ .unwrap_or_default()
+ }
+ GhacVersion::V2 => {
+ // Only RESULTS_URL is used for v2
+ env::var(ACTIONS_RESULTS_URL).unwrap_or_default()
+ }
+ }
+}
+
+/// Parse prost decode error into opendal::Error.
+pub fn new_prost_decode_error(e: prost::DecodeError) -> Error {
+ Error::new(ErrorKind::Unexpected, "deserialize protobuf").set_source(e)
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct GhacQueryResponse {
+ // Not used fields.
+ // cache_key: String,
+ // scope: String,
+ pub archive_location: String,
+}
+
+#[derive(Serialize)]
+pub struct GhacReserveRequest {
+ pub key: String,
+ pub version: String,
+}
+
+#[derive(Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub struct GhacReserveResponse {
+ pub cache_id: i64,
+}
+
+#[derive(Serialize)]
+pub struct GhacCommitRequest {
+ pub size: u64,
+}
diff --git a/core/src/services/ghac/error.rs b/core/src/services/ghac/error.rs
index 7efafe1f6..bc5ce64c0 100644
--- a/core/src/services/ghac/error.rs
+++ b/core/src/services/ghac/error.rs
@@ -15,11 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-use http::Response;
-use http::StatusCode;
-
use crate::raw::*;
use crate::*;
+use http::Response;
+use http::StatusCode;
/// Parse error response into Error.
pub(super) fn parse_error(resp: Response<Buffer>) -> Error {
diff --git a/core/src/services/ghac/mod.rs b/core/src/services/ghac/mod.rs
index fabc5bcc1..878f4c8c8 100644
--- a/core/src/services/ghac/mod.rs
+++ b/core/src/services/ghac/mod.rs
@@ -25,5 +25,8 @@ mod backend;
#[cfg(feature = "services-ghac")]
pub use backend::GhacBuilder as Ghac;
+#[cfg(feature = "services-ghac")]
+mod core;
+
mod config;
pub use config::GhacConfig;
diff --git a/core/src/services/ghac/writer.rs b/core/src/services/ghac/writer.rs
index c5d49a17a..797bda7b9 100644
--- a/core/src/services/ghac/writer.rs
+++ b/core/src/services/ghac/writer.rs
@@ -15,47 +15,119 @@
// specific language governing permissions and limitations
// under the License.
-use super::backend::GhacBackend;
+use super::core::*;
use super::error::parse_error;
use crate::raw::*;
+use crate::services::core::AzblobCore;
+use crate::services::writer::AzblobWriter;
use crate::*;
+use http::header::{ACCEPT, AUTHORIZATION, CONTENT_LENGTH, CONTENT_RANGE, CONTENT_TYPE};
+use http::Request;
+use std::str::FromStr;
+use std::sync::Arc;
-pub struct GhacWriter {
- backend: GhacBackend,
-
- cache_id: i64,
- size: u64,
-}
+pub type GhacWriter = TwoWays<GhacWriterV1, GhacWriterV2>;
impl GhacWriter {
- pub fn new(backend: GhacBackend, cache_id: i64) -> Self {
- GhacWriter {
- backend,
- cache_id,
- size: 0,
+ /// TODO: maybe we can move the signed url logic to azblob service instead.
+ pub fn new(core: Arc<GhacCore>, write_path: String, url: String) -> Result<Self> {
+ match core.service_version {
+ GhacVersion::V1 => Ok(TwoWays::One(GhacWriterV1 {
+ core,
+ path: write_path,
+ url,
+ size: 0,
+ })),
+ GhacVersion::V2 => {
+ let uri = http::Uri::from_str(&url)
+ .map_err(new_http_uri_invalid_error)?
+ .into_parts();
+ let (Some(scheme), Some(authority), Some(pq)) =
+ (uri.scheme, uri.authority, uri.path_and_query)
+ else {
+ return Err(Error::new(
+ ErrorKind::Unexpected,
+ "ghac returns invalid signed url",
+ )
+ .with_context("url", &url));
+ };
+ let endpoint = format!("{scheme}://{authority}");
+ let Some((container, path)) = pq.path().trim_matches('/').split_once("/") else {
+ return Err(Error::new(
+ ErrorKind::Unexpected,
+ "ghac returns invalid signed url that bucket or path is missing",
+ )
+ .with_context("url", &url));
+ };
+ let Some(query) = pq.query() else {
+ return Err(Error::new(
+ ErrorKind::Unexpected,
+ "ghac returns invalid signed url that sas is missing",
+ )
+ .with_context("url", &url));
+ };
+ let azure_core = Arc::new(AzblobCore {
+ container: container.to_string(),
+ root: "/".to_string(),
+ endpoint,
+ encryption_key: None,
+ encryption_key_sha256: None,
+ encryption_algorithm: None,
+ client: core.http_client.clone(),
+ loader: {
+ let config = reqsign::AzureStorageConfig {
+ sas_token: Some(query.to_string()),
+ ..Default::default()
+ };
+ reqsign::AzureStorageLoader::new(config)
+ },
+ signer: { reqsign::AzureStorageSigner::new() },
+ });
+ let w = AzblobWriter::new(azure_core, OpWrite::default(), path.to_string());
+ let writer = oio::BlockWriter::new(w, None, 4);
+ Ok(TwoWays::Two(GhacWriterV2 {
+ core,
+ writer,
+ path: write_path,
+ url,
+ size: 0,
+ }))
+ }
}
}
}
-impl oio::Write for GhacWriter {
+pub struct GhacWriterV1 {
+ core: Arc<GhacCore>,
+
+ path: String,
+ url: String,
+ size: u64,
+}
+
+impl oio::Write for GhacWriterV1 {
async fn write(&mut self, bs: Buffer) -> Result<()> {
- let size = bs.len();
+ let size = bs.len() as u64;
let offset = self.size;
- let req = self.backend.ghac_upload(
- self.cache_id,
- offset,
- size as u64,
- Buffer::from(bs.to_bytes()),
- )?;
-
- let resp = self.backend.client.send(req).await?;
+ let mut req = Request::patch(&self.url);
+ req = req.header(AUTHORIZATION, format!("Bearer {}", self.core.catch_token));
+ req = req.header(ACCEPT, CACHE_HEADER_ACCEPT);
+ req = req.header(CONTENT_LENGTH, size);
+ req = req.header(CONTENT_TYPE, "application/octet-stream");
+ req = req.header(
+ CONTENT_RANGE,
+ BytesContentRange::default()
+ .with_range(offset, offset + size - 1)
+ .to_header(),
+ );
+ let req = req.body(bs).map_err(new_request_build_error)?;
+ let resp = self.core.http_client.send(req).await?;
if !resp.status().is_success() {
return Err(parse_error(resp).map(|err| err.with_operation("Backend::ghac_upload")));
}
-
- self.size += size as u64;
+ self.size += size;
Ok(())
}
@@ -64,13 +136,38 @@ impl oio::Write for GhacWriter {
}
async fn close(&mut self) -> Result<()> {
- let req = self.backend.ghac_commit(self.cache_id, self.size)?;
- let resp = self.backend.client.send(req).await?;
+ self.core
+ .ghac_finalize_upload(&self.path, &self.url, self.size)
+ .await
+ }
+}
- if resp.status().is_success() {
- Ok(())
- } else {
- Err(parse_error(resp).map(|err| err.with_operation("Backend::ghac_commit")))
- }
+pub struct GhacWriterV2 {
+ core: Arc<GhacCore>,
+ writer: oio::BlockWriter<AzblobWriter>,
+
+ path: String,
+ url: String,
+ size: u64,
+}
+
+impl oio::Write for GhacWriterV2 {
+ async fn write(&mut self, bs: Buffer) -> Result<()> {
+ let size = bs.len() as u64;
+
+ self.writer.write(bs).await?;
+ self.size += size;
+ Ok(())
+ }
+
+ async fn close(&mut self) -> Result<()> {
+ self.writer.close().await?;
+ self.core
+ .ghac_finalize_upload(&self.path, &self.url, self.size)
+ .await
+ }
+
+ async fn abort(&mut self) -> Result<()> {
+ Ok(())
}
}