You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iggy.apache.org by pi...@apache.org on 2025/06/11 18:47:02 UTC

(iggy) branch connectors_api updated (75b7dfdc -> 4337c096)

This is an automated email from the ASF dual-hosted git repository.

piotr pushed a change to branch connectors_api
in repository https://gitbox.apache.org/repos/asf/iggy.git


 discard 75b7dfdc add initial version of connectors http api
     new 4337c096 add initial version of connectors http api

This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version.  This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:

 * -- * -- B -- O -- O -- O   (75b7dfdc)
            \
             N -- N -- N   refs/heads/connectors_api (4337c096)

You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.

Any revisions marked "omit" are not gone; other references still
refer to them.  Any revisions marked "discard" are gone forever.

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "add" were already present in the repository and have only
been added to this reference.


Summary of changes:
 core/connectors/runtime/README.md             |  4 +++-
 core/connectors/runtime/config.toml           |  8 ++++----
 core/connectors/runtime/src/api/models.rs     | 23 ++++++++++++++++++++---
 core/connectors/runtime/src/api/sink.rs       | 26 +++++++++++++++++++++++++-
 core/connectors/runtime/src/api/source.rs     | 26 +++++++++++++++++++++++++-
 core/connectors/runtime/src/configs.rs        | 12 ++++++------
 core/connectors/runtime/src/context.rs        |  4 ++++
 core/connectors/runtime/src/manager/sink.rs   |  4 +++-
 core/connectors/runtime/src/manager/source.rs |  4 +++-
 core/connectors/runtime/src/sink.rs           | 21 ++++++++++++---------
 core/connectors/runtime/src/source.rs         |  7 ++++---
 core/connectors/runtime/src/transform.rs      | 19 +++++++++----------
 core/connectors/sdk/src/transforms/mod.rs     | 11 ++++++++---
 core/connectors/sinks/README.md               |  2 +-
 core/connectors/sources/README.md             |  4 ++--
 15 files changed, 129 insertions(+), 46 deletions(-)


(iggy) 01/01: add initial version of connectors http api

Posted by pi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.

piotr pushed a commit to branch connectors_api
in repository https://gitbox.apache.org/repos/asf/iggy.git

commit 4337c096d81974f2ba9233e46447e7dea2f3a927
Author: spetz <pi...@gmail.com>
AuthorDate: Tue Jun 10 20:31:04 2025 +0200

    add initial version of connectors http api
---
 Cargo.lock                                       |   6 ++
 core/connectors/README.md                        |   3 +-
 core/connectors/runtime/Cargo.toml               |   6 ++
 core/connectors/runtime/README.md                |  38 +++++++
 core/connectors/runtime/config.toml              |  38 +++++--
 core/connectors/runtime/src/api/auth.rs          |  63 +++++++++++
 core/connectors/runtime/src/api/config.rs        | 131 +++++++++++++++++++++++
 core/connectors/runtime/src/api/error.rs         |  74 +++++++++++++
 core/connectors/runtime/src/api/mod.rs           | 110 +++++++++++++++++++
 core/connectors/runtime/src/api/models.rs        | 120 +++++++++++++++++++++
 core/connectors/runtime/src/api/sink.rs          | 113 +++++++++++++++++++
 core/connectors/runtime/src/api/source.rs        | 113 +++++++++++++++++++
 core/connectors/runtime/src/configs.rs           |  56 +++++++---
 core/connectors/runtime/src/context.rs           | 105 ++++++++++++++++++
 core/connectors/runtime/src/error.rs             |  20 ++++
 core/connectors/runtime/src/main.rs              |  30 ++++--
 core/connectors/runtime/src/manager/mod.rs       |  20 ++++
 core/connectors/runtime/src/manager/sink.rs      |  67 ++++++++++++
 core/connectors/runtime/src/manager/source.rs    |  67 ++++++++++++
 core/connectors/runtime/src/sink.rs              |  29 +++--
 core/connectors/runtime/src/source.rs            |  15 ++-
 core/connectors/runtime/src/transform.rs         |  19 ++--
 core/connectors/sdk/src/sink.rs                  |   2 +
 core/connectors/sdk/src/source.rs                |   2 +
 core/connectors/sdk/src/transforms/mod.rs        |  11 +-
 core/connectors/sinks/README.md                  |   4 +-
 core/connectors/sinks/quickwit_sink/src/lib.rs   |   4 +-
 core/connectors/sources/README.md                |   6 +-
 core/connectors/sources/random_source/src/lib.rs |   7 +-
 29 files changed, 1212 insertions(+), 67 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 76292c4b..d4944b74 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3859,6 +3859,8 @@ dependencies = [
 name = "iggy_connector_runtime"
 version = "0.1.0"
 dependencies = [
+ "axum 0.8.4",
+ "axum-server",
  "config",
  "dashmap",
  "dlopen2",
@@ -3871,8 +3873,12 @@ dependencies = [
  "postcard",
  "serde",
  "serde_json",
+ "serde_yml",
+ "strum",
  "thiserror 2.0.12",
  "tokio",
+ "toml",
+ "tower-http",
  "tracing",
  "tracing-subscriber",
 ]
diff --git a/core/connectors/README.md b/core/connectors/README.md
index 68051e50..a9a1b319 100644
--- a/core/connectors/README.md
+++ b/core/connectors/README.md
@@ -25,7 +25,8 @@ The highly performant and modular runtime for statically typed, yet dynamically
 4. Start the Iggy server and invoke the following commands via Iggy CLI to create the example streams and topics used by the sample connectors.
 
 ```
-iggy --username iggy --password iggy stream create example
+iggy --username iggy --password iggy stream create example_stream
+iggy --username iggy --password iggy topic create example_stream example_topic 1 none 1d
 iggy --username iggy --password iggy stream create qw
 iggy --username iggy --password iggy topic create qw records 1 none 1d
 ```
diff --git a/core/connectors/runtime/Cargo.toml b/core/connectors/runtime/Cargo.toml
index a975d0ae..a8e3de60 100644
--- a/core/connectors/runtime/Cargo.toml
+++ b/core/connectors/runtime/Cargo.toml
@@ -29,6 +29,8 @@ repository = "https://github.com/apache/iggy"
 readme = "../../README.md"
 
 [dependencies]
+axum = { workspace = true }
+axum-server = { workspace = true }
 config = { workspace = true }
 dashmap = { workspace = true }
 dlopen2 = { workspace = true }
@@ -41,7 +43,11 @@ once_cell = { workspace = true }
 postcard = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
+serde_yml = { workspace = true }
+strum = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true }
+toml = { workspace = true }
+tower-http = { workspace = true }
 tracing = { workspace = true }
 tracing-subscriber = { workspace = true }
diff --git a/core/connectors/runtime/README.md b/core/connectors/runtime/README.md
index 23a35c11..b502bf67 100644
--- a/core/connectors/runtime/README.md
+++ b/core/connectors/runtime/README.md
@@ -23,3 +23,41 @@ password = "iggy"
 All the other config sections start either with `sources` or `sinks` depending on the connector type.
 
 Keep in mind that either of `toml`, `yaml`, or `json` formats are supported for the configuration file. The path to the configuration can be overriden by `IGGY_CONNECTORS_RUNTIME_CONFIG_PATH` environment variable. Each configuration section can be also additionally updated by using the following convention `IGGY_CONNECTORS_SECTION_NAME.KEY_NAME` e.g. `IGGY_CONNECTORS_IGGY_USERNAME` and so on.
+
+## HTTP API
+
+Connector runtime has an optional HTTP API that can be enabled by setting the `enabled` flag to `true` in the `[http_api]` section.
+
+```toml
+[http_api] # Optional HTTP API configuration
+enabled = true
+address = "127.0.0.1:8081"
+# api_key = "secret" # Optional API key for authentication to be passed as `api-key` header
+
+[http_api.cors] # Optional CORS configuration for HTTP API
+enabled = false
+allowed_methods = ["GET", "POST", "PUT", "DELETE"]
+allowed_origins = ["*"]
+allowed_headers = ["content-type"]
+exposed_headers = [""]
+allow_credentials = false
+allow_private_network = false
+
+[http_api.tls] # Optional TLS configuration for HTTP API
+enabled = false
+cert_file = "certs/iggy_cert.pem"
+key_file = "certs/iggy_key.pem"
+```
+
+Currently, it does expose the following endpoints:
+
+- `GET /`: welcome message.
+- `GET /health`: health status of the runtime.
+- `GET /sinks`: list of sinks.
+- `GET /sinks/{key}`: sink details.
+- `GET /sinks/{key}/config`: sink config, including the optional `format` query parameter to specify the config format.
+- `GET /sinks/{key}/transforms`: sink transforms to be applied to the fields.
+- `GET /sources`: list of sources.
+- `GET /sources/{key}`: source details.
+- `GET /sources/{key}/config`: source config, including the optional `format` query parameter to specify the config format.
+- `GET /sources/{key}/transforms`: source transforms to be applied to the fields.
diff --git a/core/connectors/runtime/config.toml b/core/connectors/runtime/config.toml
index d7361472..171e3a32 100644
--- a/core/connectors/runtime/config.toml
+++ b/core/connectors/runtime/config.toml
@@ -15,6 +15,25 @@
 # specific language governing permissions and limitations
 # under the License.
 
+[http_api] # Optional HTTP API configuration
+enabled = true
+address = "127.0.0.1:8081"
+# api_key = "secret" # Optional API key for authentication to be passed as `api-key` header
+
+[http_api.cors] # Optional CORS configuration for HTTP API
+enabled = false
+allowed_methods = ["GET", "POST", "PUT", "DELETE"]
+allowed_origins = ["*"]
+allowed_headers = ["content-type"]
+exposed_headers = [""]
+allow_credentials = false
+allow_private_network = false
+
+[http_api.tls] # Optional TLS configuration for HTTP API
+enabled = false
+cert_file = "certs/iggy_cert.pem"
+key_file = "certs/iggy_key.pem"
+
 [iggy]
 address = "localhost:8090"
 username = "iggy"
@@ -30,9 +49,12 @@ path = "target/release/libiggy_connector_stdout_sink"
 stream = "example_stream"
 topics = ["example_topic"]
 schema = "json"
-batch_size = 100
+batch_length = 100
 poll_interval = "5ms"
-consumer_group = "qw_sink_connector"
+consumer_group = "stdout_sink_connector"
+
+[sinks.stdout.config]
+print_payload = false
 
 [sinks.stdout.transforms.add_fields]
 enabled = true
@@ -45,13 +67,14 @@ value.static = "hello"
 enabled = true
 name = "Random source"
 path = "target/release/libiggy_connector_random_source"
+config_format = "json"
 
 [[sources.random.streams]]
-stream = "example"
-topic = "topic1"
+stream = "example_stream"
+topic = "example_topic"
 schema = "json"
-batch_size = 1000
-send_interval = "5ms"
+batch_length = 1000
+linger_time = "5ms"
 
 [sources.random.config]
 interval = "100ms"
@@ -70,12 +93,13 @@ value.static = "hello!"
 enabled = true
 name = "Quickwit sink 1"
 path = "target/release/libiggy_connector_quickwit_sink"
+config_format = "yaml"
 
 [[sinks.quickwit.streams]]
 stream = "qw"
 topics = ["records"]
 schema = "json"
-batch_size = 1000
+batch_length = 1000
 poll_interval = "5ms"
 consumer_group = "qw_sink_connector"
 
diff --git a/core/connectors/runtime/src/api/auth.rs b/core/connectors/runtime/src/api/auth.rs
new file mode 100644
index 00000000..8fb84a3d
--- /dev/null
+++ b/core/connectors/runtime/src/api/auth.rs
@@ -0,0 +1,63 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::context::RuntimeContext;
+use axum::body::Body;
+use axum::extract::State;
+use axum::{
+    http::{Request, StatusCode},
+    middleware::Next,
+    response::Response,
+};
+use std::sync::Arc;
+
+const API_KEY_HEADER: &str = "api-key";
+
+const PUBLIC_PATHS: &[&str] = &["/", "/health"];
+
+pub async fn resolve_api_key(
+    State(context): State<Arc<RuntimeContext>>,
+    request: Request<Body>,
+    next: Next,
+) -> Result<Response, StatusCode> {
+    if PUBLIC_PATHS.contains(&request.uri().path()) {
+        return Ok(next.run(request).await);
+    }
+
+    let Some(expected_api_key) = context.api_key.as_ref() else {
+        return Ok(next.run(request).await);
+    };
+
+    if expected_api_key.is_empty() {
+        return Ok(next.run(request).await);
+    }
+
+    let Some(api_key) = request
+        .headers()
+        .get(API_KEY_HEADER)
+        .and_then(|value| value.to_str().ok())
+    else {
+        return Err(StatusCode::UNAUTHORIZED);
+    };
+
+    if api_key != expected_api_key {
+        return Err(StatusCode::UNAUTHORIZED);
+    }
+
+    Ok(next.run(request).await)
+}
diff --git a/core/connectors/runtime/src/api/config.rs b/core/connectors/runtime/src/api/config.rs
new file mode 100644
index 00000000..8337495e
--- /dev/null
+++ b/core/connectors/runtime/src/api/config.rs
@@ -0,0 +1,131 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::{configs::ConfigFormat, error::RuntimeError};
+use axum::http::{HeaderValue, Method};
+use serde::{Deserialize, Serialize};
+use tower_http::cors::{AllowOrigin, CorsLayer};
+use tracing::error;
+
+pub const JSON_HEADER: HeaderValue = HeaderValue::from_static("application/json");
+pub const YAML_HEADER: HeaderValue = HeaderValue::from_static("application/yaml");
+pub const TOML_HEADER: HeaderValue = HeaderValue::from_static("application/toml");
+pub const TEXT_HEADER: HeaderValue = HeaderValue::from_static("text/plain");
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
+pub struct HttpApiConfig {
+    pub enabled: bool,
+    pub address: String,
+    pub api_key: Option<String>,
+    pub cors: Option<HttpCorsConfig>,
+    pub tls: Option<HttpTlsConfig>,
+}
+
+#[derive(Debug, Deserialize, Serialize, Clone)]
+pub struct HttpCorsConfig {
+    pub enabled: bool,
+    pub allowed_methods: Vec<String>,
+    pub allowed_origins: Vec<String>,
+    pub allowed_headers: Vec<String>,
+    pub exposed_headers: Vec<String>,
+    pub allow_credentials: bool,
+    pub allow_private_network: bool,
+}
+
+#[derive(Debug, Deserialize, Serialize, Clone)]
+pub struct HttpTlsConfig {
+    pub enabled: bool,
+    pub cert_file: String,
+    pub key_file: String,
+}
+
+pub fn map_connector_config(
+    config: &serde_json::Value,
+    format: ConfigFormat,
+) -> Result<(HeaderValue, String), RuntimeError> {
+    match format {
+        ConfigFormat::Json => Ok((JSON_HEADER, config.to_string())),
+        ConfigFormat::Yaml => {
+            let config = serde_yml::to_value(config).map_err(|error| {
+                error!("Failed to convert configuration to YAML. {error}");
+                RuntimeError::CannotConvertConfiguration
+            })?;
+            let config = serde_yml::to_string(&config).map_err(|error| {
+                error!("Failed to serialize YAML configuration. {error}");
+                RuntimeError::CannotConvertConfiguration
+            })?;
+            Ok((YAML_HEADER, config))
+        }
+        ConfigFormat::Toml => {
+            let config = toml::to_string(config).map_err(|error| {
+                error!("Failed to convert configuration to TOML. {error}");
+                RuntimeError::CannotConvertConfiguration
+            })?;
+            Ok((TOML_HEADER, config))
+        }
+        ConfigFormat::Text => Ok((TEXT_HEADER, config.to_string())),
+    }
+}
+
+pub fn configure_cors(config: &HttpCorsConfig) -> CorsLayer {
+    let allowed_origins = match &config.allowed_origins {
+        origins if origins.is_empty() => AllowOrigin::default(),
+        origins if origins.first().unwrap() == "*" => AllowOrigin::any(),
+        origins => AllowOrigin::list(origins.iter().map(|s| s.parse().unwrap())),
+    };
+
+    let allowed_headers = config
+        .allowed_headers
+        .iter()
+        .filter(|s| !s.is_empty())
+        .map(|s| s.parse().unwrap())
+        .collect::<Vec<_>>();
+
+    let exposed_headers = config
+        .exposed_headers
+        .iter()
+        .filter(|s| !s.is_empty())
+        .map(|s| s.parse().unwrap())
+        .collect::<Vec<_>>();
+
+    let allowed_methods = config
+        .allowed_methods
+        .iter()
+        .filter(|s| !s.is_empty())
+        .map(|s| match s.to_uppercase().as_str() {
+            "GET" => Method::GET,
+            "POST" => Method::POST,
+            "PUT" => Method::PUT,
+            "DELETE" => Method::DELETE,
+            "HEAD" => Method::HEAD,
+            "OPTIONS" => Method::OPTIONS,
+            "CONNECT" => Method::CONNECT,
+            "PATCH" => Method::PATCH,
+            "TRACE" => Method::TRACE,
+            _ => panic!("Invalid HTTP method: {}", s),
+        })
+        .collect::<Vec<_>>();
+
+    CorsLayer::new()
+        .allow_methods(allowed_methods)
+        .allow_origin(allowed_origins)
+        .allow_headers(allowed_headers)
+        .expose_headers(exposed_headers)
+        .allow_credentials(config.allow_credentials)
+        .allow_private_network(config.allow_private_network)
+}
diff --git a/core/connectors/runtime/src/api/error.rs b/core/connectors/runtime/src/api/error.rs
new file mode 100644
index 00000000..b14d301a
--- /dev/null
+++ b/core/connectors/runtime/src/api/error.rs
@@ -0,0 +1,74 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use axum::{Json, http::StatusCode, response::IntoResponse};
+use serde::Serialize;
+use thiserror::Error;
+use tracing::error;
+
+use crate::error::RuntimeError;
+
+#[derive(Debug, Error)]
+pub enum ApiError {
+    #[error(transparent)]
+    Error(#[from] RuntimeError),
+    #[error(transparent)]
+    JsonError(#[from] serde_json::Error),
+}
+
+#[derive(Debug, Serialize)]
+pub struct ErrorResponse {
+    pub code: String,
+    pub reason: String,
+}
+
+impl IntoResponse for ApiError {
+    fn into_response(self) -> axum::response::Response {
+        match self {
+            ApiError::Error(error) => {
+                error!("There was an error: {error}");
+                let status_code = match error {
+                    RuntimeError::MissingIggyCredentials => StatusCode::BAD_REQUEST,
+                    RuntimeError::InvalidConfiguration(_) => StatusCode::BAD_REQUEST,
+                    RuntimeError::CannotConvertConfiguration => StatusCode::BAD_REQUEST,
+                    RuntimeError::SinkNotFound(_) => StatusCode::NOT_FOUND,
+                    RuntimeError::SourceNotFound(_) => StatusCode::NOT_FOUND,
+                    _ => StatusCode::INTERNAL_SERVER_ERROR,
+                };
+                (
+                    status_code,
+                    Json(ErrorResponse {
+                        code: error.as_code().to_owned(),
+                        reason: error.to_string(),
+                    }),
+                )
+            }
+            ApiError::JsonError(error) => {
+                error!("There was a JSON error: {error}");
+                (
+                    StatusCode::INTERNAL_SERVER_ERROR,
+                    Json(ErrorResponse {
+                        code: "json_error".to_owned(),
+                        reason: error.to_string(),
+                    }),
+                )
+            }
+        }
+        .into_response()
+    }
+}
diff --git a/core/connectors/runtime/src/api/mod.rs b/core/connectors/runtime/src/api/mod.rs
new file mode 100644
index 00000000..b6987d1d
--- /dev/null
+++ b/core/connectors/runtime/src/api/mod.rs
@@ -0,0 +1,110 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::context::RuntimeContext;
+use auth::resolve_api_key;
+use axum::{Json, Router, middleware, routing::get};
+use axum_server::tls_rustls::RustlsConfig;
+use config::{HttpApiConfig, configure_cors};
+use std::{net::SocketAddr, path::PathBuf, sync::Arc};
+use tokio::spawn;
+use tracing::{error, info};
+
+mod auth;
+pub mod config;
+mod error;
+mod models;
+mod sink;
+mod source;
+
+const NAME: &str = env!("CARGO_PKG_NAME");
+
+pub async fn init(config: &HttpApiConfig, context: Arc<RuntimeContext>) {
+    if !config.enabled {
+        info!("{NAME} HTTP API is disabled");
+        return;
+    }
+
+    let mut app = Router::new()
+        .route("/", get(|| async { "Connector Runtime API" }))
+        .route(
+            "/health",
+            get(|| async { Json(serde_json::json!({ "status": "healthy" })) }),
+        )
+        .merge(sink::router(context.clone()))
+        .merge(source::router(context.clone()));
+
+    app = app.layer(middleware::from_fn_with_state(
+        context.clone(),
+        resolve_api_key,
+    ));
+
+    if let Some(cors) = &config.cors {
+        if cors.enabled {
+            app = app.layer(configure_cors(cors));
+        }
+    }
+
+    let tls_enabled = config
+        .tls
+        .as_ref()
+        .map(|tls| tls.enabled)
+        .unwrap_or_default();
+    if !tls_enabled {
+        let listener = tokio::net::TcpListener::bind(&config.address)
+            .await
+            .unwrap_or_else(|_| panic!("Failed to bind to HTTP address {}", config.address));
+        let address = listener
+            .local_addr()
+            .expect("Failed to get local address for HTTP server");
+        info!("Started {NAME} on: {address}");
+        spawn(async move {
+            if let Err(error) = axum::serve(
+                listener,
+                app.into_make_service_with_connect_info::<SocketAddr>(),
+            )
+            .await
+            {
+                error!("Failed to start {NAME} server, error {}", error);
+            }
+        });
+        return;
+    }
+
+    let tls = config.tls.as_ref().expect("TLS configuration is required");
+    let tls_config =
+        RustlsConfig::from_pem_file(PathBuf::from(&tls.cert_file), PathBuf::from(&tls.key_file))
+            .await
+            .unwrap();
+
+    let listener = std::net::TcpListener::bind(&config.address).unwrap();
+    let address = listener
+        .local_addr()
+        .expect("Failed to get local address for HTTPS / TLS server");
+
+    info!("Started {NAME} on: {address}");
+
+    spawn(async move {
+        if let Err(error) = axum_server::from_tcp_rustls(listener, tls_config)
+            .serve(app.into_make_service_with_connect_info::<SocketAddr>())
+            .await
+        {
+            error!("Failed to start {NAME} server, error: {}", error);
+        }
+    });
+}
diff --git a/core/connectors/runtime/src/api/models.rs b/core/connectors/runtime/src/api/models.rs
new file mode 100644
index 00000000..ff25722d
--- /dev/null
+++ b/core/connectors/runtime/src/api/models.rs
@@ -0,0 +1,120 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use std::sync::Arc;
+
+use iggy_connector_sdk::transforms::TransformType;
+use serde::{Deserialize, Serialize};
+
+use crate::{
+    configs::{ConfigFormat, StreamConsumerConfig, StreamProducerConfig},
+    manager::{
+        sink::{SinkDetails, SinkInfo},
+        source::{SourceDetails, SourceInfo},
+    },
+};
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct SinkInfoResponse {
+    pub id: u32,
+    pub key: String,
+    pub name: String,
+    pub path: String,
+    pub enabled: bool,
+    pub running: bool,
+    pub config_format: Option<ConfigFormat>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct SinkDetailsResponse {
+    #[serde(flatten)]
+    pub info: SinkInfoResponse,
+    pub streams: Arc<Vec<StreamConsumerConfig>>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct SourceInfoResponse {
+    pub id: u32,
+    pub key: String,
+    pub name: String,
+    pub path: String,
+    pub enabled: bool,
+    pub running: bool,
+    pub config_format: Option<ConfigFormat>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct SourceDetailsResponse {
+    #[serde(flatten)]
+    pub info: SourceInfoResponse,
+    pub streams: Arc<Vec<StreamProducerConfig>>,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+pub struct TransformResponse {
+    pub r#type: TransformType,
+    pub config: Arc<serde_json::Value>,
+}
+
+impl From<&SinkInfo> for SinkInfoResponse {
+    fn from(sink: &SinkInfo) -> Self {
+        SinkInfoResponse {
+            id: sink.id,
+            key: sink.key.to_owned(),
+            name: sink.name.to_owned(),
+            path: sink.path.to_owned(),
+            enabled: sink.enabled,
+            running: sink.running,
+            config_format: sink.config_format,
+        }
+    }
+}
+
+impl From<&SinkDetails> for SinkDetailsResponse {
+    fn from(sink: &SinkDetails) -> Self {
+        let info = &sink.info;
+        SinkDetailsResponse {
+            info: info.into(),
+            streams: sink.streams.clone(),
+        }
+    }
+}
+
+impl From<&SourceInfo> for SourceInfoResponse {
+    fn from(source: &SourceInfo) -> Self {
+        SourceInfoResponse {
+            id: source.id,
+            key: source.key.to_owned(),
+            name: source.name.to_owned(),
+            path: source.path.to_owned(),
+            enabled: source.enabled,
+            running: source.running,
+            config_format: source.config_format,
+        }
+    }
+}
+
+impl From<&SourceDetails> for SourceDetailsResponse {
+    fn from(source: &SourceDetails) -> Self {
+        let info = &source.info;
+        SourceDetailsResponse {
+            info: info.into(),
+            streams: source.streams.clone(),
+        }
+    }
+}
diff --git a/core/connectors/runtime/src/api/sink.rs b/core/connectors/runtime/src/api/sink.rs
new file mode 100644
index 00000000..fdc8bcc8
--- /dev/null
+++ b/core/connectors/runtime/src/api/sink.rs
@@ -0,0 +1,113 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use super::{
+    config::map_connector_config,
+    error::ApiError,
+    models::{SinkDetailsResponse, SinkInfoResponse, TransformResponse},
+};
+use crate::{configs::ConfigFormat, context::RuntimeContext, error::RuntimeError};
+use axum::{
+    Json, Router,
+    extract::{Path, Query, State},
+    http::{HeaderMap, StatusCode, header},
+    response::IntoResponse,
+    routing::get,
+};
+use serde::Deserialize;
+use std::sync::Arc;
+
+pub fn router(state: Arc<RuntimeContext>) -> Router {
+    Router::new()
+        .route("/sinks", get(get_sinks))
+        .route("/sinks/{key}", get(get_sink))
+        .route("/sinks/{key}/config", get(get_sink_config))
+        .route("/sinks/{key}/transforms", get(get_sink_transforms))
+        .with_state(state)
+}
+
+async fn get_sinks(
+    State(context): State<Arc<RuntimeContext>>,
+) -> Result<Json<Vec<SinkInfoResponse>>, ApiError> {
+    let sinks = context
+        .sinks
+        .get_all()
+        .into_values()
+        .map(|sink| sink.into())
+        .collect::<Vec<_>>();
+    Ok(Json(sinks))
+}
+
+async fn get_sink(
+    State(context): State<Arc<RuntimeContext>>,
+    Path(key): Path<String>,
+) -> Result<Json<SinkDetailsResponse>, ApiError> {
+    let Some(sink) = context.sinks.get(&key) else {
+        return Err(ApiError::Error(RuntimeError::SinkNotFound(key)));
+    };
+    Ok(Json(sink.into()))
+}
+
+async fn get_sink_config(
+    State(context): State<Arc<RuntimeContext>>,
+    Path(key): Path<String>,
+    Query(query): Query<GetSinkConfig>,
+) -> Result<impl IntoResponse, ApiError> {
+    let Some(sink) = context.sinks.get(&key) else {
+        return Err(ApiError::Error(RuntimeError::SinkNotFound(key)));
+    };
+    let Some(config) = sink.config.clone() else {
+        return Ok(StatusCode::NOT_FOUND.into_response());
+    };
+
+    let format = query
+        .format
+        .unwrap_or(sink.info.config_format.unwrap_or_default());
+    let (content_type, config) = map_connector_config(&config, format)?;
+    let mut headers = HeaderMap::new();
+    headers.insert(header::CONTENT_TYPE, content_type);
+    Ok((headers, config).into_response())
+}
+
+#[derive(Debug, Deserialize)]
+struct GetSinkConfig {
+    format: Option<ConfigFormat>,
+}
+
+async fn get_sink_transforms(
+    State(context): State<Arc<RuntimeContext>>,
+    Path(key): Path<String>,
+) -> Result<Json<Vec<TransformResponse>>, ApiError> {
+    let Some(sink) = context.sinks.get(&key) else {
+        return Err(ApiError::Error(RuntimeError::SinkNotFound(key)));
+    };
+    let Some(transforms) = sink.transforms.clone() else {
+        return Ok(Json(vec![]));
+    };
+
+    Ok(Json(
+        transforms
+            .transforms
+            .iter()
+            .map(|transform| TransformResponse {
+                r#type: *transform.0,
+                config: transform.1.clone(),
+            })
+            .collect(),
+    ))
+}
diff --git a/core/connectors/runtime/src/api/source.rs b/core/connectors/runtime/src/api/source.rs
new file mode 100644
index 00000000..31820cf4
--- /dev/null
+++ b/core/connectors/runtime/src/api/source.rs
@@ -0,0 +1,113 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use super::{
+    config::map_connector_config,
+    error::ApiError,
+    models::{SourceDetailsResponse, SourceInfoResponse, TransformResponse},
+};
+use crate::{configs::ConfigFormat, context::RuntimeContext, error::RuntimeError};
+use axum::{
+    Json, Router,
+    extract::{Path, Query, State},
+    http::{HeaderMap, StatusCode, header},
+    response::IntoResponse,
+    routing::get,
+};
+use serde::Deserialize;
+use std::sync::Arc;
+
+pub fn router(state: Arc<RuntimeContext>) -> Router {
+    Router::new()
+        .route("/sources", get(get_sources))
+        .route("/sources/{key}", get(get_source))
+        .route("/sources/{key}/config", get(get_source_config))
+        .route("/sources/{key}/transforms", get(get_source_transforms))
+        .with_state(state)
+}
+
+async fn get_sources(
+    State(context): State<Arc<RuntimeContext>>,
+) -> Result<Json<Vec<SourceInfoResponse>>, ApiError> {
+    let sources = context
+        .sources
+        .get_all()
+        .into_values()
+        .map(|source| source.into())
+        .collect::<Vec<_>>();
+    Ok(Json(sources))
+}
+
+async fn get_source(
+    State(context): State<Arc<RuntimeContext>>,
+    Path(key): Path<String>,
+) -> Result<Json<SourceDetailsResponse>, ApiError> {
+    let Some(source) = context.sources.get(&key) else {
+        return Err(ApiError::Error(RuntimeError::SourceNotFound(key)));
+    };
+    Ok(Json(source.into()))
+}
+
+async fn get_source_config(
+    State(context): State<Arc<RuntimeContext>>,
+    Path(key): Path<String>,
+    Query(query): Query<GetSourceConfig>,
+) -> Result<impl IntoResponse, ApiError> {
+    let Some(source) = context.sources.get(&key) else {
+        return Err(ApiError::Error(RuntimeError::SourceNotFound(key)));
+    };
+    let Some(config) = source.config.clone() else {
+        return Ok(StatusCode::NOT_FOUND.into_response());
+    };
+
+    let format = query
+        .format
+        .unwrap_or(source.info.config_format.unwrap_or_default());
+    let (content_type, config) = map_connector_config(&config, format)?;
+    let mut headers = HeaderMap::new();
+    headers.insert(header::CONTENT_TYPE, content_type);
+    Ok((headers, config).into_response())
+}
+
+#[derive(Debug, Deserialize)]
+struct GetSourceConfig {
+    format: Option<ConfigFormat>,
+}
+
+async fn get_source_transforms(
+    State(context): State<Arc<RuntimeContext>>,
+    Path(key): Path<String>,
+) -> Result<Json<Vec<TransformResponse>>, ApiError> {
+    let Some(source) = context.sources.get(&key) else {
+        return Err(ApiError::Error(RuntimeError::SourceNotFound(key)));
+    };
+    let Some(transforms) = source.transforms.clone() else {
+        return Ok(Json(vec![]));
+    };
+
+    Ok(Json(
+        transforms
+            .transforms
+            .iter()
+            .map(|transform| TransformResponse {
+                r#type: *transform.0,
+                config: transform.1.clone(),
+            })
+            .collect(),
+    ))
+}
diff --git a/core/connectors/runtime/src/configs.rs b/core/connectors/runtime/src/configs.rs
index aafc9a48..05547d03 100644
--- a/core/connectors/runtime/src/configs.rs
+++ b/core/connectors/runtime/src/configs.rs
@@ -18,16 +18,36 @@
 
 use iggy_connector_sdk::{Schema, transforms::TransformType};
 use serde::{Deserialize, Serialize};
-use std::collections::HashMap;
+use std::{collections::HashMap, sync::Arc};
+use strum::Display;
 
-#[derive(Debug, Deserialize, Serialize)]
+use crate::api::config::HttpApiConfig;
+
+#[derive(
+    Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Deserialize, Serialize, Display,
+)]
+#[serde(rename_all = "lowercase")]
+pub enum ConfigFormat {
+    #[strum(to_string = "json")]
+    Json,
+    #[strum(to_string = "yaml")]
+    Yaml,
+    #[strum(to_string = "toml")]
+    Toml,
+    #[default]
+    #[strum(to_string = "text")]
+    Text,
+}
+
+#[derive(Debug, Clone, Deserialize, Serialize)]
 pub struct RuntimeConfig {
+    pub http_api: HttpApiConfig,
     pub iggy: IggyConfig,
     pub sinks: HashMap<String, SinkConfig>,
     pub sources: HashMap<String, SourceConfig>,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct IggyConfig {
     pub address: String,
     pub username: Option<String>,
@@ -35,27 +55,28 @@ pub struct IggyConfig {
     pub token: Option<String>,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct SinkConfig {
     pub enabled: bool,
     pub name: String,
     pub path: String,
-    pub transforms: Option<TransformsConfig>,
-    pub streams: Vec<StreamConsumerConfig>,
-    pub config: Option<serde_json::Value>,
+    pub transforms: Option<Arc<TransformsConfig>>,
+    pub streams: Arc<Vec<StreamConsumerConfig>>,
+    pub config_format: Option<ConfigFormat>,
+    pub config: Option<Arc<serde_json::Value>>,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct StreamConsumerConfig {
     pub stream: String,
     pub topics: Vec<String>,
     pub schema: Schema,
-    pub batch_size: Option<u32>,
+    pub batch_length: Option<u32>,
     pub poll_interval: Option<String>,
     pub consumer_group: Option<String>,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct StreamProducerConfig {
     pub stream: String,
     pub topic: String,
@@ -64,23 +85,24 @@ pub struct StreamProducerConfig {
     pub linger_time: Option<String>,
 }
 
-#[derive(Debug, Serialize, Deserialize)]
+#[derive(Debug, Clone, Serialize, Deserialize)]
 pub struct SourceConfig {
     pub enabled: bool,
     pub name: String,
     pub path: String,
-    pub transforms: Option<TransformsConfig>,
-    pub streams: Vec<StreamProducerConfig>,
-    pub config: Option<serde_json::Value>,
+    pub transforms: Option<Arc<TransformsConfig>>,
+    pub streams: Arc<Vec<StreamProducerConfig>>,
+    pub config_format: Option<ConfigFormat>,
+    pub config: Option<Arc<serde_json::Value>>,
 }
 
-#[derive(Debug, Default, Serialize, Deserialize)]
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct TransformsConfig {
     #[serde(flatten)]
-    pub transforms: HashMap<TransformType, serde_json::Value>,
+    pub transforms: HashMap<TransformType, Arc<serde_json::Value>>,
 }
 
-#[derive(Debug, Default, Serialize, Deserialize)]
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
 pub struct SharedTransformConfig {
     pub enabled: bool,
 }
diff --git a/core/connectors/runtime/src/context.rs b/core/connectors/runtime/src/context.rs
new file mode 100644
index 00000000..e129f626
--- /dev/null
+++ b/core/connectors/runtime/src/context.rs
@@ -0,0 +1,105 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::{
+    SinkConnectorWrapper, SourceConnectorWrapper,
+    configs::RuntimeConfig,
+    manager::{
+        sink::{SinkDetails, SinkInfo, SinkManager},
+        source::{SourceDetails, SourceInfo, SourceManager},
+    },
+};
+
+use tracing::error;
+
+pub struct RuntimeContext {
+    pub sinks: SinkManager,
+    pub sources: SourceManager,
+    pub api_key: Option<String>,
+}
+
+pub fn init(
+    config: &RuntimeConfig,
+    sink_wrappers: &[SinkConnectorWrapper],
+    source_wrappers: &[SourceConnectorWrapper],
+) -> RuntimeContext {
+    RuntimeContext {
+        sinks: SinkManager::new(map_sinks(config, sink_wrappers)),
+        sources: SourceManager::new(map_sources(config, source_wrappers)),
+        api_key: config.http_api.api_key.clone(),
+    }
+}
+
+fn map_sinks(config: &RuntimeConfig, sink_wrappers: &[SinkConnectorWrapper]) -> Vec<SinkDetails> {
+    let mut sinks = vec![];
+    for sink_wrapper in sink_wrappers.iter() {
+        for sink_plugin in sink_wrapper.plugins.iter() {
+            let Some(sink_config) = config.sinks.get(&sink_plugin.key) else {
+                error!("Missing sink config for: {}", sink_plugin.key);
+                continue;
+            };
+
+            sinks.push(SinkDetails {
+                info: SinkInfo {
+                    id: sink_plugin.id,
+                    key: sink_plugin.key.to_owned(),
+                    name: sink_plugin.name.to_owned(),
+                    path: sink_plugin.path.to_owned(),
+                    enabled: sink_config.enabled,
+                    running: sink_config.enabled,
+                    config_format: sink_plugin.config_format,
+                },
+                config: sink_config.config.clone(),
+                transforms: sink_config.transforms.clone(),
+                streams: sink_config.streams.clone(),
+            });
+        }
+    }
+    sinks
+}
+
+fn map_sources(
+    config: &RuntimeConfig,
+    source_wrappers: &[SourceConnectorWrapper],
+) -> Vec<SourceDetails> {
+    let mut sources = vec![];
+    for source_wrapper in source_wrappers.iter() {
+        for source_plugin in source_wrapper.plugins.iter() {
+            let Some(source_config) = config.sources.get(&source_plugin.key) else {
+                error!("Missing source config for: {}", source_plugin.key);
+                continue;
+            };
+
+            sources.push(SourceDetails {
+                info: SourceInfo {
+                    id: source_plugin.id,
+                    key: source_plugin.key.to_owned(),
+                    name: source_plugin.name.to_owned(),
+                    path: source_plugin.path.to_owned(),
+                    enabled: source_config.enabled,
+                    running: source_config.enabled,
+                    config_format: source_plugin.config_format,
+                },
+                config: source_config.config.clone(),
+                transforms: source_config.transforms.clone(),
+                streams: source_config.streams.clone(),
+            });
+        }
+    }
+    sources
+}
diff --git a/core/connectors/runtime/src/error.rs b/core/connectors/runtime/src/error.rs
index 702353e8..c622feab 100644
--- a/core/connectors/runtime/src/error.rs
+++ b/core/connectors/runtime/src/error.rs
@@ -38,4 +38,24 @@ pub enum RuntimeError {
     IggyError(#[from] iggy::prelude::IggyError),
     #[error("Missing Iggy credentials")]
     MissingIggyCredentials,
+    #[error("JSON error")]
+    JsonError(#[from] serde_json::Error),
+    #[error("Sink not found with key: {0}")]
+    SinkNotFound(String),
+    #[error("Source not found with key: {0}")]
+    SourceNotFound(String),
+    #[error("Cannot convert configuration")]
+    CannotConvertConfiguration,
+}
+
+impl RuntimeError {
+    pub fn as_code(&self) -> &'static str {
+        match self {
+            RuntimeError::SinkNotFound(_) => "sink_not_found",
+            RuntimeError::SourceNotFound(_) => "source_not_found",
+            RuntimeError::MissingIggyCredentials => "invalid_configuration",
+            RuntimeError::InvalidConfiguration(_) => "invalid_configuration",
+            _ => "error",
+        }
+    }
 }
diff --git a/core/connectors/runtime/src/main.rs b/core/connectors/runtime/src/main.rs
index f87057bf..67bf303a 100644
--- a/core/connectors/runtime/src/main.rs
+++ b/core/connectors/runtime/src/main.rs
@@ -17,7 +17,7 @@
  */
 
 use config::{Config, Environment, File};
-use configs::RuntimeConfig;
+use configs::{ConfigFormat, RuntimeConfig};
 use dlopen2::wrapper::{Container, WrapperApi};
 use error::RuntimeError;
 use iggy::prelude::{Client, IggyClient, IggyClientBuilder, IggyConsumer, IggyProducer};
@@ -36,8 +36,11 @@ use std::{
 use tracing::{debug, error, info};
 use tracing_subscriber::{EnvFilter, Registry, layer::SubscriberExt, util::SubscriberInitExt};
 
+mod api;
 pub(crate) mod configs;
+pub(crate) mod context;
 pub(crate) mod error;
+mod manager;
 mod sink;
 mod source;
 mod transform;
@@ -91,10 +94,11 @@ async fn main() -> Result<(), RuntimeError> {
         .try_deserialize()
         .expect("Failed to deserialize runtime config");
 
-    let iggy_address = config.iggy.address;
-    let iggy_username = config.iggy.username;
-    let iggy_password = config.iggy.password;
-    let iggy_token = config.iggy.token;
+    let iggy_config = config.iggy.clone();
+    let iggy_address = iggy_config.address;
+    let iggy_username = iggy_config.username;
+    let iggy_password = iggy_config.password;
+    let iggy_token = iggy_config.token;
     let consumer_client = create_iggy_client(
         &iggy_address,
         iggy_username.as_deref(),
@@ -112,8 +116,8 @@ async fn main() -> Result<(), RuntimeError> {
     .await?;
     producer_client.connect().await?;
 
-    let sources = source::init(config.sources, &producer_client).await?;
-    let sinks = sink::init(config.sinks, &consumer_client).await?;
+    let sources = source::init(config.sources.clone(), &producer_client).await?;
+    let sinks = sink::init(config.sinks.clone(), &consumer_client).await?;
 
     let mut sink_wrappers = vec![];
     let mut sink_with_plugins = HashMap::new();
@@ -149,6 +153,10 @@ async fn main() -> Result<(), RuntimeError> {
         );
     }
 
+    let context = context::init(&config, &sink_wrappers, &source_wrappers);
+    let context = Arc::new(context);
+    api::init(&config.http_api, context).await;
+
     source::handle(source_wrappers);
     sink::consume(sink_wrappers);
     info!("All sources and sinks spawned.");
@@ -261,6 +269,10 @@ struct SinkConnector {
 
 struct SinkConnectorPlugin {
     id: u32,
+    key: String,
+    name: String,
+    path: String,
+    config_format: Option<ConfigFormat>,
     consumers: Vec<SinkConnectorConsumer>,
 }
 
@@ -288,6 +300,10 @@ struct SourceConnector {
 
 struct SourceConnectorPlugin {
     id: u32,
+    key: String,
+    name: String,
+    path: String,
+    config_format: Option<ConfigFormat>,
     transforms: Vec<Arc<dyn Transform>>,
     producer: Option<SourceConnectorProducer>,
 }
diff --git a/core/connectors/runtime/src/manager/mod.rs b/core/connectors/runtime/src/manager/mod.rs
new file mode 100644
index 00000000..8efc27b1
--- /dev/null
+++ b/core/connectors/runtime/src/manager/mod.rs
@@ -0,0 +1,20 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+pub mod sink;
+pub mod source;
diff --git a/core/connectors/runtime/src/manager/sink.rs b/core/connectors/runtime/src/manager/sink.rs
new file mode 100644
index 00000000..e72514da
--- /dev/null
+++ b/core/connectors/runtime/src/manager/sink.rs
@@ -0,0 +1,67 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use std::{collections::HashMap, sync::Arc};
+
+use crate::configs::{ConfigFormat, StreamConsumerConfig, TransformsConfig};
+
+#[derive(Debug)]
+pub struct SinkManager {
+    sinks: HashMap<String, SinkDetails>,
+}
+
+impl SinkManager {
+    pub fn new(sinks: Vec<SinkDetails>) -> Self {
+        Self {
+            sinks: sinks
+                .into_iter()
+                .map(|sink| (sink.info.key.to_owned(), sink))
+                .collect(),
+        }
+    }
+
+    pub fn get(&self, key: &str) -> Option<&SinkDetails> {
+        self.sinks.get(key)
+    }
+
+    pub fn get_all(&self) -> HashMap<&String, &SinkInfo> {
+        self.sinks
+            .iter()
+            .map(|(key, sink)| (key, &sink.info))
+            .collect()
+    }
+}
+
+#[derive(Debug)]
+pub struct SinkInfo {
+    pub id: u32,
+    pub key: String,
+    pub name: String,
+    pub path: String,
+    pub enabled: bool,
+    pub running: bool,
+    pub config_format: Option<ConfigFormat>,
+}
+
+#[derive(Debug)]
+pub struct SinkDetails {
+    pub info: SinkInfo,
+    pub transforms: Option<Arc<TransformsConfig>>,
+    pub streams: Arc<Vec<StreamConsumerConfig>>,
+    pub config: Option<Arc<serde_json::Value>>,
+}
diff --git a/core/connectors/runtime/src/manager/source.rs b/core/connectors/runtime/src/manager/source.rs
new file mode 100644
index 00000000..251bbc32
--- /dev/null
+++ b/core/connectors/runtime/src/manager/source.rs
@@ -0,0 +1,67 @@
+/* Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use std::{collections::HashMap, sync::Arc};
+
+use crate::configs::{ConfigFormat, StreamProducerConfig, TransformsConfig};
+
+#[derive(Debug)]
+pub struct SourceManager {
+    sources: HashMap<String, SourceDetails>,
+}
+
+impl SourceManager {
+    pub fn new(sources: Vec<SourceDetails>) -> Self {
+        Self {
+            sources: sources
+                .into_iter()
+                .map(|source| (source.info.key.to_owned(), source))
+                .collect(),
+        }
+    }
+
+    pub fn get(&self, key: &str) -> Option<&SourceDetails> {
+        self.sources.get(key)
+    }
+
+    pub fn get_all(&self) -> HashMap<&String, &SourceInfo> {
+        self.sources
+            .iter()
+            .map(|(key, source)| (key, &source.info))
+            .collect()
+    }
+}
+
+#[derive(Debug)]
+pub struct SourceInfo {
+    pub id: u32,
+    pub key: String,
+    pub name: String,
+    pub path: String,
+    pub enabled: bool,
+    pub running: bool,
+    pub config_format: Option<ConfigFormat>,
+}
+
+#[derive(Debug)]
+pub struct SourceDetails {
+    pub info: SourceInfo,
+    pub transforms: Option<Arc<TransformsConfig>>,
+    pub streams: Arc<Vec<StreamProducerConfig>>,
+    pub config: Option<Arc<serde_json::Value>>,
+}
diff --git a/core/connectors/runtime/src/sink.rs b/core/connectors/runtime/src/sink.rs
index 78008011..8c721514 100644
--- a/core/connectors/runtime/src/sink.rs
+++ b/core/connectors/runtime/src/sink.rs
@@ -63,6 +63,10 @@ pub async fn init(
             );
             container.plugins.push(SinkConnectorPlugin {
                 id: plugin_id,
+                key: key.to_owned(),
+                name: name.to_owned(),
+                path: path.to_owned(),
+                config_format: config.config_format,
                 consumers: vec![],
             });
         } else {
@@ -76,6 +80,10 @@ pub async fn init(
                     container,
                     plugins: vec![SinkConnectorPlugin {
                         id: plugin_id,
+                        key: key.to_owned(),
+                        name: name.to_owned(),
+                        path: path.to_owned(),
+                        config_format: config.config_format,
                         consumers: vec![],
                     }],
                 },
@@ -88,7 +96,8 @@ pub async fn init(
         PLUGIN_ID.fetch_add(1, Ordering::Relaxed);
 
         let transforms = if let Some(transforms_config) = config.transforms {
-            let transforms = transform::load(transforms_config).expect("Failed to load transforms");
+            let transforms =
+                transform::load(&transforms_config).expect("Failed to load transforms");
             let types = transforms
                 .iter()
                 .map(|t| t.r#type().into())
@@ -109,30 +118,32 @@ pub async fn init(
             .find(|p| p.id == plugin_id)
             .expect("Failed to get sink plugin");
 
-        for stream in config.streams {
+        for stream in config.streams.iter() {
             let poll_interval =
-                IggyDuration::from_str(&stream.poll_interval.unwrap_or("5ms".to_owned()))
+                IggyDuration::from_str(stream.poll_interval.as_deref().unwrap_or("5ms"))
                     .expect("Invalid poll interval");
+            let default_consumer_group = format!("iggy-connect-sink-{key}");
             let consumer_group = stream
                 .consumer_group
-                .unwrap_or(format!("iggy-connect-{key}"));
-            let batch_size = stream.batch_size.unwrap_or(1000);
-            for topic in stream.topics {
+                .as_deref()
+                .unwrap_or(&default_consumer_group);
+            let batch_length = stream.batch_length.unwrap_or(1000);
+            for topic in stream.topics.iter() {
                 let mut consumer = iggy_client
-                    .consumer_group(&consumer_group, &stream.stream, &topic)?
+                    .consumer_group(consumer_group, &stream.stream, topic)?
                     .auto_commit(AutoCommit::When(AutoCommitWhen::PollingMessages))
                     .create_consumer_group_if_not_exists()
                     .auto_join_consumer_group()
                     .polling_strategy(PollingStrategy::next())
                     .poll_interval(poll_interval)
-                    .batch_length(batch_size)
+                    .batch_length(batch_length)
                     .build();
 
                 consumer.init().await?;
                 plugin.consumers.push(SinkConnectorConsumer {
                     consumer,
                     decoder: stream.schema.decoder(),
-                    batch_size,
+                    batch_size: batch_length,
                     transforms: transforms.clone(),
                 });
             }
diff --git a/core/connectors/runtime/src/source.rs b/core/connectors/runtime/src/source.rs
index b6cdeb4d..5fba0130 100644
--- a/core/connectors/runtime/src/source.rs
+++ b/core/connectors/runtime/src/source.rs
@@ -65,6 +65,10 @@ pub async fn init(
             );
             container.plugins.push(SourceConnectorPlugin {
                 id: plugin_id,
+                key: key.to_owned(),
+                name: name.to_owned(),
+                path: path.to_owned(),
+                config_format: config.config_format,
                 producer: None,
                 transforms: vec![],
             });
@@ -79,6 +83,10 @@ pub async fn init(
                     container,
                     plugins: vec![SourceConnectorPlugin {
                         id: plugin_id,
+                        key: key.to_owned(),
+                        name: name.to_owned(),
+                        path: path.to_owned(),
+                        config_format: config.config_format,
                         producer: None,
                         transforms: vec![],
                     }],
@@ -92,7 +100,8 @@ pub async fn init(
         PLUGIN_ID.fetch_add(1, Ordering::Relaxed);
 
         let transforms = if let Some(transforms_config) = config.transforms {
-            let transforms = transform::load(transforms_config).expect("Failed to load transforms");
+            let transforms =
+                transform::load(&transforms_config).expect("Failed to load transforms");
             let types = transforms
                 .iter()
                 .map(|t| t.r#type().into())
@@ -113,9 +122,9 @@ pub async fn init(
             .find(|p| p.id == plugin_id)
             .expect("Failed to get source plugin");
 
-        for stream in config.streams {
+        for stream in config.streams.iter() {
             let linger_time =
-                IggyDuration::from_str(&stream.linger_time.unwrap_or("5ms".to_owned()))
+                IggyDuration::from_str(stream.linger_time.as_deref().unwrap_or("5ms"))
                     .expect("Invalid send interval");
             let batch_length = stream.batch_length.unwrap_or(1000);
             let producer = iggy_client
diff --git a/core/connectors/runtime/src/transform.rs b/core/connectors/runtime/src/transform.rs
index 0cee67cc..fc76dad3 100644
--- a/core/connectors/runtime/src/transform.rs
+++ b/core/connectors/runtime/src/transform.rs
@@ -21,28 +21,27 @@ use crate::{
     configs::{SharedTransformConfig, TransformsConfig},
 };
 use iggy_connector_sdk::transforms::Transform;
+use serde::Deserialize;
 use std::sync::Arc;
 
-pub fn load(config: TransformsConfig) -> Result<Vec<Arc<dyn Transform>>, RuntimeError> {
+pub fn load(config: &TransformsConfig) -> Result<Vec<Arc<dyn Transform>>, RuntimeError> {
     let mut transforms: Vec<Arc<dyn Transform>> = vec![];
-    for (r#type, transform_config) in config.transforms {
+    for (r#type, transform_config) in config.transforms.iter() {
         let shared_config = if transform_config.is_null() {
             SharedTransformConfig::default()
         } else {
-            serde_json::from_value::<SharedTransformConfig>(transform_config.clone()).map_err(
-                |err| {
-                    RuntimeError::InvalidConfiguration(format!(
-                        "Failed to parse transform config: {err}",
-                    ))
-                },
-            )?
+            SharedTransformConfig::deserialize(transform_config.as_ref()).map_err(|error| {
+                RuntimeError::InvalidConfiguration(format!(
+                    "Failed to parse transform config. {error}",
+                ))
+            })?
         };
 
         if !shared_config.enabled {
             continue;
         }
 
-        let transform = iggy_connector_sdk::transforms::load(r#type, transform_config)?;
+        let transform = iggy_connector_sdk::transforms::load(*r#type, transform_config)?;
         transforms.push(transform);
     }
 
diff --git a/core/connectors/sdk/src/sink.rs b/core/connectors/sdk/src/sink.rs
index 7c8e1dcd..a2582fa6 100644
--- a/core/connectors/sdk/src/sink.rs
+++ b/core/connectors/sdk/src/sink.rs
@@ -66,10 +66,12 @@ impl<T: Sink + std::fmt::Debug> SinkContainer<T> {
             _ = fmt::try_init();
             let slice = std::slice::from_raw_parts(config_ptr, config_len);
             let Ok(config_str) = std::str::from_utf8(slice) else {
+                error!("Failed to read configuration for sink connector with ID: {id}",);
                 return -1;
             };
 
             let Ok(config) = serde_json::from_str(config_str) else {
+                error!("Failed to parse configuration for sink connector with ID: {id}",);
                 return -1;
             };
 
diff --git a/core/connectors/sdk/src/source.rs b/core/connectors/sdk/src/source.rs
index 09a80c67..5b569260 100644
--- a/core/connectors/sdk/src/source.rs
+++ b/core/connectors/sdk/src/source.rs
@@ -71,10 +71,12 @@ impl<T: Source + std::fmt::Debug + 'static> SourceContainer<T> {
             _ = fmt::try_init();
             let slice = std::slice::from_raw_parts(config_ptr, config_len);
             let Ok(config_str) = std::str::from_utf8(slice) else {
+                error!("Failed to read configuration for source connector with ID: {id}",);
                 return -1;
             };
 
             let Ok(config) = serde_json::from_str(config_str) else {
+                error!("Failed to parse configuration for source connector with ID: {id}",);
                 return -1;
             };
 
diff --git a/core/connectors/sdk/src/transforms/mod.rs b/core/connectors/sdk/src/transforms/mod.rs
index bbe67122..6413b956 100644
--- a/core/connectors/sdk/src/transforms/mod.rs
+++ b/core/connectors/sdk/src/transforms/mod.rs
@@ -53,10 +53,15 @@ pub enum TransformType {
     DeleteFields,
 }
 
-pub fn load(r#type: TransformType, config: serde_json::Value) -> Result<Arc<dyn Transform>, Error> {
+pub fn load(
+    r#type: TransformType,
+    config: &serde_json::Value,
+) -> Result<Arc<dyn Transform>, Error> {
     Ok(match r#type {
-        TransformType::AddFields => Arc::new(AddFields::new(as_config(r#type, config)?)),
-        TransformType::DeleteFields => Arc::new(DeleteFields::new(as_config(r#type, config)?)),
+        TransformType::AddFields => Arc::new(AddFields::new(as_config(r#type, config.to_owned())?)),
+        TransformType::DeleteFields => {
+            Arc::new(DeleteFields::new(as_config(r#type, config.to_owned())?))
+        }
     })
 }
 
diff --git a/core/connectors/sinks/README.md b/core/connectors/sinks/README.md
index 78541e30..61c380be 100644
--- a/core/connectors/sinks/README.md
+++ b/core/connectors/sinks/README.md
@@ -37,6 +37,7 @@ pub struct SinkConfig {
     pub path: String,
     pub transforms: Option<TransformsConfig>,
     pub streams: Vec<StreamConsumerConfig>,
+    pub config_format: ConfigFormat,
     pub config: Option<serde_json::Value>,
 }
 ```
@@ -49,13 +50,14 @@ Below is the example configuration for a sink connector, using `stdout` as it's
 enabled = true
 name = "Stdout sink"
 path = "target/release/libiggy_connector_stdout_sink"
+config_format = "toml"
 
 # Collection of the streams from which messages are consumed
 [[sinks.stdout.streams]]
 stream = "example_stream"
 topics = ["example_topic"]
 schema = "json"
-batch_size = 100
+batch_length = 100
 poll_interval = "5ms"
 consumer_group = "stdout_sink_connector"
 
diff --git a/core/connectors/sinks/quickwit_sink/src/lib.rs b/core/connectors/sinks/quickwit_sink/src/lib.rs
index b92a8b41..e290b3e0 100644
--- a/core/connectors/sinks/quickwit_sink/src/lib.rs
+++ b/core/connectors/sinks/quickwit_sink/src/lib.rs
@@ -163,7 +163,7 @@ impl QuickwitSink {
 impl Sink for QuickwitSink {
     async fn open(&mut self) -> Result<(), Error> {
         info!(
-            "Initialized QuickwitSink with ID: {} for URL: {}",
+            "Opened Quickwit sink connector with ID: {} for URL: {}",
             self.id, self.config.url
         );
         if !self.has_index().await? {
@@ -204,7 +204,7 @@ impl Sink for QuickwitSink {
     }
 
     async fn close(&mut self) -> Result<(), Error> {
-        info!("Quickwit sink with ID: {} is shutting down", self.id);
+        info!("Quickwit sink connector with ID: {} is closed.", self.id);
         Ok(())
     }
 }
diff --git a/core/connectors/sources/README.md b/core/connectors/sources/README.md
index f4d39cae..7fc6152a 100644
--- a/core/connectors/sources/README.md
+++ b/core/connectors/sources/README.md
@@ -31,6 +31,7 @@ pub struct SourceConfig {
     pub path: String,
     pub transforms: Option<TransformsConfig>,
     pub streams: Vec<StreamProducerConfig>,
+    pub config_format: ConfigFormat,
     pub config: Option<serde_json::Value>,
 }
 ```
@@ -43,14 +44,15 @@ Below is the example configuration for a source connector, using `random` as it'
 enabled = true # Toggle source on/off
 name = "Random source" # Name of the source
 path = "libiggy_connector_random_source" # Path to the source connector
+config_format = "toml"
 
 # Collection of the streams to which the produced messages are sent
 [[sources.random.streams]]
 stream = "example_stream"
 topic = "example_topic"
 schema = "json"
-batch_size = 100
-send_interval = "5ms"
+batch_length = 100
+linger_time = "5ms"
 
 # Custom configuration for the source connector, deserialized to type T from `config` field
 [sources.random.config]
diff --git a/core/connectors/sources/random_source/src/lib.rs b/core/connectors/sources/random_source/src/lib.rs
index 489a2f26..d92f7b74 100644
--- a/core/connectors/sources/random_source/src/lib.rs
+++ b/core/connectors/sources/random_source/src/lib.rs
@@ -117,7 +117,7 @@ impl RandomSource {
 impl Source for RandomSource {
     async fn open(&mut self) -> Result<(), iggy_connector_sdk::Error> {
         info!(
-            "Initialized random source connector with ID: {}. Interval: {:#?}, max offset: {:#?}, messages range: {} - {}, payload size: {}",
+            "Opened random source connector with ID: {}. Interval: {:#?}, max offset: {:#?}, messages range: {} - {}, payload size: {}",
             self.id,
             self.interval,
             self.max_count,
@@ -158,10 +158,7 @@ impl Source for RandomSource {
     }
 
     async fn close(&mut self) -> Result<(), Error> {
-        info!(
-            "Random source connector with ID: {} is shutting down",
-            self.id
-        );
+        info!("Random source connector with ID: {} is closed.", self.id);
         Ok(())
     }
 }