You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@airavata.apache.org by la...@apache.org on 2025/09/22 13:51:22 UTC
(airavata-custos) branch amie-decoder created (now 4e4093189)
This is an automated email from the ASF dual-hosted git repository.
lahirujayathilake pushed a change to branch amie-decoder
in repository https://gitbox.apache.org/repos/asf/airavata-custos.git
at 4e4093189 protobuf models for AMIE packets processing events
This branch includes the following new commits:
new 2cb48c149 initial AMIE decoder service implementation (protobuf models, and database structure)
new 4e4093189 protobuf models for AMIE packets processing events
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(airavata-custos) 01/02: initial AMIE decoder service implementation (protobuf models, and database structure)
Posted by la...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lahirujayathilake pushed a commit to branch amie-decoder
in repository https://gitbox.apache.org/repos/asf/airavata-custos.git
commit 2cb48c1499e546fa6eeb17089d6fa005ac4101ba
Author: lahiruj <la...@gmail.com>
AuthorDate: Sun Sep 21 23:19:17 2025 -0400
initial AMIE decoder service implementation (protobuf models, and database structure)
---
amie-decoder/pom.xml | 75 +++++++++
.../main/assembly/amie-decoder-bin-assembly.xml | 90 +++++++++++
.../apache/custos/amie/AmieDecoderApplication.java | 32 ++++
amie-decoder/src/main/proto/amie_packets.proto | 170 +++++++++++++++++++++
amie-decoder/src/main/resources/application.yml | 44 ++++++
.../db.migration/V1__initial_migration.sql | 81 ++++++++++
.../resources/distribution/bin/amie-decoder.sh | 70 +++++++++
.../resources/distribution/bin/custos-daemon.sh | 112 ++++++++++++++
.../src/main/resources/distribution/bin/setenv.sh | 46 ++++++
.../resources/distribution/conf/application.yml | 44 ++++++
.../resources/distribution/conf/logback-spring.xml | 53 +++++++
amie-decoder/src/main/resources/logback-spring.xml | 53 +++++++
pom.xml | 2 +
13 files changed, 872 insertions(+)
diff --git a/amie-decoder/pom.xml b/amie-decoder/pom.xml
new file mode 100644
index 000000000..5e8eb2ed6
--- /dev/null
+++ b/amie-decoder/pom.xml
@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.custos</groupId>
+ <artifactId>custos</artifactId>
+ <version>1.2-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>amie-decoder-service</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.springframework.boot</groupId>
+ <artifactId>spring-boot-starter-web</artifactId>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-to-slf4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jul-to-slf4j</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jul-to-slf4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>mysql</groupId>
+ <artifactId>mysql-connector-java</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.custos</groupId>
+ <artifactId>custos-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.springframework.vault</groupId>
+ <artifactId>spring-vault-core</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>${maven.assembly.plugin}</version>
+ <executions>
+ <execution>
+ <id>signer-distribution-package</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ <configuration>
+ <tarLongFileMode>posix</tarLongFileMode>
+ <finalName>${amie.service.dist.name}</finalName>
+ <descriptors>
+ <descriptor>src/main/assembly/amie-decoder-bin-assembly.xml</descriptor>
+ </descriptors>
+ <attach>false</attach>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
\ No newline at end of file
diff --git a/amie-decoder/src/main/assembly/amie-decoder-bin-assembly.xml b/amie-decoder/src/main/assembly/amie-decoder-bin-assembly.xml
new file mode 100644
index 000000000..43edf6f98
--- /dev/null
+++ b/amie-decoder/src/main/assembly/amie-decoder-bin-assembly.xml
@@ -0,0 +1,90 @@
+
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+<!DOCTYPE assembly [
+ <!ELEMENT assembly (id|includeBaseDirectory|baseDirectory|formats|fileSets|dependencySets)*>
+ <!ELEMENT id (#PCDATA)>
+ <!ELEMENT includeBaseDirectory (#PCDATA)>
+ <!ELEMENT baseDirectory (#PCDATA)>
+ <!ELEMENT formats (format)*>
+ <!ELEMENT format (#PCDATA)>
+ <!ELEMENT fileSets (fileSet)*>
+ <!ELEMENT fileSet (directory|outputDirectory|fileMode|includes)*>
+ <!ELEMENT directory (#PCDATA)>
+ <!ELEMENT outputDirectory (#PCDATA)>
+ <!ELEMENT includes (include)*>
+ <!ELEMENT include (#PCDATA)>
+ <!ELEMENT dependencySets (dependencySet)*>
+ <!ELEMENT dependencySet (outputDirectory|outputFileNameMapping|includes)*>
+ ]>
+<assembly>
+ <id>bin</id>
+ <includeBaseDirectory>true</includeBaseDirectory>
+ <baseDirectory>${custos.dist.name}</baseDirectory>
+ <formats>
+ <format>tar.gz</format>
+ <format>zip</format>
+ </formats>
+
+ <fileSets>
+ <fileSet>
+ <directory>src/main/resources/distribution/bin</directory>
+ <outputDirectory>bin</outputDirectory>
+ <fileMode>777</fileMode>
+ <includes>
+ <include>*.sh</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>src/main/resources/distribution/conf</directory>
+ <outputDirectory>conf</outputDirectory>
+ <includes>
+ <include>application.yml</include>
+ <include>logback-spring.xml</include>
+ </includes>
+ </fileSet>
+ <fileSet>
+ <directory>./</directory>
+ <outputDirectory>logs</outputDirectory>
+ <excludes>
+ <exclude>*/**</exclude>
+ </excludes>
+ </fileSet>
+ <fileSet>
+ <directory>target</directory>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>*.jar</include>
+ </includes>
+ </fileSet>
+ </fileSets>
+
+ <dependencySets>
+ <dependencySet>
+ <useProjectArtifact>false</useProjectArtifact>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>*</include>
+ </includes>
+ </dependencySet>
+ </dependencySets>
+
+</assembly>
diff --git a/amie-decoder/src/main/java/org/apache/custos/amie/AmieDecoderApplication.java b/amie-decoder/src/main/java/org/apache/custos/amie/AmieDecoderApplication.java
new file mode 100644
index 000000000..c3d6813ab
--- /dev/null
+++ b/amie-decoder/src/main/java/org/apache/custos/amie/AmieDecoderApplication.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.custos.amie;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.data.jpa.repository.config.EnableJpaAuditing;
+
+@SpringBootApplication
+@EnableJpaAuditing
+public class AmieDecoderApplication {
+ public static void main(String[] args) {
+ SpringApplication.run(AmieDecoderApplication.class, args);
+ }
+}
diff --git a/amie-decoder/src/main/proto/amie_packets.proto b/amie-decoder/src/main/proto/amie_packets.proto
new file mode 100644
index 000000000..23c51d3ab
--- /dev/null
+++ b/amie-decoder/src/main/proto/amie_packets.proto
@@ -0,0 +1,170 @@
+syntax = "proto3";
+
+package org.apache.custos.amie.v1;
+
+import "google/protobuf/struct.proto";
+import "google/protobuf/timestamp.proto";
+
+// AMIE packet types that the service must handle from the Client API
+enum PacketType {
+ PACKET_TYPE_UNSPECIFIED = 0;
+ REQUEST_PROJECT_CREATE = 1;
+ REQUEST_PROJECT_INACTIVATE = 2;
+ REQUEST_PROJECT_REACTIVATE = 3;
+ REQUEST_ACCOUNT_CREATE = 4;
+ REQUEST_ACCOUNT_INACTIVATE = 5;
+ REQUEST_USER_MODIFY = 6;
+ REQUEST_PERSON_MERGE = 7;
+}
+
+// Expected reply descriptor from the AMIE header
+message ExpectedReply {
+ string type = 1; // The type of the expected reply packet, eg, "notify_project_create"
+ int64 timeout = 2; // Timeout value as provided by AMIE
+}
+
+// Packet header as returned by the Client API
+message PacketHeader {
+ repeated ExpectedReply expected_reply_list = 1;
+ int64 packet_id = 2;
+ int64 trans_rec_id = 3;
+ int64 transaction_id = 4;
+ string remote_site_name = 5;
+ string local_site_name = 6;
+ string originating_site_name = 7;
+ bool outgoing_flag = 8;
+ string transaction_state = 9;
+ int64 packet_rec_id = 10; // Stable, unique per packet
+ string client_state = 11; // Site-defined state
+ string packet_state = 12; // AMIE-side packet state
+ google.protobuf.Timestamp packet_timestamp = 13;
+ google.protobuf.Struct client_json = 14;
+}
+
+
+// request_project_create
+message RequestProjectCreate {
+ string allocated_resource = 1; // "AllocatedResource"
+ repeated string pi_requested_login_list = 2; // "PiRequestedLoginList"
+ repeated string resource_list = 3; // "ResourceList"
+
+ string service_units_allocated = 4; // "ServiceUnitsAllocated"
+ string start_date = 5; // "StartDate" (YYYY-MM-DD as string)
+ string end_date = 6; // "EndDate"
+
+ // Often present in NPC replies, sometimes seen in requests:
+ string project_id = 7; // "ProjectID"
+ string pi_person_id = 8; // "PersonID" or "PiPersonID"
+ string pi_global_id = 9; // "PiGlobalID"
+ google.protobuf.Struct extra_fields = 99;
+}
+
+// request_project_inactivate
+message RequestProjectInactivate {
+ string project_id = 1; // "ProjectID"
+ string person_id = 2; // "PersonID" (PI)
+ repeated string resource_list = 3; // "ResourceList"
+ string allocated_resource = 4; // "AllocatedResource"
+ string comment = 5; // "Comment" if provided
+ string start_date = 6;
+ string end_date = 7;
+ string service_units_allocated = 8;
+ string service_units_remaining = 9;
+
+ google.protobuf.Struct extra_fields = 99;
+}
+
+// request_project_reactivate
+message RequestProjectReactivate {
+ string project_id = 1;
+ string person_id = 2; // PI only should be reactivated
+ repeated string resource_list = 3;
+ string allocated_resource = 4;
+ string start_date = 5;
+ string end_date = 6;
+ string service_units_allocated = 7;
+ string service_units_remaining = 8;
+ string grant_number = 9;
+
+ google.protobuf.Struct extra_fields = 99;
+}
+
+// request_account_create
+message RequestAccountCreate {
+ string project_id = 1;
+ string person_id = 2; // Target user
+ repeated string resource_list = 3; // "ResourceList"
+ string allocated_resource = 4;
+
+ // Some sites provide preferred login(s) here
+ repeated string user_requested_login_list = 5; // "UserRequestedLoginList"
+
+ google.protobuf.Struct extra_fields = 99;
+}
+
+// request_account_inactivate
+message RequestAccountInactivate {
+ string project_id = 1; // "ProjectID"
+ string person_id = 2; // "PersonID"
+ repeated string resource_list = 3; // "ResourceList"
+ string allocated_resource = 4;
+ string comment = 5;
+
+ google.protobuf.Struct extra_fields = 99;
+}
+
+// request_user_modify
+message RequestUserModify {
+ string action_type = 1; // "ActionType" (e.g., "replace" or "delete")
+ string person_id = 2; // "PersonID"
+
+ string first_name = 3;
+ string middle_name = 4;
+ string last_name = 5;
+ string title = 6;
+ string email = 7;
+ string organization = 8;
+ string org_code = 9;
+ string business_phone_number = 10;
+ string street_address = 11;
+ string street_address2 = 12;
+ string city = 13;
+ string state = 14;
+ string zip = 15;
+ string country = 16;
+
+ repeated string dn_list = 17; // "DnList"
+ google.protobuf.Struct extra_fields = 99;
+}
+
+// request_person_merge
+message RequestPersonMerge {
+ // AMIE sends identifiers to merge duplicate people
+ string surviving_person_id = 1;
+ string retiring_person_id = 2;
+ // Keep flexible to capture the resto of the fields
+ google.protobuf.Struct extra_fields = 99;
+}
+
+// Wrapper for any AMIE packet received by the service
+message Packet {
+ PacketType type = 1;
+ PacketHeader header = 2;
+
+ oneof body {
+ RequestProjectCreate request_project_create = 10;
+ RequestProjectInactivate request_project_inactivate = 11;
+ RequestProjectReactivate request_project_reactivate = 12;
+ RequestAccountCreate request_account_create = 13;
+ RequestAccountInactivate request_account_inactivate = 14;
+ RequestUserModify request_user_modify = 15;
+ RequestPersonMerge request_person_merge = 16;
+
+ // A fallback for any packet types this schema doesn't support
+ google.protobuf.Struct unknown_body = 50;
+ }
+
+ // Complete unmodified raw JSON of the packet (header + body) as a string
+ // For auditing, debugging, and reprocessing
+ string raw_json_payload = 98;
+}
diff --git a/amie-decoder/src/main/resources/application.yml b/amie-decoder/src/main/resources/application.yml
new file mode 100644
index 000000000..163c6c485
--- /dev/null
+++ b/amie-decoder/src/main/resources/application.yml
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+server:
+ port: 8083
+
+spring:
+ application:
+ name: amie-decoder-service
+ datasource:
+ url: "jdbc:mysql://localhost:3306/amie_decoder"
+ username: admin
+ password: admin
+ driver-class-name: com.mysql.cj.jdbc.Driver
+ jpa:
+ hibernate:
+ ddl-auto: update
+ show-sql: false
+ database-platform: org.hibernate.dialect.MySQL8Dialect
+
+logging:
+ pattern:
+ level: '%5p [${spring.application.name:},%X{traceId:-},%X{spanId:-}]'
+ level:
+ root: info
+
+springdoc:
+ swagger-ui:
+ title: CUSTOS AMIE DECODER API
+ oauth:
+ use-pkce-with-authorization-code-grant: true
\ No newline at end of file
diff --git a/amie-decoder/src/main/resources/db.migration/V1__initial_migration.sql b/amie-decoder/src/main/resources/db.migration/V1__initial_migration.sql
new file mode 100644
index 000000000..c8bad2577
--- /dev/null
+++ b/amie-decoder/src/main/resources/db.migration/V1__initial_migration.sql
@@ -0,0 +1,81 @@
+-- =================================================================
+-- Flyway Migration Script for AMIE Packet Decoder
+-- Target Database: MySQL / MariaDB
+-- =================================================================
+
+SET NAMES utf8mb4;
+SET
+time_zone = '+00:00';
+
+-- -----------------------------
+-- TABLE: packets
+-- Stores each unique AMIE packet received from the polling endpoint
+-- -----------------------------
+CREATE TABLE packets
+(
+ id VARCHAR(255) NOT NULL,
+ amie_id BIGINT NOT NULL, -- packet_rec_id from AMIE
+ type VARCHAR(64) NOT NULL, -- eg. request_account_create
+ status VARCHAR(32) NOT NULL, -- NEW, DECODED, PROCESSED, FAILED
+ raw_json JSON NOT NULL,
+ received_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+ decoded_at TIMESTAMP(6) NULL,
+ processed_at TIMESTAMP(6) NULL,
+ retries INT NOT NULL DEFAULT 0,
+ last_error TEXT NULL,
+
+ PRIMARY KEY (id),
+ UNIQUE KEY uq_packets_amie_id (amie_id),
+ KEY idx_packets_status (status),
+ KEY idx_packets_received_at (received_at)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+-- -----------------------------
+-- TABLE: processing_events
+-- Individual processing steps for a given packet
+-- -----------------------------
+CREATE TABLE processing_events
+(
+ id VARCHAR(255) NOT NULL,
+ packet_id VARCHAR(255) NOT NULL,
+ type VARCHAR(64) NOT NULL, -- DECODE_PACKET, SYNC_USER, ACK, etc.
+ status VARCHAR(32) NOT NULL, -- PENDING, RUNNING, DONE, FAILED
+ attempts INT NOT NULL DEFAULT 0,
+ payload LONGBLOB NOT NULL, -- Protobuf serialized DTO
+ created_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+ started_at TIMESTAMP(6) NULL,
+ finished_at TIMESTAMP(6) NULL,
+ last_error TEXT NULL,
+
+ PRIMARY KEY (id),
+ CONSTRAINT fk_events_packet FOREIGN KEY (packet_id) REFERENCES packets (id) ON DELETE CASCADE,
+ UNIQUE KEY uq_events_packet_type (packet_id, type),
+
+ KEY idx_events_status (status),
+ KEY idx_events_packet_id (packet_id),
+ KEY idx_events_type (type)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
+
+-- -----------------------------
+-- TABLE: processing_errors
+-- Processing error logs
+-- -----------------------------
+CREATE TABLE processing_errors
+(
+ id BIGINT NOT NULL AUTO_INCREMENT,
+ packet_id VARCHAR(255) NULL,
+ event_id VARCHAR(255) NULL,
+ occurred_at TIMESTAMP(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6),
+ summary VARCHAR(255) NOT NULL,
+ detail TEXT NULL,
+
+ PRIMARY KEY (id),
+ CONSTRAINT fk_errors_packet FOREIGN KEY (packet_id) REFERENCES packets (id) ON DELETE SET NULL,
+ CONSTRAINT fk_errors_event FOREIGN KEY (event_id) REFERENCES processing_events (id) ON DELETE SET NULL,
+
+ CHECK (packet_id IS NOT NULL OR event_id IS NOT NULL),
+
+ KEY idx_errors_packet_id (packet_id),
+ KEY idx_errors_event_id (event_id),
+ KEY idx_errors_occurred_at (occurred_at)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;
diff --git a/amie-decoder/src/main/resources/distribution/bin/amie-decoder.sh b/amie-decoder/src/main/resources/distribution/bin/amie-decoder.sh
new file mode 100755
index 000000000..b8e37fb8f
--- /dev/null
+++ b/amie-decoder/src/main/resources/distribution/bin/amie-decoder.sh
@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+. `dirname $0`/setenv.sh
+# Capture user's working dir before changing directory
+CWD="$PWD"
+cd ${AIRAVATA_HOME}/bin
+LOGO_FILE="logo.txt"
+
+JAVA_OPTS="-Dspring.config.location=${AIRAVATA_HOME}/conf/ -Dairavata.home=${AIRAVATA_HOME} -Dlogback.configurationFile=file:${AIRAVATA_HOME}/conf/logback-spring.xml"
+AIRAVATA_COMMAND=""
+EXTRA_ARGS=""
+SERVERS=""
+IS_SUBSET=false
+SUBSET=""
+DEFAULT_LOG_FILE="${AIRAVATA_HOME}/logs/amie-decoder.out"
+LOG_FILE=$DEFAULT_LOG_FILE
+
+# parse command arguments
+for var in "$@"
+do
+ case ${var} in
+ -xdebug)
+ AIRAVATA_COMMAND="${AIRAVATA_COMMAND}"
+ JAVA_OPTS="$JAVA_OPTS -Xdebug -Xnoagent -Xrunjdwp:transport=dt_socket,server=y,address=*:8000"
+ shift
+ ;;
+ -log)
+ shift
+ LOG_FILE="$1"
+ shift
+ # If relative path, expand to absolute path using the user's $CWD
+ if [ -z "`echo "$LOG_FILE" | egrep "^/"`" ]; then
+ LOG_FILE="${CWD}/${LOG_FILE}"
+ fi
+ ;;
+ -h)
+ echo "Usage: custos.sh"
+
+ echo "command options:"
+ echo " -xdebug Start Custos Service under JPDA debugger"
+ echo " -h Display this help and exit"
+ shift
+ exit 0
+ ;;
+ *)
+ EXTRA_ARGS="${EXTRA_ARGS} ${var}"
+ shift
+ ;;
+ esac
+done
+
+java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
+ org.apache.custos.amie.AmieDecoderApplication ${AIRAVATA_COMMAND} $*
\ No newline at end of file
diff --git a/amie-decoder/src/main/resources/distribution/bin/custos-daemon.sh b/amie-decoder/src/main/resources/distribution/bin/custos-daemon.sh
new file mode 100755
index 000000000..d8b74d189
--- /dev/null
+++ b/amie-decoder/src/main/resources/distribution/bin/custos-daemon.sh
@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+. `dirname $0`/setenv.sh
+# Capture user's working dir before changing directory
+CWD="$PWD"
+cd ${AIRAVATA_HOME}/bin
+LOGO_FILE="logo.txt"
+
+JAVA_OPTS="-Dspring.config.location=${AIRAVATA_HOME}/conf/ -Dairavata.home=${AIRAVATA_HOME} -Dlogback.configurationFile=file:${AIRAVATA_HOME}/conf/logback-spring.xml"
+AIRAVATA_COMMAND=""
+EXTRA_ARGS=""
+SERVERS=""
+LOGO=true
+IS_SUBSET=false
+SUBSET=""
+DEFAULT_LOG_FILE="${AIRAVATA_HOME}/logs/amie-decoder-daemon.out"
+LOG_FILE=$DEFAULT_LOG_FILE
+
+SERVICE_NAME="Custos AMIE Decoder Service"
+PID_PATH_NAME="${AIRAVATA_HOME}/bin/service-pid"
+
+case $1 in
+ start)
+ echo "Starting $SERVICE_NAME ..."
+ if [ ! -f $PID_PATH_NAME ]; then
+ nohup java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
+ org.apache.custos.CustosApplication ${AIRAVATA_COMMAND} $* > $LOG_FILE 2>&1 &
+ echo $! > $PID_PATH_NAME
+ echo "$SERVICE_NAME started ..."
+ else
+ echo "$SERVICE_NAME is already running ..."
+ fi
+ ;;
+ stop)
+ if [ -f $PID_PATH_NAME ]; then
+ PID=$(cat $PID_PATH_NAME);
+ echo "$SERVICE_NAME stopping ..."
+ kill $PID;
+ RETRY=0
+ while kill -0 $PID 2> /dev/null; do
+ echo "Waiting for the process $PID to be stopped"
+ RETRY=`expr ${RETRY} + 1`
+ if [ "${RETRY}" -gt "20" ]
+ then
+ echo "Forcefully killing the process as it is not responding ..."
+ kill -9 $PID
+ fi
+ sleep 1
+ done
+ echo "$SERVICE_NAME stopped ..."
+ rm $PID_PATH_NAME
+ else
+ echo "$SERVICE_NAME is not running ..."
+ fi
+ ;;
+ restart)
+ if [ -f $PID_PATH_NAME ]; then
+ PID=$(cat $PID_PATH_NAME);
+ echo "$SERVICE_NAME stopping ...";
+ kill $PID;
+ RETRY=0
+ while kill -0 $PID 2> /dev/null; do
+ echo "Waiting for the process $PID to be stopped"
+ RETRY=`expr ${RETRY} + 1`
+ if [ "${RETRY}" -gt "20" ]
+ then
+ echo "Forcefully killing the process as it is not responding ..."
+ kill -9 $PID
+ fi
+ sleep 1
+ done
+ echo "$SERVICE_NAME stopped ...";
+ rm $PID_PATH_NAME
+ echo "$SERVICE_NAME starting ..."
+ nohup java ${JAVA_OPTS} -classpath "${AIRAVATA_CLASSPATH}" \
+ org.apache.custos.CustosApplication ${AIRAVATA_COMMAND} $* > $LOG_FILE 2>&1 &
+ echo $! > $PID_PATH_NAME
+ echo "$SERVICE_NAME started ..."
+ else
+ echo "$SERVICE_NAME is not running ..."
+ fi
+ ;;
+ -h)
+ echo "Usage: custos-daemon.sh"
+
+ echo "command options:"
+ echo " start Start server in daemon mode"
+ echo " stop Stop server running in daemon mode"
+ echo " restart Restart server in daemon mode"
+ echo " -log <LOG_FILE> Where to redirect stdout/stderr (defaults to $DEFAULT_LOG_FILE)"
+ echo " -h Display this help and exit"
+ shift
+ exit 0
+ ;;
+esac
\ No newline at end of file
diff --git a/amie-decoder/src/main/resources/distribution/bin/setenv.sh b/amie-decoder/src/main/resources/distribution/bin/setenv.sh
new file mode 100755
index 000000000..3f1d7632b
--- /dev/null
+++ b/amie-decoder/src/main/resources/distribution/bin/setenv.sh
@@ -0,0 +1,46 @@
+#!/bin/sh
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# resolve links - $0 may be a softlink
+PRG="$0"
+
+while [ -h "$PRG" ]; do
+ ls=`ls -ld "$PRG"`
+ link=`expr "$ls" : '.*-> \(.*\)$'`
+ if expr "$link" : '.*/.*' > /dev/null; then
+ PRG="$link"
+ else
+ PRG=`dirname "$PRG"`/"$link"
+ fi
+done
+
+PRGDIR=`dirname "$PRG"`
+
+# Only set AIRAVATA_HOME if not already set
+[ -z "$AIRAVATA_HOME" ] && AIRAVATA_HOME=`cd "$PRGDIR/.." ; pwd`
+
+AIRAVATA_CLASSPATH=""
+
+for f in "$AIRAVATA_HOME"/lib/*.jar
+do
+ AIRAVATA_CLASSPATH="$AIRAVATA_CLASSPATH":$f
+done
+
+export AIRAVATA_HOME
+export AIRAVATA_CLASSPATH
\ No newline at end of file
diff --git a/amie-decoder/src/main/resources/distribution/conf/application.yml b/amie-decoder/src/main/resources/distribution/conf/application.yml
new file mode 100644
index 000000000..163c6c485
--- /dev/null
+++ b/amie-decoder/src/main/resources/distribution/conf/application.yml
@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+server:
+ port: 8083
+
+spring:
+ application:
+ name: amie-decoder-service
+ datasource:
+ url: "jdbc:mysql://localhost:3306/amie_decoder"
+ username: admin
+ password: admin
+ driver-class-name: com.mysql.cj.jdbc.Driver
+ jpa:
+ hibernate:
+ ddl-auto: update
+ show-sql: false
+ database-platform: org.hibernate.dialect.MySQL8Dialect
+
+logging:
+ pattern:
+ level: '%5p [${spring.application.name:},%X{traceId:-},%X{spanId:-}]'
+ level:
+ root: info
+
+springdoc:
+ swagger-ui:
+ title: CUSTOS AMIE DECODER API
+ oauth:
+ use-pkce-with-authorization-code-grant: true
\ No newline at end of file
diff --git a/amie-decoder/src/main/resources/distribution/conf/logback-spring.xml b/amie-decoder/src/main/resources/distribution/conf/logback-spring.xml
new file mode 100644
index 000000000..68c977528
--- /dev/null
+++ b/amie-decoder/src/main/resources/distribution/conf/logback-spring.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+<configuration>
+ <include resource="org/springframework/boot/logging/logback/base.xml"/>
+ <springProperty scope="context" name="appName" source="spring.application.name"/>
+ <!-- Rolling File Appender -->
+ <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>logs/custos-amie-decoder.log</file>
+ <encoder>
+ <pattern>%date{YYYY-MM-dd HH:mm:ss.SSS} app=${appName}, host=${HOSTNAME}, traceID=%X{traceId:-NONE}, level=%-5level, [%thread] %logger{36} - %msg%n</pattern>
+ </encoder>
+ <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+ <fileNamePattern>logs/%d{yyyy-MM}/custos-amie-decoder-%d{MM-dd-yyyy}-%i.log.gz</fileNamePattern>
+ <maxFileSize>10MB</maxFileSize>
+ <maxHistory>30</maxHistory>
+ <totalSizeCap>1GB</totalSizeCap>
+ </rollingPolicy>
+ </appender>
+
+ <appender name="ASYNC_FILE" class="ch.qos.logback.classic.AsyncAppender">
+ <appender-ref ref="FILE"/>
+ <queueSize>500</queueSize>
+ <discardingThreshold>0</discardingThreshold>
+ <includeCallerData>true</includeCallerData>
+ </appender>
+
+ <logger name="org.hibernate" level="ERROR"/>
+ <logger name="org.springframework" level="INFO"/>
+
+ <root level="INFO">
+ <appender-ref ref="ASYNC_FILE"/>
+ </root>
+</configuration>
\ No newline at end of file
diff --git a/amie-decoder/src/main/resources/logback-spring.xml b/amie-decoder/src/main/resources/logback-spring.xml
new file mode 100644
index 000000000..68c977528
--- /dev/null
+++ b/amie-decoder/src/main/resources/logback-spring.xml
@@ -0,0 +1,53 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+
+-->
+
+<configuration>
+ <include resource="org/springframework/boot/logging/logback/base.xml"/>
+ <springProperty scope="context" name="appName" source="spring.application.name"/>
+ <!-- Rolling File Appender -->
+ <appender name="FILE" class="ch.qos.logback.core.rolling.RollingFileAppender">
+ <file>logs/custos-amie-decoder.log</file>
+ <encoder>
+ <pattern>%date{YYYY-MM-dd HH:mm:ss.SSS} app=${appName}, host=${HOSTNAME}, traceID=%X{traceId:-NONE}, level=%-5level, [%thread] %logger{36} - %msg%n</pattern>
+ </encoder>
+ <rollingPolicy class="ch.qos.logback.core.rolling.SizeAndTimeBasedRollingPolicy">
+ <fileNamePattern>logs/%d{yyyy-MM}/custos-amie-decoder-%d{MM-dd-yyyy}-%i.log.gz</fileNamePattern>
+ <maxFileSize>10MB</maxFileSize>
+ <maxHistory>30</maxHistory>
+ <totalSizeCap>1GB</totalSizeCap>
+ </rollingPolicy>
+ </appender>
+
+ <appender name="ASYNC_FILE" class="ch.qos.logback.classic.AsyncAppender">
+ <appender-ref ref="FILE"/>
+ <queueSize>500</queueSize>
+ <discardingThreshold>0</discardingThreshold>
+ <includeCallerData>true</includeCallerData>
+ </appender>
+
+ <logger name="org.hibernate" level="ERROR"/>
+ <logger name="org.springframework" level="INFO"/>
+
+ <root level="INFO">
+ <appender-ref ref="ASYNC_FILE"/>
+ </root>
+</configuration>
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 0be83a925..6f6852512 100644
--- a/pom.xml
+++ b/pom.xml
@@ -46,6 +46,7 @@
<module>services</module>
<module>api</module>
<module>application</module>
+ <module>amie-decoder</module>
</modules>
<dependencyManagement>
@@ -205,6 +206,7 @@
<ssh.username>ubuntu</ssh.username>
<jakarta.ws.version>4.0.0</jakarta.ws.version>
<custos.dist.name>apache-airavata-custos-${project.version}</custos.dist.name>
+ <amie.service.dist.name>apache-airavata-custos-amie-decoder-${project.version}</amie.service.dist.name>
</properties>
<repositories>
(airavata-custos) 02/02: protobuf models for AMIE packets processing events
Posted by la...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lahirujayathilake pushed a commit to branch amie-decoder
in repository https://gitbox.apache.org/repos/asf/airavata-custos.git
commit 4e4093189e39201cb8df195a31b57a5d44525475
Author: lahiruj <la...@gmail.com>
AuthorDate: Sun Sep 21 23:45:52 2025 -0400
protobuf models for AMIE packets processing events
---
amie-decoder/src/main/proto/internal_events.proto | 158 ++++++++++++++++++++++
1 file changed, 158 insertions(+)
diff --git a/amie-decoder/src/main/proto/internal_events.proto b/amie-decoder/src/main/proto/internal_events.proto
new file mode 100644
index 000000000..8c68ee929
--- /dev/null
+++ b/amie-decoder/src/main/proto/internal_events.proto
@@ -0,0 +1,158 @@
+syntax = "proto3";
+
+package org.apache.custos.amie.internal.events.v1;
+
+import "google/protobuf/timestamp.proto";
+import "google/protobuf/struct.proto";
+import "amie_packets.proto";
+
+// Defines the high-level stages in the processing of a single AMIE packet
+enum EventType {
+ EVENT_TYPE_UNSPECIFIED = 0;
+
+ // Ingestion & persistence
+ PACKET_POLLED = 10; // Fetched from AMIE Client API
+ PACKET_PERSISTED = 11; // Stored in `packets` table
+
+ // Decode
+ DECODE_STARTED = 20;
+ DECODE_SUCCEEDED = 21;
+ DECODE_FAILED = 22;
+
+ // Provisioning (account, project, groups, home dir, etc.)
+ PROVISIONING_REQUESTED = 30;
+ PROVISIONING_SUCCEEDED = 31;
+ PROVISIONING_FAILED = 32;
+
+ // ACK back to ACCESS (eg, inform_transaction_complete - notify_*)
+ ACK_SCHEDULED = 40;
+ ACK_DISPATCHED = 41;
+ ACK_FAILED = 42;
+}
+
+// Represents the status of a specific processing event (mirrors `processing_events` column)
+enum ProcessingStatus {
+ PROCESSING_STATUS_UNSPECIFIED = 0;
+ PENDING = 1;
+ RUNNING = 2;
+ SUCCEEDED = 3;
+ FAILED = 4;
+ RETRY_SCHEDULED = 5;
+ SKIPPED = 6;
+}
+
+// Standard error payload for failures and retries
+message EventError {
+ string code = 1; // short code, eg, "AMIE_HTTP_502", "DECODE_JSON", "COMANAGE_API_TIMEOUT"
+ string summary = 2; // one-line summary
+ string detail = 3; // full error detail
+ bool retriable = 4; // indicate whether this can be retried
+}
+
+message PacketPolledPayload {
+ int64 amie_packet_rec_id = 1; // from AMIE header
+ string poll_run_id = 2;
+}
+
+message PacketPersistedPayload {
+ string packet_db_id = 1; // UUID from `packets.id`
+ int64 amie_packet_rec_id = 2;
+}
+
+message DecodeStartedPayload {
+ string packet_db_id = 1;
+ int64 amie_packet_rec_id = 2;
+}
+
+message DecodeSucceededPayload {
+ string packet_db_id = 1;
+ int64 amie_packet_rec_id = 2;
+
+ amie.v1.Packet packet = 3; // Fully typed decoded packet (from amie_packet.proto).
+}
+
+message DecodeFailedPayload {
+ string packet_db_id = 1;
+ int64 amie_packet_rec_id = 2;
+ EventError error = 3;
+}
+
+message ProvisioningRequestedPayload {
+ string packet_db_id = 1;
+ int64 amie_packet_rec_id = 2;
+
+ string operation = 3; // eg, "CREATE_UNIX_ACCOUNT", "ADD_TO_PROJECT", "INACTIVATE_ACCOUNT"
+ string unix_username = 4; // if known
+ string project_id = 5; // ACCESS ProjectID
+ repeated string resources = 6; // target resource codes from AMIE
+ google.protobuf.Struct desired_state = 7; // flexible struct containing the details needed for the operation
+}
+
+message ProvisioningSucceededPayload {
+ string packet_db_id = 1;
+ int64 amie_packet_rec_id = 2;
+ string operation = 3;
+ google.protobuf.Struct result = 4; // eg, { "uid": 12345, "home": "/home/foo" }
+}
+
+message ProvisioningFailedPayload {
+ string packet_db_id = 1;
+ int64 amie_packet_rec_id = 2;
+ string operation = 3;
+ EventError error = 4;
+}
+
+message AckScheduledPayload {
+ string packet_db_id = 1;
+ int64 amie_packet_rec_id = 2;
+ string reply_type = 3; // eg, "inform_transaction_complete"
+}
+
+message AckDispatchedPayload {
+ string packet_db_id = 1;
+ int64 amie_packet_rec_id = 2;
+ string reply_type = 3;
+ int32 http_status = 4; // response code from the ACCESS API
+}
+
+message AckFailedPayload {
+ string packet_db_id = 1;
+ int64 amie_packet_rec_id = 2;
+ string reply_type = 3;
+ EventError error = 4;
+}
+
+// This is the main envelope message that will be serialized and stored
+// in the `processing_events.payload` column in the database
+message ProcessingEvent {
+ string id = 1; // UUID for the event (`processing_events.id`)
+ string packet_db_id = 2; // `packets.id`
+ int64 amie_packet_rec_id = 3; // The original AMIE ID
+ EventType type = 4;
+ ProcessingStatus status = 5;
+ int32 attempts = 6;
+
+ google.protobuf.Timestamp created_at = 7;
+ google.protobuf.Timestamp started_at = 8;
+ google.protobuf.Timestamp finished_at = 9;
+
+ // Event-specific payloads
+ oneof payload {
+ PacketPolledPayload packet_polled = 20;
+ PacketPersistedPayload packet_persisted = 21;
+
+ DecodeStartedPayload decode_started = 30;
+ DecodeSucceededPayload decode_succeeded = 31;
+ DecodeFailedPayload decode_failed = 32;
+
+ ProvisioningRequestedPayload provisioning_requested = 40;
+ ProvisioningSucceededPayload provisioning_succeeded = 41;
+ ProvisioningFailedPayload provisioning_failed = 42;
+
+ AckScheduledPayload ack_scheduled = 50;
+ AckDispatchedPayload ack_dispatched = 51;
+ AckFailedPayload ack_failed = 52;
+
+ google.protobuf.Struct generic = 90;
+ }
+}