You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by ak...@apache.org on 2026/04/23 06:33:52 UTC
(tooling-agents) branch main updated: Parsing fixes, flags on rerun
This is an automated email from the ASF dual-hosted git repository.
andrewmusselman pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-agents.git
The following commit(s) were added to refs/heads/main by this push:
new 2332e28 Parsing fixes, flags on rerun
2332e28 is described below
commit 2332e28d50b88ed51c4aa41817bb87e46dde29d8
Author: Andrew Musselman <an...@gmail.com>
AuthorDate: Wed Apr 22 23:33:38 2026 -0700
Parsing fixes, flags on rerun
---
ASVS/README.md | 25 ++--
.../consolidate_asvs_security_audit_reports.py | 52 ++++++-
ASVS/rerun-sections.sh | 160 ++++++++++++---------
3 files changed, 156 insertions(+), 81 deletions(-)
diff --git a/ASVS/README.md b/ASVS/README.md
index ec9646a..96f27d4 100644
--- a/ASVS/README.md
+++ b/ASVS/README.md
@@ -278,20 +278,27 @@ comm -23 /tmp/asvs_sections.txt /tmp/github_reports.txt
### Re-run failed sections
-Use `rerun-sections.sh` to audit missing sections and push them to a `rerun` subdirectory under the original output directory:
+Use `rerun-sections.sh` to audit missing sections and/or re-consolidate:
-```bash
-./rerun-sections.sh <namespace> <output_repo> <output_token> <output_dir> <section> [section...]
-```
+| Mode | Command |
+|---|---|
+| Audit + consolidate (default) | `./rerun-sections.sh <namespace> <repo> <token> <dir> <section> [section...]` |
+| Audit only | `./rerun-sections.sh --no-consolidate <namespace> <repo> <token> <dir> <section> [section...]` |
+| Consolidate only | `./rerun-sections.sh --consolidate-only <repo> <token> <dir>` |
-To re-run consolidation after filling in the gaps (typically you would), add `--consolidate`:
+Examples:
```bash
-./rerun-sections.sh "files:apache/steve/v3" apache/tooling-agents ghp_xxx \
- ASVS/reports/steve/v3/d0aa7e9 --consolidate 1.3.3 1.5.1 1.5.2 1.5.3 3.5.7
+# Audit 5 missing sections then consolidate:
+./rerun-sections.sh "files:apache/steve" apache/tooling-agents ghp_xxx \
+ ASVS/reports/steve/v3/d0aa7e9 1.3.3 1.5.1 1.5.2 1.5.3 3.5.7
+
+# Re-consolidate only (e.g., after deploying a fix to the consolidation agent):
+./rerun-sections.sh --consolidate-only apache/tooling-agents ghp_xxx \
+ ASVS/reports/steve/v3/d0aa7e9
```
-The consolidator reads all subdirectories (including `rerun/`) and deduplicates findings across them.
+Audited sections are pushed to a `rerun/` subdirectory. The consolidator reads all subdirectories (including `rerun/`) and deduplicates findings across them.
### Data store inspection
@@ -340,4 +347,4 @@ for ns, count in sorted(Counter(d['namespace'] for d in docs).items()):
**Email not delivered** — The email agent uses `mail-relay.apache.org`. Only works from ASF infrastructure. Check the logs for SMTP errors.
-**Redacted report still shows Critical findings** — The redaction regex looks for `🔴 Critical` and `CRITICAL` in finding blocks. If the consolidator's output format changed, the regex may need updating.
+**Redacted report still shows Critical findings** — The redaction regex looks for `🔴 Critical` and `CRITICAL` in finding blocks. If the consolidator's output format changed, the regex may need updating.
\ No newline at end of file
diff --git a/ASVS/agents/consolidate_asvs_security_audit_reports.py b/ASVS/agents/consolidate_asvs_security_audit_reports.py
index 84d0f83..397c53a 100644
--- a/ASVS/agents/consolidate_asvs_security_audit_reports.py
+++ b/ASVS/agents/consolidate_asvs_security_audit_reports.py
@@ -9,6 +9,44 @@ async def run(input_dict, tools):
try:
import json
import re
+ import ast
+
+ def parse_llm_json(raw):
+ """Parse JSON from LLM output with multiple fallbacks for common issues."""
+ # 1. Standard JSON
+ try:
+ return json.loads(raw)
+ except json.JSONDecodeError:
+ pass
+
+ # 2. Fix single-quoted keys/values (most common Sonnet issue)
+ try:
+ # Replace single quotes used as JSON delimiters with double quotes
+ # This is naive but catches {'key': 'value'} patterns
+ fixed = re.sub(r"(?<=[{,\[])\s*'([^']+)'\s*:", r' "\1":', raw)
+ fixed = re.sub(r":\s*'([^']*)'", r': "\1"', fixed)
+ # Fix trailing commas
+ fixed = re.sub(r",\s*([}\]])", r"\1", fixed)
+ return json.loads(fixed)
+ except (json.JSONDecodeError, Exception):
+ pass
+
+ # 3. Python literal (handles True/False/None + single quotes)
+ try:
+ return ast.literal_eval(raw)
+ except Exception:
+ pass
+
+ # 4. Last resort: strip non-JSON wrapper and retry
+ try:
+ # Sometimes Sonnet wraps JSON in markdown or extra text
+ inner = re.search(r'\{[^{}]*(?:\{[^{}]*\}[^{}]*)*\}', raw)
+ if inner:
+ return json.loads(inner.group())
+ except Exception:
+ pass
+
+ raise json.JSONDecodeError("All parsing strategies failed", raw, 0)
import base64
from datetime import date
@@ -239,7 +277,7 @@ Return ONLY valid JSON:
)
json_match = re.search(r'\{[\s\S]*\}', result)
if json_match:
- extracted = json.loads(json_match.group())
+ extracted = parse_llm_json(json_match.group())
extracted["source_report"] = report_key
extraction_ns.set(report_key, extracted)
print(f"{len(extracted.get('findings', []))} findings, status: {extracted.get('asvs_status', '?')}")
@@ -473,7 +511,7 @@ Return valid JSON:
msg_tokens = count_message_tokens(messages, FAST_PROVIDER, FAST_MODEL)
limit = int(FAST_CONTEXT_WINDOW * 0.80)
- if msg_tokens > limit:
+ if msg_tokens > limit or len(rpts) > 25:
items = list(rpts.items())
mid = len(items) // 2
sub_results = []
@@ -486,11 +524,11 @@ Return valid JSON:
result, _ = await call_llm(
provider=FAST_PROVIDER, model=FAST_MODEL,
messages=sub_messages,
- parameters={**FAST_PARAMS, "max_tokens": 64000}, timeout=600,
+ parameters={**FAST_PARAMS, "max_tokens": 64000}, timeout=900,
)
json_match = re.search(r'\{[\s\S]*\}', result)
if json_match:
- sub_results.append(json.loads(json_match.group()))
+ sub_results.append(parse_llm_json(json_match.group()))
except Exception as e:
print(f" ERROR in sub-group {si+1}: {e}")
if sub_results:
@@ -508,11 +546,11 @@ Return valid JSON:
result, _ = await call_llm(
provider=FAST_PROVIDER, model=FAST_MODEL,
messages=messages,
- parameters={**FAST_PARAMS, "max_tokens": 64000}, timeout=600,
+ parameters={**FAST_PARAMS, "max_tokens": 64000}, timeout=900,
)
json_match = re.search(r'\{[\s\S]*\}', result)
if json_match:
- consolidated = json.loads(json_match.group())
+ consolidated = parse_llm_json(json_match.group())
consolidation_ns.set(domain, consolidated)
print(f" Result: {len(consolidated.get('consolidated_findings', []))} findings")
return domain, consolidated
@@ -655,7 +693,7 @@ If no duplicates: {"merges": []}"""
result, _ = await call_llm(provider=FAST_PROVIDER, model=FAST_MODEL, messages=messages, parameters=FAST_PARAMS, timeout=120)
json_match = re.search(r'\{[\s\S]*\}', result)
if json_match:
- dedup_result = json.loads(json_match.group())
+ dedup_result = parse_llm_json(json_match.group())
lookup = {f"{f['_xd_domain']}:{f.get('temp_id','?')}": f for f in group}
for merge in dedup_result.get("merges", []):
keep = lookup.get(merge.get("keep",""))
diff --git a/ASVS/rerun-sections.sh b/ASVS/rerun-sections.sh
index 3e0b5de..31bece6 100755
--- a/ASVS/rerun-sections.sh
+++ b/ASVS/rerun-sections.sh
@@ -1,55 +1,69 @@
#!/bin/bash
-# rerun-sections.sh — Re-run failed/missing ASVS sections
+# rerun-sections.sh — Re-run failed/missing ASVS sections and/or consolidate
#
# Usage:
# ./rerun-sections.sh <namespace> <output_repo> <output_token> <output_dir> <section> [section...]
-# ./rerun-sections.sh <namespace> <output_repo> <output_token> <output_dir> --consolidate <section> [section...]
+# ./rerun-sections.sh --no-consolidate <namespace> <output_repo> <output_token> <output_dir> <section> [section...]
+# ./rerun-sections.sh --consolidate-only <output_repo> <output_token> <output_dir>
#
-# Example:
-# ./rerun-sections.sh "files:apache/steve/v3" apache/tooling-agents ghp_xxx \
+# Examples:
+# # Audit 5 sections then consolidate (default):
+# ./rerun-sections.sh "files:apache/steve" apache/tooling-agents ghp_xxx \
# ASVS/reports/steve/v3/d0aa7e9 1.3.3 1.5.1 1.5.2 1.5.3 3.5.7
#
-# With re-consolidation (pass all domain directories):
-# ./rerun-sections.sh "files:apache/steve/v3" apache/tooling-agents ghp_xxx \
-# ASVS/reports/steve/v3/d0aa7e9 --consolidate 1.3.3 1.5.1 1.5.2 1.5.3 3.5.7
+# # Audit only, skip consolidation:
+# ./rerun-sections.sh --no-consolidate "files:apache/steve" apache/tooling-agents ghp_xxx \
+# ASVS/reports/steve/v3/d0aa7e9 1.3.3 1.5.1 1.5.2 1.5.3 3.5.7
+#
+# # Re-consolidate only (no auditing):
+# ./rerun-sections.sh --consolidate-only apache/tooling-agents ghp_xxx \
+# ASVS/reports/steve/v3/d0aa7e9
set -euo pipefail
API_BASE="${GOFANNON_API:-http://localhost:8000}"
-if [ $# -lt 5 ]; then
- echo "Usage: $0 <namespace> <output_repo> <output_token> <output_dir> [--consolidate] <section> [section...]"
- echo ""
- echo " namespace Data store namespace (e.g., files:apache/steve/v3)"
- echo " output_repo GitHub repo for reports (e.g., apache/tooling-agents)"
- echo " output_token GitHub PAT with write access"
- echo " output_dir Report directory (e.g., ASVS/reports/steve/v3/d0aa7e9)"
- echo " --consolidate Re-run consolidation after auditing (optional)"
- echo " section ASVS section numbers (e.g., 1.3.3 1.5.1)"
- exit 1
-fi
-
-NAMESPACE="$1"; shift
-OUTPUT_REPO="$1"; shift
-OUTPUT_TOKEN="$1"; shift
-OUTPUT_DIR="$1"; shift
+# ---- Parse mode ----
+CONSOLIDATE=true
+CONSOLIDATE_ONLY=false
-CONSOLIDATE=false
-if [ "${1:-}" = "--consolidate" ]; then
- CONSOLIDATE=true
+if [ "${1:-}" = "--consolidate-only" ]; then
+ CONSOLIDATE_ONLY=true
+ shift
+elif [ "${1:-}" = "--no-consolidate" ]; then
+ CONSOLIDATE=false
shift
fi
-SECTIONS=("$@")
-RERUN_DIR="${OUTPUT_DIR}/rerun"
+if [ "$CONSOLIDATE_ONLY" = true ]; then
+ if [ $# -lt 3 ]; then
+ echo "Usage: $0 --consolidate-only <output_repo> <output_token> <output_dir>"
+ exit 1
+ fi
+ NAMESPACE=""
+ OUTPUT_REPO="$1"; shift
+ OUTPUT_TOKEN="$1"; shift
+ OUTPUT_DIR="$1"; shift
+ SECTIONS=()
+else
+ if [ $# -lt 5 ]; then
+ echo "Usage: $0 [--no-consolidate] <namespace> <output_repo> <output_token> <output_dir> <section> [section...]"
+ echo " $0 --consolidate-only <output_repo> <output_token> <output_dir>"
+ exit 1
+ fi
+ NAMESPACE="$1"; shift
+ OUTPUT_REPO="$1"; shift
+ OUTPUT_TOKEN="$1"; shift
+ OUTPUT_DIR="$1"; shift
+
+ SECTIONS=("$@")
+ if [ ${#SECTIONS[@]} -eq 0 ]; then
+ echo "Error: No sections specified"
+ exit 1
+ fi
+fi
-echo "============================================================"
-echo "Re-running ${#SECTIONS[@]} sections"
-echo " Namespace: ${NAMESPACE}"
-echo " Output: ${OUTPUT_REPO}/${RERUN_DIR}"
-echo " Consolidate: ${CONSOLIDATE}"
-echo "============================================================"
-echo ""
+RERUN_DIR="${OUTPUT_DIR}/rerun"
call_agent() {
local agent_name="$1"
@@ -61,14 +75,23 @@ call_agent() {
-d "${input_json}"
}
+# ---- Audit phase ----
SUCCEEDED=0
FAILED=0
-for section in "${SECTIONS[@]}"; do
- echo "[${section}] Auditing..."
+if [ "$CONSOLIDATE_ONLY" = false ]; then
+ echo "============================================================"
+ echo "Auditing ${#SECTIONS[@]} sections"
+ echo " Namespace: ${NAMESPACE}"
+ echo " Output: ${OUTPUT_REPO}/${RERUN_DIR}"
+ echo " Consolidate after: ${CONSOLIDATE}"
+ echo "============================================================"
+ echo ""
- # Build audit input
- AUDIT_INPUT=$(python3 -c "
+ for section in "${SECTIONS[@]}"; do
+ echo "[${section}] Auditing..."
+
+ AUDIT_INPUT=$(python3 -c "
import json
print(json.dumps({
'inputText': json.dumps({
@@ -78,19 +101,18 @@ print(json.dumps({
}))
")
- AUDIT_RESULT=$(call_agent "run_asvs_security_audit" "${AUDIT_INPUT}")
- AUDIT_TEXT=$(echo "${AUDIT_RESULT}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('outputText',''))" 2>/dev/null || echo "")
+ AUDIT_RESULT=$(call_agent "run_asvs_security_audit" "${AUDIT_INPUT}")
+ AUDIT_TEXT=$(echo "${AUDIT_RESULT}" | python3 -c "import sys,json; print(json.load(sys.stdin).get('outputText',''))" 2>/dev/null || echo "")
- if [ -z "${AUDIT_TEXT}" ] || echo "${AUDIT_TEXT}" | grep -q "^Error:"; then
- echo " AUDIT FAILED: ${AUDIT_TEXT:0:200}"
- FAILED=$((FAILED + 1))
- continue
- fi
+ if [ -z "${AUDIT_TEXT}" ] || echo "${AUDIT_TEXT}" | grep -q "^Error:"; then
+ echo " AUDIT FAILED: ${AUDIT_TEXT:0:200}"
+ FAILED=$((FAILED + 1))
+ continue
+ fi
- echo " Audit done: ${#AUDIT_TEXT} chars"
+ echo " Audit done: ${#AUDIT_TEXT} chars"
- # Push to GitHub
- PUSH_INPUT=$(python3 -c "
+ PUSH_INPUT=$(python3 -c "
import json
print(json.dumps({
'inputText': json.dumps({
@@ -104,29 +126,37 @@ print(json.dumps({
}))
")
- PUSH_RESULT=$(call_agent "add_markdown_file_to_github_directory" "${PUSH_INPUT}")
+ PUSH_RESULT=$(call_agent "add_markdown_file_to_github_directory" "${PUSH_INPUT}")
- if echo "${PUSH_RESULT}" | python3 -c "import sys,json; d=json.load(sys.stdin).get('outputText',''); exit(0 if 'sha' in d.lower() or 'content' in d.lower() else 1)" 2>/dev/null; then
- echo " Push OK"
- SUCCEEDED=$((SUCCEEDED + 1))
- else
- echo " Push FAILED"
- FAILED=$((FAILED + 1))
- fi
+ if echo "${PUSH_RESULT}" | python3 -c "import sys,json; d=json.load(sys.stdin).get('outputText',''); exit(0 if 'sha' in d.lower() or 'content' in d.lower() else 1)" 2>/dev/null; then
+ echo " Push OK"
+ SUCCEEDED=$((SUCCEEDED + 1))
+ else
+ echo " Push FAILED"
+ FAILED=$((FAILED + 1))
+ fi
- echo ""
-done
+ echo ""
+ done
-echo "============================================================"
-echo "Complete: ${SUCCEEDED} succeeded, ${FAILED} failed"
-echo "============================================================"
+ echo "============================================================"
+ echo "Audit complete: ${SUCCEEDED} succeeded, ${FAILED} failed"
+ echo "============================================================"
+fi
+
+# ---- Consolidation phase ----
+if [ "${CONSOLIDATE}" = true ]; then
+ if [ "$CONSOLIDATE_ONLY" = false ] && [ ${SUCCEEDED} -eq 0 ]; then
+ echo "Skipping consolidation (no successful audits)"
+ exit 1
+ fi
-if [ "${CONSOLIDATE}" = true ] && [ ${SUCCEEDED} -gt 0 ]; then
echo ""
- echo "Re-running consolidation..."
+ echo "============================================================"
+ echo "Consolidating reports"
echo " Reading from: ${OUTPUT_DIR}"
+ echo "============================================================"
- # List all subdirectories in the output directory
DIRS=$(curl -s -H "Authorization: token ${OUTPUT_TOKEN}" \
"https://api.github.com/repos/${OUTPUT_REPO}/contents/${OUTPUT_DIR}" \
| python3 -c "
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org