You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tooling.apache.org by gs...@apache.org on 2026/10/13 11:18:49 UTC
(tooling-llmao) 01/02: proxy: watch for edits made outside llmao, and show them as work needed
This is an automated email from the ASF dual-hosted git repository.
gstein pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tooling-llmao.git
commit 01114f1b034a259f7efb9b45d00fbb30e5b6292b
Author: Greg Stein <gs...@gmail.com>
AuthorDate: Tue Oct 13 06:07:04 2026 -0500
proxy: watch for edits made outside llmao, and show them as work needed
llmao is the only thing meant to write the teams, keys and deployments
it made in LiteLLM. A change made in LiteLLM's own admin interface is
not seen by llmao and can take away a limit that was there for a reason.
A second runner, LiteLLMBackend.watch_outside_edits, looks for such
changes. It finds and records; it puts nothing right. Each pass asks a
manager a plain question and hands the answer to the skew store, so what
it finds shows under "Work needed" until somebody deals with it.
The first thing it looks for is a pretend project (free tier; service
with no project) that is no longer limited to our own boxes, or has no
team. A site admin gets a Restrict button on that row of the Admin page.
- TeamManager.pretend_projects_out_of_line() answers the question, and
restrict_pretend_project() puts one right. Startup is built from the
same two.
- The runner uses the team list everything else uses, up to a minute
old, rather than fetching its own.
- The skew loop moves from DeploymentManager to LiteLLMBackend, beside
the new runner. The two are kept apart by cause: the fleet changed, or
somebody edited LiteLLM.
- "Work needed" rows on the Admin page no longer print raw identifiers.
None of this has been run against a real proxy.
Co-Authored-By: Claude Opus 5.5 <no...@anthropic.com>
---
llmao/proxy/client.py | 119 +++++++++++++++++++++++++++++++++++++++------
llmao/proxy/deployments.py | 19 --------
llmao/seam.py | 14 ++++++
main.py | 3 +-
pages.py | 26 +++++++++-
templates/admin.ezt | 14 ++++--
tests/fake_transport.py | 19 ++++++--
tests/test_admin.py | 31 +++++++++++-
tests/test_notices.py | 87 ++++++++++++++++++++++++++++++++-
tests/test_seam.py | 23 +++++++++
tests/test_teams.py | 28 +++++++++++
11 files changed, 336 insertions(+), 47 deletions(-)
diff --git a/llmao/proxy/client.py b/llmao/proxy/client.py
index 2f85748..ef39706 100644
--- a/llmao/proxy/client.py
+++ b/llmao/proxy/client.py
@@ -26,6 +26,7 @@ LiteLLM opaque team_id is internal to LiteLLMBackend only.
from __future__ import annotations
+import asyncio
import datetime as _dt
import json
import logging
@@ -335,6 +336,7 @@ class LiteLLMBackend:
"""
def __init__(self, cfg: Any, fleet: Any, *, transport: Any = None):
+ self._cfg = cfg
self._transport = transport or LiteLLMTransport(cfg)
self.deployments = DeploymentManager(cfg, fleet, self._transport)
memory = MemoryStore(self._transport)
@@ -378,6 +380,73 @@ class LiteLLMBackend:
except BackendUnavailableError as e:
_LOGGER.warning("could not warm usage: %s", e)
+ # ---- the two things that run for as long as the service does --------
+ #
+ # Both compare LiteLLM with what it ought to hold, and they are kept
+ # apart by why it might not. They live here because each asks more than
+ # one manager, and no manager should have to know another.
+
+ async def run_skew(self) -> None:
+ """The fleet has changed and LiteLLM has not followed, or the reverse.
+
+ Config skew often; LiteLLM GET /health (tokens) on a long interval.
+ """
+ skew_s = float(self._cfg.fleet.skew_interval_s)
+ health_s = float(self._cfg.fleet.litellm_health_interval_s)
+ last_health = 0.0
+ first = True
+ while True:
+ try:
+ await self.deployments.check_config_skew()
+ now = time.time()
+ if first or (now - last_health) >= health_s:
+ await self.deployments.check_health_skew()
+ last_health = now
+ first = False
+ except Exception:
+ _LOGGER.exception("LiteLLM skew check failed")
+ await asyncio.sleep(skew_s)
+
+ async def watch_outside_edits(self) -> None:
+ """Somebody has changed LiteLLM where llmao should have been the one to.
+
+ llmao is the only thing meant to write the teams, keys and
+ deployments it made. An edit in LiteLLM's own admin interface is not
+ seen by llmao, may be undone by its next write, and can take away a
+ limit that was there for a reason.
+ """
+ while True:
+ try:
+ await self.check_outside_edits()
+ except Exception:
+ _LOGGER.exception("check for edits outside llmao failed")
+ await asyncio.sleep(float(self._cfg.fleet.skew_interval_s))
+
+ async def check_outside_edits(self) -> None:
+ """One pass. Finds and records; it puts nothing right.
+
+ Each manager is asked a plain question, and the answer is handed to
+ the skew store, which makes its rows match. What is found shows
+ under "Work needed" until somebody deals with it.
+ """
+ # From the same listing everything else uses, up to a minute old.
+ # This comes round every few minutes; a minute either way is nothing.
+ wrong = await self.teams.pretend_projects_out_of_line()
+ await self.skew.reconcile(
+ "project",
+ {
+ project: {
+ "what": (
+ f"The {PRETEND_PROJECTS[project]} project has no team."
+ if allows is None
+ else f"The {PRETEND_PROJECTS[project]} project allows "
+ f"{', '.join(allows) or 'any model'}. Its keys should reach only our own boxes."
+ )
+ }
+ for project, allows in wrong.items()
+ },
+ )
+
class KeyManager:
"""LiteLLM's virtual keys: listing, minting, revoking, and what each may do.
@@ -1012,25 +1081,47 @@ class TeamManager:
error: until this ran, a free key could reach a model that costs
money.
"""
- listed = {row.get("team_alias"): row for row in await self.rows()}
- for project in PRETEND_PROJECTS:
- row = listed.get(project)
- if row is None:
- await self.ensure(project)
- elif set(row.get("models") or []) != self._PRETEND_MODELS:
+ for project, allows in (await self.pretend_projects_out_of_line()).items():
+ if allows is not None:
_LOGGER.error(
"%s allowed %s, not %s; restricting it",
project,
- sorted(row.get("models") or []) or "any model",
+ allows or "any model",
sorted(self._PRETEND_MODELS),
)
- resp = await self._transport.request(
- "POST",
- "team/update",
- json={"team_id": row["team_id"], "models": sorted(self._PRETEND_MODELS)},
- )
- self._transport.raise_http(resp)
- self._team_rows_cache = None
+ await self.restrict_pretend_project(project)
+
+ async def pretend_projects_out_of_line(self) -> dict[str, list[str] | None]:
+ """The pretend projects that are not restricted as they should be.
+
+ {project: the models its team allows}, an empty list meaning any, or
+ None where the project has no team at all. Empty when all is well.
+ """
+ listed = {row.get("team_alias"): row for row in await self.rows()}
+ wrong: dict[str, list[str] | None] = {}
+ for project in PRETEND_PROJECTS:
+ row = listed.get(project)
+ if row is None:
+ wrong[project] = None
+ elif set(row.get("models") or []) != self._PRETEND_MODELS:
+ wrong[project] = sorted(row.get("models") or [])
+ return wrong
+
+ async def restrict_pretend_project(self, project: str) -> None:
+ """Limit a pretend project to our own boxes, making its team if it has none."""
+ if project not in PRETEND_PROJECTS:
+ raise BackendUnavailableError(f"{project} is not a pretend project")
+ team_id = await self.team_id(project)
+ if team_id is None:
+ await self.ensure(project)
+ return
+ resp = await self._transport.request(
+ "POST",
+ "team/update",
+ json={"team_id": team_id, "models": sorted(self._PRETEND_MODELS)},
+ )
+ self._transport.raise_http(resp)
+ self._team_rows_cache = None
async def ensure_id(self, project: str) -> str:
"""Map LDAP project (team_alias) → LiteLLM team_id; create team if needed."""
diff --git a/llmao/proxy/deployments.py b/llmao/proxy/deployments.py
index fd4f3e0..1e474c3 100644
--- a/llmao/proxy/deployments.py
+++ b/llmao/proxy/deployments.py
@@ -24,7 +24,6 @@ then in case they have drifted.
from __future__ import annotations
-import asyncio
import logging
import time
from typing import Any
@@ -236,24 +235,6 @@ class DeploymentManager:
elif srv.state in (UNHEALTHY, STALLED) and dep.in_litellm:
await self.delete_deployment(dep)
- async def run_skew(self) -> None:
- """Config skew often; LiteLLM GET /health (tokens) on a long interval."""
- skew_s = float(self._cfg.fleet.skew_interval_s)
- health_s = float(self._cfg.fleet.litellm_health_interval_s)
- last_health = 0.0
- first = True
- while True:
- try:
- await self.check_config_skew()
- now = time.time()
- if first or (now - last_health) >= health_s:
- await self.check_health_skew()
- last_health = now
- first = False
- except Exception:
- _LOGGER.exception("LiteLLM skew check failed")
- await asyncio.sleep(skew_s)
-
async def check_config_skew(self) -> None:
resp = await self._transport.request("GET", "model/info")
self._transport.raise_http(resp)
diff --git a/llmao/seam.py b/llmao/seam.py
index 6c42938..c447d6e 100644
--- a/llmao/seam.py
+++ b/llmao/seam.py
@@ -33,6 +33,7 @@ from typing import Any
from . import access_requests as _access
from .proxy.client import (
GRANTOR_FREE_TIER,
+ PRETEND_PROJECTS,
TIER_PROJECT,
TIER_SERVICE,
TIERS,
@@ -378,6 +379,19 @@ class Seam:
raise AuthzError("That is limited to site admins.")
return await self._backend.skew.open()
+ async def restrict_pretend_project(self, identity: Identity, project: str) -> None:
+ """Put a pretend project back to our own boxes only. Site admins.
+
+ The button on its "Work needed" row. The row is cleared here rather
+ than left for the next check to find gone.
+ """
+ if not identity.is_site_admin:
+ raise AuthzError("That is limited to site admins.")
+ if project not in PRETEND_PROJECTS:
+ raise AuthzError("There is no such project to restrict.")
+ await self._backend.teams.restrict_pretend_project(project)
+ await self._backend.skew.clear("project", project)
+
async def work_count(self, identity: Identity) -> int:
"""How many things are out of line that this person is the one to act on.
diff --git a/main.py b/main.py
index 3c29f2b..430cfc5 100644
--- a/main.py
+++ b/main.py
@@ -84,7 +84,8 @@ def create_app():
fleet.after_probe = backend.deployments.sync_selfhost
app.fleet = fleet
app.add_runner(fleet.run_lifecycle, name="fleet-lifecycle")
- app.add_runner(backend.deployments.run_skew, name="litellm-skew")
+ app.add_runner(backend.run_skew, name="litellm-skew")
+ app.add_runner(backend.watch_outside_edits, name="litellm-outside-edits")
# Sample queue depth on each box.
#
diff --git a/pages.py b/pages.py
index 0623775..4887805 100644
--- a/pages.py
+++ b/pages.py
@@ -1404,13 +1404,22 @@ def _stamp(at) -> str:
return f"{at[:16].replace('T', ' ')} UTC" if at else ""
+# What a site admin can press to deal with each kind of skew: (where the
+# form posts, what the button says). A kind with no entry has no button yet
+# and is put right some other way.
+_SKEW_ACTIONS = {
+ "project": ("/do-restrict-project", "Restrict"),
+}
+
+
def _skew_rows(entries) -> list:
"""Template rows for the Admin page's "Work needed"."""
return [
edict(
what=n.get("what") or "",
- # which thing: "key tok1", "deployment gemma"
- about=" ".join(str(x) for x in (n.get("kind"), n.get("subject")) if x),
+ subject=n.get("subject") or "",
+ action=_SKEW_ACTIONS.get(n.get("kind"), ("", ""))[0],
+ action_label=_SKEW_ACTIONS.get(n.get("kind"), ("", ""))[1],
# How long it has been wrong. An unchanged row is not rewritten,
# so its last write says nothing about when it was last checked.
since=_stamp(n.get("first_seen")),
@@ -1743,6 +1752,19 @@ async def do_create_other_key():
return _see_other("/keys/other")
+@APP.post("/do-restrict-project")
+@asfquart.auth.require
+async def do_restrict_project():
+ """A site admin puts a pretend project back to our own boxes only."""
+ form = await quart.request.form
+ try:
+ await APP.seam.restrict_pretend_project(await current_identity(APP.cfg), (form.get("subject") or "").strip())
+ await flash_success("Restricted to our own boxes.")
+ except (AuthzError, BackendUnavailableError) as e:
+ await flash_danger(str(e))
+ return _see_other("/admin")
+
+
@APP.post("/do-acknowledge")
@asfquart.auth.require
async def do_acknowledge():
diff --git a/templates/admin.ezt b/templates/admin.ezt
index a3adf25..fdc5d3b 100644
--- a/templates/admin.ezt
+++ b/templates/admin.ezt
@@ -12,9 +12,17 @@
<div class="card-header fw-semibold">Work needed</div>
<ul class="list-group list-group-flush">
[for skew]
- <li class="list-group-item small">
- <span class="fw-semibold">[skew.what]</span>
- <span class="text-muted">· [skew.about][if-any skew.since] · since [skew.since][end]</span>
+ <li class="list-group-item small d-flex justify-content-between align-items-start">
+ <div>
+ <span class="fw-semibold">[skew.what]</span>
+ <span class="text-muted">[if-any skew.since]· since [skew.since][end]</span>
+ </div>
+ [if-any skew.action]
+ <form method="post" action="[skew.action]" class="mb-0 ms-3">
+ <input type="hidden" name="subject" value="[skew.subject]">
+ <button type="submit" class="btn btn-warning btn-sm">[skew.action_label]</button>
+ </form>
+ [end]
</li>
[end]
</ul>
diff --git a/tests/fake_transport.py b/tests/fake_transport.py
index 225412d..9e095b2 100644
--- a/tests/fake_transport.py
+++ b/tests/fake_transport.py
@@ -60,11 +60,12 @@ class FakeTransport:
return None
-def fake_memory(*, down: bool = False):
- """A FakeTransport that behaves like LiteLLM's /v1/memory, and what it holds.
+def memory_answers():
+ """How LiteLLM's /v1/memory answers, and what it holds.
- Returns (transport, store). `store` is the rows, key to value, for a
- test to seed or to look at. Any call that is not to /v1/memory fails.
+ Returns (answer, store), for a test whose stand-in LiteLLM has to answer
+ other calls as well. `store` is the rows, key to value. The answer
+ function fails on any call that is not to /v1/memory.
"""
store = {}
@@ -83,4 +84,14 @@ def fake_memory(*, down: bool = False):
return {"deleted": True} if store.pop(key, None) is not None else (404, {})
raise AssertionError(f"unexpected call: {call}")
+ return answer, store
+
+
+def fake_memory(*, down: bool = False):
+ """A FakeTransport that behaves like LiteLLM's /v1/memory, and what it holds.
+
+ Returns (transport, store). `store` is the rows, key to value, for a
+ test to seed or to look at.
+ """
+ answer, store = memory_answers()
return FakeTransport(answer, down=down), store
diff --git a/tests/test_admin.py b/tests/test_admin.py
index 97e8d58..e9d329a 100644
--- a/tests/test_admin.py
+++ b/tests/test_admin.py
@@ -138,16 +138,43 @@ def test_proxy_version_is_unavailable_when_litellm_cannot_be_reached():
def test_admin_shows_work_needed():
"""Skew stays until it is put right, so it has no button to dismiss it."""
pin = pinned_litellm_version()
- skew = [edict(what="limits differ from the service tier", about="key t1", since="2026-10-12 09:00 UTC")]
+ skew = [
+ edict(
+ what="limits differ from the service tier",
+ subject="t1",
+ action="",
+ action_label="",
+ since="2026-10-12 09:00 UTC",
+ )
+ ]
out = _render(running=pin, expected=pin, skew=skew)
assert "Work needed" in out
assert "limits differ from the service tier" in out
- assert "key t1" in out
assert "since 2026-10-12 09:00 UTC" in out
assert "/do-acknowledge" not in out
+ assert "<form" not in out[out.index("Work needed") : out.index("list-group-item-action")]
assert "Work needed" not in _render(running=pin, expected=pin)
+def test_work_with_a_way_to_do_it_has_a_button_that_does_it():
+ pin = pinned_litellm_version()
+ skew = [
+ edict(
+ what="The free tier project allows any model.",
+ subject="llmao:project:free-tier",
+ action="/do-restrict-project",
+ action_label="Restrict",
+ since="",
+ )
+ ]
+ out = _render(running=pin, expected=pin, skew=skew)
+ assert 'action="/do-restrict-project"' in out
+ assert 'name="subject" value="llmao:project:free-tier"' in out
+ assert ">Restrict</button>" in out
+ # the identifier is for the form; the sentence is what a person reads
+ assert out.count("llmao:project:free-tier") == 1
+
+
def test_something_to_see_is_shown_on_the_page_and_can_be_acknowledged():
"""The same partial on every page, so there is one place a person is told things."""
pin = pinned_litellm_version()
diff --git a/tests/test_notices.py b/tests/test_notices.py
index 178967b..c96338a 100644
--- a/tests/test_notices.py
+++ b/tests/test_notices.py
@@ -23,8 +23,17 @@ entry is something to be seen, and goes when it has been.
import asyncio
import json
-from llmao.proxy.client import LogManager, MemoryStore, SkewManager
-from tests.fake_transport import fake_memory
+from easydict import EasyDict as edict # noqa: N813
+
+from llmao.proxy.client import (
+ PROJECT_FREE_TIER,
+ PROJECT_SERVICE,
+ LiteLLMBackend,
+ LogManager,
+ MemoryStore,
+ SkewManager,
+)
+from tests.fake_transport import FakeTransport, fake_memory, memory_answers
def _run(coro):
@@ -141,3 +150,77 @@ def test_an_entry_can_be_checked_against_who_is_acknowledging_it():
assert log.is_for(entry.key, "alice")
assert not log.is_for(entry.key, "bob")
assert not log.is_for(entry.key)
+
+
+# ---------------------------------------------------------------------------
+# Watching for edits made outside llmao
+#
+# llmao is the only thing meant to write what it made in LiteLLM. The watcher
+# asks each manager a plain question and hands the answer to the skew store.
+# It finds and records; it puts nothing right.
+# ---------------------------------------------------------------------------
+
+
+def _backend(teams):
+ """A real backend over a LiteLLM that has these teams and a memory store."""
+ memory, store = memory_answers()
+
+ def litellm(call):
+ if call.path.startswith("v1/memory"):
+ return memory(call)
+ if call.path == "team/list":
+ return {"teams": teams}
+ raise AssertionError(f"the watcher changed something: {call}")
+
+ return LiteLLMBackend(edict(), None, transport=FakeTransport(litellm)), store
+
+
+def _team(project, models):
+ return {"team_alias": project, "team_id": f"id-{project}", "models": models}
+
+
+def test_a_pretend_project_that_has_been_widened_becomes_work_needed():
+ teams = [_team(PROJECT_FREE_TIER, []), _team(PROJECT_SERVICE, ["llmao:self-hosted"])]
+ backend, store = _backend(teams)
+ _run(backend.check_outside_edits())
+ assert list(store) == [f"llmao:skew:project:{PROJECT_FREE_TIER}"]
+ (row,) = _run(backend.skew.open())
+ assert row.what == "The free tier project allows any model. Its keys should reach only our own boxes."
+
+
+def test_it_says_what_the_project_was_opened_to():
+ teams = [
+ _team(PROJECT_FREE_TIER, ["llmao:self-hosted", "claude-opus-5"]),
+ _team(PROJECT_SERVICE, ["llmao:self-hosted"]),
+ ]
+ backend, _ = _backend(teams)
+ _run(backend.check_outside_edits())
+ (row,) = _run(backend.skew.open())
+ assert "allows claude-opus-5, llmao:self-hosted." in row.what
+
+
+def test_the_row_goes_once_the_project_is_as_it_should_be():
+ teams = [_team(PROJECT_FREE_TIER, []), _team(PROJECT_SERVICE, ["llmao:self-hosted"])]
+ backend, store = _backend(teams)
+ _run(backend.check_outside_edits())
+ teams[0]["models"] = ["llmao:self-hosted"]
+ backend.teams._team_rows_cache = None # a minute later
+ _run(backend.check_outside_edits())
+ assert store == {}
+
+
+def test_nothing_is_recorded_when_nothing_is_wrong():
+ teams = [_team(PROJECT_FREE_TIER, ["llmao:self-hosted"]), _team(PROJECT_SERVICE, ["llmao:self-hosted"])]
+ backend, store = _backend(teams)
+ _run(backend.check_outside_edits())
+ assert store == {}
+
+
+def test_the_watcher_does_not_ask_for_its_own_copy_of_the_team_list():
+ """It comes round every few minutes; the listing everything else uses will do."""
+ teams = [_team(PROJECT_FREE_TIER, []), _team(PROJECT_SERVICE, ["llmao:self-hosted"])]
+ backend, _ = _backend(teams)
+ _run(backend.teams.rows())
+ _run(backend.check_outside_edits())
+ _run(backend.check_outside_edits())
+ assert [c.path for c in backend._transport.calls].count("team/list") == 1
diff --git a/tests/test_seam.py b/tests/test_seam.py
index 3a76224..4cb33c7 100644
--- a/tests/test_seam.py
+++ b/tests/test_seam.py
@@ -547,3 +547,26 @@ def test_work_is_counted_for_whoever_is_the_one_to_do_it():
assert await seam.work_count(Identity(uid="carol", projects=[], committees=[])) == 0
asyncio.run(run())
+
+
+def test_a_site_admin_puts_a_pretend_project_back_and_the_work_is_gone():
+ async def run():
+ seam, backend, store = _with_log()
+ restricted = []
+
+ async def restrict(project):
+ restricted.append(project)
+
+ backend.teams.restrict_pretend_project = restrict
+ await backend.skew.reconcile("project", {PROJECT_FREE_TIER: {"what": "allows any model"}})
+
+ for who, project in ((ALICE, PROJECT_FREE_TIER), (ROOT, "airflow")):
+ with pytest.raises(AuthzError):
+ await seam.restrict_pretend_project(who, project)
+ assert restricted == [] and len(store) == 1
+
+ await seam.restrict_pretend_project(ROOT, PROJECT_FREE_TIER)
+ assert restricted == [PROJECT_FREE_TIER]
+ assert store == {}
+
+ asyncio.run(run())
diff --git a/tests/test_teams.py b/tests/test_teams.py
index dc82e2f..86d1a60 100644
--- a/tests/test_teams.py
+++ b/tests/test_teams.py
@@ -249,6 +249,34 @@ def test_backfill_skips_malformed_rows_without_stopping():
assert _sent(b)[0][1]["team_id"] == "t1"
+def test_asking_which_pretend_projects_are_out_of_line_changes_nothing():
+ """The question a watcher asks. Finding is separate from putting right."""
+ teams = _litellm_with({PROJECT_FREE_TIER: [], PROJECT_SERVICE: [GROUP, "claude-opus-5"]})
+ assert _run(teams.pretend_projects_out_of_line()) == {
+ PROJECT_FREE_TIER: [],
+ PROJECT_SERVICE: ["claude-opus-5", GROUP],
+ }
+ assert _updated(teams) == [] and _made(teams) == []
+ assert _run(_litellm_with([PROJECT_FREE_TIER, PROJECT_SERVICE]).pretend_projects_out_of_line()) == {}
+ # one with no team at all is out of line too
+ assert _run(_litellm_with([PROJECT_FREE_TIER]).pretend_projects_out_of_line()) == {PROJECT_SERVICE: None}
+
+
+def test_restricting_one_pretend_project_leaves_the_other_alone():
+ teams = _litellm_with({PROJECT_FREE_TIER: [], PROJECT_SERVICE: []})
+ _run(teams.restrict_pretend_project(PROJECT_FREE_TIER))
+ assert _updated(teams) == [(f"id-{PROJECT_FREE_TIER}", [GROUP])]
+ assert _run(teams.pretend_projects_out_of_line()) == {PROJECT_SERVICE: []}
+
+
+def test_only_a_pretend_project_can_be_restricted_this_way():
+ """A real project's team has no list on purpose."""
+ teams = _litellm_with(["airflow"])
+ with pytest.raises(BackendUnavailableError):
+ _run(teams.restrict_pretend_project("airflow"))
+ assert _updated(teams) == []
+
+
def test_startup_stops_if_a_pretend_project_cannot_be_made_or_restricted():
"""Fail-fast: a service that cannot restrict free keys should not start."""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@tooling.apache.org
For additional commands, e-mail: commits-help@tooling.apache.org