OPNexus/backend/tests/test_api_rights_gateway_mute.py

"""1.29: missing API rights become visible (OpnsenseForbiddenError, `denied` in the sample, api_rights_missing alert,
dashboard, metrics) and the gateway alert can be switched off per gateway."""
import uuid

import httpx
import pytest
from sqlalchemy import select

from app.database import SessionLocal
from app.models.audit import AuditEvent
from app.models.host import OpnsenseHost
from app.models.notification import NotificationProblem
from app.models.resource_sample import HostResourceSample
from app.services import monitor_checks, notification_delivery, opnsense_client, resource_monitor_service
from app.services.notification_service import run_notification_checks
from tests.test_monitoring_alerts import (
    STATUS_PAYLOAD, _add_samples, _gw, _gateway_problems, _only, _open_count, _set,
)
from tests.test_notifications import _create_host
from tests.test_resource_monitor import _mock_resource_endpoints, _mock_vip
from tests.helpers import delete_host


async def _host_row(host_id: str) -> OpnsenseHost:
    async with SessionLocal() as db:
        return await db.get(OpnsenseHost, uuid.UUID(host_id))


async def _clear_samples():
    async with SessionLocal() as db:
        await db.execute(HostResourceSample.__table__.delete())
        await db.commit()


async def _rights_problems():
    async with SessionLocal() as db:
        return await monitor_checks.check_api_rights_missing(db)


# ── Client: 403 is told apart from everything else (a 401 is a wrong key, not a missing right) ──────────────────────

@pytest.mark.asyncio
@pytest.mark.parametrize("status", [403])
async def test_403_raises_forbidden_error_with_unchanged_message(client, respx_mock, status):
    _mock_vip(respx_mock)
    host = await _host_row((await _create_host(client, "fw1", "https://fw1.test"))["id"])
    respx_mock.get("https://fw1.test/api/routes/gateway/status").respond(status, text="denied")
    with pytest.raises(opnsense_client.OpnsenseForbiddenError) as exc:
        await opnsense_client.get_gateway_status(host)
    assert isinstance(exc.value, opnsense_client.OpnsenseAPIError)  # existing `except OpnsenseAPIError` stays valid
    assert not isinstance(exc.value, opnsense_client.OpnsenseAmbiguousError)
    assert str(exc.value).startswith(f"{status} /routes/gateway/status")


@pytest.mark.asyncio
@pytest.mark.parametrize("status,ambiguous", [(500, True), (502, True), (404, False), (400, False), (401, False)])
async def test_other_errors_are_not_forbidden(client, respx_mock, status, ambiguous):
    _mock_vip(respx_mock)
    host = await _host_row((await _create_host(client, "fw1", "https://fw1.test"))["id"])
    respx_mock.get("https://fw1.test/api/routes/gateway/status").respond(status)
    with pytest.raises(opnsense_client.OpnsenseAPIError) as exc:
        await opnsense_client.get_gateway_status(host)
    assert not isinstance(exc.value, opnsense_client.OpnsenseForbiddenError)
    assert isinstance(exc.value, opnsense_client.OpnsenseAmbiguousError) is ambiguous


@pytest.mark.asyncio
async def test_gateway_endpoint_reports_missing_rights_not_a_generic_502(client, respx_mock):
    _mock_vip(respx_mock)
    host = await _create_host(client, "fw1", "https://fw1.test")
    route = respx_mock.get("https://fw1.test/api/routes/gateway/status")
    route.respond(403)
    resp = await client.get(f"/api/hosts/{host['id']}/gateways")
    assert resp.status_code == 403 and resp.json()["detail"]["code"] == "API_RIGHTS_MISSING"
    route.respond(500)
    assert (await client.get(f"/api/hosts/{host['id']}/gateways")).status_code == 502


# ── Sampler records what was refused ─────────────────────────────────────────

def _mock_optional_reads(respx_mock, *, gateways=200, services=200, swap=200, cpu=200):
    base = "https://fw1.test/api"
    respx_mock.get(f"{base}/diagnostics/system/systemSwap").respond(swap, json={"swap": []})
    respx_mock.get(f"{base}/diagnostics/cpu_usage/getCPUType").respond(cpu, json=["X (4 cores, 4 threads)"])
    respx_mock.get(f"{base}/routes/gateway/status").respond(gateways, json=STATUS_PAYLOAD)
    respx_mock.get(f"{base}/core/service/search").respond(services, json={"rows": []})


async def _sample_raw() -> dict:
    async with SessionLocal() as db:
        await resource_monitor_service.poll_all_hosts(db)
        return (await db.scalars(select(HostResourceSample).order_by(HostResourceSample.sampled_at.desc()))).first().raw


@pytest.mark.asyncio
async def test_sample_records_only_refused_reads_in_denied(client, respx_mock):
    _mock_vip(respx_mock)
    await _create_host(client, "fw1", "https://fw1.test")
    _mock_resource_endpoints(respx_mock)
    _mock_optional_reads(respx_mock, gateways=403, services=403, swap=500, cpu=401)
    raw = await _sample_raw()
    # forbidden -> denied (sorted); a server error and a missing endpoint are "unknown", no finding about rights
    assert raw["denied"] == ["gateways", "services"]
    assert raw["gateways"] is None and raw["services"] is None and raw["swap_used_pct"] is None
    assert raw["ok"] == []  # nothing went through


@pytest.mark.asyncio
async def test_sample_denied_is_an_empty_list_when_nothing_is_refused(client, respx_mock):
    _mock_vip(respx_mock)
    await _create_host(client, "fw1", "https://fw1.test")
    _mock_resource_endpoints(respx_mock)
    _mock_optional_reads(respx_mock)
    raw = await _sample_raw()
    assert raw["denied"] == [] and raw["ok"] == ["cpu", "gateways", "services", "swap"]
    # unreachable optional endpoints (connect error / timeout) are not refusals either
    _mock_optional_reads(respx_mock)
    respx_mock.get("https://fw1.test/api/routes/gateway/status").mock(side_effect=httpx.ReadTimeout("slow"))
    assert (await _sample_raw())["denied"] == []


@pytest.mark.asyncio
async def test_base_read_refusal_still_fails_the_sample(client, respx_mock):
    _mock_vip(respx_mock)
    await _create_host(client, "fw1", "https://fw1.test")
    _mock_resource_endpoints(respx_mock)
    respx_mock.get("https://fw1.test/api/diagnostics/system/systemResources").respond(403)
    async with SessionLocal() as db:
        result = await resource_monitor_service.poll_all_hosts(db)
    assert result["hosts_failed"] == 1 and result["sampled"] == 0


# ── The alert ────────────────────────────────────────────────────────────────

DENIED_GW = {"denied": ["gateways"], "ok": ["cpu", "services", "swap"], "gateways": None, "services": {"unbound": 1}, "cpu_threads": 4}
FINE = {"denied": [], "ok": ["cpu", "gateways", "services", "swap"], "gateways": [_gw()], "services": {"unbound": 1}, "cpu_threads": 4}


@pytest.mark.asyncio
async def test_api_rights_alert_needs_consecutive_samples_and_names_the_missing_right(client):
    host = await _create_host(client, "fw1", "https://fw1.test")
    hid = uuid.UUID(host["id"])
    await _add_samples(hid, [DENIED_GW, DENIED_GW, FINE])
    assert dict(await _rights_problems()) == {}  # two in a row are not enough (debounce)
    await _clear_samples()
    await _add_samples(hid, [DENIED_GW] * 3)
    problems = await _rights_problems()
    assert list(problems) == [f"{hid}:gateways"]
    ctx = problems[f"{hid}:gateways"]
    assert ctx["host_name"] == "fw1" and ctx["fehlendes_recht"] == "System: Gateways"
    assert ctx["endpunkt"] == "/api/routes/gateway/status" and "fehlt das Recht" in ctx["hinweis"]


@pytest.mark.asyncio
async def test_api_rights_alert_opens_once_resolves_and_is_delivered(client, respx_mock):
    _mock_vip(respx_mock)
    host = await _create_host(client, "fw1", "https://fw1.test")
    hid = uuid.UUID(host["id"])
    await _only("api_rights_enabled")
    await _set(webhook_enabled=True, webhook_url="https://hooks.test/n")
    hook = respx_mock.post("https://hooks.test/n").mock(return_value=httpx.Response(200))
    await _add_samples(hid, [DENIED_GW] * 3)
    async with SessionLocal() as db:
        assert (await run_notification_checks(db))["opened"] == 1
    async with SessionLocal() as db:
        assert (await run_notification_checks(db))["opened"] == 0
        assert await _open_count(db, "api_rights_missing") == 1
    assert hook.call_count == 1
    assert b"API-Rechte fehlen" in hook.calls.last.request.content and b"System: Gateways" in hook.calls.last.request.content
    await _clear_samples()
    await _add_samples(hid, [FINE] * 3)
    async with SessionLocal() as db:
        assert (await run_notification_checks(db))["resolved"] == 1
    assert hook.call_count == 2 and b"behoben" in hook.calls.last.request.content


@pytest.mark.asyncio
async def test_api_rights_alert_stays_open_through_unknown_and_flapping_samples(client):
    host = await _create_host(client, "fw1", "https://fw1.test")
    hid = uuid.UUID(host["id"])
    key = f"{hid}:gateways"
    await _only("api_rights_enabled")
    await _add_samples(hid, [DENIED_GW] * 3)
    async with SessionLocal() as db:
        await run_notification_checks(db)
    # the gateway read now times out (gateways None, not denied): no proof the right is back
    await _clear_samples()
    await _add_samples(hid, [{"denied": [], "gateways": None}] * 3)
    assert list(await _rights_problems()) == [key]
    # samples from before the denied list existed are unknown too
    await _clear_samples()
    await _add_samples(hid, [{"gateways": [_gw()]}] * 3)
    assert list(await _rights_problems()) == [key]
    # one good sample between refusals does not close it
    await _clear_samples()
    await _add_samples(hid, [DENIED_GW, FINE, DENIED_GW])
    assert list(await _rights_problems()) == [key]


@pytest.mark.asyncio
async def test_api_rights_alert_ignores_stale_probe_and_keeps_open_problem(client):
    host = await _create_host(client, "fw1", "https://fw1.test")
    hid = uuid.UUID(host["id"])
    await _only("api_rights_enabled")
    await _add_samples(hid, [DENIED_GW] * 3)
    async with SessionLocal() as db:
        await run_notification_checks(db)
    await _clear_samples()
    await _add_samples(hid, [FINE] * 3, age_minutes=30)  # the host stopped answering
    async with SessionLocal() as db:
        assert (await run_notification_checks(db))["resolved"] == 0
        assert await _open_count(db) == 1
    # a stale refusal does not open anything either
    await _clear_samples()
    async with SessionLocal() as db:
        await db.execute(NotificationProblem.__table__.delete())
        await db.commit()
    await _add_samples(hid, [DENIED_GW] * 3, age_minutes=30)
    assert dict(await _rights_problems()) == {}


@pytest.mark.asyncio
async def test_api_rights_alert_is_on_by_default_and_can_be_switched_off(client):
    data = (await client.get("/api/notifications/settings")).json()
    assert data["api_rights_enabled"] is True
    resp = await client.put("/api/notifications/settings", json={"api_rights_enabled": False})
    assert resp.status_code == 200 and resp.json()["api_rights_enabled"] is False
    host = await _create_host(client, "fw1", "https://fw1.test")
    await _add_samples(uuid.UUID(host["id"]), [DENIED_GW] * 3)
    await _only()  # everything off, including api_rights_enabled
    async with SessionLocal() as db:
        assert (await run_notification_checks(db))["opened"] == 0


def test_alert_label_exists():
    assert notification_delivery._LABELS["api_rights_missing"] == "API-Rechte fehlen"


# ── Dashboard and metrics ────────────────────────────────────────────────────

@pytest.mark.asyncio
async def test_dashboard_and_metrics_show_missing_rights(client):
    host = await _create_host(client, "fw1", "https://fw1.test", cluster_name=None)
    hid = uuid.UUID(host["id"])
    await _add_samples(hid, [{"mem": 10, "load": 0.1, **DENIED_GW, "denied": ["gateways", "services"]}])
    summary = (await client.get("/api/dashboard")).json()["hosts"][0]
    assert summary["api_rights_missing"] == ["gateways", "services"] and summary["gateways"] is None
    text = (await client.get("/api/metrics")).text
    assert 'opnexus_host_api_rights_missing{host="fw1",cluster="",read="gateways"} 1' in text
    assert 'opnexus_host_api_rights_missing{host="fw1",cluster="",read="services"} 1' in text

    await _clear_samples()
    await _add_samples(hid, [{"mem": 10, "load": 0.1, **FINE}])
    assert (await client.get("/api/dashboard")).json()["hosts"][0]["api_rights_missing"] == []
    assert 'api_rights_missing{' not in (await client.get("/api/metrics")).text.replace("# HELP", "").replace("# TYPE", "")

    await _clear_samples()
    await _add_samples(hid, [DENIED_GW], age_minutes=20)
    assert (await client.get("/api/dashboard")).json()["hosts"][0]["api_rights_missing"] is None


# ── Mute a single gateway ────────────────────────────────────────────────────

BAD = {"gateways": [_gw("offline", 100.0, None), _gw("offline", 100.0, None, name="LTE_GW")]}


@pytest.mark.asyncio
async def test_muted_gateway_raises_no_alert_but_the_others_do(client):
    host = await _create_host(client, "fw1", "https://fw1.test")
    hid = host["id"]
    await _add_samples(uuid.UUID(hid), [BAD] * 3)
    assert sorted(await _gateway_problems()) == sorted([f"{hid}:WAN_GW", f"{hid}:LTE_GW"])
    resp = await client.put(f"/api/hosts/{hid}/gateways/LTE_GW/alert", json={"enabled": False})
    assert resp.status_code == 200 and resp.json() == {"name": "LTE_GW", "alert_enabled": False}
    assert list(await _gateway_problems()) == [f"{hid}:WAN_GW"]
    # a mute is per host: the same name on another host is still watched
    other = await _create_host(client, "fw2", "https://fw2.test")
    await _add_samples(uuid.UUID(other["id"]), [BAD] * 3)
    assert f"{other['id']}:LTE_GW" in await _gateway_problems()
    # switching it on again brings the alert back
    await client.put(f"/api/hosts/{hid}/gateways/LTE_GW/alert", json={"enabled": True})
    assert f"{hid}:LTE_GW" in await _gateway_problems()


@pytest.mark.asyncio
async def test_muting_resolves_an_open_gateway_problem(client):
    host = await _create_host(client, "fw1", "https://fw1.test")
    hid = host["id"]
    await _only("gateway_down_enabled")
    await _add_samples(uuid.UUID(hid), [BAD] * 3)
    async with SessionLocal() as db:
        assert (await run_notification_checks(db))["opened"] == 2
    await client.put(f"/api/hosts/{hid}/gateways/LTE_GW/alert", json={"enabled": False})
    async with SessionLocal() as db:
        assert (await run_notification_checks(db))["resolved"] == 1
        open_keys = (await db.scalars(select(NotificationProblem.dedup_key).where(NotificationProblem.status == "open"))).all()
    assert open_keys == [f"{hid}:WAN_GW"]


@pytest.mark.asyncio
async def test_mute_endpoint_is_idempotent_validates_and_audits(client):
    host = await _create_host(client, "fw1", "https://fw1.test")
    hid = host["id"]
    for _ in range(2):
        assert (await client.put(f"/api/hosts/{hid}/gateways/WAN_DHCP6/alert", json={"enabled": False})).status_code == 200
    stored = (await client.get("/api/notifications/settings")).json()["gateway_alert_muted"]
    assert stored == [f"{hid}:WAN_DHCP6"]  # no duplicate

    assert (await client.put(f"/api/hosts/{hid}/gateways/WAN_DHCP6/alert", json={})).status_code == 422
    assert (await client.put(f"/api/hosts/{hid}/gateways/WAN_DHCP6/alert", json={"enabled": "maybe"})).status_code == 422
    for bad in ["x" * 65, "a%0Ab", "a%20b", "%E2%80%AE", "a%3Ab", "a%2Fb"]:
        resp = await client.put(f"/api/hosts/{hid}/gateways/{bad}/alert", json={"enabled": False})
        assert resp.status_code in (404, 422), (bad, resp.status_code)
        assert resp.status_code != 200
    assert (await client.get("/api/notifications/settings")).json()["gateway_alert_muted"] == stored
    missing = await client.put(f"/api/hosts/{uuid.uuid4()}/gateways/WAN_GW/alert", json={"enabled": False})
    assert missing.status_code == 404

    async with SessionLocal() as db:
        events = (await db.scalars(select(AuditEvent).where(AuditEvent.action.like("gateway_alert_%")))).all()
    assert [e.action for e in events] == ["gateway_alert_off", "gateway_alert_off"]
    assert events[0].host_id == hid and events[0].message == "Gateway WAN_DHCP6" and events[0].actor


@pytest.mark.asyncio
async def test_mute_list_is_capped(client, monkeypatch):
    from app.routers import gateways
    monkeypatch.setattr(gateways, "MAX_MUTED_GATEWAYS", 3)
    hid = (await _create_host(client, "fw1", "https://fw1.test"))["id"]
    for i in range(3):
        assert (await client.put(f"/api/hosts/{hid}/gateways/GW{i}/alert", json={"enabled": False})).status_code == 200
    resp = await client.put(f"/api/hosts/{hid}/gateways/GW3/alert", json={"enabled": False})
    assert resp.status_code == 422 and resp.json()["detail"]["code"] == "GATEWAY_MUTE_LIMIT"
    # an already muted one and un-muting still work at the limit
    assert (await client.put(f"/api/hosts/{hid}/gateways/GW0/alert", json={"enabled": False})).status_code == 200
    assert (await client.put(f"/api/hosts/{hid}/gateways/GW0/alert", json={"enabled": True})).status_code == 200


@pytest.mark.asyncio
async def test_only_admins_may_switch_a_gateway_alert(client):
    from tests.test_api_tokens import _login
    from tests.test_operator_role import PW, _operator
    hid = (await _create_host(client, "fw1", "https://fw1.test"))["id"]
    ro = await client.post("/api/users", json={"username": "reader", "password": PW, "role": "read_only"})
    assert ro.status_code == 201
    readers = [await _operator(client, "gw-op"), await _login("reader", PW)]
    try:
        for who in readers:
            resp = await who.put(f"/api/hosts/{hid}/gateways/WAN_GW/alert", json={"enabled": False})
            assert resp.status_code == 403, resp.text
    finally:
        for who in readers:
            await who.aclose()
    assert (await client.get("/api/notifications/settings")).json()["gateway_alert_muted"] == []


@pytest.mark.asyncio
async def test_gateway_list_and_dashboard_carry_the_alert_flag(client, respx_mock):
    _mock_vip(respx_mock)
    host = await _create_host(client, "fw1", "https://fw1.test")
    hid = host["id"]
    respx_mock.get("https://fw1.test/api/routes/gateway/status").respond(200, json=STATUS_PAYLOAD)
    await client.put(f"/api/hosts/{hid}/gateways/LTE_GW/alert", json={"enabled": False})
    rows = {g["name"]: g for g in (await client.get(f"/api/hosts/{hid}/gateways")).json()}
    assert rows["LTE_GW"]["alert_enabled"] is False and rows["WAN_GW"]["alert_enabled"] is True
    await _add_samples(uuid.UUID(hid), [{"gateways": [_gw(), _gw(name="LTE_GW")]}])
    dash = {g["name"]: g for g in (await client.get("/api/dashboard")).json()["hosts"][0]["gateways"]}
    assert dash["LTE_GW"]["alert_enabled"] is False and dash["WAN_GW"]["alert_enabled"] is True


@pytest.mark.asyncio
async def test_api_rights_endpoint_reads_the_newest_fresh_sample(client):
    hid = (await _create_host(client, "fw1", "https://fw1.test"))["id"]
    url = f"/api/hosts/{hid}/gateways/api-rights"
    assert (await client.get(url)).json() == {"missing": None}  # no sample at all: unknown, not "fine"
    await _add_samples(uuid.UUID(hid), [DENIED_GW, FINE])
    assert (await client.get(url)).json() == {"missing": ["gateways"]}
    await _clear_samples()
    await _add_samples(uuid.UUID(hid), [FINE, DENIED_GW])
    assert (await client.get(url)).json() == {"missing": []}
    await _clear_samples()
    await _add_samples(uuid.UUID(hid), [DENIED_GW], age_minutes=20)
    assert (await client.get(url)).json() == {"missing": None}  # stale
    assert (await client.get(f"/api/hosts/{uuid.uuid4()}/gateways/api-rights")).status_code == 404


@pytest.mark.asyncio
async def test_right_back_closes_the_alert_even_when_the_answer_is_unparsable(client):
    """cpu_threads stays None when the CPU string has no "threads"/"cores": only the `ok` list proves the read went through."""
    host = await _create_host(client, "fw1", "https://fw1.test")
    hid = uuid.UUID(host["id"])
    await _only("api_rights_enabled")
    await _add_samples(hid, [{"denied": ["cpu"], "ok": ["gateways", "services", "swap"], "cpu_threads": None}] * 3)
    assert list(await _rights_problems()) == [f"{hid}:cpu"]
    await _clear_samples()
    await _add_samples(hid, [{"denied": [], "ok": ["cpu", "gateways", "services", "swap"], "cpu_threads": None}] * 3)
    assert list(await _rights_problems()) == []


@pytest.mark.asyncio
async def test_mute_refuses_an_unknown_gateway_name_but_unmute_always_works(client):
    host = await _create_host(client, "fw1", "https://fw1.test")
    hid = uuid.UUID(host["id"])
    await _add_samples(hid, [FINE])
    assert (await client.put(f"/api/hosts/{hid}/gateways/wan_gw/alert", json={"enabled": False})).status_code == 404
    assert (await client.get("/api/notifications/settings")).json()["gateway_alert_muted"] == []
    assert (await client.put(f"/api/hosts/{hid}/gateways/{_gw()['name']}/alert", json={"enabled": False})).status_code == 200
    assert (await client.put(f"/api/hosts/{hid}/gateways/wan_gw/alert", json={"enabled": True})).status_code == 200


@pytest.mark.asyncio
async def test_deleting_a_host_removes_its_gateway_mutes(client):
    keep = await _create_host(client, "fw-keep", "https://fwkeep.test")
    gone = await _create_host(client, "fw-gone", "https://fwgone.test")
    for h in (keep, gone):
        assert (await client.put(f"/api/hosts/{h['id']}/gateways/WAN_GW/alert", json={"enabled": False})).status_code == 200
    assert (await delete_host(client, gone['id'])).status_code == 204
    assert (await client.get("/api/notifications/settings")).json()["gateway_alert_muted"] == [f"{keep['id']}:WAN_GW"]