Julian Boldt
1aba618ad5
Release / verify (push) Canceled after 0s
CI / checks (push) Canceled after 0s
CI / frontend (push) Canceled after 0s
CI / backend (push) Canceled after 0s
Release / test (push) Canceled after 0s
Release / publish (backend) (push) Canceled after 0s
Release / publish (frontend) (push) Canceled after 0s
Release / publish (proxy) (push) Canceled after 0s
Release / publish (updater) (push) Canceled after 0s
- **Fix (Produktiv-Vorfall 2026-10-20):** Das Ändern der Beschreibung eines CARP-VIP wendete das VIP neu an und ließ es auf beiden Firewalls im Zustand INIT hängen – OPNexus zeigte trotzdem „alles in Ordnung“. Nur die Beschreibung zu ändern, wendet jetzt nichts mehr an. Nach jeder anderen CARP-VIP-Änderung prüft OPNexus den Live-Zustand (MASTER/BACKUP), wendet bei Bedarf einmal erneut an und meldet sonst „Angewendet, Prüfung weicht ab“. - **CARP-Überwachung je VHID:** Neue Warnungen `vip_down` (ein VIP ist auf einem Node nicht aktiv) und `role_mismatch` (ein Node ist nur für einen Teil der VIPs MASTER). Bisher zählte der Host schon als MASTER, sobald ein einziges VIP MASTER war.
192 lines
8.7 KiB
Python
192 lines
8.7 KiB
Python
import uuid
|
|
|
|
import httpx
|
|
import pytest
|
|
from sqlalchemy import select
|
|
|
|
from app.database import SessionLocal
|
|
from app.models.carp_event import CarpEvent
|
|
from app.services import carp_event_service, notification_checks
|
|
|
|
|
|
async def _create_host(client, name: str, base_url: str, cluster_name: str) -> dict:
|
|
payload = {
|
|
"name": name, "base_url": base_url, "api_key": "key", "api_secret": "secret",
|
|
"verify_tls": False, "cluster_name": cluster_name,
|
|
}
|
|
resp = await client.post("/api/hosts", json=payload)
|
|
assert resp.status_code == 201, resp.text
|
|
return resp.json()
|
|
|
|
|
|
def _vip(status: str | None) -> dict:
|
|
rows = [{"vhid": "1", "status": status}] if status else []
|
|
return {"rows": rows, "carp": {"allow": "1", "maintenancemode": "0", "demotion": "0"}}
|
|
|
|
|
|
# ── check_carp_health ────────────────────────────────────────────────────
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_check_carp_health_detects_split_brain(client, respx_mock):
|
|
respx_mock.get("https://fw-a.test/api/diagnostics/interface/getVipStatus").mock(return_value=httpx.Response(200, json=_vip("MASTER")))
|
|
respx_mock.get("https://fw-b.test/api/diagnostics/interface/getVipStatus").mock(return_value=httpx.Response(200, json=_vip("MASTER")))
|
|
await _create_host(client, "fw-a", "https://fw-a.test", "ha")
|
|
await _create_host(client, "fw-b", "https://fw-b.test", "ha")
|
|
|
|
async with SessionLocal() as db:
|
|
problems = await notification_checks.check_carp_health(db)
|
|
|
|
assert len(problems) == 1
|
|
row = next(iter(problems.values()))
|
|
assert row["condition"] == "split_brain"
|
|
assert row["host_names"] == ["fw-a", "fw-b"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_check_carp_health_detects_no_master(client, respx_mock):
|
|
respx_mock.get("https://fw-a.test/api/diagnostics/interface/getVipStatus").mock(return_value=httpx.Response(200, json=_vip("BACKUP")))
|
|
respx_mock.get("https://fw-b.test/api/diagnostics/interface/getVipStatus").mock(return_value=httpx.Response(200, json=_vip("BACKUP")))
|
|
await _create_host(client, "fw-a", "https://fw-a.test", "ha")
|
|
await _create_host(client, "fw-b", "https://fw-b.test", "ha")
|
|
|
|
async with SessionLocal() as db:
|
|
problems = await notification_checks.check_carp_health(db)
|
|
|
|
assert len(problems) == 1
|
|
assert next(iter(problems.values()))["condition"] == "no_master"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_check_carp_health_healthy_reports_nothing(client, respx_mock):
|
|
respx_mock.get("https://fw-a.test/api/diagnostics/interface/getVipStatus").mock(return_value=httpx.Response(200, json=_vip("MASTER")))
|
|
respx_mock.get("https://fw-b.test/api/diagnostics/interface/getVipStatus").mock(return_value=httpx.Response(200, json=_vip("BACKUP")))
|
|
await _create_host(client, "fw-a", "https://fw-a.test", "ha")
|
|
await _create_host(client, "fw-b", "https://fw-b.test", "ha")
|
|
|
|
async with SessionLocal() as db:
|
|
problems = await notification_checks.check_carp_health(db)
|
|
|
|
assert problems == {}
|
|
|
|
|
|
def _vips(**states: str) -> dict:
|
|
return {"rows": [{"vhid": v, "status": st} for v, st in states.items()],
|
|
"carp": {"allow": "1", "maintenancemode": "0", "demotion": "0"}}
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_check_carp_health_detects_a_vip_that_is_down_on_one_node(client, respx_mock):
|
|
# 2026-10-20: VHID 130 was DISABLED on the primary and taken over by the secondary; per host it looked fine.
|
|
respx_mock.get("https://fw-a.test/api/diagnostics/interface/getVipStatus").mock(
|
|
return_value=httpx.Response(200, json=_vips(**{"110": "MASTER", "130": "DISABLED"})))
|
|
respx_mock.get("https://fw-b.test/api/diagnostics/interface/getVipStatus").mock(
|
|
return_value=httpx.Response(200, json=_vips(**{"110": "BACKUP", "130": "BACKUP"})))
|
|
await _create_host(client, "fw-a", "https://fw-a.test", "ha")
|
|
await _create_host(client, "fw-b", "https://fw-b.test", "ha")
|
|
|
|
async with SessionLocal() as db:
|
|
problems = await notification_checks.check_carp_health(db)
|
|
|
|
assert {p["condition"] for p in problems.values()} == {"vip_down"}
|
|
assert next(iter(problems.values()))["vhids"] == ["130"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_check_carp_health_detects_a_partial_failover(client, respx_mock):
|
|
respx_mock.get("https://fw-a.test/api/diagnostics/interface/getVipStatus").mock(
|
|
return_value=httpx.Response(200, json=_vips(**{"110": "MASTER", "130": "BACKUP"})))
|
|
respx_mock.get("https://fw-b.test/api/diagnostics/interface/getVipStatus").mock(
|
|
return_value=httpx.Response(200, json=_vips(**{"110": "BACKUP", "130": "BACKUP"})))
|
|
await _create_host(client, "fw-a", "https://fw-a.test", "ha")
|
|
await _create_host(client, "fw-b", "https://fw-b.test", "ha")
|
|
|
|
async with SessionLocal() as db:
|
|
problems = await notification_checks.check_carp_health(db)
|
|
|
|
assert [p["condition"] for p in problems.values()] == ["role_mismatch"]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_check_carp_health_ignores_single_reachable_host(client, respx_mock):
|
|
respx_mock.get("https://fw-a.test/api/diagnostics/interface/getVipStatus").mock(return_value=httpx.Response(200, json=_vip("MASTER")))
|
|
respx_mock.get("https://fw-b.test/api/diagnostics/interface/getVipStatus").mock(side_effect=httpx.ConnectError("down"))
|
|
await _create_host(client, "fw-a", "https://fw-a.test", "ha")
|
|
await _create_host(client, "fw-b", "https://fw-b.test", "ha")
|
|
|
|
async with SessionLocal() as db:
|
|
problems = await notification_checks.check_carp_health(db)
|
|
|
|
assert problems == {}
|
|
|
|
|
|
# ── carp_event_service.detect_and_record ─────────────────────────────────
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_detect_and_record_creates_event_on_first_observation(client, respx_mock):
|
|
respx_mock.get("https://fw-a.test/api/diagnostics/interface/getVipStatus").mock(return_value=httpx.Response(200, json=_vip("MASTER")))
|
|
host = await _create_host(client, "fw-a", "https://fw-a.test", "ha")
|
|
|
|
async with SessionLocal() as db:
|
|
created = await carp_event_service.detect_and_record(db)
|
|
|
|
assert len(created) == 1
|
|
assert created[0].host_id == uuid.UUID(host["id"])
|
|
assert created[0].from_status is None
|
|
assert created[0].to_status == "MASTER"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_detect_and_record_only_fires_on_transition(client, respx_mock):
|
|
route = respx_mock.get("https://fw-a.test/api/diagnostics/interface/getVipStatus")
|
|
route.mock(return_value=httpx.Response(200, json=_vip("MASTER")))
|
|
await _create_host(client, "fw-a", "https://fw-a.test", "ha")
|
|
|
|
async with SessionLocal() as db:
|
|
first = await carp_event_service.detect_and_record(db)
|
|
assert len(first) == 1
|
|
second = await carp_event_service.detect_and_record(db)
|
|
assert second == []
|
|
|
|
route.mock(return_value=httpx.Response(200, json=_vip("BACKUP")))
|
|
async with SessionLocal() as db:
|
|
third = await carp_event_service.detect_and_record(db)
|
|
assert len(third) == 1
|
|
assert third[0].from_status == "MASTER"
|
|
assert third[0].to_status == "BACKUP"
|
|
|
|
all_events = (await db.execute(select(CarpEvent))).scalars().all()
|
|
assert len(all_events) == 2
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_detect_and_record_skips_unreachable_hosts(client, respx_mock):
|
|
respx_mock.get("https://fw-a.test/api/diagnostics/interface/getVipStatus").mock(side_effect=httpx.ConnectError("down"))
|
|
await _create_host(client, "fw-a", "https://fw-a.test", "ha")
|
|
|
|
async with SessionLocal() as db:
|
|
created = await carp_event_service.detect_and_record(db)
|
|
|
|
assert created == []
|
|
|
|
|
|
# ── API ───────────────────────────────────────────────────────────────────
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_carp_events_endpoint_filters_by_cluster(client, respx_mock):
|
|
respx_mock.get("https://fw-a.test/api/diagnostics/interface/getVipStatus").mock(return_value=httpx.Response(200, json=_vip("MASTER")))
|
|
respx_mock.get("https://fw-c.test/api/diagnostics/interface/getVipStatus").mock(return_value=httpx.Response(200, json=_vip("BACKUP")))
|
|
await _create_host(client, "fw-a", "https://fw-a.test", "ha")
|
|
await _create_host(client, "fw-c", "https://fw-c.test", "other")
|
|
|
|
async with SessionLocal() as db:
|
|
await carp_event_service.detect_and_record(db)
|
|
|
|
resp = await client.get("/api/cluster/carp-events?cluster_name=ha")
|
|
assert resp.status_code == 200
|
|
data = resp.json()
|
|
assert len(data) == 1
|
|
assert data[0]["host_name"] == "fw-a"
|
|
|
|
resp_all = await client.get("/api/cluster/carp-events")
|
|
assert len(resp_all.json()) == 2
|