pablan/backend/tests/test_llm_settings.py

"""LLM endpoint configuration: bootstrapped from `.env` once, then owned by
the database, applied without a restart, and the api_key never leaves the
server (rule 12)."""

import logging

import pytest
from httpx import AsyncClient
from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession

from app.llm import client as llm_client
from app.llm import overrides
from app.llm.overrides import bootstrap_llm_settings, env_defaults, load_config
from app.log import JsonFormatter
from app.models import LLMSetting, User

pytestmark = pytest.mark.usefixtures("fake_llm")

SECRET = "sk-super-secret-key-9876"


@pytest.fixture(autouse=True)
def _clean_config():
    overrides.clear()
    llm_client.rebuild_clients()
    yield
    overrides.clear()
    llm_client.rebuild_clients()


async def _login_admin(client: AsyncClient) -> None:
    response = await client.post(
        "/api/auth/login", json={"email": "florian@test.dev", "password": "secret123"}
    )
    assert response.status_code == 200


async def test_bootstrap_copies_the_environment_once(db: AsyncSession) -> None:
    written = await bootstrap_llm_settings(db)
    assert written > 0

    rows = {row.role: row for row in (await db.execute(select(LLMSetting))).scalars()}
    for role in ("chat", "utility", "embedding"):
        defaults = env_defaults(role)
        assert rows[role].base_url == defaults.base_url
        assert rows[role].base_url_from_env is True
        assert rows[role].model_from_env is True

    # A second start changes nothing — the rows are the admin's now.
    assert await bootstrap_llm_settings(db) == 0


async def test_bootstrap_fills_a_field_an_upgrade_left_deferring_to_env(
    db: AsyncSession,
) -> None:
    """The upgrade path: before this milestone a NULL column meant "inherit
    from .env", so the migration flags those fields `*_from_env` and leaves
    them empty. Startup has to fill them in — otherwise a row whose
    overrides had been cleared comes out as an empty configuration and
    takes the endpoint down."""
    db.add(
        LLMSetting(
            role="chat",
            base_url=None,
            model="hand-picked",
            api_key=None,
            base_url_from_env=True,
            model_from_env=False,
            api_key_from_env=True,
        )
    )
    await db.commit()

    await bootstrap_llm_settings(db)

    row = (
        await db.execute(select(LLMSetting).where(LLMSetting.role == "chat"))
    ).scalar_one()
    assert row.base_url == env_defaults("chat").base_url
    # The admin's own value is never overwritten.
    assert row.model == "hand-picked"
    assert row.model_from_env is False


async def test_startup_logging_survives_the_reserved_name_trap(
    db: AsyncSession, caplog: pytest.LogCaptureFixture
) -> None:
    """`extra={"created": ...}` raises KeyError inside logging, because
    LogRecord already owns that attribute — and the process dies on startup.

    This slipped through once: the log line only builds when the logger is
    enabled for INFO, and the suite otherwise runs above that level, so
    every existing test passed while the app refused to boot.
    """
    with caplog.at_level(logging.INFO, logger="pablan.llm"):
        await bootstrap_llm_settings(db)
        await load_config(db)

    rendered = "\n".join(JsonFormatter().format(record) for record in caplog.records)
    assert "llm_bootstrap" in rendered


async def test_the_database_wins_over_a_later_env_change(
    client: AsyncClient, db: AsyncSession, seeded_admin: User
) -> None:
    """The point of bootstrap-then-DB: once a value is stored, the process
    uses it even though `.env` still says something else."""
    await _login_admin(client)
    env_url = env_defaults("chat").base_url

    saved = await client.put(
        "/api/admin/llm/settings/chat", json={"base_url": "http://stored.invalid/v1"}
    )
    assert saved.status_code == 200
    assert saved.json()["base_url"] == "http://stored.invalid/v1"
    assert saved.json()["base_url_from_env"] is False
    assert llm_client.role_config("chat")[0] == "http://stored.invalid/v1"

    # Re-running bootstrap (i.e. a restart) must not undo it.
    await bootstrap_llm_settings(db)
    await load_config(db)
    assert llm_client.role_config("chat")[0] == "http://stored.invalid/v1"
    assert env_url != "http://stored.invalid/v1"


async def test_resetting_a_field_restores_the_env_value(
    client: AsyncClient, db: AsyncSession, seeded_admin: User
) -> None:
    await _login_admin(client)
    env_model = env_defaults("chat").model

    changed = await client.put(
        "/api/admin/llm/settings/chat", json={"model": "gemma-9000"}
    )
    assert changed.json()["model"] == "gemma-9000"
    assert changed.json()["model_from_env"] is False

    reset = await client.put("/api/admin/llm/settings/chat", json={"reset_model": True})
    assert reset.json()["model_from_env"] is True
    assert reset.json()["model"] == (env_model or "")
    assert llm_client.role_config("chat")[2] == env_model


async def test_provenance_is_tracked_per_field(
    client: AsyncClient, db: AsyncSession, seeded_admin: User
) -> None:
    """Changing the model must not relabel the URL as hand-edited."""
    await _login_admin(client)
    body = (
        await client.put("/api/admin/llm/settings/chat", json={"model": "gemma-9000"})
    ).json()
    assert body["model_from_env"] is False
    assert body["base_url_from_env"] is True
    assert body["api_key_from_env"] is True


async def test_saving_rebuilds_the_client_so_no_restart_is_needed(
    client: AsyncClient, db: AsyncSession, seeded_admin: User
) -> None:
    await _login_admin(client)
    before = llm_client._client_for("chat")

    await client.put(
        "/api/admin/llm/settings/chat",
        json={"base_url": "http://elsewhere.invalid/v1"},
    )

    after = llm_client._client_for("chat")
    assert after is not before, "cached client kept the old base_url"
    assert str(after.base_url).startswith("http://elsewhere.invalid")


async def test_the_api_key_is_write_only(
    client: AsyncClient, db: AsyncSession, seeded_admin: User
) -> None:
    await _login_admin(client)
    saved = await client.put("/api/admin/llm/settings/chat", json={"api_key": SECRET})
    assert saved.status_code == 200

    # It is stored...
    row = (
        await db.execute(select(LLMSetting).where(LLMSetting.role == "chat"))
    ).scalar_one()
    assert row.api_key == SECRET
    # ...and used...
    assert llm_client.role_config("chat")[1] == SECRET

    # ...but no response body ever contains it.
    assert SECRET not in saved.text
    assert saved.json()["api_key_set"] is True
    assert saved.json()["api_key_from_env"] is False

    listing = await client.get("/api/admin/llm/settings")
    assert SECRET not in listing.text
    assert "api_key" not in listing.json()[0]


async def test_the_api_key_never_reaches_a_log_line(
    client: AsyncClient,
    db: AsyncSession,
    seeded_admin: User,
    caplog: pytest.LogCaptureFixture,
) -> None:
    await _login_admin(client)
    with caplog.at_level(logging.DEBUG):
        await client.put("/api/admin/llm/settings/chat", json={"api_key": SECRET})
        await client.post(
            "/api/admin/llm/test",
            json={"role": "chat", "api_key": SECRET, "base_url": "http://x.invalid/v1"},
        )
        await client.post(
            "/api/admin/llm/models/chat",
            json={"api_key": SECRET, "base_url": "http://x.invalid/v1"},
        )

    formatter = JsonFormatter()
    rendered = "\n".join(formatter.format(record) for record in caplog.records)
    assert SECRET not in rendered


async def test_testing_a_candidate_does_not_persist_it(
    client: AsyncClient, db: AsyncSession, seeded_admin: User
) -> None:
    """The test button must not change the running configuration."""
    await _login_admin(client)
    before = llm_client.role_config("chat")

    response = await client.post(
        "/api/admin/llm/test",
        json={"role": "chat", "base_url": "http://candidate.invalid/v1"},
    )
    assert response.status_code == 200
    assert [role["role"] for role in response.json()["roles"]] == ["chat"]

    assert llm_client.role_config("chat") == before
    assert (await db.execute(select(LLMSetting))).scalars().all() == []


async def test_available_models_come_from_the_endpoint(
    client: AsyncClient, db: AsyncSession, seeded_admin: User
) -> None:
    await _login_admin(client)
    response = await client.post("/api/admin/llm/models/chat", json={})
    assert response.status_code == 200
    body = response.json()
    assert body["supported"] is True
    assert body["models"] == ["bge-m3", "gemma-3-27b"]


async def test_an_endpoint_without_the_route_degrades_quietly(
    client: AsyncClient, db: AsyncSession, seeded_admin: User, fake_llm
) -> None:
    """Plenty of OpenAI-compatible servers do not implement /v1/models. That
    is a missing convenience, not an error worth showing."""
    fake_llm.served_models = None
    await _login_admin(client)

    body = (await client.post("/api/admin/llm/models/chat", json={})).json()
    assert body["supported"] is False
    assert body["models"] == []
    assert body["error"] is None


async def test_listing_models_does_not_persist_the_candidate(
    client: AsyncClient, db: AsyncSession, seeded_admin: User
) -> None:
    await _login_admin(client)
    await client.post(
        "/api/admin/llm/models/chat",
        json={"base_url": "http://candidate.invalid/v1", "api_key": SECRET},
    )
    assert (await db.execute(select(LLMSetting))).scalars().all() == []


async def test_settings_require_an_admin(
    client: AsyncClient, seeded_user: User
) -> None:
    await client.post(
        "/api/auth/login", json={"email": "pablo@test.dev", "password": "secret123"}
    )
    assert (await client.get("/api/admin/llm/settings")).status_code == 403
    assert (
        await client.put("/api/admin/llm/settings/chat", json={"model": "x"})
    ).status_code == 403
    assert (await client.post("/api/admin/llm/models/chat", json={})).status_code == 403