import { expect, test, type Page } from '@playwright/test'; import { login as loginAs, openChat, waitForTurnEnd } from './helpers'; // Runs against the dev stack with the REAL LLM endpoints and the seeded, // indexed corpus (make seed + backend running). Streaming answers from the // local model take seconds — generous timeouts on stream assertions. test.describe.configure({ mode: 'serial' }); test.setTimeout(120_000); async function login(page: Page) { await loginAs(page, 'pablo@pablan.dev'); } async function deleteNewestConversation(page: Page) { const list = await (await page.request.get('/api/conversations')).json(); if (list.length > 0) { await page.request.delete(`/api/conversations/${list[0].id}`); } } // Zero-residue: every test cleans up its conversation; the session goes here. test.afterEach(async ({ page }) => { await page.request.post('/api/auth/logout'); }); test('streams an answer with citations from the corpus', async ({ page }) => { await login(page); await openChat(page); await page.getByTestId('chat-input').fill('Wie beantrage ich Urlaub?'); await page.getByTestId('send-button').click(); // Retrieval progress is reported before the answer arrives. await expect(page.getByTestId('retrieval-status')).toBeVisible({ timeout: 30_000 }); const assistant = page.getByTestId('assistant-message').last(); await expect(assistant).toBeVisible({ timeout: 30_000 }); await expect(page.getByTestId('sources').last()).toContainText( 'Urlaubsanträge und Abwesenheiten', { timeout: 30_000 } ); // Wait for the stream to finish, then check we got a real answer. await waitForTurnEnd(page); const answer = (await assistant.innerText()).trim(); expect(answer.length).toBeGreaterThan(40); // Transient: the status line disappears once the turn ends. await expect(page.getByTestId('retrieval-status')).toHaveCount(0); // One badge per cited document, however many of its sections matched. // Retrieval works on chunks, so this is exactly where duplicates appear. const titles = await page .getByTestId('sources') .last() .getByTestId('source-badge') .allInnerTexts(); const documentNames = titles.map((text) => text.replace(/\d+ sections$/, '').trim()); expect(new Set(documentNames).size).toBe(documentNames.length); await deleteNewestConversation(page); }); test('a citation shows its excerpt on hover and opens the document beside the chat', async ({ page }) => { await login(page); await openChat(page); await page.getByTestId('chat-input').fill('Wie beantrage ich Urlaub?'); await page.getByTestId('send-button').click(); await expect(page.getByTestId('sources').last()).toBeVisible({ timeout: 45_000 }); await waitForTurnEnd(page); const badge = page.getByTestId('source-badge').first(); await badge.hover(); // The popover carries the cited passage as plain text. const tooltip = page.getByTestId('tooltip-content'); await expect(tooltip).toBeVisible({ timeout: 10_000 }); expect((await tooltip.innerText()).trim().length).toBeGreaterThan(20); await badge.click(); const panel = page.getByTestId('document-panel'); await expect(panel).toBeVisible(); await expect(panel).toContainText('Urlaub', { timeout: 15_000 }); await panel.getByLabel('Close document panel').click(); await expect(panel).toHaveCount(0); await deleteNewestConversation(page); }); test('an undocumented question offers to capture the knowledge', async ({ page }) => { await login(page); await openChat(page); // One of the eval-calibrated no-answer queries (tests/fixtures/ // golden_queries.yaml), so this reliably takes the low-confidence path. await page.getByTestId('chat-input').fill('Welche Gerichte gibt es in der Kantine für Veganer?'); await page.getByTestId('send-button').click(); await waitForTurnEnd(page); // The answer stands on general knowledge, but is labelled source-free // and offers to close the gap by writing it down. await expect(page.getByTestId('no-sources-note')).toBeVisible(); // The link carries the conversation, so the draft starts with the question // that exposed the gap as background. await expect(page.getByTestId('capture-gap')).toHaveAttribute( 'href', /\/documents\/new\?conversation=/ ); await page.getByTestId('capture-gap').click(); await expect(page).toHaveURL(/\/documents\/new\?conversation=/); // Only the query conversation was created; clean it up. await deleteNewestConversation(page); }); test('stop button aborts the stream and the partial answer survives reload', async ({ page }) => { await login(page); await openChat(page); await page .getByTestId('chat-input') .fill('Erkläre ausführlich und Schritt für Schritt den kompletten Reklamationsprozess.'); await page.getByTestId('send-button').click(); // Wait until answer TOKENS have arrived (the .markdown body, not the // sources badges which render first), then stop mid-stream. const assistant = page.getByTestId('assistant-message').last(); await expect(assistant).toBeVisible({ timeout: 30_000 }); const answerBody = assistant.locator('.markdown'); await expect .poll(async () => (await answerBody.innerText()).trim().length, { timeout: 40_000 }) .toBeGreaterThan(15); const stop = page.getByTestId('stop-button'); if (await stop.isVisible()) { await stop.click(); } await expect(page.getByTestId('stop-button')).toHaveCount(0, { timeout: 90_000 }); // The (partial) assistant message is persisted server-side. await page.reload(); await page.locator('body[data-hydrated]').waitFor(); // Sidebar rows are links; the only button in a row is "delete". await page.getByTestId('conversation-list').getByRole('link').first().click(); const restored = page.getByTestId('assistant-message').last(); await expect(restored).toBeVisible({ timeout: 15_000 }); expect((await restored.innerText()).trim().length).toBeGreaterThan(10); await deleteNewestConversation(page); }); test('deleting a conversation removes it from the list', async ({ page }) => { await login(page); // Create a dedicated conversation via the API so the test never deletes // pre-existing user data (and doesn't depend on earlier tests' residue). await page.request.post('/api/conversations', { data: { mode: 'query' } }); await openChat(page); const list = page.getByTestId('conversation-list'); await expect.poll(async () => list.locator('li').count(), { timeout: 15_000 }).toBeGreaterThan(0); const before = await list.locator('li').count(); // Newest first: the untitled conversation we just created. Deleting asks // first — it is destructive and the row carries no undo. const ownRow = list.locator('li').filter({ hasText: 'New conversation' }).first(); await ownRow.hover(); await ownRow.getByLabel('Delete conversation').click(); await page.getByTestId('confirm-accept').click(); await expect.poll(async () => list.locator('li').count()).toBe(before - 1); }); test('Pablan answers questions about itself, in the language they were asked', async ({ page }) => { await login(page); await openChat(page); // Deictic phrasing ("this app") and English, against German help pages: // the built-in documentation has to carry both the self-reference and // survive the language switch. await page.getByTestId('chat-input').fill('How does this app work?'); await page.getByTestId('send-button').click(); await waitForTurnEnd(page); const assistant = page.getByTestId('assistant-message').last(); await expect(assistant).toContainText('Pablan'); await expect(page.getByTestId('sources').last()).toContainText('Pablan:', { timeout: 15_000 }); // It was answered FROM the knowledge base, not from general knowledge. await expect(page.getByTestId('no-sources-note')).toHaveCount(0); await deleteNewestConversation(page); }); test('the first message moves the URL to the conversation, and Back leaves it', async ({ page }) => { await login(page); await openChat(page); await expect(page).toHaveURL('/chat'); await page.getByTestId('chat-input').fill('Wie beantrage ich Urlaub?'); await page.getByTestId('send-button').click(); // The conversation gets its own address as soon as it exists, while the // answer is still streaming. await expect(page).toHaveURL(/\/chat\/[0-9a-f-]{36}$/, { timeout: 30_000 }); const conversationUrl = page.url(); await waitForTurnEnd(page); // The stream survived the navigation: the state outlives the route. expect( (await page.getByTestId('assistant-message').last().innerText()).trim().length ).toBeGreaterThan(20); // replaceState, so Back never lands on an orphaned empty composer. await page.goBack(); await expect(page).not.toHaveURL(conversationUrl); await deleteNewestConversation(page); }); test('switching conversations abandons the previous view, and a foreign id is 404', async ({ page }) => { await login(page); // Two conversations, created via the API so nothing depends on ordering. const first = await ( await page.request.post('/api/conversations', { data: { mode: 'query' } }) ).json(); const second = await ( await page.request.post('/api/conversations', { data: { mode: 'query' } }) ).json(); await page.goto(`/chat/${first.id}`); await page.locator('body[data-hydrated]').waitFor(); await page.goto(`/chat/${second.id}`); await page.locator('body[data-hydrated]').waitFor(); // Nothing bled across: the second conversation is empty. await expect(page.getByTestId('assistant-message')).toHaveCount(0); // An id that is not yours is indistinguishable from one that does not // exist — existence must not leak. await page.goto('/chat/00000000-0000-4000-8000-000000000000'); await expect(page.locator('body')).toContainText('404'); await page.request.delete(`/api/conversations/${first.id}`); await page.request.delete(`/api/conversations/${second.id}`); });