pablan/frontend/e2e/chat.spec.ts

import { expect, test, type Page } from '@playwright/test';
import { login as loginAs, openChat, waitForTurnEnd } from './helpers';

// Runs against the dev stack with the REAL LLM endpoints and the seeded,
// indexed corpus (make seed + backend running). Streaming answers from the
// local model take seconds — generous timeouts on stream assertions.

test.describe.configure({ mode: 'serial' });
test.setTimeout(120_000);

async function login(page: Page) {
	await loginAs(page, 'pablo@pablan.dev');
}

async function deleteNewestConversation(page: Page) {
	const list = await (await page.request.get('/api/conversations')).json();
	if (list.length > 0) {
		await page.request.delete(`/api/conversations/${list[0].id}`);
	}
}

// Zero-residue: every test cleans up its conversation; the session goes here.
test.afterEach(async ({ page }) => {
	await page.request.post('/api/auth/logout');
});

test('streams an answer with citations from the corpus', async ({ page }) => {
	await login(page);
	await openChat(page);

	await page.getByTestId('chat-input').fill('Wie beantrage ich Urlaub?');
	await page.getByTestId('send-button').click();

	// Retrieval progress is reported before the answer arrives.
	await expect(page.getByTestId('retrieval-status')).toBeVisible({ timeout: 30_000 });

	const assistant = page.getByTestId('assistant-message').last();
	await expect(assistant).toBeVisible({ timeout: 30_000 });
	await expect(page.getByTestId('sources').last()).toContainText(
		'Urlaubsanträge und Abwesenheiten',
		{ timeout: 30_000 }
	);
	// Wait for the stream to finish, then check we got a real answer.
	await waitForTurnEnd(page);
	const answer = (await assistant.innerText()).trim();
	expect(answer.length).toBeGreaterThan(40);
	// Transient: the status line disappears once the turn ends.
	await expect(page.getByTestId('retrieval-status')).toHaveCount(0);

	// One badge per cited document, however many of its sections matched.
	// Retrieval works on chunks, so this is exactly where duplicates appear.
	const titles = await page
		.getByTestId('sources')
		.last()
		.getByTestId('source-badge')
		.allInnerTexts();
	const documentNames = titles.map((text) => text.replace(/\d+ sections$/, '').trim());
	expect(new Set(documentNames).size).toBe(documentNames.length);

	await deleteNewestConversation(page);
});

test('a citation shows its excerpt on hover and opens the document beside the chat', async ({
	page
}) => {
	await login(page);
	await openChat(page);

	await page.getByTestId('chat-input').fill('Wie beantrage ich Urlaub?');
	await page.getByTestId('send-button').click();
	await expect(page.getByTestId('sources').last()).toBeVisible({ timeout: 45_000 });
	await waitForTurnEnd(page);

	const badge = page.getByTestId('source-badge').first();
	await badge.hover();
	// The popover carries the cited passage as plain text.
	const tooltip = page.getByTestId('tooltip-content');
	await expect(tooltip).toBeVisible({ timeout: 10_000 });
	expect((await tooltip.innerText()).trim().length).toBeGreaterThan(20);

	await badge.click();
	const panel = page.getByTestId('document-panel');
	await expect(panel).toBeVisible();
	await expect(panel).toContainText('Urlaub', { timeout: 15_000 });

	await panel.getByLabel('Close document panel').click();
	await expect(panel).toHaveCount(0);
	await deleteNewestConversation(page);
});

test('an undocumented question offers to capture the knowledge', async ({ page }) => {
	await login(page);
	await openChat(page);

	// One of the eval-calibrated no-answer queries (tests/fixtures/
	// golden_queries.yaml), so this reliably takes the low-confidence path.
	await page.getByTestId('chat-input').fill('Welche Gerichte gibt es in der Kantine für Veganer?');
	await page.getByTestId('send-button').click();
	await waitForTurnEnd(page);

	// The answer stands on general knowledge, but is labelled source-free
	// and offers to close the gap by writing it down.
	await expect(page.getByTestId('no-sources-note')).toBeVisible();
	// The link carries the conversation, so the draft starts with the question
	// that exposed the gap as background.
	await expect(page.getByTestId('capture-gap')).toHaveAttribute(
		'href',
		/\/documents\/new\?conversation=/
	);
	await page.getByTestId('capture-gap').click();
	await expect(page).toHaveURL(/\/documents\/new\?conversation=/);

	// Only the query conversation was created; clean it up.
	await deleteNewestConversation(page);
});

test('stop button aborts the stream and the partial answer survives reload', async ({ page }) => {
	await login(page);
	await openChat(page);

	await page
		.getByTestId('chat-input')
		.fill('Erkläre ausführlich und Schritt für Schritt den kompletten Reklamationsprozess.');
	await page.getByTestId('send-button').click();

	// Wait until answer TOKENS have arrived (the .markdown body, not the
	// sources badges which render first), then stop mid-stream.
	const assistant = page.getByTestId('assistant-message').last();
	await expect(assistant).toBeVisible({ timeout: 30_000 });
	const answerBody = assistant.locator('.markdown');
	await expect
		.poll(async () => (await answerBody.innerText()).trim().length, { timeout: 40_000 })
		.toBeGreaterThan(15);
	const stop = page.getByTestId('stop-button');
	if (await stop.isVisible()) {
		await stop.click();
	}
	await expect(page.getByTestId('stop-button')).toHaveCount(0, { timeout: 90_000 });

	// The (partial) assistant message is persisted server-side.
	await page.reload();
	await page.locator('body[data-hydrated]').waitFor();
	// Sidebar rows are links; the only button in a row is "delete".
	await page.getByTestId('conversation-list').getByRole('link').first().click();
	const restored = page.getByTestId('assistant-message').last();
	await expect(restored).toBeVisible({ timeout: 15_000 });
	expect((await restored.innerText()).trim().length).toBeGreaterThan(10);
	await deleteNewestConversation(page);
});

test('deleting a conversation removes it from the list', async ({ page }) => {
	await login(page);
	// Create a dedicated conversation via the API so the test never deletes
	// pre-existing user data (and doesn't depend on earlier tests' residue).
	await page.request.post('/api/conversations', { data: { mode: 'query' } });
	await openChat(page);

	const list = page.getByTestId('conversation-list');
	await expect.poll(async () => list.locator('li').count(), { timeout: 15_000 }).toBeGreaterThan(0);
	const before = await list.locator('li').count();

	// Newest first: the untitled conversation we just created. Deleting asks
	// first — it is destructive and the row carries no undo.
	const ownRow = list.locator('li').filter({ hasText: 'New conversation' }).first();
	await ownRow.hover();
	await ownRow.getByLabel('Delete conversation').click();
	await page.getByTestId('confirm-accept').click();
	await expect.poll(async () => list.locator('li').count()).toBe(before - 1);
});

test('Pablan answers questions about itself, in the language they were asked', async ({ page }) => {
	await login(page);
	await openChat(page);

	// Deictic phrasing ("this app") and English, against German help pages:
	// the built-in documentation has to carry both the self-reference and
	// survive the language switch.
	await page.getByTestId('chat-input').fill('How does this app work?');
	await page.getByTestId('send-button').click();
	await waitForTurnEnd(page);

	const assistant = page.getByTestId('assistant-message').last();
	await expect(assistant).toContainText('Pablan');
	await expect(page.getByTestId('sources').last()).toContainText('Pablan:', {
		timeout: 15_000
	});
	// It was answered FROM the knowledge base, not from general knowledge.
	await expect(page.getByTestId('no-sources-note')).toHaveCount(0);

	await deleteNewestConversation(page);
});

test('the first message moves the URL to the conversation, and Back leaves it', async ({
	page
}) => {
	await login(page);
	await openChat(page);
	await expect(page).toHaveURL('/chat');

	await page.getByTestId('chat-input').fill('Wie beantrage ich Urlaub?');
	await page.getByTestId('send-button').click();

	// The conversation gets its own address as soon as it exists, while the
	// answer is still streaming.
	await expect(page).toHaveURL(/\/chat\/[0-9a-f-]{36}$/, { timeout: 30_000 });
	const conversationUrl = page.url();
	await waitForTurnEnd(page);
	// The stream survived the navigation: the state outlives the route.
	expect(
		(await page.getByTestId('assistant-message').last().innerText()).trim().length
	).toBeGreaterThan(20);

	// replaceState, so Back never lands on an orphaned empty composer.
	await page.goBack();
	await expect(page).not.toHaveURL(conversationUrl);

	await deleteNewestConversation(page);
});

test('switching conversations abandons the previous view, and a foreign id is 404', async ({
	page
}) => {
	await login(page);
	// Two conversations, created via the API so nothing depends on ordering.
	const first = await (
		await page.request.post('/api/conversations', { data: { mode: 'query' } })
	).json();
	const second = await (
		await page.request.post('/api/conversations', { data: { mode: 'query' } })
	).json();

	await page.goto(`/chat/${first.id}`);
	await page.locator('body[data-hydrated]').waitFor();
	await page.goto(`/chat/${second.id}`);
	await page.locator('body[data-hydrated]').waitFor();
	// Nothing bled across: the second conversation is empty.
	await expect(page.getByTestId('assistant-message')).toHaveCount(0);

	// An id that is not yours is indistinguishable from one that does not
	// exist — existence must not leak.
	await page.goto('/chat/00000000-0000-4000-8000-000000000000');
	await expect(page.locator('body')).toContainText('404');

	await page.request.delete(`/api/conversations/${first.id}`);
	await page.request.delete(`/api/conversations/${second.id}`);
});