Pablan, as it stands
Self-hosted knowledge management for SMEs: a split-screen Markdown editor whose sections an LLM refines while you write, and RAG question answering over the documents that result. FastAPI + Postgres/pgvector on the back, SvelteKit on the front, everything OpenAI-compatible and self-hostable. Squashed into a single commit; the development history stays local. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01CA43ZJda8Rbp2hKXNy8f6b
This commit is contained in:
ProfessorNova
co-authored by
Claude Opus 5
Claude Opus 5
parent
68d3a43191
commit
784b76baf7
@@ -0,0 +1,246 @@
|
||||
import { expect, test, type Page } from '@playwright/test';
|
||||
import { login as loginAs, openChat, waitForTurnEnd } from './helpers';
|
||||
|
||||
// Runs against the dev stack with the REAL LLM endpoints and the seeded,
|
||||
// indexed corpus (make seed + backend running). Streaming answers from the
|
||||
// local model take seconds — generous timeouts on stream assertions.
|
||||
|
||||
test.describe.configure({ mode: 'serial' });
|
||||
test.setTimeout(120_000);
|
||||
|
||||
async function login(page: Page) {
|
||||
await loginAs(page, 'pablo@pablan.dev');
|
||||
}
|
||||
|
||||
async function deleteNewestConversation(page: Page) {
|
||||
const list = await (await page.request.get('/api/conversations')).json();
|
||||
if (list.length > 0) {
|
||||
await page.request.delete(`/api/conversations/${list[0].id}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Zero-residue: every test cleans up its conversation; the session goes here.
|
||||
test.afterEach(async ({ page }) => {
|
||||
await page.request.post('/api/auth/logout');
|
||||
});
|
||||
|
||||
test('streams an answer with citations from the corpus', async ({ page }) => {
|
||||
await login(page);
|
||||
await openChat(page);
|
||||
|
||||
await page.getByTestId('chat-input').fill('Wie beantrage ich Urlaub?');
|
||||
await page.getByTestId('send-button').click();
|
||||
|
||||
// Retrieval progress is reported before the answer arrives.
|
||||
await expect(page.getByTestId('retrieval-status')).toBeVisible({ timeout: 30_000 });
|
||||
|
||||
const assistant = page.getByTestId('assistant-message').last();
|
||||
await expect(assistant).toBeVisible({ timeout: 30_000 });
|
||||
await expect(page.getByTestId('sources').last()).toContainText(
|
||||
'Urlaubsanträge und Abwesenheiten',
|
||||
{ timeout: 30_000 }
|
||||
);
|
||||
// Wait for the stream to finish, then check we got a real answer.
|
||||
await waitForTurnEnd(page);
|
||||
const answer = (await assistant.innerText()).trim();
|
||||
expect(answer.length).toBeGreaterThan(40);
|
||||
// Transient: the status line disappears once the turn ends.
|
||||
await expect(page.getByTestId('retrieval-status')).toHaveCount(0);
|
||||
|
||||
// One badge per cited document, however many of its sections matched.
|
||||
// Retrieval works on chunks, so this is exactly where duplicates appear.
|
||||
const titles = await page
|
||||
.getByTestId('sources')
|
||||
.last()
|
||||
.getByTestId('source-badge')
|
||||
.allInnerTexts();
|
||||
const documentNames = titles.map((text) => text.replace(/\d+ sections$/, '').trim());
|
||||
expect(new Set(documentNames).size).toBe(documentNames.length);
|
||||
|
||||
await deleteNewestConversation(page);
|
||||
});
|
||||
|
||||
test('a citation shows its excerpt on hover and opens the document beside the chat', async ({
|
||||
page
|
||||
}) => {
|
||||
await login(page);
|
||||
await openChat(page);
|
||||
|
||||
await page.getByTestId('chat-input').fill('Wie beantrage ich Urlaub?');
|
||||
await page.getByTestId('send-button').click();
|
||||
await expect(page.getByTestId('sources').last()).toBeVisible({ timeout: 45_000 });
|
||||
await waitForTurnEnd(page);
|
||||
|
||||
const badge = page.getByTestId('source-badge').first();
|
||||
await badge.hover();
|
||||
// The popover carries the cited passage as plain text.
|
||||
const tooltip = page.getByTestId('tooltip-content');
|
||||
await expect(tooltip).toBeVisible({ timeout: 10_000 });
|
||||
expect((await tooltip.innerText()).trim().length).toBeGreaterThan(20);
|
||||
|
||||
await badge.click();
|
||||
const panel = page.getByTestId('document-panel');
|
||||
await expect(panel).toBeVisible();
|
||||
await expect(panel).toContainText('Urlaub', { timeout: 15_000 });
|
||||
|
||||
await panel.getByLabel('Close document panel').click();
|
||||
await expect(panel).toHaveCount(0);
|
||||
await deleteNewestConversation(page);
|
||||
});
|
||||
|
||||
test('an undocumented question offers to capture the knowledge', async ({ page }) => {
|
||||
await login(page);
|
||||
await openChat(page);
|
||||
|
||||
// One of the eval-calibrated no-answer queries (tests/fixtures/
|
||||
// golden_queries.yaml), so this reliably takes the low-confidence path.
|
||||
await page.getByTestId('chat-input').fill('Welche Gerichte gibt es in der Kantine für Veganer?');
|
||||
await page.getByTestId('send-button').click();
|
||||
await waitForTurnEnd(page);
|
||||
|
||||
// The answer stands on general knowledge, but is labelled source-free
|
||||
// and offers to close the gap by writing it down.
|
||||
await expect(page.getByTestId('no-sources-note')).toBeVisible();
|
||||
// The link carries the conversation, so the draft starts with the question
|
||||
// that exposed the gap as background.
|
||||
await expect(page.getByTestId('capture-gap')).toHaveAttribute(
|
||||
'href',
|
||||
/\/documents\/new\?conversation=/
|
||||
);
|
||||
await page.getByTestId('capture-gap').click();
|
||||
await expect(page).toHaveURL(/\/documents\/new\?conversation=/);
|
||||
|
||||
// Only the query conversation was created; clean it up.
|
||||
await deleteNewestConversation(page);
|
||||
});
|
||||
|
||||
test('stop button aborts the stream and the partial answer survives reload', async ({ page }) => {
|
||||
await login(page);
|
||||
await openChat(page);
|
||||
|
||||
await page
|
||||
.getByTestId('chat-input')
|
||||
.fill('Erkläre ausführlich und Schritt für Schritt den kompletten Reklamationsprozess.');
|
||||
await page.getByTestId('send-button').click();
|
||||
|
||||
// Wait until answer TOKENS have arrived (the .markdown body, not the
|
||||
// sources badges which render first), then stop mid-stream.
|
||||
const assistant = page.getByTestId('assistant-message').last();
|
||||
await expect(assistant).toBeVisible({ timeout: 30_000 });
|
||||
const answerBody = assistant.locator('.markdown');
|
||||
await expect
|
||||
.poll(async () => (await answerBody.innerText()).trim().length, { timeout: 40_000 })
|
||||
.toBeGreaterThan(15);
|
||||
const stop = page.getByTestId('stop-button');
|
||||
if (await stop.isVisible()) {
|
||||
await stop.click();
|
||||
}
|
||||
await expect(page.getByTestId('stop-button')).toHaveCount(0, { timeout: 90_000 });
|
||||
|
||||
// The (partial) assistant message is persisted server-side.
|
||||
await page.reload();
|
||||
await page.locator('body[data-hydrated]').waitFor();
|
||||
// Sidebar rows are links; the only button in a row is "delete".
|
||||
await page.getByTestId('conversation-list').getByRole('link').first().click();
|
||||
const restored = page.getByTestId('assistant-message').last();
|
||||
await expect(restored).toBeVisible({ timeout: 15_000 });
|
||||
expect((await restored.innerText()).trim().length).toBeGreaterThan(10);
|
||||
await deleteNewestConversation(page);
|
||||
});
|
||||
|
||||
test('deleting a conversation removes it from the list', async ({ page }) => {
|
||||
await login(page);
|
||||
// Create a dedicated conversation via the API so the test never deletes
|
||||
// pre-existing user data (and doesn't depend on earlier tests' residue).
|
||||
await page.request.post('/api/conversations', { data: { mode: 'query' } });
|
||||
await openChat(page);
|
||||
|
||||
const list = page.getByTestId('conversation-list');
|
||||
await expect.poll(async () => list.locator('li').count(), { timeout: 15_000 }).toBeGreaterThan(0);
|
||||
const before = await list.locator('li').count();
|
||||
|
||||
// Newest first: the untitled conversation we just created. Deleting asks
|
||||
// first — it is destructive and the row carries no undo.
|
||||
const ownRow = list.locator('li').filter({ hasText: 'New conversation' }).first();
|
||||
await ownRow.hover();
|
||||
await ownRow.getByLabel('Delete conversation').click();
|
||||
await page.getByTestId('confirm-accept').click();
|
||||
await expect.poll(async () => list.locator('li').count()).toBe(before - 1);
|
||||
});
|
||||
|
||||
test('Pablan answers questions about itself, in the language they were asked', async ({ page }) => {
|
||||
await login(page);
|
||||
await openChat(page);
|
||||
|
||||
// Deictic phrasing ("this app") and English, against German help pages:
|
||||
// the built-in documentation has to carry both the self-reference and
|
||||
// survive the language switch.
|
||||
await page.getByTestId('chat-input').fill('How does this app work?');
|
||||
await page.getByTestId('send-button').click();
|
||||
await waitForTurnEnd(page);
|
||||
|
||||
const assistant = page.getByTestId('assistant-message').last();
|
||||
await expect(assistant).toContainText('Pablan');
|
||||
await expect(page.getByTestId('sources').last()).toContainText('Pablan:', {
|
||||
timeout: 15_000
|
||||
});
|
||||
// It was answered FROM the knowledge base, not from general knowledge.
|
||||
await expect(page.getByTestId('no-sources-note')).toHaveCount(0);
|
||||
|
||||
await deleteNewestConversation(page);
|
||||
});
|
||||
|
||||
test('the first message moves the URL to the conversation, and Back leaves it', async ({
|
||||
page
|
||||
}) => {
|
||||
await login(page);
|
||||
await openChat(page);
|
||||
await expect(page).toHaveURL('/chat');
|
||||
|
||||
await page.getByTestId('chat-input').fill('Wie beantrage ich Urlaub?');
|
||||
await page.getByTestId('send-button').click();
|
||||
|
||||
// The conversation gets its own address as soon as it exists, while the
|
||||
// answer is still streaming.
|
||||
await expect(page).toHaveURL(/\/chat\/[0-9a-f-]{36}$/, { timeout: 30_000 });
|
||||
const conversationUrl = page.url();
|
||||
await waitForTurnEnd(page);
|
||||
// The stream survived the navigation: the state outlives the route.
|
||||
expect(
|
||||
(await page.getByTestId('assistant-message').last().innerText()).trim().length
|
||||
).toBeGreaterThan(20);
|
||||
|
||||
// replaceState, so Back never lands on an orphaned empty composer.
|
||||
await page.goBack();
|
||||
await expect(page).not.toHaveURL(conversationUrl);
|
||||
|
||||
await deleteNewestConversation(page);
|
||||
});
|
||||
|
||||
test('switching conversations abandons the previous view, and a foreign id is 404', async ({
|
||||
page
|
||||
}) => {
|
||||
await login(page);
|
||||
// Two conversations, created via the API so nothing depends on ordering.
|
||||
const first = await (
|
||||
await page.request.post('/api/conversations', { data: { mode: 'query' } })
|
||||
).json();
|
||||
const second = await (
|
||||
await page.request.post('/api/conversations', { data: { mode: 'query' } })
|
||||
).json();
|
||||
|
||||
await page.goto(`/chat/${first.id}`);
|
||||
await page.locator('body[data-hydrated]').waitFor();
|
||||
await page.goto(`/chat/${second.id}`);
|
||||
await page.locator('body[data-hydrated]').waitFor();
|
||||
// Nothing bled across: the second conversation is empty.
|
||||
await expect(page.getByTestId('assistant-message')).toHaveCount(0);
|
||||
|
||||
// An id that is not yours is indistinguishable from one that does not
|
||||
// exist — existence must not leak.
|
||||
await page.goto('/chat/00000000-0000-4000-8000-000000000000');
|
||||
await expect(page.locator('body')).toContainText('404');
|
||||
|
||||
await page.request.delete(`/api/conversations/${first.id}`);
|
||||
await page.request.delete(`/api/conversations/${second.id}`);
|
||||
});
|
||||
Reference in New Issue
Block a user