Implement async Great Sage processing and dynamic document extraction

README.md

@@ -6,6 +6,25 @@ Designed for environments where document confidentiality is non-negotiable, Docu
 
 ---
 
+## 🚀 How to Run the Application
+
+The easiest way to run the entire stack (Poneglyph + Great Sage) on Windows is using the provided `start.ps1` orchestrator script. 
+
+1. Ensure **PostgreSQL** is running (`docker-compose up -d`)
+2. Ensure **Ollama** is running locally
+3. Open a PowerShell terminal in this directory and run:
+   ```powershell
+   .\start.ps1
+   ```
+4. Access the web app at [http://localhost:8080](http://localhost:8080) (Default login: `admin` / `admin`)
+5. In a separate terminal, start Great Sage:
+   ```powershell
+   cd ..\great-sage
+   uvicorn app.main:app --host 127.0.0.1 --port 8000
+   ```
+
+---
+
 ## Core Capabilities
 
 - **Local OCR**: Extracts text from PDFs and images locally using PyMuPDF and Tesseract. Very fast and lightweight (no heavy PyTorch models required).

cmd/api/main.go

@@ -45,6 +45,7 @@ func main() {
 
 	// Internal service-to-service routes (authenticated via X-Webhook-Secret, not JWT)
 	api.HandleFunc("/internal/webhook/analyze", handlers.AnalyzeWebhook).Methods("POST")
+	api.HandleFunc("/internal/webhook/jobs", handlers.JobWebhook).Methods("POST")
 
 	// Protected routes
 	protected := api.PathPrefix("/").Subrouter()

e2e/.gitignore

@@ -0,0 +1,8 @@
+
+# Playwright
+node_modules/
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/
+/playwright/.auth/

e2e/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "e2e",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {},
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "type": "commonjs",
+  "devDependencies": {
+    "@playwright/test": "^1.62.1",
+    "@types/node": "^26.3.0"
+  }
+}

e2e/playwright.config.ts

@@ -0,0 +1,79 @@
+import { defineConfig, devices } from '@playwright/test';
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// import dotenv from 'dotenv';
+// import path from 'path';
+// dotenv.config({ path: path.resolve(__dirname, '.env') });
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './tests',
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('')`. */
+    // baseURL: 'http://localhost:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+
+    // {
+    //   name: 'firefox',
+    //   use: { ...devices['Desktop Firefox'] },
+    // },
+
+    // {
+    //   name: 'webkit',
+    //   use: { ...devices['Desktop Safari'] },
+    // },
+
+    /* Test against mobile viewports. */
+    // {
+    //   name: 'Mobile Chrome',
+    //   use: { ...devices['Pixel 5'] },
+    // },
+    // {
+    //   name: 'Mobile Safari',
+    //   use: { ...devices['iPhone 12'] },
+    // },
+
+    /* Test against branded browsers. */
+    // {
+    //   name: 'Microsoft Edge',
+    //   use: { ...devices['Desktop Edge'], channel: 'msedge' },
+    // },
+    // {
+    //   name: 'Google Chrome',
+    //   use: { ...devices['Desktop Chrome'], channel: 'chrome' },
+    // },
+  ],
+
+  /* Run your local dev server before starting the tests */
+  // webServer: {
+  //   command: 'npm run start',
+  //   url: 'http://localhost:3000',
+  //   reuseExistingServer: !process.env.CI,
+  // },
+});

e2e/tests/auth.spec.ts

@@ -0,0 +1,35 @@
+import { test, expect } from '@playwright/test';
+
+test.describe('Authentication Flow', () => {
+  const baseURL = 'http://127.0.0.1:8080';
+
+  test('should show login page and allow admin login', async ({ page }) => {
+    await page.goto(`${baseURL}/`);
+    await expect(page.locator('text=Login').first()).toBeVisible();
+
+    const usernameInput = page.locator('input[type="text"]').first();
+    const passwordInput = page.locator('input[type="password"]').first();
+
+    await usernameInput.fill('admin');
+    await passwordInput.fill('admin'); // Seed password
+
+    await page.locator('button[type="submit"], button:has-text("Login")').first().click();
+
+    await expect(page.locator('text=DocuNest Login').first()).toBeHidden({ timeout: 10000 });
+  });
+
+  test('should reject invalid credentials', async ({ page }) => {
+    await page.goto(`${baseURL}/`);
+
+    const usernameInput = page.locator('input[type="text"]').first();
+    const passwordInput = page.locator('input[type="password"]').first();
+
+    await usernameInput.fill('admin');
+    await passwordInput.fill('wrongpassword');
+
+    await page.locator('button[type="submit"], button:has-text("Login")').first().click();
+
+    // Check for an error message or that we are still on the login page
+    await expect(page.locator('text=Login').first()).toBeVisible();
+  });
+});

e2e/tests/example.spec.ts

@@ -0,0 +1,18 @@
+import { test, expect } from '@playwright/test';
+
+test('has title', async ({ page }) => {
+  await page.goto('https://playwright.dev/');
+
+  // Expect a title "to contain" a substring.
+  await expect(page).toHaveTitle(/Playwright/);
+});
+
+test('get started link', async ({ page }) => {
+  await page.goto('https://playwright.dev/');
+
+  // Click the get started link.
+  await page.getByRole('link', { name: 'Get started' }).click();
+
+  // Expects page to have a heading with the name of Installation.
+  await expect(page.getByRole('heading', { name: 'Installation' })).toBeVisible();
+});

internal/database/db.go

@@ -97,6 +97,7 @@ func InitSchema() error {
 		document_id_number VARCHAR(100),
 		confidence FLOAT,
 		customer_id VARCHAR(50) REFERENCES customers(id),
+		job_id VARCHAR(36),
 		created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
 	);
 
@@ -132,6 +133,7 @@ func InitSchema() error {
 	DB.Exec("ALTER TABLE documents ADD COLUMN dob VARCHAR(50)")
 	DB.Exec("ALTER TABLE documents ADD COLUMN document_id_number VARCHAR(100)")
 	DB.Exec("ALTER TABLE documents ADD COLUMN customer_id VARCHAR(50) REFERENCES customers(id)")
+	DB.Exec("ALTER TABLE documents ADD COLUMN job_id VARCHAR(36)")
 	DB.Exec("ALTER TABLE audit_logs ADD COLUMN actor_id INT REFERENCES users(id)")
 
 	// 2. Rename existing columns to workspace_id where appropriate
@@ -140,6 +142,25 @@ func InitSchema() error {
 	DB.Exec("ALTER TABLE documents RENAME COLUMN user_id TO workspace_id")
 	DB.Exec("ALTER TABLE audit_logs RENAME COLUMN user_id TO workspace_id")
 
+	// 3. Add extracted_data JSONB column for flexible document extraction storage.
+	// This is the canonical source of truth for all document-type-specific extracted fields.
+	// Legacy columns (person_name, dob, document_id_number) are kept temporarily as
+	// synchronized projections for backward compatibility.
+	DB.Exec("ALTER TABLE documents ADD COLUMN extracted_data JSONB DEFAULT '{}'::jsonb")
+
+	// 4. Backfill existing identity-document rows into extracted_data.
+	// Idempotent: only touches rows where extracted_data is empty and legacy fields exist.
+	DB.Exec(`
+		UPDATE documents
+		SET extracted_data = jsonb_strip_nulls(jsonb_build_object(
+			'person_name', person_name,
+			'dob', dob,
+			'document_id_number', document_id_number
+		))
+		WHERE (extracted_data IS NULL OR extracted_data = '{}'::jsonb)
+		  AND (person_name IS NOT NULL OR dob IS NOT NULL OR document_id_number IS NOT NULL)
+	`)
+
 	log.Println("Database schema initialized")
 	return nil
 }

internal/handlers/document.go

@@ -28,14 +28,14 @@ func GetDocuments(w http.ResponseWriter, r *http.Request) {
 
 	if role == "admin" {
 		rows, err = database.DB.Query(`
-			SELECT d.id, d.filename, d.original_name, d.status, d.document_type, d.person_name, d.dob, d.document_id_number, d.confidence, d.created_at, d.ocr_text, c.name 
+			SELECT d.id, d.filename, d.original_name, d.status, d.document_type, d.extracted_data, d.person_name, d.dob, d.document_id_number, d.confidence, d.created_at, d.ocr_text, c.name 
 			FROM documents d 
 			LEFT JOIN customers c ON d.customer_id = c.id 
 			ORDER BY d.created_at DESC LIMIT 100
 		`)
 	} else {
 		rows, err = database.DB.Query(`
-			SELECT d.id, d.filename, d.original_name, d.status, d.document_type, d.person_name, d.dob, d.document_id_number, d.confidence, d.created_at, d.ocr_text, c.name 
+			SELECT d.id, d.filename, d.original_name, d.status, d.document_type, d.extracted_data, d.person_name, d.dob, d.document_id_number, d.confidence, d.created_at, d.ocr_text, c.name 
 			FROM documents d 
 			LEFT JOIN customers c ON d.customer_id = c.id 
 			WHERE d.workspace_id = $1
@@ -59,7 +59,7 @@ func GetDocuments(w http.ResponseWriter, r *http.Request) {
 		var doc DocumentWithCustomer
 		if err := rows.Scan(
 			&doc.ID, &doc.Filename, &doc.OriginalName, &doc.Status,
-			&doc.DocumentType, &doc.PersonName, &doc.DOB, &doc.DocumentIDNumber, &doc.Confidence, &doc.CreatedAt, &doc.OCRText, &doc.CustomerName,
+			&doc.DocumentType, &doc.ExtractedData, &doc.PersonName, &doc.DOB, &doc.DocumentIDNumber, &doc.Confidence, &doc.CreatedAt, &doc.OCRText, &doc.CustomerName,
 		); err != nil {
 			http.Error(w, "Failed to parse document", http.StatusInternalServerError)
 			return

internal/handlers/document_user.go

@@ -131,15 +131,15 @@ func GetCustomerDocuments(w http.ResponseWriter, r *http.Request) {
 
 	if role == "admin" {
 		rows, err = database.DB.Query(`
-			SELECT id, original_name, status, document_type, person_name, dob, document_id_number, created_at, ocr_text 
+			SELECT id, original_name, status, document_type, extracted_data, person_name, dob, document_id_number, created_at, ocr_text 
 			FROM documents 
 			WHERE customer_id = $1
 			ORDER BY created_at DESC
 			LIMIT 100
 		`, customerID)
 	} else {
 		rows, err = database.DB.Query(`
-			SELECT id, original_name, status, document_type, person_name, dob, document_id_number, created_at, ocr_text 
+			SELECT id, original_name, status, document_type, extracted_data, person_name, dob, document_id_number, created_at, ocr_text 
 			FROM documents 
 			WHERE customer_id = $1
 			ORDER BY created_at DESC
@@ -156,7 +156,7 @@ func GetCustomerDocuments(w http.ResponseWriter, r *http.Request) {
 	var documents []models.Document
 	for rows.Next() {
 		var doc models.Document
-		if err := rows.Scan(&doc.ID, &doc.OriginalName, &doc.Status, &doc.DocumentType, &doc.PersonName, &doc.DOB, &doc.DocumentIDNumber, &doc.CreatedAt, &doc.OCRText); err != nil {
+		if err := rows.Scan(&doc.ID, &doc.OriginalName, &doc.Status, &doc.DocumentType, &doc.ExtractedData, &doc.PersonName, &doc.DOB, &doc.DocumentIDNumber, &doc.CreatedAt, &doc.OCRText); err != nil {
 			http.Error(w, "Failed to scan document", http.StatusInternalServerError)
 			return
 		}