From 9a7f27de4bad2477ed9634ac54ff9243a42ddb67 Mon Sep 17 00:00:00 2001 From: Mayank Joshi Date: Fri, 4 Sep 2026 21:53:39 +0530 Subject: [PATCH] Implement async Great Sage processing and dynamic document extraction --- ARCHITECTURE_SNAPSHOT.md | 418 +++++++++++++++++++++++++++++ README.md | 19 ++ cmd/api/main.go | 1 + e2e/.gitignore | 8 + e2e/package-lock.json | 97 +++++++ e2e/package.json | 15 ++ e2e/playwright.config.ts | 79 ++++++ e2e/tests/auth.spec.ts | 35 +++ e2e/tests/example.spec.ts | 18 ++ internal/database/db.go | 21 ++ internal/handlers/document.go | 6 +- internal/handlers/document_user.go | 6 +- internal/handlers/review.go | 95 +++++-- internal/handlers/upload.go | 177 ++++++------ internal/handlers/webhook.go | 205 +++++++++++++- internal/models/models.go | 50 ++-- internal/services/greatsage.go | 71 +++++ public/index.html | 20 +- public/templates/documents.html | 28 +- public/templates/index.html | 28 +- public/templates/upload.html | 10 +- 21 files changed, 1226 insertions(+), 181 deletions(-) create mode 100644 ARCHITECTURE_SNAPSHOT.md create mode 100644 e2e/.gitignore create mode 100644 e2e/package-lock.json create mode 100644 e2e/package.json create mode 100644 e2e/playwright.config.ts create mode 100644 e2e/tests/auth.spec.ts create mode 100644 e2e/tests/example.spec.ts diff --git a/ARCHITECTURE_SNAPSHOT.md b/ARCHITECTURE_SNAPSHOT.md new file mode 100644 index 0000000..921032a --- /dev/null +++ b/ARCHITECTURE_SNAPSHOT.md @@ -0,0 +1,418 @@ +# Architecture Snapshot + +> Generated: 2026-09-01 · Scope: **poneglyph** (DocuNest) + **great-sage** (Document Intelligence Engine) + +--- + +## System Overview + +This monorepo pair implements **DocuNest** — a private, local-first document management platform for organisations that cannot tolerate data leaving their infrastructure. The system is split across two independent repositories that communicate exclusively through HTTP: + +| Repo | Language | Role | +|---|---|---| +| **poneglyph** | Go 1.26 | Core web application — user sessions, document lifecycle, PostgreSQL, human-review workflow, static UI | +| **great-sage** | Python 3 / FastAPI | Document intelligence engine — OCR (Tesseract + PyMuPDF), AI classification (Ollama), async job queue, SQLite | + +At runtime a user uploads a document through the **poneglyph** Go server; poneglyph writes the file to disk, stores a record in PostgreSQL, then calls Great Sage's REST API to submit it for processing. Great Sage performs OCR, passes the extracted text to a local Ollama LLM, and fires a webhook back to poneglyph once classification is complete. A human operator then reviews the AI-extracted fields before they are committed to the customer profile. + +--- + +## 1. Directory Trees + +### 1.1 `poneglyph` (DocuNest — Go API + Frontend) + +``` +poneglyph/ +├── cmd/ +│ └── api/ +│ └── main.go # Entry point — router wiring, middleware, server start +├── internal/ +│ ├── database/ +│ │ └── db.go # PostgreSQL connection, schema init, seed admin +│ ├── handlers/ +│ │ ├── admin.go # Admin-only: user management, DB wipe, log streaming +│ │ ├── auth.go # Login/logout, JWT issuance, AuthMiddleware, brute-force lockout +│ │ ├── dashboard.go # Stats aggregation endpoint +│ │ ├── document.go # Document listing and viewer +│ │ ├── document_user.go # Per-customer document queries +│ │ ├── events.go # SSE log streaming +│ │ ├── review.go # Human-in-the-loop review confirmation +│ │ ├── share.go # Share-link creation/revocation/view +│ │ ├── upload.go # File upload, MIME validation, Great Sage submission +│ │ ├── webhook.go # Receives Great Sage callbacks, writes AI results to DB +│ │ └── webhook_test.go # Unit tests for webhook handler +│ ├── models/ +│ │ └── models.go # Go structs: User, Customer, Document, AuditLog, DocumentShare +│ ├── services/ +│ │ ├── greatsage.go # HTTP client for submitting docs to Great Sage API +│ │ └── greatsage_test.go # Unit tests for GreatSageClient +│ └── storage/ +│ └── storage.go # Secure file persistence (MIME-validated, crypto-UUID filenames) +├── e2e/ +│ ├── tests/ +│ │ ├── auth.spec.ts # Playwright auth flow tests +│ │ └── example.spec.ts # Playwright smoke tests +│ ├── playwright.config.ts # Playwright config (Chromium, HTML reporter) +│ └── package.json # E2E dev dependencies +├── public/ +│ ├── index.html # Single-file SPA entry (vanilla JS + CSS) +│ ├── manifest.json # PWA manifest +│ ├── sw.js # Service worker (offline caching) +│ └── templates/ +│ ├── admin.html # Admin panel template +│ ├── customers.html # Customer list/dossier template +│ ├── dashboard.html # Dashboard / stats template +│ ├── documents.html # Document queue template +│ ├── index.html # Login page template +│ └── upload.html # Upload form template +├── scratch/ +│ ├── fix_compile.py # One-off compile-error fixup script +│ ├── fix_infinite_reload.py # Debug script for reload loop +│ ├── fix_redirect.py # Debug script for redirect issues +│ ├── fix_syntax.py # Syntax fixup utility +│ ├── fix_workspace.py # Workspace repair helper +│ ├── merge.py # File merge utility +│ └── update_tables.py # DB table update script +├── uploads/ # Runtime: uploaded documents stored by server (gitignored) +├── docker-compose.yml # PostgreSQL service definition +├── go.mod # Go module manifest +├── go.sum # Go dependency checksums +├── start.ps1 # Windows dev-start script +├── .env.example # Environment variable template +└── README.md +``` + +### 1.2 `great-sage` (Document Intelligence Engine — Python/FastAPI) + +``` +great-sage/ +├── app/ +│ ├── __init__.py +│ ├── auth.py # API-key verification (X-API-Key / Bearer) +│ ├── config.py # Immutable Settings dataclass loaded from env +│ ├── database.py # SQLModel engine init, session factory, schema creation +│ ├── llm.py # Ollama HTTP client, document classification, AI string sanitization +│ ├── main.py # FastAPI app, lifespan, /api/v1/analyze, /api/v2/jobs, /health +│ ├── models.py # SQLModel ORM: Job, JobFile (SQLite-backed) +│ ├── ocr.py # Text extraction: PyMuPDF (native PDF) → Tesseract fallback +│ ├── pipeline.py # End-to-end processing: OCR → LLM → DB → webhook +│ ├── schemas.py # Pydantic request/response schemas +│ ├── webhook.py # Async webhook delivery to Poneglyph +│ ├── worker.py # asyncio.Queue-backed background worker +│ └── routers/ +│ └── jobs.py # /api/v2/jobs CRUD (create, get, cancel, delete) +├── data/ +│ └── jobs/ # Runtime: per-job file storage (one sub-dir per UUID job) +│ └── / # Each job's uploaded files live here during processing +├── tests/ +│ ├── __init__.py +│ ├── test_app.py # pytest integration tests for API endpoints +│ └── test_v2_jobs.py # pytest tests for v2 jobs API +├── requirements.txt # Python dependency manifest +├── .env.example # Environment variable template +└── README.md +``` + +--- + +## 2. Dependency Manifests + +### 2.1 `great-sage/requirements.txt` + +```text +# Great Sage — AI/Document Intelligence Service +# Core +fastapi>=0.115.0 +uvicorn>=0.30.0 +python-multipart>=0.0.9 + +# OCR & Document Processing +PyMuPDF>=1.25.0 +Pillow>=10.0.0 +pytesseract>=0.3.10 + +# HTTP client (for Ollama + webhook) +httpx>=0.27.0 + +# Pydantic (pulled by FastAPI, pinned for clarity) +pydantic>=2.9.0 + +# Testing +pytest>=8.0.0 +pytest-asyncio>=0.24.0 +sqlmodel>=0.0.22 +``` + +### 2.2 `poneglyph/go.mod` + +```go +module docunest + +go 1.26.3 + +require ( + github.com/alexedwards/argon2id v1.0.0 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/gorilla/mux v1.8.1 // indirect + github.com/joho/godotenv v1.5.1 // indirect + github.com/lib/pq v1.12.3 // indirect + golang.org/x/crypto v0.55.0 // indirect + golang.org/x/sys v0.47.0 // indirect +) +``` + +### 2.3 `poneglyph/go.sum` (full contents) + +``` +github.com/alexedwards/argon2id v1.0.0 h1:wJzDx66hqWX7siL/SRUmgz3F8YMrd/nfX/xHHcQQP0w= +github.com/alexedwards/argon2id v1.0.0/go.mod h1:tYKkqIjzXvZdzPvADMWOEZ+l6+BD6CtBXMj5fnJppiw= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/lib/pq v1.12.3 h1:tTWxr2YLKwIvK90ZXEw8GP7UFHtcbTtty8zsI+YjrfQ= +github.com/lib/pq v1.12.3/go.mod h1:/p+8NSbOcwzAEI7wiMXFlgydTwcgTr3OSKMsD2BitpA= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.55.0 h1:+KWHjbgOaAQ66dh/YlkZKHlz9ZUlq61AFirAR9ntP8M= +golang.org/x/crypto v0.55.0/go.mod h1:uq0V9dE/fzQuJtbnL+2EhWOE63vo164FY8xqEnV9xis= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +``` + +### 2.4 `poneglyph/docker-compose.yml` + +```yaml +version: '3.8' + +services: + db: + image: postgres:15-alpine + container_name: docunest_db + environment: + POSTGRES_USER: ${DB_USER:-docunest} + POSTGRES_PASSWORD: ${DB_PASSWORD:?DB_PASSWORD must be set} + POSTGRES_DB: ${DB_NAME:-docunest} + ports: + # Bind only to localhost — do not expose PostgreSQL to external interfaces + - "127.0.0.1:5432:5432" + volumes: + - pgdata:/var/lib/postgresql/data + # Resource limits to prevent a runaway container from exhausting host memory + deploy: + resources: + limits: + memory: 512m + restart: unless-stopped + +volumes: + pgdata: +``` + +### 2.5 `poneglyph/e2e/package.json` + +```json +{ + "name": "e2e", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": {}, + "keywords": [], + "author": "", + "license": "ISC", + "type": "commonjs", + "devDependencies": { + "@playwright/test": "^1.62.1", + "@types/node": "^26.3.0" + } +} +``` + +--- + +## 3. Module / Folder Summaries + +### `poneglyph` (DocuNest — Go Application) + +#### `cmd/api/` +The application entry point. `main.go` bootstraps the entire server: it loads `.env` via `godotenv`, calls `handlers.InitAuth()` to validate the JWT secret at startup (refusing to start with a default/empty value), connects to PostgreSQL and runs schema migrations, seeds the admin user if absent, and then wires all routes through a `gorilla/mux` router — public login and share-view routes first, then a JWT-protected subrouter for normal users, and finally an admin subrouter behind an additional role check. Static files in `./public` are served last by a catch-all file server on `:8080`. + +#### `internal/database/` +Owns the global `*sql.DB` connection and all schema DDL. `ConnectDB()` constructs the PostgreSQL DSN from environment variables (with safe defaults for host/port/user), pings the database to confirm connectivity, and exposes the shared `DB` variable. `InitSchema()` executes the `CREATE TABLE IF NOT EXISTS` statements for `users`, `customers`, `documents`, `audit_logs`, and `document_shares`, and creates the necessary indexes. `SeedAdminUser()` inserts the initial admin account if the `users` table is empty. + +#### `internal/handlers/` +The HTTP handler layer — ten handler files covering every route group. `auth.go` implements Argon2id login, JWT cookie issuance with short expiry, `AuthMiddleware`, IP-based brute-force lockout (10 attempts triggers a 15-minute ban), and `AdminMiddleware`. `upload.go` validates file MIME types via binary inspection, delegates persistent storage to `storage.SaveFile`, records the document in PostgreSQL, and asynchronously submits it to Great Sage via `services.GreatSageClient`. `webhook.go` receives Great Sage callbacks (`POST /api/internal/webhook/analyze` and `/jobs`), verifies the shared webhook secret header, and writes OCR and AI classification results back to the `documents` table, setting status to `needs_review`. `review.go` allows a logged-in user to confirm or correct AI-extracted fields before committing them to a customer profile. `admin.go` provides user CRUD, account disabling, password reset, and the destructive `WipeDatabase` endpoint. `share.go` issues single-use or time-limited share tokens, while `events.go` and `dashboard.go` support real-time log streaming via Server-Sent Events and dashboard stats aggregation respectively. + +#### `internal/models/` +Defines the canonical Go structs — `User`, `Customer`, `Document`, `AuditLog`, `ReviewRequest`, and `DocumentShare` — with `json` field tags used across the entire application. `Document` carries all AI-extracted fields (`DocumentType`, `PersonName`, `DOB`, `DocumentIDNumber`, `Confidence`) as nullable pointers so that unprocessed records have clean zero-values without interfering with JSON serialisation. + +#### `internal/services/` +Houses `GreatSageClient`, a thin HTTP client that reads `GREAT_SAGE_URL` and `GREAT_SAGE_API_KEY` from the environment and exposes `SubmitDocument(docID, filePath)` (multipart POST to `/api/v1/analyze`) and `SubmitJob(jobID, filePaths, context, webhookURL)` (multipart POST to `/api/v2/jobs`). Timeouts are set to 30 seconds for submission only — Great Sage returns `202 Accepted` immediately and delivers results asynchronously via a webhook callback. + +#### `internal/storage/` +Secure file persistence layer. `SaveFile(reader, mimeType)` generates a cryptographically random 32-character hex ID, appends the canonical extension for the validated MIME type (PDF, JPEG, or PNG only — never the user-supplied filename), performs a path-traversal safety check on the resolved absolute path, and writes the file with permissions `0640`. Extension spoofing and path traversal are explicitly blocked by design and enforced at the only point where extensions are decided. + +#### `e2e/` +Playwright end-to-end test suite targeting the running Go server. `auth.spec.ts` covers login and logout flows; `example.spec.ts` provides a basic smoke test. The Playwright configuration runs only Chromium in development mode (Firefox and WebKit are commented out) with HTML reporting and two retries on CI. + +#### `public/` +The self-contained frontend — a vanilla-JS single-page application served directly by the Go file server. `index.html` is the main SPA shell; `templates/` holds six HTML page templates (login, dashboard, customers, documents, upload, admin) rendered client-side. A PWA `manifest.json` and `sw.js` service worker enable offline caching and browser installability. + +#### `scratch/` +A collection of one-off Python utility scripts used during development and debugging (compile fixes, merge helpers, DB table updates). These are not part of the production system and are excluded from deployments. + +#### `uploads/` +Runtime upload directory. All user-uploaded files are written here by `storage.SaveFile` under randomised cryptographic filenames. This directory is gitignored and is created automatically at startup by `storage.init()`. + +--- + +### `great-sage` (Document Intelligence Engine — Python/FastAPI) + +#### `app/` +The entire FastAPI application package. Exposes three route groups: `POST /api/v1/analyze` (legacy single-file endpoint, backward-compatible with poneglyph v1), the v2 jobs API (via `app/routers/jobs.py`), and `GET /health`. The `lifespan` context manager initialises the SQLite database and starts the background worker on startup, then gracefully drains the queue on shutdown. + +#### `app/auth.py` +Validates inbound API keys sent by poneglyph. Accepts either an `X-API-Key` header or a `Bearer` token in `Authorization`. Returns `HTTP 401` on mismatch. API key checking is skipped entirely when `GREAT_SAGE_API_KEY` is left empty in the environment, which is useful for local development without inter-service authentication. + +#### `app/config.py` +A frozen `dataclasses.dataclass` (`Settings`) that reads all configuration from environment variables at startup using `python-dotenv`. Covers the inbound API key, Ollama endpoint and model name, outbound webhook URL and secret, upload/OCR/LLM character-size limits, and worker queue capacity. Exposes a `validate()` method that returns a list of configuration errors so the application can fail-fast with a clear diagnostic message. + +#### `app/database.py` +Wraps SQLModel and SQLite: creates the engine pointing at `./data/great_sage.db`, defines `init_db()` (calls `SQLModel.metadata.create_all` to provision tables), and provides a `get_session()` FastAPI dependency that yields a scoped `Session` for each request. + +#### `app/llm.py` +HTTP client for a locally-running Ollama instance. `classify_document(text, context, settings)` sends a structured prompt to the configured model (default: `qwen2.5`) requesting a JSON object with `document_type`, `person_name`, `dob`, `document_id_number`, and `confidence`. `sanitize_ai_string()` strips null bytes and enforces a maximum length on every AI-returned field before it is written to the database. `check_ollama()` performs a lightweight reachability check consumed by the `/health` endpoint. + +#### `app/ocr.py` +Text extraction with a dual-strategy approach: first attempts to read embedded text from PDFs using PyMuPDF (fast, lossless, requires no model inference); if the extracted text falls below a minimum threshold — indicating a scanned or image-only PDF, or an image file — it falls back to Tesseract OCR with Pillow-based image pre-processing (grayscale conversion and unsharp-mask sharpening). Platform detection at import time sets the Tesseract binary path for Windows development versus Linux production. `check_tesseract()` is used by the `/health` endpoint. + +#### `app/pipeline.py` +Orchestrates the end-to-end processing flow for a single document or a full batch job: (1) extract text via `ocr.extract_text`, (2) classify via `llm.classify_document` with any job-level context string, (3) persist results to the SQLite `job_files` table, (4) update the parent `job` status to `completed` or `failed`, and (5) deliver a webhook to poneglyph if a `webhook_url` was provided. Errors at each step are caught individually so a failure in one file does not abort the rest of the batch. + +#### `app/worker.py` +An `asyncio.Queue`-backed background consumer (`Worker` class). `start()` spawns a single long-running `asyncio.Task` (`_consume`) that waits for job UUIDs, fetches the full `Job` record from SQLite, and calls `pipeline.process_job`. `stop()` drains the queue gracefully and cancels the consumer task. Queue capacity is configurable via `worker_queue_size` (default 64); `enqueue_job_id()` raises immediately if the queue is full, allowing the API layer to return `503` to the caller. + +#### `app/models.py` +Two SQLModel ORM tables backed by SQLite: `Job` (UUID primary key, `status`, optional `context` string, `webhook_url`, `legacy_document_id`, timestamps, and a cascading `files` relationship) and `JobFile` (UUID PK, foreign key to `Job`, `filename`, `filepath`, per-file `status`, `ocr_text`, `ai_result`, `error_message`). Job status lifecycle: `in_queue → processing → completed | failed | cancelled`. + +#### `app/schemas.py` +Pydantic v2 request and response schemas used for API validation and JSON serialisation: `AcceptedResponse`, `HealthResponse`, `ClassificationResult`, `WebhookPayload`, `JobWebhookPayload`, `FileResult`, `JobResponse`, and `JobFileResponse`. + +#### `app/webhook.py` +Asynchronous webhook delivery using `httpx.AsyncClient`. Signs outbound requests with the shared `PONEGLYPH_WEBHOOK_SECRET` in the `X-Webhook-Secret` header. Returns `True` on a `2xx` response and `False` on any error — webhook failures are logged but never re-raised, ensuring the worker continues processing remaining jobs regardless of delivery outcome. A 30-second timeout is applied to each delivery attempt. + +#### `app/routers/jobs.py` +FastAPI `APIRouter` mounted at `/api/v2/jobs`. Implements four endpoints: `POST /` (create job, persist files to `data/jobs//`, enqueue for async processing, return `202`), `GET /{job_id}` (status and results poll), `PUT /{job_id}/cancel` (soft-cancel if job is not already in a terminal state), and `DELETE /{job_id}` (hard-delete the job record and remove files from disk). + +#### `data/` +Runtime storage root for great-sage. `great_sage.db` (SQLite) lives at the top level; `jobs/` holds one subdirectory per job UUID, containing all uploaded files for that job during and after processing. + +#### `tests/` +`pytest` suite covering the FastAPI application. `test_app.py` tests the legacy v1 `analyze` endpoint, health checks, authentication rejection, and webhook delivery callbacks. `test_v2_jobs.py` covers the v2 batch jobs API — job creation, status polling, cancellation, and deletion — using FastAPI's `TestClient`. + +--- + +## 4. Inter-Service Communication + +``` +poneglyph (Go :8080) great-sage (Python :8000) + | | + | POST /api/v1/analyze (multipart, X-API-Key)| + |---------------------------------------------->| + | 202 Accepted {job_id} | + |<----------------------------------------------| + | | <- asyncio worker processes + | | OCR -> Ollama -> DB + | POST /api/internal/webhook/analyze | + |<----------------------------------------------| + | (X-Webhook-Secret, JSON result) | + | | + | POST /api/v2/jobs (multipart batch) | + |---------------------------------------------->| + | 202 Accepted {job_id} | + |<----------------------------------------------| + | | <- processes all files + | POST /api/internal/webhook/jobs | + |<----------------------------------------------| +``` + +Both services share two secrets via environment variables: +- `GREAT_SAGE_API_KEY` / `X-API-Key` — poneglyph to great-sage authentication +- `PONEGLYPH_WEBHOOK_SECRET` / `X-Webhook-Secret` — great-sage to poneglyph callback authentication + +--- + +## 5. Environment Variables Reference + +### `great-sage` + +| Variable | Purpose | Default | +|---|---|---| +| `GREAT_SAGE_API_KEY` | Shared API key for inbound requests from poneglyph | *(empty = auth disabled)* | +| `PONEGLYPH_WEBHOOK_URL` | URL to POST results back to poneglyph | — | +| `PONEGLYPH_WEBHOOK_SECRET` | Secret header value for signing webhook callbacks | — | +| `OLLAMA_URL` | Local Ollama base URL | `http://127.0.0.1:11434` | +| `OLLAMA_MODEL` | LLM model name | `qwen2.5` | +| `OLLAMA_TIMEOUT_SECONDS` | LLM call timeout | `180` | +| `MAX_UPLOAD_BYTES` | Per-file upload size limit | `15728640` (15 MB) | +| `WORKER_QUEUE_SIZE` | Maximum number of queued jobs | `64` | + +### `poneglyph` + +| Variable | Purpose | Default | +|---|---|---| +| `DB_HOST` | PostgreSQL host | `localhost` | +| `DB_PORT` | PostgreSQL port | `5432` | +| `DB_USER` | PostgreSQL user | `postgres` | +| `DB_PASSWORD` | PostgreSQL password | *(required)* | +| `DB_NAME` | PostgreSQL database name | `postgres` | +| `DB_SSLMODE` | PostgreSQL SSL mode | `disable` | +| `JWT_SECRET` | HMAC secret for JWT signing and verification | *(required — server refuses to start if missing or default)* | +| `GREAT_SAGE_URL` | Base URL of the Great Sage service | — | +| `GREAT_SAGE_API_KEY` | API key sent to Great Sage in `X-API-Key` header | — | +| `WEBHOOK_SECRET` | Secret validated on incoming Great Sage webhook callbacks | — | +| `ADMIN_USERNAME` | Username for the seeded initial admin account | — | +| `ADMIN_PASSWORD` | Password for the seeded initial admin account | — | + +--- + +*This snapshot was generated on 2026-09-01 without modifying any source files.* diff --git a/README.md b/README.md index a0a3657..91ee1c2 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,25 @@ Designed for environments where document confidentiality is non-negotiable, Docu --- +## 🚀 How to Run the Application + +The easiest way to run the entire stack (Poneglyph + Great Sage) on Windows is using the provided `start.ps1` orchestrator script. + +1. Ensure **PostgreSQL** is running (`docker-compose up -d`) +2. Ensure **Ollama** is running locally +3. Open a PowerShell terminal in this directory and run: + ```powershell + .\start.ps1 + ``` +4. Access the web app at [http://localhost:8080](http://localhost:8080) (Default login: `admin` / `admin`) +5. In a separate terminal, start Great Sage: + ```powershell + cd ..\great-sage + uvicorn app.main:app --host 127.0.0.1 --port 8000 + ``` + +--- + ## Core Capabilities - **Local OCR**: Extracts text from PDFs and images locally using PyMuPDF and Tesseract. Very fast and lightweight (no heavy PyTorch models required). diff --git a/cmd/api/main.go b/cmd/api/main.go index 98c7fe6..e5bde64 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -45,6 +45,7 @@ func main() { // Internal service-to-service routes (authenticated via X-Webhook-Secret, not JWT) api.HandleFunc("/internal/webhook/analyze", handlers.AnalyzeWebhook).Methods("POST") + api.HandleFunc("/internal/webhook/jobs", handlers.JobWebhook).Methods("POST") // Protected routes protected := api.PathPrefix("/").Subrouter() diff --git a/e2e/.gitignore b/e2e/.gitignore new file mode 100644 index 0000000..335bd46 --- /dev/null +++ b/e2e/.gitignore @@ -0,0 +1,8 @@ + +# Playwright +node_modules/ +/test-results/ +/playwright-report/ +/blob-report/ +/playwright/.cache/ +/playwright/.auth/ diff --git a/e2e/package-lock.json b/e2e/package-lock.json new file mode 100644 index 0000000..1223e6b --- /dev/null +++ b/e2e/package-lock.json @@ -0,0 +1,97 @@ +{ + "name": "e2e", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "e2e", + "version": "1.0.0", + "license": "ISC", + "devDependencies": { + "@playwright/test": "^1.62.1", + "@types/node": "^26.3.0" + } + }, + "node_modules/@playwright/test": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.62.1.tgz", + "integrity": "sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@types/node": { + "version": "26.3.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-26.3.0.tgz", + "integrity": "sha512-L3fgrnchriRC2ExBflb8j4uZZURHZfQsmQeyVzhjcHW4kkwVyo8/0h1B2MVzMTrYUJYu6G7EWs14hW/L9putqw==", + "dev": true, + "license": "MIT", + "dependencies": { + "undici-types": "~8.3.0" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/playwright": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz", + "integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==", + "dev": true, + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.62.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=20" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.62.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz", + "integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/undici-types": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-8.3.0.tgz", + "integrity": "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ==", + "dev": true, + "license": "MIT" + } + } +} diff --git a/e2e/package.json b/e2e/package.json new file mode 100644 index 0000000..97595c9 --- /dev/null +++ b/e2e/package.json @@ -0,0 +1,15 @@ +{ + "name": "e2e", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": {}, + "keywords": [], + "author": "", + "license": "ISC", + "type": "commonjs", + "devDependencies": { + "@playwright/test": "^1.62.1", + "@types/node": "^26.3.0" + } +} diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts new file mode 100644 index 0000000..2bf3143 --- /dev/null +++ b/e2e/playwright.config.ts @@ -0,0 +1,79 @@ +import { defineConfig, devices } from '@playwright/test'; + +/** + * Read environment variables from file. + * https://github.com/motdotla/dotenv + */ +// import dotenv from 'dotenv'; +// import path from 'path'; +// dotenv.config({ path: path.resolve(__dirname, '.env') }); + +/** + * See https://playwright.dev/docs/test-configuration. + */ +export default defineConfig({ + testDir: './tests', + /* Run tests in files in parallel */ + fullyParallel: true, + /* Fail the build on CI if you accidentally left test.only in the source code. */ + forbidOnly: !!process.env.CI, + /* Retry on CI only */ + retries: process.env.CI ? 2 : 0, + /* Opt out of parallel tests on CI. */ + workers: process.env.CI ? 1 : undefined, + /* Reporter to use. See https://playwright.dev/docs/test-reporters */ + reporter: 'html', + /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */ + use: { + /* Base URL to use in actions like `await page.goto('')`. */ + // baseURL: 'http://localhost:3000', + + /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */ + trace: 'on-first-retry', + }, + + /* Configure projects for major browsers */ + projects: [ + { + name: 'chromium', + use: { ...devices['Desktop Chrome'] }, + }, + + // { + // name: 'firefox', + // use: { ...devices['Desktop Firefox'] }, + // }, + + // { + // name: 'webkit', + // use: { ...devices['Desktop Safari'] }, + // }, + + /* Test against mobile viewports. */ + // { + // name: 'Mobile Chrome', + // use: { ...devices['Pixel 5'] }, + // }, + // { + // name: 'Mobile Safari', + // use: { ...devices['iPhone 12'] }, + // }, + + /* Test against branded browsers. */ + // { + // name: 'Microsoft Edge', + // use: { ...devices['Desktop Edge'], channel: 'msedge' }, + // }, + // { + // name: 'Google Chrome', + // use: { ...devices['Desktop Chrome'], channel: 'chrome' }, + // }, + ], + + /* Run your local dev server before starting the tests */ + // webServer: { + // command: 'npm run start', + // url: 'http://localhost:3000', + // reuseExistingServer: !process.env.CI, + // }, +}); diff --git a/e2e/tests/auth.spec.ts b/e2e/tests/auth.spec.ts new file mode 100644 index 0000000..192576f --- /dev/null +++ b/e2e/tests/auth.spec.ts @@ -0,0 +1,35 @@ +import { test, expect } from '@playwright/test'; + +test.describe('Authentication Flow', () => { + const baseURL = 'http://127.0.0.1:8080'; + + test('should show login page and allow admin login', async ({ page }) => { + await page.goto(`${baseURL}/`); + await expect(page.locator('text=Login').first()).toBeVisible(); + + const usernameInput = page.locator('input[type="text"]').first(); + const passwordInput = page.locator('input[type="password"]').first(); + + await usernameInput.fill('admin'); + await passwordInput.fill('admin'); // Seed password + + await page.locator('button[type="submit"], button:has-text("Login")').first().click(); + + await expect(page.locator('text=DocuNest Login').first()).toBeHidden({ timeout: 10000 }); + }); + + test('should reject invalid credentials', async ({ page }) => { + await page.goto(`${baseURL}/`); + + const usernameInput = page.locator('input[type="text"]').first(); + const passwordInput = page.locator('input[type="password"]').first(); + + await usernameInput.fill('admin'); + await passwordInput.fill('wrongpassword'); + + await page.locator('button[type="submit"], button:has-text("Login")').first().click(); + + // Check for an error message or that we are still on the login page + await expect(page.locator('text=Login').first()).toBeVisible(); + }); +}); diff --git a/e2e/tests/example.spec.ts b/e2e/tests/example.spec.ts new file mode 100644 index 0000000..54a906a --- /dev/null +++ b/e2e/tests/example.spec.ts @@ -0,0 +1,18 @@ +import { test, expect } from '@playwright/test'; + +test('has title', async ({ page }) => { + await page.goto('https://playwright.dev/'); + + // Expect a title "to contain" a substring. + await expect(page).toHaveTitle(/Playwright/); +}); + +test('get started link', async ({ page }) => { + await page.goto('https://playwright.dev/'); + + // Click the get started link. + await page.getByRole('link', { name: 'Get started' }).click(); + + // Expects page to have a heading with the name of Installation. + await expect(page.getByRole('heading', { name: 'Installation' })).toBeVisible(); +}); diff --git a/internal/database/db.go b/internal/database/db.go index 831b4ab..8cc032e 100644 --- a/internal/database/db.go +++ b/internal/database/db.go @@ -97,6 +97,7 @@ func InitSchema() error { document_id_number VARCHAR(100), confidence FLOAT, customer_id VARCHAR(50) REFERENCES customers(id), + job_id VARCHAR(36), created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ); @@ -132,6 +133,7 @@ func InitSchema() error { DB.Exec("ALTER TABLE documents ADD COLUMN dob VARCHAR(50)") DB.Exec("ALTER TABLE documents ADD COLUMN document_id_number VARCHAR(100)") DB.Exec("ALTER TABLE documents ADD COLUMN customer_id VARCHAR(50) REFERENCES customers(id)") + DB.Exec("ALTER TABLE documents ADD COLUMN job_id VARCHAR(36)") DB.Exec("ALTER TABLE audit_logs ADD COLUMN actor_id INT REFERENCES users(id)") // 2. Rename existing columns to workspace_id where appropriate @@ -140,6 +142,25 @@ func InitSchema() error { DB.Exec("ALTER TABLE documents RENAME COLUMN user_id TO workspace_id") DB.Exec("ALTER TABLE audit_logs RENAME COLUMN user_id TO workspace_id") + // 3. Add extracted_data JSONB column for flexible document extraction storage. + // This is the canonical source of truth for all document-type-specific extracted fields. + // Legacy columns (person_name, dob, document_id_number) are kept temporarily as + // synchronized projections for backward compatibility. + DB.Exec("ALTER TABLE documents ADD COLUMN extracted_data JSONB DEFAULT '{}'::jsonb") + + // 4. Backfill existing identity-document rows into extracted_data. + // Idempotent: only touches rows where extracted_data is empty and legacy fields exist. + DB.Exec(` + UPDATE documents + SET extracted_data = jsonb_strip_nulls(jsonb_build_object( + 'person_name', person_name, + 'dob', dob, + 'document_id_number', document_id_number + )) + WHERE (extracted_data IS NULL OR extracted_data = '{}'::jsonb) + AND (person_name IS NOT NULL OR dob IS NOT NULL OR document_id_number IS NOT NULL) + `) + log.Println("Database schema initialized") return nil } diff --git a/internal/handlers/document.go b/internal/handlers/document.go index 85a20c2..452e88f 100644 --- a/internal/handlers/document.go +++ b/internal/handlers/document.go @@ -28,14 +28,14 @@ func GetDocuments(w http.ResponseWriter, r *http.Request) { if role == "admin" { rows, err = database.DB.Query(` - SELECT d.id, d.filename, d.original_name, d.status, d.document_type, d.person_name, d.dob, d.document_id_number, d.confidence, d.created_at, d.ocr_text, c.name + SELECT d.id, d.filename, d.original_name, d.status, d.document_type, d.extracted_data, d.person_name, d.dob, d.document_id_number, d.confidence, d.created_at, d.ocr_text, c.name FROM documents d LEFT JOIN customers c ON d.customer_id = c.id ORDER BY d.created_at DESC LIMIT 100 `) } else { rows, err = database.DB.Query(` - SELECT d.id, d.filename, d.original_name, d.status, d.document_type, d.person_name, d.dob, d.document_id_number, d.confidence, d.created_at, d.ocr_text, c.name + SELECT d.id, d.filename, d.original_name, d.status, d.document_type, d.extracted_data, d.person_name, d.dob, d.document_id_number, d.confidence, d.created_at, d.ocr_text, c.name FROM documents d LEFT JOIN customers c ON d.customer_id = c.id WHERE d.workspace_id = $1 @@ -59,7 +59,7 @@ func GetDocuments(w http.ResponseWriter, r *http.Request) { var doc DocumentWithCustomer if err := rows.Scan( &doc.ID, &doc.Filename, &doc.OriginalName, &doc.Status, - &doc.DocumentType, &doc.PersonName, &doc.DOB, &doc.DocumentIDNumber, &doc.Confidence, &doc.CreatedAt, &doc.OCRText, &doc.CustomerName, + &doc.DocumentType, &doc.ExtractedData, &doc.PersonName, &doc.DOB, &doc.DocumentIDNumber, &doc.Confidence, &doc.CreatedAt, &doc.OCRText, &doc.CustomerName, ); err != nil { http.Error(w, "Failed to parse document", http.StatusInternalServerError) return diff --git a/internal/handlers/document_user.go b/internal/handlers/document_user.go index 87937ca..9af60c2 100644 --- a/internal/handlers/document_user.go +++ b/internal/handlers/document_user.go @@ -131,7 +131,7 @@ func GetCustomerDocuments(w http.ResponseWriter, r *http.Request) { if role == "admin" { rows, err = database.DB.Query(` - SELECT id, original_name, status, document_type, person_name, dob, document_id_number, created_at, ocr_text + SELECT id, original_name, status, document_type, extracted_data, person_name, dob, document_id_number, created_at, ocr_text FROM documents WHERE customer_id = $1 ORDER BY created_at DESC @@ -139,7 +139,7 @@ func GetCustomerDocuments(w http.ResponseWriter, r *http.Request) { `, customerID) } else { rows, err = database.DB.Query(` - SELECT id, original_name, status, document_type, person_name, dob, document_id_number, created_at, ocr_text + SELECT id, original_name, status, document_type, extracted_data, person_name, dob, document_id_number, created_at, ocr_text FROM documents WHERE customer_id = $1 ORDER BY created_at DESC @@ -156,7 +156,7 @@ func GetCustomerDocuments(w http.ResponseWriter, r *http.Request) { var documents []models.Document for rows.Next() { var doc models.Document - if err := rows.Scan(&doc.ID, &doc.OriginalName, &doc.Status, &doc.DocumentType, &doc.PersonName, &doc.DOB, &doc.DocumentIDNumber, &doc.CreatedAt, &doc.OCRText); err != nil { + if err := rows.Scan(&doc.ID, &doc.OriginalName, &doc.Status, &doc.DocumentType, &doc.ExtractedData, &doc.PersonName, &doc.DOB, &doc.DocumentIDNumber, &doc.CreatedAt, &doc.OCRText); err != nil { http.Error(w, "Failed to scan document", http.StatusInternalServerError) return } diff --git a/internal/handlers/review.go b/internal/handlers/review.go index d96ba08..949e409 100644 --- a/internal/handlers/review.go +++ b/internal/handlers/review.go @@ -12,6 +12,18 @@ import ( "github.com/gorilla/mux" ) +// resolveCustomerName extracts a suitable customer display name from extracted_data. +// It checks for the most likely entity name field depending on document type: +// person_name (identity docs), vendor (invoices), merchant (receipts). +func resolveCustomerName(extractedData map[string]interface{}) string { + for _, key := range []string{"person_name", "vendor", "merchant"} { + if v, ok := extractedData[key].(string); ok && v != "" { + return v + } + } + return "" +} + func ConfirmDocument(w http.ResponseWriter, r *http.Request) { workspaceID, _ := r.Context().Value(WorkspaceIDKey).(int) userID, ok := r.Context().Value(UserIDKey).(int) @@ -61,36 +73,69 @@ func ConfirmDocument(w http.ResponseWriter, r *http.Request) { } // Input validation - if len(req.PersonName) == 0 || len(req.PersonName) > 255 { - http.Error(w, "Person name must be between 1 and 255 characters", http.StatusBadRequest) - return - } if len(req.DocumentType) == 0 || len(req.DocumentType) > 100 { http.Error(w, "Document type must be between 1 and 100 characters", http.StatusBadRequest) return } - if len(req.DOB) > 50 { - http.Error(w, "DOB must be under 50 characters", http.StatusBadRequest) + if len(req.CustomerID) > 50 { + http.Error(w, "Invalid customer ID", http.StatusBadRequest) return } - if len(req.DocumentIDNumber) > 100 { - http.Error(w, "Document ID Number must be under 100 characters", http.StatusBadRequest) - return + + // Resolve extracted_data: prefer canonical ExtractedData, fall back to legacy fields + var extractedMap map[string]interface{} + if len(req.ExtractedData) > 0 { + if err := json.Unmarshal(req.ExtractedData, &extractedMap); err != nil { + http.Error(w, "Invalid extracted_data JSON", http.StatusBadRequest) + return + } + } else { + // Legacy fallback: build extracted_data from old fields + extractedMap = make(map[string]interface{}) + if req.PersonName != "" { + extractedMap["person_name"] = req.PersonName + } + if req.DOB != "" { + extractedMap["dob"] = req.DOB + } + if req.DocumentIDNumber != "" { + extractedMap["document_id_number"] = req.DocumentIDNumber + } } - if len(req.CustomerID) > 50 { - http.Error(w, "Invalid customer ID", http.StatusBadRequest) + + extractedJSON, err := json.Marshal(extractedMap) + if err != nil { + http.Error(w, "Failed to serialize extracted data", http.StatusInternalServerError) return } + // Project legacy columns from extracted_data (single source of truth) + var personName, dob, docIDNumber *string + if v, ok := extractedMap["person_name"].(string); ok { + personName = &v + } + if v, ok := extractedMap["dob"].(string); ok { + dob = &v + } + if v, ok := extractedMap["document_id_number"].(string); ok { + docIDNumber = &v + } + + // Determine customer name for new customer creation + customerName := resolveCustomerName(extractedMap) + if customerName == "" { + customerName = "Customer" + } + finalCustomerID := req.CustomerID if finalCustomerID == "new" || finalCustomerID == "" { // Create a new customer using a cryptographic UUID from the DB serial - // The name is user-provided, not AI-provided — user has already verified it + // The name is derived from extracted_data — user has already verified it var newID string err = database.DB.QueryRow( "INSERT INTO customers (id, workspace_id, name) VALUES (gen_random_uuid()::text, $1, $2) RETURNING id", - workspaceID, req.PersonName, + workspaceID, customerName, ).Scan(&newID) if err != nil { log.Printf("Failed to create new customer: %v", err) @@ -114,15 +159,15 @@ func ConfirmDocument(w http.ResponseWriter, r *http.Request) { if role == "admin" { _, err = database.DB.Exec(` UPDATE documents - SET document_type = $1, person_name = $2, dob = $3, document_id_number = $4, customer_id = $5, status = 'completed' - WHERE id = $6 - `, req.DocumentType, req.PersonName, req.DOB, req.DocumentIDNumber, finalCustomerID, docID) + SET document_type = $1, extracted_data = $2, person_name = $3, dob = $4, document_id_number = $5, customer_id = $6, status = 'completed' + WHERE id = $7 + `, req.DocumentType, extractedJSON, personName, dob, docIDNumber, finalCustomerID, docID) } else { _, err = database.DB.Exec(` UPDATE documents - SET document_type = $1, person_name = $2, dob = $3, document_id_number = $4, customer_id = $5, status = 'completed' - WHERE id = $6 AND workspace_id = $7 - `, req.DocumentType, req.PersonName, req.DOB, req.DocumentIDNumber, finalCustomerID, docID, workspaceID) + SET document_type = $1, extracted_data = $2, person_name = $3, dob = $4, document_id_number = $5, customer_id = $6, status = 'completed' + WHERE id = $7 AND workspace_id = $8 + `, req.DocumentType, extractedJSON, personName, dob, docIDNumber, finalCustomerID, docID, workspaceID) } if err != nil { log.Printf("Failed to update document %d: %v", docID, err) @@ -130,14 +175,12 @@ func ConfirmDocument(w http.ResponseWriter, r *http.Request) { return } - // Audit log — record the human review action + // Audit log — record the human review action with canonical extracted_data LogEvent(workspaceID, userID, "confirm_ai_review", map[string]interface{}{ - "document_id": docID, - "person_name": req.PersonName, - "document_type": req.DocumentType, - "dob": req.DOB, - "document_id_number": req.DocumentIDNumber, - "customer_id": finalCustomerID, + "document_id": docID, + "document_type": req.DocumentType, + "extracted_data": extractedMap, + "customer_id": finalCustomerID, }) w.Header().Set("Content-Type", "application/json") diff --git a/internal/handlers/upload.go b/internal/handlers/upload.go index 66072a1..8c5ce1e 100644 --- a/internal/handlers/upload.go +++ b/internal/handlers/upload.go @@ -2,6 +2,7 @@ package handlers import ( "encoding/json" + "fmt" "io" "log" "net/http" @@ -72,29 +73,9 @@ func UploadDocument(w http.ResponseWriter, r *http.Request) { return } - file, handler, err := r.FormFile("document") - if err != nil { - http.Error(w, "Error retrieving the file", http.StatusBadRequest) - return - } - defer file.Close() - - // MIME Validation: read first 512 bytes to determine real content type - buffer := make([]byte, 512) - n, err := file.Read(buffer) - if err != nil && err != io.EOF { - http.Error(w, "Failed to read file", http.StatusBadRequest) - return - } - buffer = buffer[:n] - if _, err := file.Seek(0, 0); err != nil { - http.Error(w, "Failed to process file", http.StatusInternalServerError) - return - } - - contentType := http.DetectContentType(buffer) - if contentType != "application/pdf" && contentType != "image/jpeg" && contentType != "image/png" { - http.Error(w, "Invalid file type. Only PDF, JPEG, and PNG are accepted.", http.StatusBadRequest) + files := r.MultipartForm.File["documents"] + if len(files) == 0 { + http.Error(w, "No documents provided", http.StatusBadRequest) return } @@ -133,79 +114,125 @@ func UploadDocument(w http.ResponseWriter, r *http.Request) { } } - // Save file — extension is derived from MIME type, never from user-supplied filename - newFilename, filePath, err := storage.SaveFile(file, contentType) - if err != nil { - log.Printf("Failed to save uploaded file: %v", err) - http.Error(w, "Failed to save file", http.StatusInternalServerError) - return - } + var docInfos []services.DocumentInfo + var docIDs []int - // Use the sanitized original filename for display only (never for storage) - originalName := handler.Filename - if len(originalName) > 255 { - originalName = originalName[:255] - } + for _, handler := range files { + file, err := handler.Open() + if err != nil { + log.Printf("Failed to open uploaded file: %v", err) + continue + } + + // MIME Validation: read first 512 bytes to determine real content type + buffer := make([]byte, 512) + n, err := file.Read(buffer) + if err != nil && err != io.EOF { + file.Close() + continue + } + buffer = buffer[:n] + if _, err := file.Seek(0, 0); err != nil { + file.Close() + continue + } - var docID int - var cID *string - if customerType == "existing" { - cID = &customerID + contentType := http.DetectContentType(buffer) + if contentType != "application/pdf" && contentType != "image/jpeg" && contentType != "image/png" { + file.Close() + continue // Skip invalid file types + } + + // Save file + newFilename, filePath, err := storage.SaveFile(file, contentType) + file.Close() + if err != nil { + log.Printf("Failed to save uploaded file: %v", err) + continue + } + + // Use the sanitized original filename for display only + originalName := handler.Filename + if len(originalName) > 255 { + originalName = originalName[:255] + } + + var docID int + var cID *string + if customerType == "existing" { + cID = &customerID + } + + query := `INSERT INTO documents (workspace_id, filename, filepath, original_name, status, customer_id) + VALUES ($1, $2, $3, $4, $5, $6) RETURNING id` + err = database.DB.QueryRow(query, workspaceID, newFilename, filePath, originalName, "uploaded", cID).Scan(&docID) + if err != nil { + os.Remove(filePath) + log.Printf("Failed to create document record: %v", err) + continue + } + + // Log the upload event + userID := r.Context().Value(UserIDKey).(int) + LogEvent(workspaceID, userID, "document_uploaded", map[string]interface{}{ + "document_id": docID, + "filename": originalName, + }) + + docInfos = append(docInfos, services.DocumentInfo{ + ID: docID, + FilePath: filePath, + }) + docIDs = append(docIDs, docID) } - query := `INSERT INTO documents (workspace_id, filename, filepath, original_name, status, customer_id) - VALUES ($1, $2, $3, $4, $5, $6) RETURNING id` - err = database.DB.QueryRow(query, workspaceID, newFilename, filePath, originalName, "uploaded", cID).Scan(&docID) - if err != nil { - // Clean up the saved file if we couldn't create the DB record - os.Remove(filePath) - log.Printf("Failed to create document record: %v", err) - http.Error(w, "Failed to create database record", http.StatusInternalServerError) + if len(docInfos) == 0 { + http.Error(w, "No valid documents were uploaded", http.StatusBadRequest) return } - // Log the upload event - userID := r.Context().Value(UserIDKey).(int) - LogEvent(workspaceID, userID, "document_uploaded", map[string]interface{}{ - "document_id": docID, - "filename": originalName, - }) - - // Submit the document to Great Sage for asynchronous OCR + AI processing. - // The goroutine updates the status to "processing" on success or "failed" on error. - go sendToGreatSage(docID, filePath) + // Submit the batch of documents to Great Sage's V2 API + sendJobToGreatSage(docInfos) w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(map[string]interface{}{ - "message": "File uploaded successfully", - "id": docID, + "message": fmt.Sprintf("%d files uploaded successfully", len(docIDs)), + "ids": docIDs, }) } -// sendToGreatSage submits a document to the Great Sage intelligence service -// for asynchronous OCR and AI classification processing. -// -// On successful submission (HTTP 202), the document status is set to "processing". -// On failure, the document status is set to "failed" to prevent it from being -// permanently stuck. -func sendToGreatSage(docID int, filePath string) { +// sendJobToGreatSage submits a batch of documents to Great Sage +// for asynchronous OCR and AI classification processing using V2 APIs. +func sendJobToGreatSage(docs []services.DocumentInfo) { client, err := services.NewGreatSageClient() if err != nil { - log.Printf("Document %d: Great Sage client error: %v", docID, err) - database.DB.Exec("UPDATE documents SET status = 'failed' WHERE id = $1", docID) + log.Printf("Great Sage client error: %v", err) + for _, doc := range docs { + database.DB.Exec("UPDATE documents SET status = 'failed' WHERE id = $1", doc.ID) + } return } - err = client.SubmitDocument(docID, filePath) + webhookURL := os.Getenv("API_BASE_URL") + if webhookURL == "" { + webhookURL = "http://backend:8080" // default for local docker + } + webhookURL = webhookURL + "/api/internal/webhook/jobs" + + jobID, err := client.SubmitJob(docs, webhookURL) if err != nil { - log.Printf("Document %d: failed to submit to Great Sage: %v", docID, err) - database.DB.Exec("UPDATE documents SET status = 'failed' WHERE id = $1", docID) + log.Printf("Failed to submit job to Great Sage: %v", err) + for _, doc := range docs { + database.DB.Exec("UPDATE documents SET status = 'failed' WHERE id = $1", doc.ID) + } return } - // Great Sage accepted the document — mark as processing - _, err = database.DB.Exec("UPDATE documents SET status = 'processing' WHERE id = $1", docID) - if err != nil { - log.Printf("Document %d: failed to update status to processing: %v", docID, err) + // Job submitted — mark all as processing and store the job ID + for _, doc := range docs { + _, err = database.DB.Exec("UPDATE documents SET status = 'processing', job_id = $1 WHERE id = $2", jobID, doc.ID) + if err != nil { + log.Printf("Document %d: failed to update status to processing: %v", doc.ID, err) + } } } diff --git a/internal/handlers/webhook.go b/internal/handlers/webhook.go index 086dd93..a7e6279 100644 --- a/internal/handlers/webhook.go +++ b/internal/handlers/webhook.go @@ -1,11 +1,14 @@ package handlers import ( + "crypto/subtle" "encoding/json" "io" "log" "net/http" "os" + "strconv" + "strings" "docunest/internal/database" ) @@ -20,10 +23,64 @@ type greatSageWebhookPayload struct { } type greatSageClassification struct { - DocumentType *string `json:"document_type"` - PersonName *string `json:"person_name"` - DOB *string `json:"dob"` - DocumentIDNumber *string `json:"document_id_number"` + DocumentType *string `json:"document_type"` + ExtractedData map[string]interface{} `json:"extracted_data"` // Canonical: flexible key-value extraction + // Legacy identity fields — accepted for backward compatibility with older Great Sage versions. + PersonName *string `json:"person_name,omitempty"` + DOB *string `json:"dob,omitempty"` + DocumentIDNumber *string `json:"document_id_number,omitempty"` +} + +type jobWebhookFileResult struct { + Filename string `json:"filename"` + Status string `json:"status"` + OCRText *string `json:"ocr_text"` + Classification greatSageClassification `json:"classification"` + ErrorMessage *string `json:"error_message"` +} + +type jobWebhookPayload struct { + JobID string `json:"job_id"` + Status string `json:"status"` + Files []jobWebhookFileResult `json:"files"` +} + +// resolveExtractedData builds the canonical extracted_data JSON from a classification result. +// If the classification already contains an ExtractedData map, it is used directly. +// Otherwise, legacy identity fields are assembled into extracted_data for backward compatibility. +// Returns the JSON bytes for JSONB storage and the legacy projection values. +func resolveExtractedData(c greatSageClassification) (extractedJSON []byte, personName, dob, docIDNumber *string) { + data := c.ExtractedData + if data == nil { + // Build extracted_data from legacy fields + data = make(map[string]interface{}) + if c.PersonName != nil { + data["person_name"] = *c.PersonName + } + if c.DOB != nil { + data["dob"] = *c.DOB + } + if c.DocumentIDNumber != nil { + data["document_id_number"] = *c.DocumentIDNumber + } + } + + // Project legacy columns from extracted_data (single source of truth) + if v, ok := data["person_name"].(string); ok { + personName = &v + } + if v, ok := data["dob"].(string); ok { + dob = &v + } + if v, ok := data["document_id_number"].(string); ok { + docIDNumber = &v + } + + extractedJSON, err := json.Marshal(data) + if err != nil { + extractedJSON = []byte("{}") + } + return extractedJSON, personName, dob, docIDNumber } // AnalyzeWebhook receives asynchronous processing results from Great Sage. @@ -44,7 +101,7 @@ func AnalyzeWebhook(w http.ResponseWriter, r *http.Request) { } receivedSecret := r.Header.Get("X-Webhook-Secret") - if receivedSecret == "" || receivedSecret != expectedSecret { + if receivedSecret == "" || subtle.ConstantTimeCompare([]byte(receivedSecret), []byte(expectedSecret)) != 1 { http.Error(w, "Unauthorized", http.StatusUnauthorized) return } @@ -98,21 +155,25 @@ func AnalyzeWebhook(w http.ResponseWriter, r *http.Request) { // --- Apply the result --- if payload.Status == "success" { + extractedJSON, personName, dob, docIDNumber := resolveExtractedData(payload.Classification) + _, err = database.DB.Exec(` UPDATE documents SET ocr_text = $1, document_type = $2, - person_name = $3, - dob = $4, - document_id_number = $5, + extracted_data = $3, + person_name = $4, + dob = $5, + document_id_number = $6, status = 'needs_review' - WHERE id = $6 AND status IN ('processing', 'uploaded') + WHERE id = $7 AND status IN ('processing', 'uploaded') `, payload.OCRText, payload.Classification.DocumentType, - payload.Classification.PersonName, - payload.Classification.DOB, - payload.Classification.DocumentIDNumber, + extractedJSON, + personName, + dob, + docIDNumber, payload.DocumentID, ) if err != nil { @@ -158,3 +219,123 @@ func AnalyzeWebhook(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(map[string]string{"message": "ok"}) } + +// JobWebhook receives asynchronous processing results from Great Sage's V2 jobs API. +// It applies the individual file classifications to their respective documents in Poneglyph. +func JobWebhook(w http.ResponseWriter, r *http.Request) { + expectedSecret := os.Getenv("PONEGLYPH_WEBHOOK_SECRET") + if expectedSecret == "" { + http.Error(w, "Server misconfiguration", http.StatusInternalServerError) + return + } + receivedSecret := r.Header.Get("X-Webhook-Secret") + if receivedSecret == "" || subtle.ConstantTimeCompare([]byte(receivedSecret), []byte(expectedSecret)) != 1 { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + r.Body = http.MaxBytesReader(w, r.Body, 20<<20) // 20 MB limit for batch results + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "Failed to read request body", http.StatusBadRequest) + return + } + + var payload jobWebhookPayload + if err := json.Unmarshal(body, &payload); err != nil { + http.Error(w, "Invalid JSON payload", http.StatusBadRequest) + return + } + + if payload.JobID == "" { + http.Error(w, "Invalid or missing job_id", http.StatusBadRequest) + return + } + + for _, fileResult := range payload.Files { + // filename format: "_" + parts := strings.SplitN(fileResult.Filename, "_", 2) + if len(parts) < 2 { + log.Printf("Webhook: invalid filename format '%s'", fileResult.Filename) + continue + } + + docID, err := strconv.Atoi(parts[0]) + if err != nil { + log.Printf("Webhook: failed to parse document_id from '%s'", fileResult.Filename) + continue + } + + var currentStatus string + var workspaceID int + err = database.DB.QueryRow( + "SELECT status, workspace_id FROM documents WHERE id = $1 AND job_id = $2", + docID, payload.JobID, + ).Scan(¤tStatus, &workspaceID) + if err != nil { + log.Printf("Webhook: document %d not found for job %s: %v", docID, payload.JobID, err) + continue + } + + if currentStatus != "processing" && currentStatus != "uploaded" { + continue + } + + if fileResult.Status == "success" { + extractedJSON, personName, dob, docIDNumber := resolveExtractedData(fileResult.Classification) + + _, err = database.DB.Exec(` + UPDATE documents + SET ocr_text = $1, + document_type = $2, + extracted_data = $3, + person_name = $4, + dob = $5, + document_id_number = $6, + status = 'needs_review' + WHERE id = $7 + `, + fileResult.OCRText, + fileResult.Classification.DocumentType, + extractedJSON, + personName, + dob, + docIDNumber, + docID, + ) + if err != nil { + log.Printf("Webhook: failed to update document %d: %v", docID, err) + continue + } + LogEvent(workspaceID, 0, "document_processing_completed", map[string]interface{}{ + "document_id": docID, + "status": "needs_review", + }) + } else { + _, err = database.DB.Exec(` + UPDATE documents + SET ocr_text = COALESCE($1, ocr_text), + status = 'failed' + WHERE id = $2 + `, + fileResult.OCRText, + docID, + ) + if err != nil { + log.Printf("Webhook: failed to update document %d failure: %v", docID, err) + continue + } + errMsg := "unknown error" + if fileResult.ErrorMessage != nil { + errMsg = *fileResult.ErrorMessage + } + LogEvent(workspaceID, 0, "document_processing_failed", map[string]interface{}{ + "document_id": docID, + "error_message": errMsg, + }) + } + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(map[string]string{"message": "ok"}) +} diff --git a/internal/models/models.go b/internal/models/models.go index b079ba7..62a39de 100644 --- a/internal/models/models.go +++ b/internal/models/models.go @@ -1,6 +1,9 @@ package models -import "time" +import ( + "encoding/json" + "time" +) type User struct { ID int `json:"id"` @@ -19,20 +22,23 @@ type Customer struct { } type Document struct { - ID int `json:"id"` - UserID int `json:"user_id"` - Filename string `json:"filename"` - Filepath string `json:"-"` - OriginalName string `json:"original_name"` - Status string `json:"status"` // uploaded, processing, needs_review, completed, failed - OCRText *string `json:"ocr_text,omitempty"` - DocumentType *string `json:"document_type,omitempty"` - PersonName *string `json:"person_name,omitempty"` - DOB *string `json:"dob,omitempty"` - DocumentIDNumber *string `json:"document_id_number,omitempty"` - Confidence *float64 `json:"confidence,omitempty"` - CustomerID *string `json:"customer_id,omitempty"` - CreatedAt time.Time `json:"created_at"` + ID int `json:"id"` + UserID int `json:"user_id"` + Filename string `json:"filename"` + Filepath string `json:"-"` + OriginalName string `json:"original_name"` + Status string `json:"status"` // uploaded, processing, needs_review, completed, failed + OCRText *string `json:"ocr_text,omitempty"` + DocumentType *string `json:"document_type,omitempty"` + ExtractedData json.RawMessage `json:"extracted_data"` // Canonical source of truth for all extracted fields + Confidence *float64 `json:"confidence,omitempty"` + CustomerID *string `json:"customer_id,omitempty"` + CreatedAt time.Time `json:"created_at"` + // Legacy projection fields — synchronized mirrors of identity-document fields + // inside extracted_data. Kept temporarily for backward compatibility. + PersonName *string `json:"person_name,omitempty"` + DOB *string `json:"dob,omitempty"` + DocumentIDNumber *string `json:"document_id_number,omitempty"` } type AuditLog struct { @@ -45,13 +51,17 @@ type AuditLog struct { } type ReviewRequest struct { - PersonName string `json:"person_name"` - DocumentType string `json:"document_type"` - DOB string `json:"dob"` - DocumentIDNumber string `json:"document_id_number"` - CustomerID string `json:"customer_id"` + DocumentType string `json:"document_type"` + CustomerID string `json:"customer_id"` + ExtractedData json.RawMessage `json:"extracted_data"` // Canonical: dynamic fields from the review form + // Legacy fields — accepted from older clients for backward compatibility. + // If ExtractedData is provided, these are ignored. + PersonName string `json:"person_name,omitempty"` + DOB string `json:"dob,omitempty"` + DocumentIDNumber string `json:"document_id_number,omitempty"` } + type DocumentShare struct { Token string `json:"token"` DocumentID int `json:"document_id"` diff --git a/internal/services/greatsage.go b/internal/services/greatsage.go index c39b68f..5939798 100644 --- a/internal/services/greatsage.go +++ b/internal/services/greatsage.go @@ -2,6 +2,7 @@ package services import ( "bytes" + "encoding/json" "fmt" "io" "log" @@ -98,3 +99,73 @@ func (c *GreatSageClient) SubmitDocument(docID int, filePath string) error { return fmt.Errorf("Great Sage returned HTTP %d: %s", resp.StatusCode, string(errBody)) } + +// DocumentInfo holds information about a document to submit to Great Sage. +type DocumentInfo struct { + ID int + FilePath string +} + +// SubmitJob sends a batch of documents to Great Sage's V2 jobs API. +// It returns the Job ID returned by Great Sage, or an error. +func (c *GreatSageClient) SubmitJob(docs []DocumentInfo, webhookURL string) (string, error) { + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + + for _, doc := range docs { + file, err := os.Open(doc.FilePath) + if err != nil { + return "", fmt.Errorf("failed to open file %s: %w", doc.FilePath, err) + } + defer file.Close() + + // Include the Poneglyph document ID in the filename so it can be parsed from the webhook + filename := fmt.Sprintf("%d_%s", doc.ID, filepath.Base(doc.FilePath)) + part, err := writer.CreateFormFile("files", filename) + if err != nil { + return "", fmt.Errorf("failed to create form file for %s: %w", filename, err) + } + if _, err = io.Copy(part, file); err != nil { + return "", fmt.Errorf("failed to copy file content for %s: %w", filename, err) + } + } + + if webhookURL != "" { + if err := writer.WriteField("webhook_url", webhookURL); err != nil { + return "", fmt.Errorf("failed to write webhook_url field: %w", err) + } + } + + writer.Close() + + url := c.BaseURL + "/api/v2/jobs" + req, err := http.NewRequest("POST", url, body) + if err != nil { + return "", fmt.Errorf("failed to create request: %w", err) + } + req.Header.Set("Content-Type", writer.FormDataContentType()) + req.Header.Set("X-API-Key", c.APIKey) + + resp, err := c.HTTPClient.Do(req) + if err != nil { + return "", fmt.Errorf("Great Sage unreachable: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusAccepted || resp.StatusCode == http.StatusOK { + var result struct { + ID string `json:"id"` + } + if err := json.NewDecoder(resp.Body).Decode(&result); err == nil { + log.Printf("Job submitted to Great Sage successfully, Job ID: %s", result.ID) + return result.ID, nil + } + return "", fmt.Errorf("failed to parse Great Sage response") + } + + errBody := make([]byte, 512) + n, _ := io.ReadAtLeast(resp.Body, errBody, 1) + errBody = errBody[:n] + + return "", fmt.Errorf("Great Sage returned HTTP %d: %s", resp.StatusCode, string(errBody)) +} diff --git a/public/index.html b/public/index.html index 1e27438..6546bea 100644 --- a/public/index.html +++ b/public/index.html @@ -416,16 +416,16 @@

Upload a Document

- +
-
-
+
+
- - +
@@ -958,7 +958,7 @@

Create User

searchQuery: '', customers: [], selectedCustomer: null, - file: null, + files: [], uploading: false, uploadSuccess: false, uploadError: '', @@ -983,18 +983,18 @@

Create User

handleDrop(e) { if (e.dataTransfer.files.length > 0) { - this.file = e.dataTransfer.files[0]; + this.files = Array.from(e.dataTransfer.files); } }, handleFileSelect(e) { if (e.target.files.length > 0) { - this.file = e.target.files[0]; + this.files = Array.from(e.target.files); } }, uploadFile() { - if (!this.file) return; + if (this.files.length === 0) return; if (this.customerType === 'existing' && !this.selectedCustomer) { this.uploadError = "Please select an existing customer"; return; @@ -1005,7 +1005,7 @@

Create User

this.uploadSuccess = false; const formData = new FormData(); - formData.append('document', this.file); + this.files.forEach(f => formData.append('documents', f)); formData.append('customer_type', this.customerType); if (this.selectedCustomer) { formData.append('customer_id', this.selectedCustomer.id); diff --git a/public/templates/documents.html b/public/templates/documents.html index db24b77..6d3bf4d 100644 --- a/public/templates/documents.html +++ b/public/templates/documents.html @@ -21,7 +21,7 @@

Recent Documents

Document Status Type - Person + Entity Customer Date Actions @@ -47,7 +47,7 @@

Recent Documents

- + @@ -86,21 +86,15 @@

Review AI

-
- - -
-
- - -
-
- - -
+ +
diff --git a/public/templates/index.html b/public/templates/index.html index 8086281..d9b36ce 100644 --- a/public/templates/index.html +++ b/public/templates/index.html @@ -280,7 +280,7 @@

DocuNest

searchQuery: '', customers: [], selectedCustomer: null, - file: null, + files: [], uploading: false, uploadSuccess: false, uploadError: '', @@ -305,18 +305,18 @@

DocuNest

handleDrop(e) { if (e.dataTransfer.files.length > 0) { - this.file = e.dataTransfer.files[0]; + this.files = Array.from(e.dataTransfer.files); } }, handleFileSelect(e) { if (e.target.files.length > 0) { - this.file = e.target.files[0]; + this.files = Array.from(e.target.files); } }, uploadFile() { - if (!this.file) return; + if (this.files.length === 0) return; if (this.customerType === 'existing' && !this.selectedCustomer) { this.uploadError = "Please select an existing customer"; return; @@ -327,7 +327,7 @@

DocuNest

this.uploadSuccess = false; const formData = new FormData(); - formData.append('document', this.file); + this.files.forEach(f => formData.append('documents', f)); formData.append('customer_type', this.customerType); if (this.selectedCustomer) { formData.append('customer_id', this.selectedCustomer.id); @@ -340,7 +340,10 @@

DocuNest

.then(res => { if (res.ok) { this.uploadSuccess = true; - this.file = null; + this.files = []; + if (this.$refs && this.$refs.fileInput) { + this.$refs.fileInput.value = ''; + } setTimeout(() => { this.uploadSuccess = false; }, 3000); @@ -386,7 +389,14 @@

DocuNest

}, openReviewModal(doc) { - this.reviewDoc = { ...doc }; + this.reviewDoc = { + ...doc, + _extracted_data: doc.extracted_data ? { ...doc.extracted_data } : { + person_name: doc.person_name || '', + dob: doc.dob || '', + document_id_number: doc.document_id_number || '' + } + }; this.reviewCustomerType = 'new'; this.reviewSearchQuery = ''; this.reviewSelectedCustomer = null; @@ -419,10 +429,8 @@

DocuNest

method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ - person_name: this.reviewDoc.person_name, document_type: this.reviewDoc.document_type, - dob: this.reviewDoc.dob || "", - document_id_number: this.reviewDoc.document_id_number || "", + extracted_data: this.reviewDoc._extracted_data, customer_id: custId }) }) diff --git a/public/templates/upload.html b/public/templates/upload.html index 8f59a52..e81a2c5 100644 --- a/public/templates/upload.html +++ b/public/templates/upload.html @@ -62,16 +62,16 @@

Upload a Document

- +
-
-
+
+
- - +