From 2a17f2d3336d8cb2da5bb1c696702b9c2ccc2123 Mon Sep 17 00:00:00 2001 From: Nils Schneider Date: Wed, 29 Oct 2025 11:52:19 +0100 Subject: [PATCH] backend dry --- backend/src/config.rs | 7 ++ backend/src/workers/index.rs | 38 ++++--- backend/src/workers/mod.rs | 66 ++++++++++++ backend/src/workers/ocr.rs | 19 ++-- backend/src/workers/thumbnails.rs | 25 +++-- frontend/src/detail/DetailPanel.jsx | 117 +++------------------- frontend/src/documents/DocumentsTable.jsx | 12 +-- frontend/src/index.jsx | 35 ++++++- frontend/src/preview/PreviewWorkspace.jsx | 20 ++-- frontend/src/styles.css | 52 +--------- 10 files changed, 180 insertions(+), 211 deletions(-) diff --git a/backend/src/config.rs b/backend/src/config.rs index 48ff6da..a74d245 100644 --- a/backend/src/config.rs +++ b/backend/src/config.rs @@ -56,6 +56,8 @@ pub struct AppConfig { pub quickwit_index: Option, #[serde(default = "default_tenant_slug")] pub default_tenant_slug: String, + #[serde(default = "default_worker_max_document_bytes")] + pub worker_max_document_bytes: u64, } impl AppConfig { @@ -68,6 +70,7 @@ impl AppConfig { pool_size = config.database_max_pool_size, quickwit_enabled = config.quickwit_endpoint.is_some(), s3_bucket = %config.s3_bucket, + worker_max_document_bytes = config.worker_max_document_bytes, "loaded backend configuration" ); Ok(config) @@ -149,6 +152,10 @@ fn default_tenant_slug() -> String { "admin".to_string() } +fn default_worker_max_document_bytes() -> u64 { + 200 * 1024 * 1024 +} + fn redact_database_url(raw: &str) -> String { match Url::parse(raw) { Ok(mut parsed) => { diff --git a/backend/src/workers/index.rs b/backend/src/workers/index.rs index bc88c17..5a04e8e 100644 --- a/backend/src/workers/index.rs +++ b/backend/src/workers/index.rs @@ -18,7 +18,13 @@ use crate::{ storage::TenantStorage, }; -use super::{ocr::OCR_TEXT_ASSET_TYPE, JobExecution, JobHandler}; +use super::{ + fetch_version_object, + handle_fetch_error, + ocr::OCR_TEXT_ASSET_TYPE, + JobExecution, + JobHandler, +}; #[derive(Debug, Deserialize)] struct IndexPayload { @@ -114,21 +120,23 @@ impl JobHandler for IndexDocumentTextJob { } let s3_key = context.text_s3_key.unwrap(); - let text = match storage.get_object(&s3_key).await { - Ok(bytes) => match String::from_utf8(bytes) { - Ok(text) => text, - Err(err) => { - warn!(job_id = %job.id, error = %err, "ocr text not valid UTF-8"); - return JobExecution::Failed { - error: "ocr text not valid UTF-8".into(), - }; - } - }, + let bytes = match fetch_version_object( + &context.version, + &storage, + &s3_key, + state.config.worker_max_document_bytes, + ) + .await + { + Ok(bytes) => bytes, + Err(err) => return handle_fetch_error(job, err, "failed to download ocr text"), + }; + let text = match String::from_utf8(bytes) { + Ok(text) => text, Err(err) => { - warn!(job_id = %job.id, error = %err, "failed to download ocr text"); - return JobExecution::Retry { - delay: Duration::from_secs(30), - error: err.to_string(), + warn!(job_id = %job.id, error = %err, "ocr text not valid UTF-8"); + return JobExecution::Failed { + error: "ocr text not valid UTF-8".into(), }; } }; diff --git a/backend/src/workers/mod.rs b/backend/src/workers/mod.rs index 6ea7ce5..121861b 100644 --- a/backend/src/workers/mod.rs +++ b/backend/src/workers/mod.rs @@ -1,5 +1,6 @@ use std::{collections::HashMap, sync::Arc, time::Duration}; +use anyhow::Error as AnyhowError; use async_trait::async_trait; use tokio::time::sleep; use tracing::{error, info, warn}; @@ -147,3 +148,68 @@ pub fn default_handlers() -> Vec> { Arc::new(index::IndexDocumentTextJob::new()), ] } + +pub(crate) fn check_worker_document_limit( + size_bytes: i64, + limit_bytes: u64, +) -> Result<(), (u64, u64)> { + let size = size_bytes.max(0) as u64; + if size > limit_bytes { + Err((size, limit_bytes)) + } else { + Ok(()) + } +} + +pub(crate) enum FetchVersionError { + TooLarge { size: u64, limit: u64 }, + Storage(AnyhowError), +} + +pub(crate) async fn fetch_version_object( + version: &crate::models::DocumentVersion, + storage: &TenantStorage, + s3_key: &str, + limit_bytes: u64, +) -> Result, FetchVersionError> { + check_worker_document_limit(version.size_bytes, limit_bytes).map_err(|(size, limit)| { + FetchVersionError::TooLarge { + size, + limit, + } + })?; + + storage + .get_object(s3_key) + .await + .map_err(FetchVersionError::Storage) +} + +pub(crate) fn handle_fetch_error( + job_id: crate::models::Job, + err: FetchVersionError, + message: &str, +) -> JobExecution { + match err { + FetchVersionError::TooLarge { size, limit } => { + warn!( + job_id = %job_id.id, + size_bytes = size, + limit_bytes = limit, + "document exceeds worker size limit" + ); + JobExecution::Failed { + error: format!( + "document size {size} bytes exceeds worker limit of {limit} bytes" + ), + } + } + FetchVersionError::Storage(err) => { + warn!(job_id = %job_id.id, error = %err, "{message}"); + JobExecution::Retry { + delay: Duration::from_secs(30), + error: err.to_string(), + } + } + } +} diff --git a/backend/src/workers/ocr.rs b/backend/src/workers/ocr.rs index 9905ccd..877e0ce 100644 --- a/backend/src/workers/ocr.rs +++ b/backend/src/workers/ocr.rs @@ -30,7 +30,7 @@ use crate::{ utils::storage_paths::document_asset_object_prefix, }; -use super::{JobExecution, JobHandler}; +use super::{fetch_version_object, handle_fetch_error, JobExecution, JobHandler}; pub const OCR_TEXT_ASSET_TYPE: &str = "ocr-text"; const MIN_TEXT_LENGTH: usize = 50; @@ -99,15 +99,16 @@ impl JobHandler for GenerateOcrTextJob { return JobExecution::Success; } - let bytes = match storage.get_object(&context.version.s3_key).await { + let bytes = match fetch_version_object( + &context.version, + &storage, + &context.version.s3_key, + state.config.worker_max_document_bytes, + ) + .await + { Ok(bytes) => bytes, - Err(err) => { - warn!(job_id = %job.id, error = %err, "failed to fetch document for ocr"); - return JobExecution::Retry { - delay: Duration::from_secs(30), - error: err.to_string(), - }; - } + Err(err) => return handle_fetch_error(job, err, "failed to fetch document for ocr"), }; let doc_meta = PdfDocumentMeta { diff --git a/backend/src/workers/thumbnails.rs b/backend/src/workers/thumbnails.rs index f205ade..0fa0a0c 100644 --- a/backend/src/workers/thumbnails.rs +++ b/backend/src/workers/thumbnails.rs @@ -24,7 +24,13 @@ use crate::{ utils::storage_paths::document_asset_object_key, }; -use super::{analyze::determine_thumbnail_support, JobExecution, JobHandler}; +use super::{ + analyze::determine_thumbnail_support, + fetch_version_object, + handle_fetch_error, + JobExecution, + JobHandler, +}; const THUMBNAIL_WIDTH: u32 = 512; const THUMBNAIL_HEIGHT: u32 = 512; @@ -96,15 +102,16 @@ impl JobHandler for GenerateThumbnailsJob { return JobExecution::Success; } - let bytes = match storage.get_object(&initial.version.s3_key).await { + let bytes = match fetch_version_object( + &initial.version, + &storage, + &initial.version.s3_key, + state.config.worker_max_document_bytes, + ) + .await + { Ok(bytes) => bytes, - Err(err) => { - warn!(job_id = %job.id, error = %err, "thumbnail fetch failed; will retry"); - return JobExecution::Retry { - delay: Duration::from_secs(30), - error: err.to_string(), - }; - } + Err(err) => return handle_fetch_error(job, err, "thumbnail fetch failed; will retry"), }; let generation = match generate_preview_and_thumbnail(&initial.document, &bytes) { diff --git a/frontend/src/detail/DetailPanel.jsx b/frontend/src/detail/DetailPanel.jsx index 3b18863..3e75aef 100644 --- a/frontend/src/detail/DetailPanel.jsx +++ b/frontend/src/detail/DetailPanel.jsx @@ -1,5 +1,4 @@ import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; -import { createPortal } from 'react-dom'; import { DownloadIcon, EditIcon, @@ -15,6 +14,7 @@ import { formatFileSize } from '../utils/format'; import { resolveDocumentAssetUrl, createAssetView } from '../asset_manager'; import { useAssetNavigator } from '../hooks/useAssetNavigator'; import PreviewZoomOverlay from './PreviewZoomOverlay'; +import { openOcrTextInNewTab } from '../utils/ocr'; const MAX_PREVIEW_STACK_ITEMS = 15; @@ -300,6 +300,7 @@ const DetailPanel = ({ onUpdateTitle = async () => false, ensureAssetUrl = null, getDocumentAsset = () => null, + ensurePreviewData = () => Promise.resolve(), correspondents = [], onCorrespondentAdd, onCorrespondentRemove, @@ -329,10 +330,6 @@ const DetailPanel = ({ const [titleDraft, setTitleDraft] = useState(''); const [titleSaving, setTitleSaving] = useState(false); const [titleError, setTitleError] = useState(null); - const [ocrOpen, setOcrOpen] = useState(false); - const [ocrUrl, setOcrUrl] = useState(null); - const [ocrLoading, setOcrLoading] = useState(false); - const [ocrError, setOcrError] = useState(null); const [zoomedPreview, setZoomedPreview] = useState(null); useEffect(() => { @@ -341,10 +338,6 @@ const DetailPanel = ({ setTitleDraft(''); setTitleError(null); setTitleSaving(false); - setOcrOpen(false); - setOcrLoading(false); - setOcrError(null); - setOcrUrl(null); return; } @@ -356,13 +349,6 @@ const DetailPanel = ({ } }, [singleDoc, titleEditDocId]); - useEffect(() => { - setOcrOpen(false); - setOcrLoading(false); - setOcrError(null); - setOcrUrl(null); - }, [singleDoc?.id]); - useEffect(() => { setZoomedPreview(null); }, [selectionKey]); @@ -420,59 +406,21 @@ const DetailPanel = ({ [singleDoc, getDocumentAsset], ); - const loadOcrUrl = useCallback(async () => { + const openOcr = useCallback(async () => { if (!singleDoc) { return; } - const asset = getDocumentAsset(singleDoc, 'ocr-text'); - if (!asset) { - setOcrError('No OCR text available for this document.'); - setOcrUrl(null); - return; - } - setOcrLoading(true); - setOcrError(null); try { - let entry = asset; - let url = entry?.url || - resolveDocumentAssetUrl(singleDoc, 'ocr-text', { - ensureAssetUrl, - getAsset: getDocumentAsset, - }); - if (!url && typeof ensureAssetUrl === 'function') { - const ensured = await ensureAssetUrl(singleDoc.id, asset, { force: false }); - if (ensured) { - entry = ensured; - } - url = entry?.url || - resolveDocumentAssetUrl(singleDoc, 'ocr-text', { - ensureAssetUrl, - getAsset: getDocumentAsset, - }); - } - if (!url) { - throw new Error('OCR text URL is unavailable.'); - } - setOcrUrl(url); + await openOcrTextInNewTab({ + document: singleDoc, + ensurePreviewData, + getDocumentAsset, + ensureAssetUrl, + }); } catch (error) { - setOcrError(error.message || 'Failed to load OCR text.'); - setOcrUrl(null); - } finally { - setOcrLoading(false); + /* noop */ } - }, [singleDoc, ensureAssetUrl, getDocumentAsset]); - - const openOcrModal = useCallback(() => { - if (!singleDoc) { - return; - } - setOcrOpen(true); - loadOcrUrl(); - }, [singleDoc, loadOcrUrl]); - - const closeOcrModal = useCallback(() => { - setOcrOpen(false); - }, []); + }, [singleDoc, ensurePreviewData, getDocumentAsset, ensureAssetUrl]); const singlePreviewNavigator = useAssetNavigator({ document: singleDoc, @@ -1138,47 +1086,6 @@ const DetailPanel = ({ 
{JSON.stringify(metadata, null, 2)}
)} - {hasOcrAsset && ocrOpen - ? createPortal( -
-
event.stopPropagation()} - > -
-

OCR Text

- -
-
- {ocrLoading ? ( -
Loading OCR text…
- ) : ocrError ? ( -
{ocrError}
- ) : ocrUrl ? ( -