nils/papercrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

backend dry

Browse Source
This commit is contained in:
nils
2025-10-29 11:52:19 +01:00
parent 788858f0c1
commit 2a17f2d333
10 changed files with 180 additions and 211 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/config.rs
+7
View File
@@ -56,6 +56,8 @@ pub struct AppConfig {
pub quickwit_index: Option<String>,
#[serde(default = "default_tenant_slug")]
pub default_tenant_slug: String,
#[serde(default = "default_worker_max_document_bytes")]
pub worker_max_document_bytes: u64,
}
impl AppConfig {
@@ -68,6 +70,7 @@ impl AppConfig {
pool_size = config.database_max_pool_size,
quickwit_enabled = config.quickwit_endpoint.is_some(),
s3_bucket = %config.s3_bucket,
worker_max_document_bytes = config.worker_max_document_bytes,
"loaded backend configuration"
);
Ok(config)
@@ -149,6 +152,10 @@ fn default_tenant_slug() -> String {
"admin".to_string()
}
fn default_worker_max_document_bytes() -> u64 {
200 * 1024 * 1024
}
fn redact_database_url(raw: &str) -> String {
match Url::parse(raw) {
Ok(mut parsed) => {
backend/src/workers/index.rs
+23 -15
View File
@@ -18,7 +18,13 @@ use crate::{
storage::TenantStorage,
};
use super::{ocr::OCR_TEXT_ASSET_TYPE, JobExecution, JobHandler};
use super::{
fetch_version_object,
handle_fetch_error,
ocr::OCR_TEXT_ASSET_TYPE,
JobExecution,
JobHandler,
};
#[derive(Debug, Deserialize)]
struct IndexPayload {
@@ -114,21 +120,23 @@ impl JobHandler for IndexDocumentTextJob {
}
let s3_key = context.text_s3_key.unwrap();
let text = match storage.get_object(&s3_key).await {
Ok(bytes) => match String::from_utf8(bytes) {
Ok(text) => text,
Err(err) => {
warn!(job_id = %job.id, error = %err, "ocr text not valid UTF-8");
return JobExecution::Failed {
error: "ocr text not valid UTF-8".into(),
};
}
},
let bytes = match fetch_version_object(
&context.version,
&storage,
&s3_key,
state.config.worker_max_document_bytes,
)
.await
{
Ok(bytes) => bytes,
Err(err) => return handle_fetch_error(job, err, "failed to download ocr text"),
};
let text = match String::from_utf8(bytes) {
Ok(text) => text,
Err(err) => {
warn!(job_id = %job.id, error = %err, "failed to download ocr text");
return JobExecution::Retry {
delay: Duration::from_secs(30),
error: err.to_string(),
warn!(job_id = %job.id, error = %err, "ocr text not valid UTF-8");
return JobExecution::Failed {
error: "ocr text not valid UTF-8".into(),
};
}
};
backend/src/workers/mod.rs
+66
View File
@@ -1,5 +1,6 @@
use std::{collections::HashMap, sync::Arc, time::Duration};
use anyhow::Error as AnyhowError;
use async_trait::async_trait;
use tokio::time::sleep;
use tracing::{error, info, warn};
@@ -147,3 +148,68 @@ pub fn default_handlers() -> Vec<Arc<dyn JobHandler>> {
Arc::new(index::IndexDocumentTextJob::new()),
]
}
pub(crate) fn check_worker_document_limit(
size_bytes: i64,
limit_bytes: u64,
) -> Result<(), (u64, u64)> {
let size = size_bytes.max(0) as u64;
if size > limit_bytes {
Err((size, limit_bytes))
} else {
Ok(())
}
}
pub(crate) enum FetchVersionError {
TooLarge { size: u64, limit: u64 },
Storage(AnyhowError),
}
pub(crate) async fn fetch_version_object(
version: &crate::models::DocumentVersion,
storage: &TenantStorage,
s3_key: &str,
limit_bytes: u64,
) -> Result<Vec<u8>, FetchVersionError> {
check_worker_document_limit(version.size_bytes, limit_bytes).map_err(|(size, limit)| {
FetchVersionError::TooLarge {
size,
limit,
}
})?;
storage
.get_object(s3_key)
.await
.map_err(FetchVersionError::Storage)
}
pub(crate) fn handle_fetch_error(
job_id: crate::models::Job,
err: FetchVersionError,
message: &str,
) -> JobExecution {
match err {
FetchVersionError::TooLarge { size, limit } => {
warn!(
job_id = %job_id.id,
size_bytes = size,
limit_bytes = limit,
"document exceeds worker size limit"
);
JobExecution::Failed {
error: format!(
"document size {size} bytes exceeds worker limit of {limit} bytes"
),
}
}
FetchVersionError::Storage(err) => {
warn!(job_id = %job_id.id, error = %err, "{message}");
JobExecution::Retry {
delay: Duration::from_secs(30),
error: err.to_string(),
}
}
}
}
backend/src/workers/ocr.rs
+10 -9
View File
@@ -30,7 +30,7 @@ use crate::{
utils::storage_paths::document_asset_object_prefix,
};
use super::{JobExecution, JobHandler};
use super::{fetch_version_object, handle_fetch_error, JobExecution, JobHandler};
pub const OCR_TEXT_ASSET_TYPE: &str = "ocr-text";
const MIN_TEXT_LENGTH: usize = 50;
@@ -99,15 +99,16 @@ impl JobHandler for GenerateOcrTextJob {
return JobExecution::Success;
}
let bytes = match storage.get_object(&context.version.s3_key).await {
let bytes = match fetch_version_object(
&context.version,
&storage,
&context.version.s3_key,
state.config.worker_max_document_bytes,
)
.await
{
Ok(bytes) => bytes,
Err(err) => {
warn!(job_id = %job.id, error = %err, "failed to fetch document for ocr");
return JobExecution::Retry {
delay: Duration::from_secs(30),
error: err.to_string(),
};
}
Err(err) => return handle_fetch_error(job, err, "failed to fetch document for ocr"),
};
let doc_meta = PdfDocumentMeta {
backend/src/workers/thumbnails.rs
+16 -9
View File
@@ -24,7 +24,13 @@ use crate::{
utils::storage_paths::document_asset_object_key,
};
use super::{analyze::determine_thumbnail_support, JobExecution, JobHandler};
use super::{
analyze::determine_thumbnail_support,
fetch_version_object,
handle_fetch_error,
JobExecution,
JobHandler,
};
const THUMBNAIL_WIDTH: u32 = 512;
const THUMBNAIL_HEIGHT: u32 = 512;
@@ -96,15 +102,16 @@ impl JobHandler for GenerateThumbnailsJob {
return JobExecution::Success;
}
let bytes = match storage.get_object(&initial.version.s3_key).await {
let bytes = match fetch_version_object(
&initial.version,
&storage,
&initial.version.s3_key,
state.config.worker_max_document_bytes,
)
.await
{
Ok(bytes) => bytes,
Err(err) => {
warn!(job_id = %job.id, error = %err, "thumbnail fetch failed; will retry");
return JobExecution::Retry {
delay: Duration::from_secs(30),
error: err.to_string(),
};
}
Err(err) => return handle_fetch_error(job, err, "thumbnail fetch failed; will retry"),
};
let generation = match generate_preview_and_thumbnail(&initial.document, &bytes) {
frontend/src/detail/DetailPanel.jsx
+12 -105
View File
@@ -1,5 +1,4 @@
import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react';
import { createPortal } from 'react-dom';
import {
DownloadIcon,
EditIcon,
@@ -15,6 +14,7 @@ import { formatFileSize } from '../utils/format';
import { resolveDocumentAssetUrl, createAssetView } from '../asset_manager';
import { useAssetNavigator } from '../hooks/useAssetNavigator';
import PreviewZoomOverlay from './PreviewZoomOverlay';
import { openOcrTextInNewTab } from '../utils/ocr';
const MAX_PREVIEW_STACK_ITEMS = 15;
@@ -300,6 +300,7 @@ const DetailPanel = ({
onUpdateTitle = async () => false,
ensureAssetUrl = null,
getDocumentAsset = () => null,
ensurePreviewData = () => Promise.resolve(),
correspondents = [],
onCorrespondentAdd,
onCorrespondentRemove,
@@ -329,10 +330,6 @@ const DetailPanel = ({
const [titleDraft, setTitleDraft] = useState('');
const [titleSaving, setTitleSaving] = useState(false);
const [titleError, setTitleError] = useState(null);
const [ocrOpen, setOcrOpen] = useState(false);
const [ocrUrl, setOcrUrl] = useState(null);
const [ocrLoading, setOcrLoading] = useState(false);
const [ocrError, setOcrError] = useState(null);
const [zoomedPreview, setZoomedPreview] = useState(null);
useEffect(() => {
@@ -341,10 +338,6 @@ const DetailPanel = ({
setTitleDraft('');
setTitleError(null);
setTitleSaving(false);
setOcrOpen(false);
setOcrLoading(false);
setOcrError(null);
setOcrUrl(null);
return;
}
@@ -356,13 +349,6 @@ const DetailPanel = ({
}
}, [singleDoc, titleEditDocId]);
useEffect(() => {
setOcrOpen(false);
setOcrLoading(false);
setOcrError(null);
setOcrUrl(null);
}, [singleDoc?.id]);
useEffect(() => {
setZoomedPreview(null);
}, [selectionKey]);
@@ -420,59 +406,21 @@ const DetailPanel = ({
[singleDoc, getDocumentAsset],
);
const loadOcrUrl = useCallback(async () => {
const openOcr = useCallback(async () => {
if (!singleDoc) {
return;
}
const asset = getDocumentAsset(singleDoc, 'ocr-text');
if (!asset) {
setOcrError('No OCR text available for this document.');
setOcrUrl(null);
return;
}
setOcrLoading(true);
setOcrError(null);
try {
let entry = asset;
let url = entry?.url ||
resolveDocumentAssetUrl(singleDoc, 'ocr-text', {
ensureAssetUrl,
getAsset: getDocumentAsset,
});
if (!url && typeof ensureAssetUrl === 'function') {
const ensured = await ensureAssetUrl(singleDoc.id, asset, { force: false });
if (ensured) {
entry = ensured;
}
url = entry?.url ||
resolveDocumentAssetUrl(singleDoc, 'ocr-text', {
ensureAssetUrl,
getAsset: getDocumentAsset,
});
}
if (!url) {
throw new Error('OCR text URL is unavailable.');
}
setOcrUrl(url);
await openOcrTextInNewTab({
document: singleDoc,
ensurePreviewData,
getDocumentAsset,
ensureAssetUrl,
});
} catch (error) {
setOcrError(error.message || 'Failed to load OCR text.');
setOcrUrl(null);
} finally {
setOcrLoading(false);
/* noop */
}
}, [singleDoc, ensureAssetUrl, getDocumentAsset]);
const openOcrModal = useCallback(() => {
if (!singleDoc) {
return;
}
setOcrOpen(true);
loadOcrUrl();
}, [singleDoc, loadOcrUrl]);
const closeOcrModal = useCallback(() => {
setOcrOpen(false);
}, []);
}, [singleDoc, ensurePreviewData, getDocumentAsset, ensureAssetUrl]);
const singlePreviewNavigator = useAssetNavigator({
document: singleDoc,
@@ -1138,47 +1086,6 @@ const DetailPanel = ({
<pre className="detail-metadata__block">{JSON.stringify(metadata, null, 2)}</pre>
</div>
)}
{hasOcrAsset && ocrOpen
? createPortal(
<div className="modal-backdrop" role="presentation" onClick={closeOcrModal}>
<div
className="modal modal--ocr"
role="dialog"
aria-modal="true"
aria-label="OCR text"
onClick={(event) => event.stopPropagation()}
>
<div className="modal__header">
<h3>OCR Text</h3>
<button
type="button"
className="icon-button ghost"
onClick={closeOcrModal}
aria-label="Close OCR text"
>
×
</button>
</div>
<div className="modal__body detail-ocr__content">
{ocrLoading ? (
<div className="meta">Loading OCR text…</div>
) : ocrError ? (
<div className="status-inline error">{ocrError}</div>
) : ocrUrl ? (
<iframe
title="OCR text"
src={ocrUrl}
className="detail-ocr__frame"
/>
) : (
<div className="meta">OCR text empty.</div>
)}
</div>
</div>
</div>,
document.body,
)
: null}
</>
);
};
@@ -1349,7 +1256,7 @@ const DetailPanel = ({
className="icon-button ghost"
onClick={(event) => {
event.stopPropagation();
openOcrModal();
openOcr();
}}
aria-label="View OCR text"
title="View OCR text"
frontend/src/documents/DocumentsTable.jsx
+2 -10
View File
@@ -1127,6 +1127,7 @@ export const createDocumentsSurface = ({
onNavigateParent,
renderSidebarToggle,
detailProps,
detailOpen = false,
}) => {
const {
currentFolderName,
@@ -1176,16 +1177,7 @@ export const createDocumentsSurface = ({
)
: null;
const detail = (() => {
if (!detailProps) {
return null;
}
const count = detailProps.selectedDocuments?.length || 0;
if (!count) {
return null;
}
return <DetailPanel {...detailProps} />;
})();
const detail = detailOpen && detailProps ? <DetailPanel {...detailProps} /> : null;
return {
key: 'documents',
frontend/src/index.jsx
+33 -2
View File
@@ -735,6 +735,14 @@ const AppLayout = () => {
[selectedRowKeys],
);
const [detailPanelOpen, setDetailPanelOpen] = useState(() => selectedDocumentIds.length > 0);
useEffect(() => {
if (selectedDocumentIds.length > 0 && !detailPanelOpen) {
setDetailPanelOpen(true);
}
}, [selectedDocumentIds, detailPanelOpen]);
const resetWorkspaceState = useCallback(() => {
const rootNode = createRootNode();
setFolderNodes(new Map([[rootNode.id, rootNode]]));
@@ -762,6 +770,7 @@ const AppLayout = () => {
setActiveTagFilters([]);
setDropOverlayState({ active: false, folderName: DEFAULT_FOLDER_NAME });
setActivePreviewId(null);
setDetailPanelOpen(false);
assetManager.reset();
setPreviewEntries(() => new Map());
previewInflightRef.current = new Map();
@@ -4845,6 +4854,11 @@ const AppLayout = () => {
const lastScrolledDetailDocRef = useRef(null);
useEffect(() => {
if (!detailPanelOpen) {
lastScrolledDetailDocRef.current = null;
return;
}
if (!orderedSelectedDocuments.length) {
lastScrolledDetailDocRef.current = null;
return;
@@ -4870,6 +4884,10 @@ const AppLayout = () => {
return;
}
if (typeof document === 'undefined') {
return;
}
const targetElement =
document.getElementById(`document-row-${lastSelectedId}`) ||
document.getElementById(`document-card-${lastSelectedId}`);
@@ -4882,7 +4900,7 @@ const AppLayout = () => {
requestAnimationFrame(() => {
targetElement.scrollIntoView({ block: 'nearest', behavior: 'smooth' });
});
}, [orderedSelectedDocuments, selectionOrder, documentsViewMode]);
}, [orderedSelectedDocuments, selectionOrder, documentsViewMode, detailPanelOpen]);
const { breadcrumbs, missingBreadcrumbAncestors } = useMemo(() => {
const chain = [];
@@ -5267,6 +5285,11 @@ const AppLayout = () => {
[sidebarProps, openTagsModal, openCorrespondentsModal, collapseSidebar],
);
const handleDetailPanelClose = useCallback(() => {
setDetailPanelOpen(false);
clearDocumentSelection();
}, [clearDocumentSelection]);
const detailPanelProps = {
selectedDocuments: orderedSelectedDocuments,
tags,
@@ -5286,12 +5309,13 @@ const AppLayout = () => {
onUpdateTitle: handleDocumentTitleUpdate,
ensureAssetUrl,
getDocumentAsset,
ensurePreviewData,
correspondents,
onCorrespondentAdd: handleCorrespondentAdd,
onCorrespondentRemove: handleCorrespondentRemove,
resolveApiPath,
onFolderNavigate: selectFolder,
onClose: clearDocumentSelection,
onClose: handleDetailPanelClose,
resolveFolderPath,
};
@@ -5369,6 +5393,8 @@ const AppLayout = () => {
openTagsModal,
openCorrespondentsModal,
openSettingsModal,
detailPanelOpen,
setDetailPanelOpen,
}),
[
token,
@@ -5410,6 +5436,8 @@ const AppLayout = () => {
openTagsModal,
openCorrespondentsModal,
openSettingsModal,
detailPanelOpen,
setDetailPanelOpen,
],
);
@@ -5575,6 +5603,7 @@ const DocumentsRoute = () => {
sidebarProps,
documentsTableProps,
detailPanelProps,
detailPanelOpen,
workspaceMode,
skeuoWorkspaceProps,
openTagsModal,
@@ -5653,6 +5682,7 @@ const DocumentsRoute = () => {
onNavigateParent: parentBreadcrumb ? handleNavigateParent : null,
renderSidebarToggle,
detailProps: detailPanelProps,
detailOpen: detailPanelOpen,
});
}, [
documentsTableProps,
@@ -5660,6 +5690,7 @@ const DocumentsRoute = () => {
handleNavigateParent,
renderSidebarToggle,
detailPanelProps,
detailPanelOpen,
]);
const previewSurface = useMemo(() => {
frontend/src/preview/PreviewWorkspace.jsx
+7 -13
View File
@@ -1,7 +1,7 @@
import React, { useMemo } from 'react';
import { resolveDocumentAssetUrl } from '../asset_manager';
import { formatFileSize } from '../utils/format';
import { DownloadIcon, TextScanIcon, AnalyzeIcon, CloseIcon } from '../ui/icons';
import { openOcrTextInNewTab } from '../utils/ocr';
const PreviewWorkspace = ({
document,
@@ -106,21 +106,15 @@ export const createPreviewWorkspaceHeaderActions = ({
const hasOcr = Boolean(ocrAsset);
const handleOcrClick = async () => {
if (!ocrAsset) {
return;
}
try {
const ensured = await ensureAssetUrl(document.id, ocrAsset, { force: false });
const entry = ensured || ocrAsset;
const url = entry?.url
|| resolveDocumentAssetUrl(document, 'ocr-text', {
ensureAssetUrl,
getAsset: getDocumentAsset,
});
if (!url) {
const success = await openOcrTextInNewTab({
document,
getDocumentAsset,
ensureAssetUrl,
});
if (!success) {
throw new Error('OCR text URL unavailable.');
}
window.open(url, '_blank', 'noopener,noreferrer');
} catch (error) {
notifyApiError(error, 'Unable to open OCR text.');
}
frontend/src/styles.css
+4 -48
View File
@@ -2120,54 +2120,6 @@ button.danger:hover:not([disabled]) {
}
.detail-ocr__content {
max-height: 60vh;
overflow: auto;
padding: 0.75rem;
background: var(--surface-soft);
border-radius: 6px;
box-shadow: inset 0 1px 2px var(--shadow-faint);
}
.detail-ocr__content pre {
margin: 0;
white-space: pre-wrap;
word-break: break-word;
font-family: var(--font-mono);
font-size: 0.85rem;
line-height: 1.4;
}
.detail-ocr__frame {
width: 100%;
min-height: 360px;
height: 60vh;
background: var(--surface-subtle);
border-radius: 4px;
box-shadow: inset 0 1px 2px var(--shadow-faint);
}
.modal__body {
width: 100%;
}
.modal--ocr {
width: min(720px, 100%);
max-height: 80vh;
gap: 0.75rem;
}
.modal--ocr .modal__header {
display: flex;
align-items: center;
justify-content: space-between;
gap: 0.75rem;
}
.modal--ocr .modal__body {
padding: 0;
}
.preview-pane {
margin-top: 0.4rem;
border-radius: 0;
@@ -2743,6 +2695,10 @@ form.inline {
padding: 0;
}
.modal__body {
width: 100%;
}
.modal h3 {
margin: 0;
font-size: 1.15rem;