nils/papercrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cleanup

Browse Source
This commit is contained in:
nils
2025-10-31 01:52:51 +01:00
parent 722c2f9a5c
commit 17db827e3a
11 changed files with 220 additions and 334 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/workers/mod.rs
+1
View File
@@ -13,6 +13,7 @@ use crate::{
};
pub mod analyze;
pub mod common;
pub mod index;
pub mod ocr;
pub mod tenants;
backend/src/workers/ocr.rs
+27 -44
View File
@@ -24,13 +24,16 @@ use crate::{
Document, DocumentAsset, DocumentAssetObject, DocumentVersion, NewDocumentAsset,
NewDocumentAssetObject,
},
schema::{document_asset_objects, document_assets, document_versions, documents},
schema::{document_asset_objects, document_assets, documents},
state::AppState,
storage::TenantStorage,
utils::storage_paths::document_asset_object_prefix,
};
use super::{fetch_version_object, handle_fetch_error, JobExecution, JobHandler};
use super::{
common::{load_document_version, load_version_assets},
fetch_version_object, handle_fetch_error, JobExecution, JobHandler,
};
pub const OCR_TEXT_ASSET_TYPE: &str = "ocr-text";
const MIN_TEXT_LENGTH: usize = 50;
@@ -234,54 +237,34 @@ struct OcrGeneration {
}
fn load_ocr_context(state: Arc<AppState>, payload: &OcrPayload) -> Result<OcrContext, String> {
let mut base_conn = state.db_unscoped().map_err(|err| format!("{err:?}"))?;
let version: DocumentVersion = document_versions::table
.find(payload.document_version_id)
.first(&mut base_conn)
.map_err(|err| format!("{err:?}"))?;
if version.document_id != payload.document_id {
return Err("document/version mismatch".into());
}
let document: Document = documents::table
.find(payload.document_id)
.first(&mut base_conn)
.map_err(|err| format!("{err:?}"))?;
let tenant_id = document.tenant_id;
drop(base_conn);
let base = load_document_version(
state.as_ref(),
payload.document_id,
payload.document_version_id,
)?;
let mut conn = state
.db_for_tenant(tenant_id)
.db_for_tenant(base.tenant_id)
.map_err(|err| format!("{err:?}"))?;
let existing_asset: Option<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq(payload.document_version_id))
.filter(document_assets::asset_type.eq(OCR_TEXT_ASSET_TYPE))
.filter(document_assets::tenant_id.eq(tenant_id))
.first(&mut conn)
.optional()
.map_err(|err| format!("{err:?}"))?;
let mut assets = load_version_assets(
&mut conn,
base.tenant_id,
base.version.id,
&[OCR_TEXT_ASSET_TYPE],
)?;
let existing_objects: Vec<DocumentAssetObject> = if let Some(asset) = &existing_asset {
document_asset_objects::table
.filter(document_asset_objects::asset_id.eq(asset.id))
.filter(document_asset_objects::tenant_id.eq(tenant_id))
.order(document_asset_objects::ordinal.asc())
.load(&mut conn)
.map_err(|err| format!("{err:?}"))?
} else {
Vec::new()
};
let (existing_asset, existing_objects) = assets
.remove(OCR_TEXT_ASSET_TYPE)
.map(|entry| (Some(entry.asset), entry.objects))
.unwrap_or((None, Vec::new()));
let is_pdf = document_is_pdf(&document);
let is_pdf = document_is_pdf(&base.document);
if !is_pdf {
return Ok(OcrContext {
document,
version,
existing_asset: existing_asset,
document: base.document,
version: base.version,
existing_asset,
existing_objects,
skip: true,
});
@@ -290,8 +273,8 @@ fn load_ocr_context(state: Arc<AppState>, payload: &OcrPayload) -> Result<OcrCon
let skip = existing_asset.is_some() && !payload.force;
Ok(OcrContext {
document,
version,
document: base.document,
version: base.version,
existing_asset,
existing_objects,
skip,
backend/src/workers/thumbnails.rs
+28 -56
View File
@@ -25,8 +25,9 @@ use crate::{
};
use super::{
analyze::determine_thumbnail_support, fetch_version_object, handle_fetch_error, JobExecution,
JobHandler,
analyze::determine_thumbnail_support,
common::{load_document_version, load_version_assets},
fetch_version_object, handle_fetch_error, JobExecution, JobHandler,
};
const THUMBNAIL_WIDTH: u32 = 512;
@@ -406,68 +407,39 @@ fn load_thumbnail_context(
state: Arc<AppState>,
payload: &ThumbnailPayload,
) -> Result<ThumbnailContext, String> {
let mut conn = state.db_unscoped().map_err(|err| format!("{err:?}"))?;
let base = load_document_version(
state.as_ref(),
payload.document_id,
payload.document_version_id,
)?;
let version: DocumentVersion = document_versions::table
.find(payload.document_version_id)
.first(&mut conn)
let mut conn = state
.db_for_tenant(base.tenant_id)
.map_err(|err| format!("{err:?}"))?;
if version.document_id != payload.document_id {
return Err("document/version mismatch".into());
}
let mut assets = load_version_assets(
&mut conn,
base.tenant_id,
base.version.id,
&[THUMBNAIL_ASSET_TYPE, PREVIEW_ASSET_TYPE],
)?;
let document: Document = documents::table
.find(payload.document_id)
.first(&mut conn)
.map_err(|err| format!("{err:?}"))?;
let (existing_thumbnail, existing_thumbnail_objects) = assets
.remove(THUMBNAIL_ASSET_TYPE)
.map(|entry| (Some(entry.asset), entry.objects))
.unwrap_or((None, Vec::new()));
let tenant_id = document.tenant_id;
let (existing_preview, existing_preview_objects) = assets
.remove(PREVIEW_ASSET_TYPE)
.map(|entry| (Some(entry.asset), entry.objects))
.unwrap_or((None, Vec::new()));
let existing_assets: Vec<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq(payload.document_version_id))
.filter(document_assets::asset_type.eq_any(vec![
THUMBNAIL_ASSET_TYPE.to_string(),
PREVIEW_ASSET_TYPE.to_string(),
]))
.filter(document_assets::tenant_id.eq(tenant_id))
.load(&mut conn)
.map_err(|err| format!("{err:?}"))?;
let mut existing_thumbnail = None;
let mut existing_thumbnail_objects: Vec<DocumentAssetObject> = Vec::new();
let mut existing_preview = None;
let mut existing_preview_objects: Vec<DocumentAssetObject> = Vec::new();
for asset in existing_assets {
match asset.asset_type.as_str() {
THUMBNAIL_ASSET_TYPE => {
existing_thumbnail_objects = document_asset_objects::table
.filter(document_asset_objects::asset_id.eq(asset.id))
.filter(document_asset_objects::tenant_id.eq(tenant_id))
.order(document_asset_objects::ordinal.asc())
.load(&mut conn)
.map_err(|err| format!("{err:?}"))?;
existing_thumbnail = Some(asset);
}
PREVIEW_ASSET_TYPE => {
existing_preview_objects = document_asset_objects::table
.filter(document_asset_objects::asset_id.eq(asset.id))
.filter(document_asset_objects::tenant_id.eq(tenant_id))
.order(document_asset_objects::ordinal.asc())
.load(&mut conn)
.map_err(|err| format!("{err:?}"))?;
existing_preview = Some(asset);
}
_ => {}
}
}
let (supported, _) = determine_thumbnail_support(&document);
let (supported, _) = determine_thumbnail_support(&base.document);
if !supported {
return Err("thumbnail generation not supported for this document".into());
}
let expected_cardinality = expected_asset_cardinality(&document, &version);
let expected_cardinality = expected_asset_cardinality(&base.document, &base.version);
let preview_cardinality = existing_preview
.as_ref()
.and_then(|asset| asset.cardinality)
@@ -488,8 +460,8 @@ fn load_thumbnail_context(
&& !needs_regeneration;
Ok(ThumbnailContext {
document,
version,
document: base.document,
version: base.version,
existing_thumbnail,
existing_thumbnail_objects,
existing_preview,