nils/papercrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

backend: split document assets into assets and objects

Browse Source
This commit is contained in:
nils
2025-10-19 19:46:16 +02:00
parent 0689e680fa
commit ab06205744
9 changed files with 397 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/workers/index.rs
+13 -8
View File
@@ -12,8 +12,8 @@ use uuid::Uuid;
use crate::{
jobs::JOB_INDEX_DOCUMENT_TEXT,
models::{Document, DocumentAsset, DocumentVersion},
schema::{document_assets, document_versions, documents},
models::{Document, DocumentVersion},
schema::{document_asset_objects, document_assets, document_versions, documents},
state::AppState,
};
@@ -87,15 +87,15 @@ impl JobHandler for IndexDocumentTextJob {
}
};
if context.text_asset.is_none() {
if context.text_s3_key.is_none() {
warn!(job_id = %job.id, "missing OCR text asset; failing indexing job");
return JobExecution::Failed {
error: "missing OCR text asset".into(),
};
}
let asset = context.text_asset.unwrap();
let text = match state.storage.get_object(&asset.s3_key).await {
let s3_key = context.text_s3_key.unwrap();
let text = match state.storage.get_object(&s3_key).await {
Ok(bytes) => match String::from_utf8(bytes) {
Ok(text) => text,
Err(err) => {
@@ -169,7 +169,7 @@ impl JobHandler for IndexDocumentTextJob {
struct IndexContext {
document: Document,
version: DocumentVersion,
text_asset: Option<DocumentAsset>,
text_s3_key: Option<String>,
}
fn load_context(state: Arc<AppState>, payload: &IndexPayload) -> Result<IndexContext, String> {
@@ -189,9 +189,14 @@ fn load_context(state: Arc<AppState>, payload: &IndexPayload) -> Result<IndexCon
.first(&mut conn)
.map_err(|err| format!("{err:?}"))?;
let text_asset: Option<DocumentAsset> = document_assets::table
let text_s3_key: Option<String> = document_asset_objects::table
.inner_join(
document_assets::table.on(document_asset_objects::asset_id.eq(document_assets::id)),
)
.filter(document_assets::document_version_id.eq(payload.document_version_id))
.filter(document_assets::asset_type.eq(OCR_TEXT_ASSET_TYPE))
.filter(document_asset_objects::ordinal.eq(1))
.select(document_asset_objects::s3_key)
.first(&mut conn)
.optional()
.map_err(|err| format!("{err:?}"))?;
@@ -199,6 +204,6 @@ fn load_context(state: Arc<AppState>, payload: &IndexPayload) -> Result<IndexCon
Ok(IndexContext {
document,
version,
text_asset,
text_s3_key,
})
}
backend/src/workers/ocr.rs
+71 -13
View File
@@ -19,8 +19,11 @@ use uuid::Uuid;
use crate::{
jobs::{enqueue_job, JOB_GENERATE_OCR_TEXT, JOB_INDEX_DOCUMENT_TEXT},
models::{Document, DocumentAsset, DocumentVersion, NewDocumentAsset},
schema::{document_assets, document_versions, documents},
models::{
Document, DocumentAsset, DocumentAssetObject, DocumentVersion, NewDocumentAsset,
NewDocumentAssetObject,
},
schema::{document_asset_objects, document_assets, document_versions, documents},
state::AppState,
};
@@ -123,11 +126,15 @@ impl JobHandler for GenerateOcrTextJob {
};
};
let asset_id = context
.existing_asset
.as_ref()
.map(|asset| asset.id)
.unwrap_or_else(Uuid::new_v4);
if context.existing_asset.is_some() {
for object in &context.existing_objects {
if let Err(err) = state.storage.delete_object(&object.s3_key).await {
warn!(job_id = %job.id, error = %err, s3_key = %object.s3_key, "failed to delete existing ocr asset object");
}
}
}
let asset_id = Uuid::new_v4();
let s3_key = format!(
"documents/{}/v{}/assets/{}/{}",
@@ -193,6 +200,7 @@ struct OcrContext {
document: Document,
version: DocumentVersion,
existing_asset: Option<DocumentAsset>,
existing_objects: Vec<DocumentAssetObject>,
skip: bool,
}
@@ -218,29 +226,41 @@ fn load_ocr_context(state: Arc<AppState>, payload: &OcrPayload) -> Result<OcrCon
.first(&mut conn)
.map_err(|err| format!("{err:?}"))?;
let existing: Option<DocumentAsset> = document_assets::table
let existing_asset: Option<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq(payload.document_version_id))
.filter(document_assets::asset_type.eq(OCR_TEXT_ASSET_TYPE))
.first(&mut conn)
.optional()
.map_err(|err| format!("{err:?}"))?;
let existing_objects: Vec<DocumentAssetObject> = if let Some(asset) = &existing_asset {
document_asset_objects::table
.filter(document_asset_objects::asset_id.eq(asset.id))
.order(document_asset_objects::ordinal.asc())
.load(&mut conn)
.map_err(|err| format!("{err:?}"))?
} else {
Vec::new()
};
let is_pdf = document_is_pdf(&document);
if !is_pdf {
return Ok(OcrContext {
document,
version,
existing_asset: existing,
existing_asset: existing_asset,
existing_objects,
skip: true,
});
}
let skip = existing.is_some() && !payload.force;
let skip = existing_asset.is_some() && !payload.force;
Ok(OcrContext {
document,
version,
existing_asset: existing,
existing_asset,
existing_objects,
skip,
})
}
@@ -371,16 +391,22 @@ fn persist_ocr_metadata(
) -> Result<(), String> {
let mut conn = state.db().map_err(|err| format!("{err:?}"))?;
if let Some(existing_asset) = &context.existing_asset {
diesel::delete(document_assets::table.filter(document_assets::id.eq(existing_asset.id)))
.execute(&mut conn)
.map_err(|err| format!("{err:?}"))?;
}
let new_asset = NewDocumentAsset {
id: asset_id,
document_version_id: context.version.id,
asset_type: OCR_TEXT_ASSET_TYPE.to_string(),
s3_key: s3_key.to_string(),
mime_type: "text/plain".to_string(),
metadata: json!({
"generated_at": Utc::now().to_rfc3339(),
"source": source,
}),
cardinality: Some(1),
};
diesel::insert_into(document_assets::table)
@@ -391,9 +417,41 @@ fn persist_ocr_metadata(
))
.do_update()
.set((
document_assets::s3_key.eq(excluded(document_assets::s3_key)),
document_assets::mime_type.eq(excluded(document_assets::mime_type)),
document_assets::metadata.eq(excluded(document_assets::metadata)),
document_assets::cardinality.eq(excluded(document_assets::cardinality)),
))
.execute(&mut conn)
.map_err(|err| format!("{err:?}"))?;
let existing_object_id: Option<Uuid> = document_asset_objects::table
.filter(document_asset_objects::asset_id.eq(asset_id))
.filter(document_asset_objects::ordinal.eq(1))
.select(document_asset_objects::id)
.first(&mut conn)
.optional()
.map_err(|err| format!("{err:?}"))?;
let object_id = existing_object_id.unwrap_or_else(Uuid::new_v4);
let new_object = NewDocumentAssetObject {
id: object_id,
asset_id,
ordinal: 1,
s3_key: s3_key.to_string(),
metadata: json!({}),
};
diesel::insert_into(document_asset_objects::table)
.values(&new_object)
.on_conflict((
document_asset_objects::asset_id,
document_asset_objects::ordinal,
))
.do_update()
.set((
document_asset_objects::s3_key.eq(excluded(document_asset_objects::s3_key)),
document_asset_objects::metadata.eq(excluded(document_asset_objects::metadata)),
))
.execute(&mut conn)
.map_err(|err| format!("{err:?}"))?;
backend/src/workers/thumbnails.rs
+113 -18
View File
@@ -13,8 +13,11 @@ use uuid::Uuid;
use crate::{
jobs::JOB_GENERATE_THUMBNAILS,
models::{Document, DocumentAsset, DocumentVersion, NewDocumentAsset},
schema::{document_assets, document_versions, documents},
models::{
Document, DocumentAsset, DocumentAssetObject, DocumentVersion, NewDocumentAsset,
NewDocumentAssetObject,
},
schema::{document_asset_objects, document_assets, document_versions, documents},
state::AppState,
};
@@ -142,11 +145,33 @@ impl JobHandler for GenerateThumbnailsJob {
}
}
let thumbnail_asset_id = initial
.existing_thumbnail
.as_ref()
.map(|asset| asset.id)
.unwrap_or_else(Uuid::new_v4);
if initial.existing_preview.is_some() {
for object in &initial.existing_preview_objects {
if let Err(err) = state.storage.delete_object(&object.s3_key).await {
warn!(
job_id = %job.id,
error = %err,
s3_key = %object.s3_key,
"failed to delete existing preview object"
);
}
}
}
if initial.existing_thumbnail.is_some() {
for object in &initial.existing_thumbnail_objects {
if let Err(err) = state.storage.delete_object(&object.s3_key).await {
warn!(
job_id = %job.id,
error = %err,
s3_key = %object.s3_key,
"failed to delete existing thumbnail object"
);
}
}
}
let thumbnail_asset_id = Uuid::new_v4();
let thumbnail_s3_key = format!(
"documents/{}/v{}/assets/{}/{}",
initial.document.id,
@@ -155,11 +180,7 @@ impl JobHandler for GenerateThumbnailsJob {
thumbnail_asset_id
);
let preview_asset_id = initial
.existing_preview
.as_ref()
.map(|asset| asset.id)
.unwrap_or_else(Uuid::new_v4);
let preview_asset_id = Uuid::new_v4();
let preview_s3_key = format!(
"documents/{}/v{}/assets/{}/{}",
initial.document.id,
@@ -250,7 +271,9 @@ struct ThumbnailContext {
document: Document,
version: DocumentVersion,
existing_thumbnail: Option<DocumentAsset>,
existing_thumbnail_objects: Vec<DocumentAssetObject>,
existing_preview: Option<DocumentAsset>,
existing_preview_objects: Vec<DocumentAssetObject>,
skip: bool,
}
@@ -303,11 +326,27 @@ fn load_thumbnail_context(
.map_err(|err| format!("{err:?}"))?;
let mut existing_thumbnail = None;
let mut existing_thumbnail_objects: Vec<DocumentAssetObject> = Vec::new();
let mut existing_preview = None;
let mut existing_preview_objects: Vec<DocumentAssetObject> = Vec::new();
for asset in existing_assets {
match asset.asset_type.as_str() {
THUMBNAIL_ASSET_TYPE => existing_thumbnail = Some(asset),
PREVIEW_ASSET_TYPE => existing_preview = Some(asset),
THUMBNAIL_ASSET_TYPE => {
existing_thumbnail_objects = document_asset_objects::table
.filter(document_asset_objects::asset_id.eq(asset.id))
.order(document_asset_objects::ordinal.asc())
.load(&mut conn)
.map_err(|err| format!("{err:?}"))?;
existing_thumbnail = Some(asset);
}
PREVIEW_ASSET_TYPE => {
existing_preview_objects = document_asset_objects::table
.filter(document_asset_objects::asset_id.eq(asset.id))
.order(document_asset_objects::ordinal.asc())
.load(&mut conn)
.map_err(|err| format!("{err:?}"))?;
existing_preview = Some(asset);
}
_ => {}
}
}
@@ -323,7 +362,9 @@ fn load_thumbnail_context(
document,
version,
existing_thumbnail,
existing_thumbnail_objects,
existing_preview,
existing_preview_objects,
skip,
})
}
@@ -462,18 +503,30 @@ fn persist_assets_metadata(
) -> Result<(), String> {
let mut conn = state.db().map_err(|err| format!("{err:?}"))?;
if let Some(existing_preview) = &context.existing_preview {
diesel::delete(document_assets::table.filter(document_assets::id.eq(existing_preview.id)))
.execute(&mut conn)
.map_err(|err| format!("{err:?}"))?;
}
if let Some(existing_thumbnail) = &context.existing_thumbnail {
diesel::delete(
document_assets::table.filter(document_assets::id.eq(existing_thumbnail.id)),
)
.execute(&mut conn)
.map_err(|err| format!("{err:?}"))?;
}
for asset in assets {
let new_asset = NewDocumentAsset {
id: asset.asset_id,
document_version_id: context.version.id,
asset_type: asset.asset_type.to_string(),
s3_key: asset.s3_key.to_string(),
mime_type: "image/png".to_string(),
metadata: json!({
"generated_at": Utc::now().to_rfc3339(),
"width": asset.generated.width,
"height": asset.generated.height,
}),
cardinality: Some(1),
};
diesel::insert_into(document_assets::table)
@@ -484,9 +537,51 @@ fn persist_assets_metadata(
))
.do_update()
.set((
document_assets::s3_key.eq(excluded(document_assets::s3_key)),
document_assets::mime_type.eq(excluded(document_assets::mime_type)),
document_assets::metadata.eq(excluded(document_assets::metadata)),
document_assets::cardinality.eq(excluded(document_assets::cardinality)),
))
.execute(&mut conn)
.map_err(|err| format!("{err:?}"))?;
let existing_object_id: Option<Uuid> = document_asset_objects::table
.filter(document_asset_objects::asset_id.eq(asset.asset_id))
.filter(document_asset_objects::ordinal.eq(1))
.select(document_asset_objects::id)
.first(&mut conn)
.optional()
.map_err(|err| format!("{err:?}"))?;
let object_id = existing_object_id.unwrap_or_else(Uuid::new_v4);
let mut metadata_map = Map::new();
if let Some(width) = asset.generated.width {
metadata_map.insert("width".to_string(), Value::from(width));
}
if let Some(height) = asset.generated.height {
metadata_map.insert("height".to_string(), Value::from(height));
}
let object_metadata = Value::Object(metadata_map);
let new_object = NewDocumentAssetObject {
id: object_id,
asset_id: asset.asset_id,
ordinal: 1,
s3_key: asset.s3_key.to_string(),
metadata: object_metadata,
};
diesel::insert_into(document_asset_objects::table)
.values(&new_object)
.on_conflict((
document_asset_objects::asset_id,
document_asset_objects::ordinal,
))
.do_update()
.set((
document_asset_objects::s3_key.eq(excluded(document_asset_objects::s3_key)),
document_asset_objects::metadata.eq(excluded(document_asset_objects::metadata)),
))
.execute(&mut conn)
.map_err(|err| format!("{err:?}"))?;