nils/papercrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

backend: generate high resolution preview

Browse Source
This commit is contained in:
nils
2025-10-13 02:39:02 +02:00
parent 86c75cea8e
commit 159577dff9
5 changed files with 552 additions and 224 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/routes/documents.rs
+91 -22
View File
@@ -93,6 +93,7 @@ pub struct DocumentResponse {
pub metadata: Value,
pub tags: Vec<TagResponse>,
pub thumbnail: Option<DocumentAssetResponse>,
pub preview: Option<DocumentAssetResponse>,
pub download_path: String,
}
@@ -118,6 +119,12 @@ pub struct DocumentAssetResponse {
pub created_at: String,
}
#[derive(Clone, Default)]
pub struct PrimaryDocumentAssets {
pub thumbnail: Option<DocumentAssetResponse>,
pub preview: Option<DocumentAssetResponse>,
}
#[derive(Serialize)]
pub struct DocumentDetailResponse {
pub document: DocumentResponse,
@@ -239,18 +246,23 @@ pub async fn list_documents(
let tags_map = load_tags_for_documents(&mut conn, &doc_ids)?;
drop(conn);
let thumbnails = load_primary_thumbnails(&state, &docs).await?;
let primary_assets = load_primary_assets(&state, &docs).await?;
let mut response = Vec::with_capacity(doc_ids.len());
for doc in docs {
let tags = tags_map.get(&doc.id).cloned();
let thumbnail = thumbnails.get(&doc.id).cloned();
let (thumbnail, preview) = if let Some(assets) = primary_assets.get(&doc.id) {
(assets.thumbnail.clone(), assets.preview.clone())
} else {
(None, None)
};
response.push(to_document_response(
&state,
user.user_id,
doc,
tags,
thumbnail,
preview,
)?);
}
@@ -283,6 +295,10 @@ pub async fn get_document(
.iter()
.find(|asset| asset.asset_type == "thumbnail")
.cloned();
let preview = assets
.iter()
.find(|asset| asset.asset_type == "preview")
.cloned();
Ok(Json(DocumentDetailResponse {
document: to_document_response(
@@ -291,6 +307,7 @@ pub async fn get_document(
doc,
tags_map.get(&document_id).cloned(),
thumbnail,
preview,
)?,
current_version: to_version_response(current_version),
assets,
@@ -682,6 +699,10 @@ pub async fn update_document(
.iter()
.find(|asset| asset.asset_type == "thumbnail")
.cloned();
let preview = assets
.iter()
.find(|asset| asset.asset_type == "preview")
.cloned();
Ok(Json(DocumentDetailResponse {
document: to_document_response(
@@ -690,6 +711,7 @@ pub async fn update_document(
document,
tags_map.get(&document_id).cloned(),
thumbnail,
preview,
)?,
current_version: to_version_response(current_version),
assets,
@@ -973,6 +995,10 @@ async fn process_upload(
.iter()
.find(|asset| asset.asset_type == "thumbnail")
.cloned();
let preview = assets
.iter()
.find(|asset| asset.asset_type == "preview")
.cloned();
info!(
document_id = %document.id,
@@ -982,7 +1008,9 @@ async fn process_upload(
return Ok(UploadOutcome {
detail: DocumentDetailResponse {
document: to_document_response(state, user_id, document, tags, thumbnail)?,
document: to_document_response(
state, user_id, document, tags, thumbnail, preview,
)?,
current_version: to_version_response(version),
assets,
},
@@ -1053,7 +1081,7 @@ async fn process_upload(
};
let detail = DocumentDetailResponse {
document: to_document_response(state, user_id, document, None, None)?,
document: to_document_response(state, user_id, document, None, None, None)?,
current_version: to_version_response(version.clone()),
assets: Vec::new(),
};
@@ -1112,14 +1140,20 @@ pub(crate) fn load_tags_for_documents(
Ok(map)
}
pub(crate) async fn load_primary_thumbnails(
pub(crate) async fn load_primary_assets(
state: &AppState,
documents: &[Document],
) -> AppResult<HashMap<Uuid, DocumentAssetResponse>> {
) -> AppResult<HashMap<Uuid, PrimaryDocumentAssets>> {
if documents.is_empty() {
return Ok(HashMap::new());
}
#[derive(Default)]
struct RawPrimaryAssets {
thumbnail: Option<DocumentAsset>,
preview: Option<DocumentAsset>,
}
let mut current_versions: HashMap<Uuid, i32> = HashMap::new();
let mut doc_ids = Vec::with_capacity(documents.len());
for doc in documents {
@@ -1151,33 +1185,66 @@ pub(crate) async fn load_primary_thumbnails(
let assets: Vec<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq_any(&version_ids))
.filter(document_assets::asset_type.eq("thumbnail"))
.filter(
document_assets::asset_type
.eq_any(vec!["thumbnail".to_string(), "preview".to_string()]),
)
.order((
document_assets::document_version_id.asc(),
document_assets::created_at.asc(),
))
.load(&mut conn)?;
drop(conn);
let mut first_assets: HashMap<Uuid, DocumentAsset> = HashMap::new();
let mut raw_assets: HashMap<Uuid, RawPrimaryAssets> = HashMap::new();
for asset in assets {
if let Some(doc_id) = doc_id_by_version.get(&asset.document_version_id) {
first_assets.entry(*doc_id).or_insert(asset);
let entry = raw_assets.entry(*doc_id).or_default();
let asset_type = asset.asset_type.clone();
match asset_type.as_str() {
"thumbnail" => {
if entry.thumbnail.is_none() {
entry.thumbnail = Some(asset);
}
}
"preview" => {
if entry.preview.is_none() {
entry.preview = Some(asset);
}
}
_ => {}
}
}
}
let mut responses = HashMap::with_capacity(first_assets.len());
for (doc_id, asset) in first_assets {
let url = state
.storage
.presign_get_object(
&asset.s3_key,
Duration::from_secs(PRESIGNED_URL_EXPIRY_SECONDS),
)
.await
.map_err(|err| AppError::internal(format!("failed to sign asset URL: {err}")))?;
responses.insert(doc_id, to_asset_response(asset, url));
drop(conn);
let mut responses = HashMap::with_capacity(raw_assets.len());
for (doc_id, raw) in raw_assets {
let mut assets = PrimaryDocumentAssets::default();
if let Some(asset) = raw.preview {
let s3_key = asset.s3_key.clone();
let url = state
.storage
.presign_get_object(&s3_key, Duration::from_secs(PRESIGNED_URL_EXPIRY_SECONDS))
.await
.map_err(|err| AppError::internal(format!("failed to sign asset URL: {err}")))?;
assets.preview = Some(to_asset_response(asset, url));
}
if let Some(asset) = raw.thumbnail {
let s3_key = asset.s3_key.clone();
let url = state
.storage
.presign_get_object(&s3_key, Duration::from_secs(PRESIGNED_URL_EXPIRY_SECONDS))
.await
.map_err(|err| AppError::internal(format!("failed to sign asset URL: {err}")))?;
assets.thumbnail = Some(to_asset_response(asset, url));
}
if assets.thumbnail.is_some() || assets.preview.is_some() {
responses.insert(doc_id, assets);
}
}
Ok(responses)
@@ -1189,6 +1256,7 @@ pub(crate) fn to_document_response(
doc: Document,
tags: Option<Vec<Tag>>,
thumbnail: Option<DocumentAssetResponse>,
preview: Option<DocumentAssetResponse>,
) -> AppResult<DocumentResponse> {
let download_path = build_download_path(state, doc.id, user_id)?;
@@ -1211,6 +1279,7 @@ pub(crate) fn to_document_response(
.map(TagResponse::from)
.collect(),
thumbnail,
preview,
download_path,
})
}
backend/src/routes/folders.rs
+15 -6
View File
@@ -19,8 +19,7 @@ use crate::{
};
use super::documents::{
load_primary_thumbnails, load_tags_for_documents, to_document_response, to_iso,
DocumentResponse,
load_primary_assets, load_tags_for_documents, to_document_response, to_iso, DocumentResponse,
};
const QUICKWIT_MAX_HITS: usize = 200;
@@ -216,18 +215,23 @@ pub async fn list_folder_contents(
let tags_map = load_tags_for_documents(&mut conn, &doc_ids)?;
drop(conn);
let thumbnails = load_primary_thumbnails(&state, &docs).await?;
let primary_assets = load_primary_assets(&state, &docs).await?;
let mut documents = Vec::with_capacity(doc_ids.len());
for doc in docs {
let tags = tags_map.get(&doc.id).cloned();
let thumbnail = thumbnails.get(&doc.id).cloned();
let (thumbnail, preview) = if let Some(assets) = primary_assets.get(&doc.id) {
(assets.thumbnail.clone(), assets.preview.clone())
} else {
(None, None)
};
documents.push(to_document_response(
&state,
user.user_id,
doc,
tags,
thumbnail,
preview,
)?);
}
@@ -400,17 +404,22 @@ pub async fn search_documents(
let tags_map = load_tags_for_documents(&mut conn, &doc_ids)?;
drop(conn);
let thumbnails = load_primary_thumbnails(&state, &docs).await?;
let primary_assets = load_primary_assets(&state, &docs).await?;
let mut response = Vec::with_capacity(doc_ids.len());
for doc in docs {
let tags = tags_map.get(&doc.id).cloned();
let thumbnail = thumbnails.get(&doc.id).cloned();
let (thumbnail, preview) = if let Some(assets) = primary_assets.get(&doc.id) {
(assets.thumbnail.clone(), assets.preview.clone())
} else {
(None, None)
};
response.push(to_document_response(
&state,
user.user_id,
doc,
tags,
thumbnail,
preview,
)?);
}
backend/src/workers/thumbnails.rs
+184 -82
View File
@@ -3,9 +3,7 @@ use std::{io::Cursor, panic, sync::Arc, time::Duration};
use async_trait::async_trait;
use chrono::Utc;
use diesel::{pg::upsert::excluded, prelude::*};
use image::{
codecs::png::PngEncoder, ColorType, GenericImageView, ImageEncoder, ImageFormat, ImageReader,
};
use image::{GenericImageView, ImageFormat, ImageReader};
use pdfium_render::prelude::*;
use serde::Deserialize;
use serde_json::json;
@@ -24,7 +22,10 @@ use super::{analyze::determine_thumbnail_support, JobExecution, JobHandler};
const THUMBNAIL_WIDTH: u32 = 512;
const THUMBNAIL_HEIGHT: u32 = 512;
const PREVIEW_WIDTH: u32 = THUMBNAIL_WIDTH * 4;
const PREVIEW_HEIGHT: u32 = THUMBNAIL_HEIGHT * 4;
const THUMBNAIL_ASSET_TYPE: &str = "thumbnail";
const PREVIEW_ASSET_TYPE: &str = "preview";
#[derive(Debug, Deserialize)]
struct ThumbnailPayload {
@@ -95,28 +96,61 @@ impl JobHandler for GenerateThumbnailsJob {
}
};
let generation = match generate_thumbnail(&initial.document, &bytes) {
let generation = match generate_preview_and_thumbnail(&initial.document, &bytes) {
Ok(result) => result,
Err(err) => {
return JobExecution::Failed { error: err };
}
};
let asset_id = initial
.existing_asset
let thumbnail_asset_id = initial
.existing_thumbnail
.as_ref()
.map(|asset| asset.id)
.unwrap_or_else(Uuid::new_v4);
let s3_key = format!(
let thumbnail_s3_key = format!(
"documents/{}/v{}/assets/{}/{}",
initial.document.id, initial.version.version_number, THUMBNAIL_ASSET_TYPE, asset_id
initial.document.id,
initial.version.version_number,
THUMBNAIL_ASSET_TYPE,
thumbnail_asset_id
);
let preview_asset_id = initial
.existing_preview
.as_ref()
.map(|asset| asset.id)
.unwrap_or_else(Uuid::new_v4);
let preview_s3_key = format!(
"documents/{}/v{}/assets/{}/{}",
initial.document.id,
initial.version.version_number,
PREVIEW_ASSET_TYPE,
preview_asset_id
);
if let Err(err) = state
.storage
.put_object(
&s3_key,
generation.image_bytes.clone(),
&preview_s3_key,
generation.preview.image_bytes.clone(),
Some("image/png".into()),
None,
)
.await
{
warn!(job_id = %job.id, error = %err, "failed to upload preview; retrying");
return JobExecution::Retry {
delay: Duration::from_secs(30),
error: err.to_string(),
};
}
if let Err(err) = state
.storage
.put_object(
&thumbnail_s3_key,
generation.thumbnail.image_bytes.clone(),
Some("image/png".into()),
None,
)
@@ -131,7 +165,24 @@ impl JobHandler for GenerateThumbnailsJob {
let state_clone = state.clone();
match task::spawn_blocking(move || {
persist_thumbnail_metadata(state_clone, &initial, &generation, asset_id, &s3_key)
persist_assets_metadata(
state_clone,
&initial,
&[
AssetPersistence {
asset_type: PREVIEW_ASSET_TYPE,
asset_id: preview_asset_id,
s3_key: &preview_s3_key,
generated: &generation.preview,
},
AssetPersistence {
asset_type: THUMBNAIL_ASSET_TYPE,
asset_id: thumbnail_asset_id,
s3_key: &thumbnail_s3_key,
generated: &generation.thumbnail,
},
],
)
})
.await
{
@@ -159,16 +210,29 @@ impl JobHandler for GenerateThumbnailsJob {
struct ThumbnailContext {
document: Document,
version: DocumentVersion,
existing_asset: Option<DocumentAsset>,
existing_thumbnail: Option<DocumentAsset>,
existing_preview: Option<DocumentAsset>,
skip: bool,
}
struct GeneratedThumbnail {
struct GeneratedImage {
image_bytes: Vec<u8>,
width: Option<i32>,
height: Option<i32>,
}
struct GeneratedAssets {
thumbnail: GeneratedImage,
preview: GeneratedImage,
}
struct AssetPersistence<'a> {
asset_type: &'static str,
asset_id: Uuid,
s3_key: &'a str,
generated: &'a GeneratedImage,
}
fn load_thumbnail_context(
state: Arc<AppState>,
payload: &ThumbnailPayload,
@@ -189,29 +253,45 @@ fn load_thumbnail_context(
.first(&mut conn)
.map_err(|err| format!("{err:?}"))?;
let existing: Option<DocumentAsset> = document_assets::table
let existing_assets: Vec<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq(payload.document_version_id))
.filter(document_assets::asset_type.eq(THUMBNAIL_ASSET_TYPE))
.first(&mut conn)
.optional()
.filter(document_assets::asset_type.eq_any(vec![
THUMBNAIL_ASSET_TYPE.to_string(),
PREVIEW_ASSET_TYPE.to_string(),
]))
.load(&mut conn)
.map_err(|err| format!("{err:?}"))?;
let mut existing_thumbnail = None;
let mut existing_preview = None;
for asset in existing_assets {
match asset.asset_type.as_str() {
THUMBNAIL_ASSET_TYPE => existing_thumbnail = Some(asset),
PREVIEW_ASSET_TYPE => existing_preview = Some(asset),
_ => {}
}
}
let (supported, _) = determine_thumbnail_support(&document);
if !supported {
return Err("thumbnail generation not supported for this document".into());
}
let skip = existing.is_some() && !payload.force;
let skip = existing_thumbnail.is_some() && existing_preview.is_some() && !payload.force;
Ok(ThumbnailContext {
document,
version,
existing_asset: existing,
existing_thumbnail,
existing_preview,
skip,
})
}
fn generate_thumbnail(document: &Document, bytes: &[u8]) -> Result<GeneratedThumbnail, String> {
fn generate_preview_and_thumbnail(
document: &Document,
bytes: &[u8],
) -> Result<GeneratedAssets, String> {
let is_pdf = document
.content_type
.as_deref()
@@ -225,38 +305,41 @@ fn generate_thumbnail(document: &Document, bytes: &[u8]) -> Result<GeneratedThum
.unwrap_or(false)
});
let (png_bytes, width, height) = if is_pdf {
generate_pdf_thumbnail(bytes)?
let (preview, thumbnail) = if is_pdf {
generate_pdf_assets(bytes)?
} else {
generate_image_thumbnail(bytes)?
generate_image_assets(bytes)?
};
Ok(GeneratedThumbnail {
image_bytes: png_bytes,
width,
height,
})
Ok(GeneratedAssets { preview, thumbnail })
}
fn generate_image_thumbnail(bytes: &[u8]) -> Result<(Vec<u8>, Option<i32>, Option<i32>), String> {
fn generate_image_assets(bytes: &[u8]) -> Result<(GeneratedImage, GeneratedImage), String> {
let reader = ImageReader::new(Cursor::new(bytes))
.with_guessed_format()
.map_err(|err| err.to_string())?;
let mut image = reader.decode().map_err(|err| err.to_string())?;
if image.width() > THUMBNAIL_WIDTH || image.height() > THUMBNAIL_HEIGHT {
image = image.thumbnail(THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT);
}
let image = reader.decode().map_err(|err| err.to_string())?;
let (width, height) = image.dimensions();
let mut cursor = Cursor::new(Vec::new());
image
.write_to(&mut cursor, ImageFormat::Png)
.map_err(|err| err.to_string())?;
let buffer = cursor.into_inner();
Ok((buffer, Some(width as i32), Some(height as i32)))
let preview_image = if image.width() > PREVIEW_WIDTH || image.height() > PREVIEW_HEIGHT {
image.thumbnail(PREVIEW_WIDTH, PREVIEW_HEIGHT)
} else {
image.clone()
};
let thumbnail_image =
if preview_image.width() > THUMBNAIL_WIDTH || preview_image.height() > THUMBNAIL_HEIGHT {
preview_image.thumbnail(THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT)
} else {
preview_image.clone()
};
let preview = encode_dynamic_image(preview_image)?;
let thumbnail = encode_dynamic_image(thumbnail_image)?;
Ok((preview, thumbnail))
}
fn generate_pdf_thumbnail(bytes: &[u8]) -> Result<(Vec<u8>, Option<i32>, Option<i32>), String> {
fn generate_pdf_assets(bytes: &[u8]) -> Result<(GeneratedImage, GeneratedImage), String> {
let pdfium = panic::catch_unwind(|| Pdfium::default())
.map_err(|_| "failed to initialize PDFium".to_string())?;
@@ -270,8 +353,8 @@ fn generate_pdf_thumbnail(bytes: &[u8]) -> Result<(Vec<u8>, Option<i32>, Option<
.map_err(|err| format!("load first page: {err}"))?;
let render_config = PdfRenderConfig::new()
.set_target_width(THUMBNAIL_WIDTH as i32)
.set_maximum_height(THUMBNAIL_HEIGHT as i32)
.set_target_width(PREVIEW_WIDTH as i32)
.set_maximum_height(PREVIEW_HEIGHT as i32)
.render_form_data(true)
.rotate_if_landscape(PdfPageRenderRotation::None, true);
@@ -279,54 +362,73 @@ fn generate_pdf_thumbnail(bytes: &[u8]) -> Result<(Vec<u8>, Option<i32>, Option<
.render_with_config(&render_config)
.map_err(|err| format!("render pdf page: {err}"))?;
let image = bitmap.as_image().to_rgb8();
let (width, height) = image.dimensions();
let mut cursor = Cursor::new(Vec::new());
PngEncoder::new(&mut cursor)
.write_image(image.as_raw(), width, height, ColorType::Rgb8.into())
.map_err(|err| format!("encode pdf thumbnail: {err}"))?;
let preview_buffer = bitmap.as_image().to_rgb8();
let preview_image = image::DynamicImage::ImageRgb8(preview_buffer);
Ok((cursor.into_inner(), Some(width as i32), Some(height as i32)))
let thumbnail_image =
if preview_image.width() > THUMBNAIL_WIDTH || preview_image.height() > THUMBNAIL_HEIGHT {
preview_image.thumbnail(THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT)
} else {
preview_image.clone()
};
let preview = encode_dynamic_image(preview_image)?;
let thumbnail = encode_dynamic_image(thumbnail_image)?;
Ok((preview, thumbnail))
}
fn persist_thumbnail_metadata(
fn encode_dynamic_image(image: image::DynamicImage) -> Result<GeneratedImage, String> {
let (width, height) = image.dimensions();
let mut cursor = Cursor::new(Vec::new());
image
.write_to(&mut cursor, ImageFormat::Png)
.map_err(|err| err.to_string())?;
Ok(GeneratedImage {
image_bytes: cursor.into_inner(),
width: Some(width as i32),
height: Some(height as i32),
})
}
fn persist_assets_metadata(
state: Arc<AppState>,
context: &ThumbnailContext,
generated: &GeneratedThumbnail,
asset_id: Uuid,
s3_key: &str,
assets: &[AssetPersistence<'_>],
) -> Result<(), String> {
let mut conn = state.db().map_err(|err| format!("{err:?}"))?;
let new_asset = NewDocumentAsset {
id: asset_id,
document_version_id: context.version.id,
asset_type: THUMBNAIL_ASSET_TYPE.to_string(),
s3_key: s3_key.to_string(),
mime_type: "image/png".to_string(),
width: generated.width,
height: generated.height,
metadata: json!({
"generated_at": Utc::now().to_rfc3339(),
}),
};
for asset in assets {
let new_asset = NewDocumentAsset {
id: asset.asset_id,
document_version_id: context.version.id,
asset_type: asset.asset_type.to_string(),
s3_key: asset.s3_key.to_string(),
mime_type: "image/png".to_string(),
width: asset.generated.width,
height: asset.generated.height,
metadata: json!({
"generated_at": Utc::now().to_rfc3339(),
}),
};
diesel::insert_into(document_assets::table)
.values(&new_asset)
.on_conflict((
document_assets::document_version_id,
document_assets::asset_type,
))
.do_update()
.set((
document_assets::s3_key.eq(excluded(document_assets::s3_key)),
document_assets::mime_type.eq(excluded(document_assets::mime_type)),
document_assets::width.eq(excluded(document_assets::width)),
document_assets::height.eq(excluded(document_assets::height)),
document_assets::metadata.eq(excluded(document_assets::metadata)),
))
.execute(&mut conn)
.map_err(|err| format!("{err:?}"))?;
diesel::insert_into(document_assets::table)
.values(&new_asset)
.on_conflict((
document_assets::document_version_id,
document_assets::asset_type,
))
.do_update()
.set((
document_assets::s3_key.eq(excluded(document_assets::s3_key)),
document_assets::mime_type.eq(excluded(document_assets::mime_type)),
document_assets::width.eq(excluded(document_assets::width)),
document_assets::height.eq(excluded(document_assets::height)),
document_assets::metadata.eq(excluded(document_assets::metadata)),
))
.execute(&mut conn)
.map_err(|err| format!("{err:?}"))?;
}
Ok(())
}