no preview assets anymore

This commit is contained in:
2025-11-16 01:46:58 +01:00
parent 39787f9552
commit 42f3917881
4 changed files with 151 additions and 148 deletions
+73 -8
View File
@@ -28,7 +28,7 @@ use papercrate::{
user_memberships, users, user_memberships, users,
}, },
storage::{ObjectStorage, S3Storage, TenantStorage}, storage::{ObjectStorage, S3Storage, TenantStorage},
tenants::TenantService, tenants::{apply_tenant_guc, clear_tenant_context, TenantService},
utils::{text::normalize_identifier, tracing::init_tracing}, utils::{text::normalize_identifier, tracing::init_tracing},
workers::tenants::{build_delete_proof_message, sign_delete_proof, DeleteAction}, workers::tenants::{build_delete_proof_message, sign_delete_proof, DeleteAction},
}; };
@@ -86,6 +86,10 @@ enum Command {
ListTenants, ListTenants,
DeleteAssets { DeleteAssets {
tenant_id: Uuid, tenant_id: Uuid,
#[arg(long = "asset-type")]
asset_type: Option<String>,
#[arg(long = "all", help = "Confirm deleting every asset for the tenant")]
delete_all: bool,
}, },
QuickwitCreate { QuickwitCreate {
tenant_id: Uuid, tenant_id: Uuid,
@@ -192,8 +196,17 @@ async fn main() -> Result<()> {
} => remove_user_from_tenant(&pool, &username, tenant_id)?, } => remove_user_from_tenant(&pool, &username, tenant_id)?,
Command::ReanalyzeDocuments { tenant_id } => reanalyze_documents(&pool, tenant_id)?, Command::ReanalyzeDocuments { tenant_id } => reanalyze_documents(&pool, tenant_id)?,
Command::ListTenants => list_tenants(&pool)?, Command::ListTenants => list_tenants(&pool)?,
Command::DeleteAssets { tenant_id } => { Command::DeleteAssets {
delete_assets_for_tenant(&config, &pool, tenant_id).await? tenant_id,
asset_type,
delete_all,
} => {
let asset_type = asset_type.as_deref();
if asset_type.is_none() && !delete_all {
bail!("refusing to delete all assets without --all confirmation");
}
delete_assets_for_tenant(&config, &pool, tenant_id, asset_type).await?
} }
Command::QuickwitCreate { tenant_id } => { Command::QuickwitCreate { tenant_id } => {
quickwit_index(&config, &pool, tenant_id, Method::POST).await? quickwit_index(&config, &pool, tenant_id, Method::POST).await?
@@ -486,6 +499,9 @@ fn enqueue_delete_tenant_job_internal(
.optional()? .optional()?
.ok_or_else(|| anyhow!("tenant '{}' not found", tenant_id))?; .ok_or_else(|| anyhow!("tenant '{}' not found", tenant_id))?;
apply_tenant_guc(&mut conn, tenant.id)
.map_err(|err| anyhow!("failed to set tenant context for {}: {err:?}", tenant.name))?;
if tenant.name != expected_name { if tenant.name != expected_name {
bail!( bail!(
"tenant name mismatch: expected '{}', database has '{}'", "tenant name mismatch: expected '{}', database has '{}'",
@@ -548,6 +564,14 @@ fn enqueue_delete_tenant_job_internal(
"delete-tenant job enqueued for '{}' (remove_tenant={}, final_status={})", "delete-tenant job enqueued for '{}' (remove_tenant={}, final_status={})",
tenant.name, remove_tenant, status_label tenant.name, remove_tenant, status_label
); );
clear_tenant_context(&mut conn).map_err(|err| {
anyhow!(
"failed to clear tenant context for {}: {err:?}",
tenant.name
)
})?;
Ok(()) Ok(())
} }
@@ -705,6 +729,7 @@ async fn delete_assets_for_tenant(
config: &AppConfig, config: &AppConfig,
pool: &PgPool, pool: &PgPool,
tenant_id: Uuid, tenant_id: Uuid,
asset_type: Option<&str>,
) -> Result<()> { ) -> Result<()> {
let bucket = s3::build_bucket(config)?; let bucket = s3::build_bucket(config)?;
let storage: Arc<dyn ObjectStorage> = Arc::new(S3Storage::new(bucket)); let storage: Arc<dyn ObjectStorage> = Arc::new(S3Storage::new(bucket));
@@ -716,25 +741,50 @@ async fn delete_assets_for_tenant(
.optional()? .optional()?
.ok_or_else(|| anyhow!("tenant '{}' not found", tenant_id))?; .ok_or_else(|| anyhow!("tenant '{}' not found", tenant_id))?;
apply_tenant_guc(&mut conn, tenant.id)
.map_err(|err| anyhow!("failed to set tenant context for {}: {err:?}", tenant.name))?;
let result = async {
let tenant_storage = TenantStorage::new(Arc::clone(&storage), &tenant) let tenant_storage = TenantStorage::new(Arc::clone(&storage), &tenant)
.with_context(|| format!("missing storage root for tenant {}", tenant.name))?; .with_context(|| format!("missing storage root for tenant {}", tenant.name))?;
let assets: Vec<DocumentAsset> = document_assets::table let mut asset_query = document_assets::table
.filter(document_assets::tenant_id.eq(tenant.id)) .filter(document_assets::tenant_id.eq(tenant.id))
.into_boxed();
if let Some(asset_type) = asset_type {
asset_query = asset_query.filter(document_assets::asset_type.eq(asset_type));
}
let assets: Vec<DocumentAsset> = asset_query
.load(&mut conn) .load(&mut conn)
.with_context(|| format!("failed to load assets for tenant {}", tenant.name))?; .with_context(|| format!("failed to load assets for tenant {}", tenant.name))?;
if assets.is_empty() { if assets.is_empty() {
println!("Tenant {}: no assets", tenant.name); match asset_type {
Some(asset_type) => {
println!("Tenant {}: no assets of type '{}'", tenant.name, asset_type)
}
None => println!("Tenant {}: no assets", tenant.name),
}
return Ok(()); return Ok(());
} }
println!( match asset_type {
Some(asset_type) => println!(
"Tenant {} ({}): deleting {} '{}' assets…",
tenant.name,
tenant.id,
assets.len(),
asset_type
),
None => println!(
"Tenant {} ({}): deleting {} assets…", "Tenant {} ({}): deleting {} assets…",
tenant.name, tenant.name,
tenant.id, tenant.id,
assets.len() assets.len()
); ),
}
let asset_ids: Vec<Uuid> = assets.iter().map(|asset| asset.id).collect(); let asset_ids: Vec<Uuid> = assets.iter().map(|asset| asset.id).collect();
@@ -761,12 +811,27 @@ async fn delete_assets_for_tenant(
.execute(&mut conn) .execute(&mut conn)
.with_context(|| format!("failed to remove asset objects for tenant {}", tenant.name))?; .with_context(|| format!("failed to remove asset objects for tenant {}", tenant.name))?;
diesel::delete(document_assets::table.filter(document_assets::tenant_id.eq(tenant.id))) diesel::delete(
document_assets::table
.filter(document_assets::tenant_id.eq(tenant.id))
.filter(document_assets::id.eq_any(&asset_ids)),
)
.execute(&mut conn) .execute(&mut conn)
.with_context(|| format!("failed to remove asset records for tenant {}", tenant.name))?; .with_context(|| format!("failed to remove asset records for tenant {}", tenant.name))?;
println!("Tenant {}: asset records deleted.", tenant.name); println!("Tenant {}: asset records deleted.", tenant.name);
Ok(()) Ok(())
}
.await;
clear_tenant_context(&mut conn).map_err(|err| {
anyhow!(
"failed to clear tenant context for {}: {err:?}",
tenant.name
)
})?;
result
} }
async fn quickwit_index( async fn quickwit_index(
+4 -4
View File
@@ -103,13 +103,13 @@ mod tests {
); );
assert_eq!( assert_eq!(
document_asset_object_prefix(document_id, 3, "preview", asset_id), document_asset_object_prefix(document_id, 3, "thumbnail", asset_id),
format!("documents/{document_id}/v3/assets/preview/{asset_id}") format!("documents/{document_id}/v3/assets/thumbnail/{asset_id}")
); );
assert_eq!( assert_eq!(
document_asset_object_key(document_id, 3, "preview", asset_id, 2), document_asset_object_key(document_id, 3, "thumbnail", asset_id, 2),
format!("documents/{document_id}/v3/assets/preview/{asset_id}/2") format!("documents/{document_id}/v3/assets/thumbnail/{asset_id}/2")
); );
} }
} }
+30 -86
View File
@@ -29,10 +29,9 @@ use super::{
pub const THUMBNAIL_WIDTH: u32 = 512; pub const THUMBNAIL_WIDTH: u32 = 512;
pub const THUMBNAIL_HEIGHT: u32 = 512; pub const THUMBNAIL_HEIGHT: u32 = 512;
const PREVIEW_WIDTH: u32 = THUMBNAIL_WIDTH * 4; const RENDER_WIDTH: u32 = THUMBNAIL_WIDTH * 4;
const PREVIEW_HEIGHT: u32 = THUMBNAIL_HEIGHT * 4; const RENDER_HEIGHT: u32 = THUMBNAIL_HEIGHT * 4;
pub const THUMBNAIL_ASSET_TYPE: &str = "thumbnail"; pub const THUMBNAIL_ASSET_TYPE: &str = "thumbnail";
pub const PREVIEW_ASSET_TYPE: &str = "preview";
pub struct GenerateThumbnailsTask { pub struct GenerateThumbnailsTask {
force: bool, force: bool,
@@ -59,8 +58,7 @@ impl Task<DocumentVersionTaskContext> for GenerateThumbnailsTask {
} }
let bytes = ctx.buffered_object().await?; let bytes = ctx.buffered_object().await?;
let generation = let generation = generate_thumbnails(&context.document, bytes).map_err(TaskError::fail)?;
generate_preview_and_thumbnail(&context.document, bytes).map_err(TaskError::fail)?;
if let Some(page_count) = generation.page_count { if let Some(page_count) = generation.page_count {
let state = ctx.state().clone(); let state = ctx.state().clone();
@@ -80,19 +78,10 @@ impl Task<DocumentVersionTaskContext> for GenerateThumbnailsTask {
.map_err(|err| TaskError::retry(Duration::from_secs(30), err))?; .map_err(|err| TaskError::retry(Duration::from_secs(30), err))?;
} }
remove_existing_assets(ctx, &context).await; remove_existing_thumbnail_assets(ctx, &context).await;
let preview_asset_id = Uuid::new_v4();
let thumbnail_asset_id = Uuid::new_v4(); let thumbnail_asset_id = Uuid::new_v4();
let preview_objects = upload_generated_objects(
ctx,
&context,
PREVIEW_ASSET_TYPE,
preview_asset_id,
&generation.preview,
)
.await?;
let thumbnail_objects = upload_generated_objects( let thumbnail_objects = upload_generated_objects(
ctx, ctx,
&context, &context,
@@ -102,18 +91,11 @@ impl Task<DocumentVersionTaskContext> for GenerateThumbnailsTask {
) )
.await?; .await?;
let asset_persistences = vec![ let asset_persistences = vec![AssetPersistence {
AssetPersistence {
asset_type: PREVIEW_ASSET_TYPE,
asset_id: preview_asset_id,
objects: preview_objects,
},
AssetPersistence {
asset_type: THUMBNAIL_ASSET_TYPE, asset_type: THUMBNAIL_ASSET_TYPE,
asset_id: thumbnail_asset_id, asset_id: thumbnail_asset_id,
objects: thumbnail_objects, objects: thumbnail_objects,
}, }];
];
let state = ctx.state().clone(); let state = ctx.state().clone();
let tenant_id = context.document.tenant_id; let tenant_id = context.document.tenant_id;
@@ -150,8 +132,6 @@ async fn build_thumbnail_context(
version, version,
existing_thumbnail: None, existing_thumbnail: None,
existing_thumbnail_objects: Vec::new(), existing_thumbnail_objects: Vec::new(),
existing_preview: None,
existing_preview_objects: Vec::new(),
skip: true, skip: true,
tenant_id, tenant_id,
}); });
@@ -162,45 +142,30 @@ async fn build_thumbnail_context(
.get(THUMBNAIL_ASSET_TYPE) .get(THUMBNAIL_ASSET_TYPE)
.map(|entry| (Some(entry.asset.clone()), entry.objects.clone())) .map(|entry| (Some(entry.asset.clone()), entry.objects.clone()))
.unwrap_or((None, Vec::new())); .unwrap_or((None, Vec::new()));
let (existing_preview, existing_preview_objects) = assets
.get(PREVIEW_ASSET_TYPE)
.map(|entry| (Some(entry.asset.clone()), entry.objects.clone()))
.unwrap_or((None, Vec::new()));
let expected_cardinality = expected_asset_cardinality(&document, &version); let expected_cardinality = expected_asset_cardinality(&document, &version);
let preview_cardinality = existing_preview
.as_ref()
.and_then(|asset| asset.cardinality)
.unwrap_or_else(|| existing_preview_objects.len() as i32);
let thumbnail_cardinality = existing_thumbnail let thumbnail_cardinality = existing_thumbnail
.as_ref() .as_ref()
.and_then(|asset| asset.cardinality) .and_then(|asset| asset.cardinality)
.unwrap_or_else(|| existing_thumbnail_objects.len() as i32); .unwrap_or_else(|| existing_thumbnail_objects.len() as i32);
let needs_regeneration = preview_cardinality < expected_cardinality let needs_regeneration = thumbnail_cardinality < expected_cardinality
|| thumbnail_cardinality < expected_cardinality
|| (existing_preview_objects.len() as i32) < expected_cardinality
|| (existing_thumbnail_objects.len() as i32) < expected_cardinality; || (existing_thumbnail_objects.len() as i32) < expected_cardinality;
let skip = let skip = existing_thumbnail.is_some() && !force && !needs_regeneration;
existing_thumbnail.is_some() && existing_preview.is_some() && !force && !needs_regeneration;
Ok(ThumbnailContext { Ok(ThumbnailContext {
document, document,
version, version,
existing_thumbnail, existing_thumbnail,
existing_thumbnail_objects, existing_thumbnail_objects,
existing_preview,
existing_preview_objects,
skip, skip,
tenant_id, tenant_id,
}) })
} }
async fn remove_existing_assets(ctx: &DocumentVersionTaskContext, context: &ThumbnailContext) { async fn remove_existing_thumbnail_assets(
if let Some(existing_preview) = &context.existing_preview { ctx: &DocumentVersionTaskContext,
delete_asset_with_objects(ctx, existing_preview, &context.existing_preview_objects).await; context: &ThumbnailContext,
} ) {
if let Some(existing_thumbnail) = &context.existing_thumbnail { if let Some(existing_thumbnail) = &context.existing_thumbnail {
delete_asset_with_objects(ctx, existing_thumbnail, &context.existing_thumbnail_objects) delete_asset_with_objects(ctx, existing_thumbnail, &context.existing_thumbnail_objects)
.await; .await;
@@ -300,8 +265,6 @@ struct ThumbnailContext {
version: DocumentVersion, version: DocumentVersion,
existing_thumbnail: Option<DocumentAsset>, existing_thumbnail: Option<DocumentAsset>,
existing_thumbnail_objects: Vec<DocumentAssetObject>, existing_thumbnail_objects: Vec<DocumentAssetObject>,
existing_preview: Option<DocumentAsset>,
existing_preview_objects: Vec<DocumentAssetObject>,
skip: bool, skip: bool,
tenant_id: Uuid, tenant_id: Uuid,
} }
@@ -318,7 +281,6 @@ struct GeneratedAsset {
struct GeneratedAssets { struct GeneratedAssets {
thumbnail: GeneratedAsset, thumbnail: GeneratedAsset,
preview: GeneratedAsset,
page_count: Option<u32>, page_count: Option<u32>,
} }
@@ -335,61 +297,49 @@ struct AssetPersistence {
objects: Vec<AssetObjectPersistence>, objects: Vec<AssetObjectPersistence>,
} }
fn generate_preview_and_thumbnail( fn generate_thumbnails(document: &Document, bytes: &[u8]) -> Result<GeneratedAssets, String> {
document: &Document,
bytes: &[u8],
) -> Result<GeneratedAssets, String> {
if document_is_pdf(document) { if document_is_pdf(document) {
let pdf_assets = generate_pdf_assets(bytes)?; let pdf_assets = generate_pdf_assets(bytes)?;
Ok(GeneratedAssets { Ok(GeneratedAssets {
preview: pdf_assets.preview,
thumbnail: pdf_assets.thumbnail, thumbnail: pdf_assets.thumbnail,
page_count: Some(pdf_assets.page_count), page_count: Some(pdf_assets.page_count),
}) })
} else { } else {
let (preview, thumbnail) = generate_image_assets(bytes)?; let thumbnail = generate_image_assets(bytes)?;
Ok(GeneratedAssets { Ok(GeneratedAssets {
preview,
thumbnail, thumbnail,
page_count: None, page_count: None,
}) })
} }
} }
fn generate_image_assets(bytes: &[u8]) -> Result<(GeneratedAsset, GeneratedAsset), String> { fn generate_image_assets(bytes: &[u8]) -> Result<GeneratedAsset, String> {
let reader = ImageReader::new(Cursor::new(bytes)) let reader = ImageReader::new(Cursor::new(bytes))
.with_guessed_format() .with_guessed_format()
.map_err(|err| err.to_string())?; .map_err(|err| err.to_string())?;
let image = reader.decode().map_err(|err| err.to_string())?; let image = reader.decode().map_err(|err| err.to_string())?;
let preview_image = if image.width() > PREVIEW_WIDTH || image.height() > PREVIEW_HEIGHT { let render_image = if image.width() > RENDER_WIDTH || image.height() > RENDER_HEIGHT {
image.thumbnail(PREVIEW_WIDTH, PREVIEW_HEIGHT) image.thumbnail(RENDER_WIDTH, RENDER_HEIGHT)
} else { } else {
image.clone() image.clone()
}; };
let thumbnail_image = let thumbnail_image =
if preview_image.width() > THUMBNAIL_WIDTH || preview_image.height() > THUMBNAIL_HEIGHT { if render_image.width() > THUMBNAIL_WIDTH || render_image.height() > THUMBNAIL_HEIGHT {
preview_image.thumbnail(THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT) render_image.thumbnail(THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT)
} else { } else {
preview_image.clone() render_image.clone()
}; };
let preview = encode_dynamic_image(preview_image)?;
let thumbnail = encode_dynamic_image(thumbnail_image)?; let thumbnail = encode_dynamic_image(thumbnail_image)?;
Ok(( Ok(GeneratedAsset {
GeneratedAsset {
objects: vec![preview],
},
GeneratedAsset {
objects: vec![thumbnail], objects: vec![thumbnail],
}, })
))
} }
struct PdfGeneratedAssets { struct PdfGeneratedAssets {
preview: GeneratedAsset,
thumbnail: GeneratedAsset, thumbnail: GeneratedAsset,
page_count: u32, page_count: u32,
} }
@@ -406,12 +356,11 @@ fn generate_pdf_assets(bytes: &[u8]) -> Result<PdfGeneratedAssets, String> {
let total_pages = pages.len() as usize; let total_pages = pages.len() as usize;
let render_config = PdfRenderConfig::new() let render_config = PdfRenderConfig::new()
.set_target_width(PREVIEW_WIDTH as i32) .set_target_width(RENDER_WIDTH as i32)
.set_maximum_height(PREVIEW_HEIGHT as i32) .set_maximum_height(RENDER_HEIGHT as i32)
.render_form_data(true) .render_form_data(true)
.rotate_if_landscape(PdfPageRenderRotation::None, true); .rotate_if_landscape(PdfPageRenderRotation::None, true);
let mut preview_objects: Vec<GeneratedImage> = Vec::with_capacity(total_pages);
let mut thumbnail_objects: Vec<GeneratedImage> = Vec::with_capacity(total_pages); let mut thumbnail_objects: Vec<GeneratedImage> = Vec::with_capacity(total_pages);
for page_index in 0..total_pages { for page_index in 0..total_pages {
@@ -423,18 +372,16 @@ fn generate_pdf_assets(bytes: &[u8]) -> Result<PdfGeneratedAssets, String> {
.render_with_config(&render_config) .render_with_config(&render_config)
.map_err(|err| format!("render pdf page {page_index}: {err}"))?; .map_err(|err| format!("render pdf page {page_index}: {err}"))?;
let preview_buffer = bitmap.as_image().to_rgb8(); let render_buffer = bitmap.as_image().to_rgb8();
let preview_image = image::DynamicImage::ImageRgb8(preview_buffer); let render_image = image::DynamicImage::ImageRgb8(render_buffer);
let thumbnail_image = if preview_image.width() > THUMBNAIL_WIDTH let thumbnail_image =
|| preview_image.height() > THUMBNAIL_HEIGHT if render_image.width() > THUMBNAIL_WIDTH || render_image.height() > THUMBNAIL_HEIGHT {
{ render_image.thumbnail(THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT)
preview_image.thumbnail(THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT)
} else { } else {
preview_image.clone() render_image.clone()
}; };
preview_objects.push(encode_dynamic_image(preview_image)?);
thumbnail_objects.push(encode_dynamic_image(thumbnail_image)?); thumbnail_objects.push(encode_dynamic_image(thumbnail_image)?);
} }
@@ -443,9 +390,6 @@ fn generate_pdf_assets(bytes: &[u8]) -> Result<PdfGeneratedAssets, String> {
.map_err(|_| "page count exceeds supported range".to_string())?; .map_err(|_| "page count exceeds supported range".to_string())?;
Ok(PdfGeneratedAssets { Ok(PdfGeneratedAssets {
preview: GeneratedAsset {
objects: preview_objects,
},
thumbnail: GeneratedAsset { thumbnail: GeneratedAsset {
objects: thumbnail_objects, objects: thumbnail_objects,
}, },
+2 -8
View File
@@ -57,11 +57,6 @@ struct SignupStartResponse {
challenge: RegistrationChallengeResponse, challenge: RegistrationChallengeResponse,
} }
#[derive(Deserialize)]
struct TenantListResponse {
tenants: Vec<TenantSummary>,
}
#[derive(Deserialize)] #[derive(Deserialize)]
struct TenantSummary { struct TenantSummary {
id: Uuid, id: Uuid,
@@ -549,11 +544,10 @@ async fn login_returns_tenant_selection_when_multiple_memberships() -> Result<()
let tenants_response = app.get("/api/tenants", Some(&login.access_token)).await?; let tenants_response = app.get("/api/tenants", Some(&login.access_token)).await?;
assert_eq!(tenants_response.status(), StatusCode::OK); assert_eq!(tenants_response.status(), StatusCode::OK);
let tenants_body = body_to_vec(tenants_response.into_body()).await?; let tenants_body = body_to_vec(tenants_response.into_body()).await?;
let tenant_list: TenantListResponse = serde_json::from_slice(&tenants_body)?; let tenant_list: Vec<TenantSummary> = serde_json::from_slice(&tenants_body)?;
assert!(tenant_list.tenants.len() >= 2); assert!(tenant_list.len() >= 2);
let secondary = tenant_list let secondary = tenant_list
.tenants
.iter() .iter()
.find(|tenant| tenant.name == secondary_name) .find(|tenant| tenant.name == secondary_name)
.map(|t| t.id) .map(|t| t.id)