nils/papercrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

no preview assets anymore

Browse Source
This commit is contained in:
nils
2025-11-16 01:46:58 +01:00
parent 39787f9552
commit 42f3917881
4 changed files with 151 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/bin/admin.rs
+109 -44
View File
@@ -28,7 +28,7 @@ use papercrate::{
user_memberships, users,
},
storage::{ObjectStorage, S3Storage, TenantStorage},
tenants::TenantService,
tenants::{apply_tenant_guc, clear_tenant_context, TenantService},
utils::{text::normalize_identifier, tracing::init_tracing},
workers::tenants::{build_delete_proof_message, sign_delete_proof, DeleteAction},
};
@@ -86,6 +86,10 @@ enum Command {
ListTenants,
DeleteAssets {
tenant_id: Uuid,
#[arg(long = "asset-type")]
asset_type: Option<String>,
#[arg(long = "all", help = "Confirm deleting every asset for the tenant")]
delete_all: bool,
},
QuickwitCreate {
tenant_id: Uuid,
@@ -192,8 +196,17 @@ async fn main() -> Result<()> {
} => remove_user_from_tenant(&pool, &username, tenant_id)?,
Command::ReanalyzeDocuments { tenant_id } => reanalyze_documents(&pool, tenant_id)?,
Command::ListTenants => list_tenants(&pool)?,
Command::DeleteAssets { tenant_id } => {
delete_assets_for_tenant(&config, &pool, tenant_id).await?
Command::DeleteAssets {
tenant_id,
asset_type,
delete_all,
} => {
let asset_type = asset_type.as_deref();
if asset_type.is_none() && !delete_all {
bail!("refusing to delete all assets without --all confirmation");
}
delete_assets_for_tenant(&config, &pool, tenant_id, asset_type).await?
}
Command::QuickwitCreate { tenant_id } => {
quickwit_index(&config, &pool, tenant_id, Method::POST).await?
@@ -486,6 +499,9 @@ fn enqueue_delete_tenant_job_internal(
.optional()?
.ok_or_else(|| anyhow!("tenant '{}' not found", tenant_id))?;
apply_tenant_guc(&mut conn, tenant.id)
.map_err(|err| anyhow!("failed to set tenant context for {}: {err:?}", tenant.name))?;
if tenant.name != expected_name {
bail!(
"tenant name mismatch: expected '{}', database has '{}'",
@@ -548,6 +564,14 @@ fn enqueue_delete_tenant_job_internal(
"delete-tenant job enqueued for '{}' (remove_tenant={}, final_status={})",
tenant.name, remove_tenant, status_label
);
clear_tenant_context(&mut conn).map_err(|err| {
anyhow!(
"failed to clear tenant context for {}: {err:?}",
tenant.name
)
})?;
Ok(())
}
@@ -705,6 +729,7 @@ async fn delete_assets_for_tenant(
config: &AppConfig,
pool: &PgPool,
tenant_id: Uuid,
asset_type: Option<&str>,
) -> Result<()> {
let bucket = s3::build_bucket(config)?;
let storage: Arc<dyn ObjectStorage> = Arc::new(S3Storage::new(bucket));
@@ -716,57 +741,97 @@ async fn delete_assets_for_tenant(
.optional()?
.ok_or_else(|| anyhow!("tenant '{}' not found", tenant_id))?;
let tenant_storage = TenantStorage::new(Arc::clone(&storage), &tenant)
.with_context(|| format!("missing storage root for tenant {}", tenant.name))?;
apply_tenant_guc(&mut conn, tenant.id)
.map_err(|err| anyhow!("failed to set tenant context for {}: {err:?}", tenant.name))?;
let assets: Vec<DocumentAsset> = document_assets::table
.filter(document_assets::tenant_id.eq(tenant.id))
.load(&mut conn)
.with_context(|| format!("failed to load assets for tenant {}", tenant.name))?;
let result = async {
let tenant_storage = TenantStorage::new(Arc::clone(&storage), &tenant)
.with_context(|| format!("missing storage root for tenant {}", tenant.name))?;
if assets.is_empty() {
println!("Tenant {}: no assets", tenant.name);
return Ok(());
}
let mut asset_query = document_assets::table
.filter(document_assets::tenant_id.eq(tenant.id))
.into_boxed();
println!(
"Tenant {} ({}): deleting {} assets…",
tenant.name,
tenant.id,
assets.len()
);
let asset_ids: Vec<Uuid> = assets.iter().map(|asset| asset.id).collect();
let objects: Vec<DocumentAssetObject> = document_asset_objects::table
.filter(document_asset_objects::tenant_id.eq(tenant.id))
.filter(document_asset_objects::asset_id.eq_any(&asset_ids))
.load(&mut conn)
.with_context(|| format!("failed to load asset objects for tenant {}", tenant.name))?;
for object in &objects {
if let Err(err) = tenant_storage.delete_object(&object.s3_key).await {
eprintln!(
"Failed to delete object {} (tenant {}): {err}",
object.s3_key, tenant.name
);
if let Some(asset_type) = asset_type {
asset_query = asset_query.filter(document_assets::asset_type.eq(asset_type));
}
}
diesel::delete(
document_asset_objects::table
let assets: Vec<DocumentAsset> = asset_query
.load(&mut conn)
.with_context(|| format!("failed to load assets for tenant {}", tenant.name))?;
if assets.is_empty() {
match asset_type {
Some(asset_type) => {
println!("Tenant {}: no assets of type '{}'", tenant.name, asset_type)
}
None => println!("Tenant {}: no assets", tenant.name),
}
return Ok(());
}
match asset_type {
Some(asset_type) => println!(
"Tenant {} ({}): deleting {} '{}' assets…",
tenant.name,
tenant.id,
assets.len(),
asset_type
),
None => println!(
"Tenant {} ({}): deleting {} assets…",
tenant.name,
tenant.id,
assets.len()
),
}
let asset_ids: Vec<Uuid> = assets.iter().map(|asset| asset.id).collect();
let objects: Vec<DocumentAssetObject> = document_asset_objects::table
.filter(document_asset_objects::tenant_id.eq(tenant.id))
.filter(document_asset_objects::asset_id.eq_any(&asset_ids)),
)
.execute(&mut conn)
.with_context(|| format!("failed to remove asset objects for tenant {}", tenant.name))?;
.filter(document_asset_objects::asset_id.eq_any(&asset_ids))
.load(&mut conn)
.with_context(|| format!("failed to load asset objects for tenant {}", tenant.name))?;
diesel::delete(document_assets::table.filter(document_assets::tenant_id.eq(tenant.id)))
for object in &objects {
if let Err(err) = tenant_storage.delete_object(&object.s3_key).await {
eprintln!(
"Failed to delete object {} (tenant {}): {err}",
object.s3_key, tenant.name
);
}
}
diesel::delete(
document_asset_objects::table
.filter(document_asset_objects::tenant_id.eq(tenant.id))
.filter(document_asset_objects::asset_id.eq_any(&asset_ids)),
)
.execute(&mut conn)
.with_context(|| format!("failed to remove asset objects for tenant {}", tenant.name))?;
diesel::delete(
document_assets::table
.filter(document_assets::tenant_id.eq(tenant.id))
.filter(document_assets::id.eq_any(&asset_ids)),
)
.execute(&mut conn)
.with_context(|| format!("failed to remove asset records for tenant {}", tenant.name))?;
println!("Tenant {}: asset records deleted.", tenant.name);
Ok(())
println!("Tenant {}: asset records deleted.", tenant.name);
Ok(())
}
.await;
clear_tenant_context(&mut conn).map_err(|err| {
anyhow!(
"failed to clear tenant context for {}: {err:?}",
tenant.name
)
})?;
result
}
async fn quickwit_index(
backend/src/utils/storage_paths.rs
+4 -4
View File
@@ -103,13 +103,13 @@ mod tests {
);
assert_eq!(
document_asset_object_prefix(document_id, 3, "preview", asset_id),
format!("documents/{document_id}/v3/assets/preview/{asset_id}")
document_asset_object_prefix(document_id, 3, "thumbnail", asset_id),
format!("documents/{document_id}/v3/assets/thumbnail/{asset_id}")
);
assert_eq!(
document_asset_object_key(document_id, 3, "preview", asset_id, 2),
format!("documents/{document_id}/v3/assets/preview/{asset_id}/2")
document_asset_object_key(document_id, 3, "thumbnail", asset_id, 2),
format!("documents/{document_id}/v3/assets/thumbnail/{asset_id}/2")
);
}
}
backend/src/workers/thumbnails.rs
+36 -92
View File
@@ -29,10 +29,9 @@ use super::{
pub const THUMBNAIL_WIDTH: u32 = 512;
pub const THUMBNAIL_HEIGHT: u32 = 512;
const PREVIEW_WIDTH: u32 = THUMBNAIL_WIDTH * 4;
const PREVIEW_HEIGHT: u32 = THUMBNAIL_HEIGHT * 4;
const RENDER_WIDTH: u32 = THUMBNAIL_WIDTH * 4;
const RENDER_HEIGHT: u32 = THUMBNAIL_HEIGHT * 4;
pub const THUMBNAIL_ASSET_TYPE: &str = "thumbnail";
pub const PREVIEW_ASSET_TYPE: &str = "preview";
pub struct GenerateThumbnailsTask {
force: bool,
@@ -59,8 +58,7 @@ impl Task<DocumentVersionTaskContext> for GenerateThumbnailsTask {
}
let bytes = ctx.buffered_object().await?;
let generation =
generate_preview_and_thumbnail(&context.document, bytes).map_err(TaskError::fail)?;
let generation = generate_thumbnails(&context.document, bytes).map_err(TaskError::fail)?;
if let Some(page_count) = generation.page_count {
let state = ctx.state().clone();
@@ -80,19 +78,10 @@ impl Task<DocumentVersionTaskContext> for GenerateThumbnailsTask {
.map_err(|err| TaskError::retry(Duration::from_secs(30), err))?;
}
remove_existing_assets(ctx, &context).await;
remove_existing_thumbnail_assets(ctx, &context).await;
let preview_asset_id = Uuid::new_v4();
let thumbnail_asset_id = Uuid::new_v4();
let preview_objects = upload_generated_objects(
ctx,
&context,
PREVIEW_ASSET_TYPE,
preview_asset_id,
&generation.preview,
)
.await?;
let thumbnail_objects = upload_generated_objects(
ctx,
&context,
@@ -102,18 +91,11 @@ impl Task<DocumentVersionTaskContext> for GenerateThumbnailsTask {
)
.await?;
let asset_persistences = vec![
AssetPersistence {
asset_type: PREVIEW_ASSET_TYPE,
asset_id: preview_asset_id,
objects: preview_objects,
},
AssetPersistence {
asset_type: THUMBNAIL_ASSET_TYPE,
asset_id: thumbnail_asset_id,
objects: thumbnail_objects,
},
];
let asset_persistences = vec![AssetPersistence {
asset_type: THUMBNAIL_ASSET_TYPE,
asset_id: thumbnail_asset_id,
objects: thumbnail_objects,
}];
let state = ctx.state().clone();
let tenant_id = context.document.tenant_id;
@@ -150,8 +132,6 @@ async fn build_thumbnail_context(
version,
existing_thumbnail: None,
existing_thumbnail_objects: Vec::new(),
existing_preview: None,
existing_preview_objects: Vec::new(),
skip: true,
tenant_id,
});
@@ -162,45 +142,30 @@ async fn build_thumbnail_context(
.get(THUMBNAIL_ASSET_TYPE)
.map(|entry| (Some(entry.asset.clone()), entry.objects.clone()))
.unwrap_or((None, Vec::new()));
let (existing_preview, existing_preview_objects) = assets
.get(PREVIEW_ASSET_TYPE)
.map(|entry| (Some(entry.asset.clone()), entry.objects.clone()))
.unwrap_or((None, Vec::new()));
let expected_cardinality = expected_asset_cardinality(&document, &version);
let preview_cardinality = existing_preview
.as_ref()
.and_then(|asset| asset.cardinality)
.unwrap_or_else(|| existing_preview_objects.len() as i32);
let thumbnail_cardinality = existing_thumbnail
.as_ref()
.and_then(|asset| asset.cardinality)
.unwrap_or_else(|| existing_thumbnail_objects.len() as i32);
let needs_regeneration = preview_cardinality < expected_cardinality
|| thumbnail_cardinality < expected_cardinality
|| (existing_preview_objects.len() as i32) < expected_cardinality
let needs_regeneration = thumbnail_cardinality < expected_cardinality
|| (existing_thumbnail_objects.len() as i32) < expected_cardinality;
let skip =
existing_thumbnail.is_some() && existing_preview.is_some() && !force && !needs_regeneration;
let skip = existing_thumbnail.is_some() && !force && !needs_regeneration;
Ok(ThumbnailContext {
document,
version,
existing_thumbnail,
existing_thumbnail_objects,
existing_preview,
existing_preview_objects,
skip,
tenant_id,
})
}
async fn remove_existing_assets(ctx: &DocumentVersionTaskContext, context: &ThumbnailContext) {
if let Some(existing_preview) = &context.existing_preview {
delete_asset_with_objects(ctx, existing_preview, &context.existing_preview_objects).await;
}
async fn remove_existing_thumbnail_assets(
ctx: &DocumentVersionTaskContext,
context: &ThumbnailContext,
) {
if let Some(existing_thumbnail) = &context.existing_thumbnail {
delete_asset_with_objects(ctx, existing_thumbnail, &context.existing_thumbnail_objects)
.await;
@@ -300,8 +265,6 @@ struct ThumbnailContext {
version: DocumentVersion,
existing_thumbnail: Option<DocumentAsset>,
existing_thumbnail_objects: Vec<DocumentAssetObject>,
existing_preview: Option<DocumentAsset>,
existing_preview_objects: Vec<DocumentAssetObject>,
skip: bool,
tenant_id: Uuid,
}
@@ -318,7 +281,6 @@ struct GeneratedAsset {
struct GeneratedAssets {
thumbnail: GeneratedAsset,
preview: GeneratedAsset,
page_count: Option<u32>,
}
@@ -335,61 +297,49 @@ struct AssetPersistence {
objects: Vec<AssetObjectPersistence>,
}
fn generate_preview_and_thumbnail(
document: &Document,
bytes: &[u8],
) -> Result<GeneratedAssets, String> {
fn generate_thumbnails(document: &Document, bytes: &[u8]) -> Result<GeneratedAssets, String> {
if document_is_pdf(document) {
let pdf_assets = generate_pdf_assets(bytes)?;
Ok(GeneratedAssets {
preview: pdf_assets.preview,
thumbnail: pdf_assets.thumbnail,
page_count: Some(pdf_assets.page_count),
})
} else {
let (preview, thumbnail) = generate_image_assets(bytes)?;
let thumbnail = generate_image_assets(bytes)?;
Ok(GeneratedAssets {
preview,
thumbnail,
page_count: None,
})
}
}
fn generate_image_assets(bytes: &[u8]) -> Result<(GeneratedAsset, GeneratedAsset), String> {
fn generate_image_assets(bytes: &[u8]) -> Result<GeneratedAsset, String> {
let reader = ImageReader::new(Cursor::new(bytes))
.with_guessed_format()
.map_err(|err| err.to_string())?;
let image = reader.decode().map_err(|err| err.to_string())?;
let preview_image = if image.width() > PREVIEW_WIDTH || image.height() > PREVIEW_HEIGHT {
image.thumbnail(PREVIEW_WIDTH, PREVIEW_HEIGHT)
let render_image = if image.width() > RENDER_WIDTH || image.height() > RENDER_HEIGHT {
image.thumbnail(RENDER_WIDTH, RENDER_HEIGHT)
} else {
image.clone()
};
let thumbnail_image =
if preview_image.width() > THUMBNAIL_WIDTH || preview_image.height() > THUMBNAIL_HEIGHT {
preview_image.thumbnail(THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT)
if render_image.width() > THUMBNAIL_WIDTH || render_image.height() > THUMBNAIL_HEIGHT {
render_image.thumbnail(THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT)
} else {
preview_image.clone()
render_image.clone()
};
let preview = encode_dynamic_image(preview_image)?;
let thumbnail = encode_dynamic_image(thumbnail_image)?;
Ok((
GeneratedAsset {
objects: vec![preview],
},
GeneratedAsset {
objects: vec![thumbnail],
},
))
Ok(GeneratedAsset {
objects: vec![thumbnail],
})
}
struct PdfGeneratedAssets {
preview: GeneratedAsset,
thumbnail: GeneratedAsset,
page_count: u32,
}
@@ -406,12 +356,11 @@ fn generate_pdf_assets(bytes: &[u8]) -> Result<PdfGeneratedAssets, String> {
let total_pages = pages.len() as usize;
let render_config = PdfRenderConfig::new()
.set_target_width(PREVIEW_WIDTH as i32)
.set_maximum_height(PREVIEW_HEIGHT as i32)
.set_target_width(RENDER_WIDTH as i32)
.set_maximum_height(RENDER_HEIGHT as i32)
.render_form_data(true)
.rotate_if_landscape(PdfPageRenderRotation::None, true);
let mut preview_objects: Vec<GeneratedImage> = Vec::with_capacity(total_pages);
let mut thumbnail_objects: Vec<GeneratedImage> = Vec::with_capacity(total_pages);
for page_index in 0..total_pages {
@@ -423,18 +372,16 @@ fn generate_pdf_assets(bytes: &[u8]) -> Result<PdfGeneratedAssets, String> {
.render_with_config(&render_config)
.map_err(|err| format!("render pdf page {page_index}: {err}"))?;
let preview_buffer = bitmap.as_image().to_rgb8();
let preview_image = image::DynamicImage::ImageRgb8(preview_buffer);
let render_buffer = bitmap.as_image().to_rgb8();
let render_image = image::DynamicImage::ImageRgb8(render_buffer);
let thumbnail_image = if preview_image.width() > THUMBNAIL_WIDTH
|| preview_image.height() > THUMBNAIL_HEIGHT
{
preview_image.thumbnail(THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT)
} else {
preview_image.clone()
};
let thumbnail_image =
if render_image.width() > THUMBNAIL_WIDTH || render_image.height() > THUMBNAIL_HEIGHT {
render_image.thumbnail(THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT)
} else {
render_image.clone()
};
preview_objects.push(encode_dynamic_image(preview_image)?);
thumbnail_objects.push(encode_dynamic_image(thumbnail_image)?);
}
@@ -443,9 +390,6 @@ fn generate_pdf_assets(bytes: &[u8]) -> Result<PdfGeneratedAssets, String> {
.map_err(|_| "page count exceeds supported range".to_string())?;
Ok(PdfGeneratedAssets {
preview: GeneratedAsset {
objects: preview_objects,
},
thumbnail: GeneratedAsset {
objects: thumbnail_objects,
},
backend/tests/auth_flow.rs
+2 -8
View File
@@ -57,11 +57,6 @@ struct SignupStartResponse {
challenge: RegistrationChallengeResponse,
}
#[derive(Deserialize)]
struct TenantListResponse {
tenants: Vec<TenantSummary>,
}
#[derive(Deserialize)]
struct TenantSummary {
id: Uuid,
@@ -549,11 +544,10 @@ async fn login_returns_tenant_selection_when_multiple_memberships() -> Result<()
let tenants_response = app.get("/api/tenants", Some(&login.access_token)).await?;
assert_eq!(tenants_response.status(), StatusCode::OK);
let tenants_body = body_to_vec(tenants_response.into_body()).await?;
let tenant_list: TenantListResponse = serde_json::from_slice(&tenants_body)?;
assert!(tenant_list.tenants.len() >= 2);
let tenant_list: Vec<TenantSummary> = serde_json::from_slice(&tenants_body)?;
assert!(tenant_list.len() >= 2);
let secondary = tenant_list
.tenants
.iter()
.find(|tenant| tenant.name == secondary_name)
.map(|t| t.id)