From 7f371b2ab668799af1f2acd47de906051b2aebb4 Mon Sep 17 00:00:00 2001 From: Nils Schneider Date: Tue, 11 Nov 2025 13:27:07 +0100 Subject: [PATCH] fix --- backend/Cargo.lock | 1 + backend/Cargo.toml | 1 + .../down.sql | 10 +- .../up.sql | 10 +- backend/src/openapi.rs | 1 + backend/src/routes/documents.rs | 5 +- backend/src/services/documents.rs | 28 +- backend/tests/auth_flow.rs | 9 +- backend/tests/documents_flow.rs | 25 +- backend/tests/tenant_deletion.rs | 767 ++++++++++++++---- 10 files changed, 672 insertions(+), 185 deletions(-) rename backend/migrations/{202511050000_make_jobs_tenant_nullable => 202511110000_make_jobs_tenant_nullable}/down.sql (57%) rename backend/migrations/{202511050000_make_jobs_tenant_nullable => 202511110000_make_jobs_tenant_nullable}/up.sql (61%) diff --git a/backend/Cargo.lock b/backend/Cargo.lock index ab6c3e8..cede449 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -2130,6 +2130,7 @@ dependencies = [ "envy", "futures-util", "hex", + "hmac", "http-body-util", "hyper", "image", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index f4ce6ce..f01dc58 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -32,6 +32,7 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] } dotenv = "0.15" sha2 = "0.10" hex = "0.4" +hmac = "0.12" bytes = "1.5" async-trait = "0.1" image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp"] } diff --git a/backend/migrations/202511050000_make_jobs_tenant_nullable/down.sql b/backend/migrations/202511110000_make_jobs_tenant_nullable/down.sql similarity index 57% rename from backend/migrations/202511050000_make_jobs_tenant_nullable/down.sql rename to backend/migrations/202511110000_make_jobs_tenant_nullable/down.sql index 3bbbf91..8172ced 100644 --- a/backend/migrations/202511050000_make_jobs_tenant_nullable/down.sql +++ b/backend/migrations/202511110000_make_jobs_tenant_nullable/down.sql @@ -1,13 +1,13 @@ -ALTER TABLE jobs +ALTER TABLE shared.jobs DROP CONSTRAINT jobs_tenant_id_fkey; -ALTER TABLE jobs +ALTER TABLE shared.jobs ALTER COLUMN tenant_id SET NOT NULL; -ALTER TABLE jobs +ALTER TABLE shared.jobs ADD CONSTRAINT jobs_tenant_id_fkey FOREIGN KEY (tenant_id) - REFERENCES tenants(id); + REFERENCES shared.tenants(id); -ALTER TABLE jobs +ALTER TABLE shared.jobs DROP COLUMN result; diff --git a/backend/migrations/202511050000_make_jobs_tenant_nullable/up.sql b/backend/migrations/202511110000_make_jobs_tenant_nullable/up.sql similarity index 61% rename from backend/migrations/202511050000_make_jobs_tenant_nullable/up.sql rename to backend/migrations/202511110000_make_jobs_tenant_nullable/up.sql index 5032ea2..86be1e8 100644 --- a/backend/migrations/202511050000_make_jobs_tenant_nullable/up.sql +++ b/backend/migrations/202511110000_make_jobs_tenant_nullable/up.sql @@ -1,14 +1,14 @@ -ALTER TABLE jobs +ALTER TABLE shared.jobs ALTER COLUMN tenant_id DROP NOT NULL; -ALTER TABLE jobs +ALTER TABLE shared.jobs DROP CONSTRAINT jobs_tenant_id_fkey; -ALTER TABLE jobs +ALTER TABLE shared.jobs ADD CONSTRAINT jobs_tenant_id_fkey FOREIGN KEY (tenant_id) - REFERENCES tenants(id) + REFERENCES shared.tenants(id) ON DELETE SET NULL; -ALTER TABLE jobs +ALTER TABLE shared.jobs ADD COLUMN result JSONB; diff --git a/backend/src/openapi.rs b/backend/src/openapi.rs index d4c2b00..5b20f14 100644 --- a/backend/src/openapi.rs +++ b/backend/src/openapi.rs @@ -73,6 +73,7 @@ pub mod schemas { DocumentVersionDetailResponse, DocumentVersionResponse, }; pub use crate::documents::correspondents::DocumentCorrespondentResponse; + pub use crate::error::ApiErrorResponse; pub use crate::models::ApiCapability; pub use crate::routes::correspondents::{ CorrespondentSummary, CorrespondentUsage, CreateCorrespondentRequest, diff --git a/backend/src/routes/documents.rs b/backend/src/routes/documents.rs index 4af8fc5..96ef214 100644 --- a/backend/src/routes/documents.rs +++ b/backend/src/routes/documents.rs @@ -19,6 +19,8 @@ use crate::documents::asset::{ asset_object_disposition, DocumentAssetDetailResponse, DocumentAssetResponse, DocumentVersionDetailResponse, DocumentVersionResponse, }; +#[allow(unused_imports)] +use crate::error::ApiErrorResponse; use crate::error::{AppError, AppResult}; use crate::http::responders::{accepted_json, created_json, no_content, ok_json, JsonResponse}; use crate::models::{Document, DocumentAsset, DocumentAssetObject, DocumentVersion}; @@ -189,7 +191,7 @@ pub async fn get_document( ( status = 409, description = "Document with identical contents already exists", - body = crate::error::ApiErrorResponse + body = ApiErrorResponse ) ), tag = "Documents" @@ -1085,6 +1087,7 @@ pub async fn remove_tag( crate::documents::asset::DocumentAssetDetailResponse, crate::documents::asset::DocumentAssetObjectResponse, crate::documents::correspondents::DocumentCorrespondentResponse, + crate::error::ApiErrorResponse, )) )] pub struct DocumentsApiDoc; diff --git a/backend/src/services/documents.rs b/backend/src/services/documents.rs index 3a39fe3..07fb964 100644 --- a/backend/src/services/documents.rs +++ b/backend/src/services/documents.rs @@ -1420,12 +1420,32 @@ impl<'a> DocumentsService<'a> { checksum = %checksum_hex, "upload rejected because document already exists", ); + + let mut message = String::from("a document with the same contents already exists"); + let mut details = json!({ + "conflict_document_id": document.id, + }); + + if let Some(deleted_at) = document.deleted_at { + message.push_str( + ". Note: the existing document is currently in the trash.", + ); + if let Some(obj) = details.as_object_mut() { + obj.insert( + "conflict_document_in_trash".to_string(), + Value::Bool(true), + ); + obj.insert( + "conflict_document_deleted_at".to_string(), + Value::String(to_iso(deleted_at)), + ); + } + } + return Err( - AppError::conflict("a document with the same contents already exists") + AppError::conflict(message) .with_code("duplicate_document") - .with_details(json!({ - "conflict_document_id": document.id, - })), + .with_details(details), ); } diff --git a/backend/tests/auth_flow.rs b/backend/tests/auth_flow.rs index 8d83e14..4eb0052 100644 --- a/backend/tests/auth_flow.rs +++ b/backend/tests/auth_flow.rs @@ -13,13 +13,8 @@ use papercrate::models::{ }; use papercrate::openapi::schemas::PasskeySummary; use papercrate::schema::{ - capability_sets, - magic_tokens::dsl as magic_dsl, - tenants, - tenants::dsl as tenant_dsl, - user_memberships, - user_sessions, - users, + capability_sets, magic_tokens::dsl as magic_dsl, tenants, tenants::dsl as tenant_dsl, + user_memberships, user_sessions, users, }; use papercrate::test_support::{acquire_db_lock, body_to_vec, TestApp, TestUserRole}; use rand::{rngs::OsRng, TryRngCore}; diff --git a/backend/tests/documents_flow.rs b/backend/tests/documents_flow.rs index 6930270..61eeeaf 100644 --- a/backend/tests/documents_flow.rs +++ b/backend/tests/documents_flow.rs @@ -519,6 +519,29 @@ async fn duplicate_and_restore_document() -> Result<()> { .await?; assert_eq!(delete.status(), StatusCode::NO_CONTENT); + let trashed_conflict = app + .upload_document( + "/api/documents", + "dup.bin", + "application/octet-stream", + &payload, + None, + &token, + ) + .await?; + assert_eq!(trashed_conflict.status(), StatusCode::CONFLICT); + let trashed_body = body_to_vec(trashed_conflict.into_body()).await?; + let trashed_error: ApiErrorResponse = serde_json::from_slice(&trashed_body)?; + assert_eq!(trashed_error.code.as_deref(), Some("duplicate_document")); + assert!(trashed_error.error.contains("trash")); + let trashed_details = trashed_error.details.as_ref().expect("details present"); + assert_eq!( + trashed_details + .get("conflict_document_in_trash") + .and_then(|value| value.as_bool()), + Some(true) + ); + let third = app .upload_document_with_extras( "/api/documents", @@ -1837,7 +1860,7 @@ async fn purge_document_removes_data() -> Result<()> { let state = Arc::new(app.state.clone()); let storage = app .state - .storage_for_tenant(job.tenant_id) + .storage_for_tenant(job.tenant_id.expect("job should have tenant")) .map_err(|err| anyhow!("tenant storage unavailable: {err:?}"))?; let execution = handler.handle(state, job.clone(), storage).await; assert!(matches!(execution, JobExecution::Success)); diff --git a/backend/tests/tenant_deletion.rs b/backend/tests/tenant_deletion.rs index e70aed4..1f4aab5 100644 --- a/backend/tests/tenant_deletion.rs +++ b/backend/tests/tenant_deletion.rs @@ -1,13 +1,20 @@ -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, bail, Result}; use axum::http::StatusCode; +use chrono::{Duration as ChronoDuration, Utc}; +use diesel::dsl::{count_star, exists, select}; use diesel::prelude::*; -use papercrate::jobs::{enqueue_job, JOB_DELETE_TENANT}; +use papercrate::jobs::{ + enqueue_job, mark_job_failed, mark_job_succeeded, JOB_DELETE_TENANT, STATUS_FAILED, + STATUS_SUCCEEDED, +}; use papercrate::models::TenantStatus; -use papercrate::schema::{documents, tenants, user_memberships}; +use papercrate::schema::{documents, jobs, tenants, user_memberships}; use papercrate::test_support::{acquire_db_lock, body_to_vec, TestApp, TestUserRole}; -use papercrate::workers::tenants::DeleteTenantJob; -use papercrate::workers::JobHandler; -use serde_json::json; +use papercrate::workers::tenants::{ + build_delete_proof_message, sign_delete_proof, DeleteAction, DeleteTenantJob, +}; +use papercrate::workers::{JobExecution, JobHandler}; +use serde_json::{json, Value}; use std::sync::Arc; use uuid::Uuid; @@ -20,63 +27,39 @@ async fn delete_tenant_job_keeps_tenant_when_requested() -> Result<()> { app.insert_user("tenant-keep", TestUserRole::Owner).await?; let token = app.login_token("tenant-keep", password).await?; - let upload = app - .upload_document( - "/api/documents", - "keep.pdf", - "application/pdf", - b"keep", - None, - &token, - ) - .await?; - assert_eq!(upload.status(), StatusCode::CREATED); - body_to_vec(upload.into_body()).await?; + upload_fixture(&app, &token, "keep.pdf", b"keep").await?; - let tenant_id = app - .state - .tenants - .get_by_name("test_tenant") - .map_err(|err| anyhow!("tenant lookup failed: {err:?}"))? - .id; + let tenant_id = default_tenant_id(&app)?; set_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?; + assert_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?; + + let storage = tenant_storage(&app, tenant_id)?; + let storage_prefix = storage.root_prefix().to_string(); + let before = tenant_snapshot(&app, tenant_id, &storage_prefix).await?; + assert_eq!(before.doc_count, 1); + assert!(before.membership_count > 0); + assert!(!before.storage_keys.is_empty()); + assert_storage_keys_present(&app, &before.storage_keys).await?; let job = enqueue_delete_job(&app, tenant_id, false).await?; + let job_id = job.id; let handler = DeleteTenantJob::new(); let state = Arc::new(app.state.clone()); - let storage = app - .state - .storage_for_tenant(tenant_id) - .map_err(|err| anyhow!("storage unavailable: {err:?}"))?; - let execution = handler.handle(state, job, storage).await; - assert!(matches!( - execution, - papercrate::workers::JobExecution::Success - )); + assert_job_success(&execution); + record_job_outcome(&app, job_id, &execution).await?; + assert_eq!(fetch_job_status(&app, job_id).await?, STATUS_SUCCEEDED); - assert_eq!(app.storage().object_count().await, 0); - - let tenant_id_for_check = tenant_id; - app.with_conn(move |conn| { - use diesel::dsl::count_star; - - let doc_count: i64 = documents::table.select(count_star()).get_result(conn)?; - assert_eq!(doc_count, 0); - - let membership_count: i64 = user_memberships::table - .select(count_star()) - .get_result(conn)?; - assert!(membership_count > 0, "memberships should remain"); - - let status: TenantStatus = tenants::table - .find(tenant_id_for_check) - .select(tenants::status) - .first(conn)?; - assert_eq!(status, TenantStatus::Suspended); - Ok(()) - }) - .await?; + let after = tenant_snapshot(&app, tenant_id, &storage_prefix).await?; + assert_eq!(after.doc_count, 0); + assert!(after.membership_count > 0); + assert_storage_keys_absent(&app, &before.storage_keys).await?; + assert_eq!(storage_object_count(&app, &storage_prefix).await?, 0); + assert_eq!( + fetch_tenant_status(&app, tenant_id).await?, + TenantStatus::Suspended + ); + assert!(tenant_exists(&app, tenant_id).await?); app.cleanup().await?; Ok(()) @@ -90,64 +73,38 @@ async fn delete_tenant_job_can_reset_tenant_to_active() -> Result<()> { let password = "delete-reset"; app.insert_user("tenant-reset", TestUserRole::Owner).await?; let token = app.login_token("tenant-reset", password).await?; + upload_fixture(&app, &token, "reset.pdf", b"reset").await?; - let upload = app - .upload_document( - "/api/documents", - "reset.pdf", - "application/pdf", - b"reset", - None, - &token, - ) - .await?; - assert_eq!(upload.status(), StatusCode::CREATED); - body_to_vec(upload.into_body()).await?; - - let tenant_id = app - .state - .tenants - .get_by_name("test_tenant") - .map_err(|err| anyhow!("tenant lookup failed: {err:?}"))? - .id; + let tenant_id = default_tenant_id(&app)?; set_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?; + assert_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?; + + let storage = tenant_storage(&app, tenant_id)?; + let storage_prefix = storage.root_prefix().to_string(); + let before = tenant_snapshot(&app, tenant_id, &storage_prefix).await?; + assert_eq!(before.doc_count, 1); + assert!(before.membership_count > 0); + assert!(!before.storage_keys.is_empty()); + assert_storage_keys_present(&app, &before.storage_keys).await?; let job = enqueue_delete_job_with_status(&app, tenant_id, false, Some("active")).await?; + let job_id = job.id; let handler = DeleteTenantJob::new(); let state = Arc::new(app.state.clone()); - let storage = app - .state - .storage_for_tenant(tenant_id) - .map_err(|err| anyhow!("storage unavailable: {err:?}"))?; - let execution = handler.handle(state, job, storage).await; - assert!(matches!( - execution, - papercrate::workers::JobExecution::Success - )); + assert_job_success(&execution); + record_job_outcome(&app, job_id, &execution).await?; + assert_eq!(fetch_job_status(&app, job_id).await?, STATUS_SUCCEEDED); - assert_eq!(app.storage().object_count().await, 0); - - let tenant_id_for_check = tenant_id; - app.with_conn(move |conn| { - use diesel::dsl::count_star; - - let doc_count: i64 = documents::table.select(count_star()).get_result(conn)?; - assert_eq!(doc_count, 0); - - let membership_count: i64 = user_memberships::table - .select(count_star()) - .get_result(conn)?; - assert!(membership_count > 0, "memberships should remain"); - - let status: TenantStatus = tenants::table - .find(tenant_id_for_check) - .select(tenants::status) - .first(conn)?; - assert_eq!(status, TenantStatus::Active); - Ok(()) - }) - .await?; + let after = tenant_snapshot(&app, tenant_id, &storage_prefix).await?; + assert_eq!(after.doc_count, 0); + assert!(after.membership_count > 0); + assert_storage_keys_absent(&app, &before.storage_keys).await?; + assert_eq!(storage_object_count(&app, &storage_prefix).await?, 0); + assert_eq!( + fetch_tenant_status(&app, tenant_id).await?, + TenantStatus::Active + ); app.cleanup().await?; Ok(()) @@ -163,67 +120,269 @@ async fn delete_tenant_job_removes_tenant_entirely() -> Result<()> { .await?; let token = app.login_token("tenant-remove", password).await?; - let upload = app + upload_fixture(&app, &token, "remove-1.pdf", b"remove-1").await?; + upload_fixture(&app, &token, "remove-2.pdf", b"remove-2").await?; + + let tenant_id = default_tenant_id(&app)?; + set_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?; + assert_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?; + + let storage = tenant_storage(&app, tenant_id)?; + let storage_prefix = storage.root_prefix().to_string(); + let before = tenant_snapshot(&app, tenant_id, &storage_prefix).await?; + assert!(before.doc_count >= 2); + assert!(before.membership_count > 0); + assert!(before.storage_keys.len() >= 2); + assert_storage_keys_present(&app, &before.storage_keys).await?; + + let job = enqueue_delete_job(&app, tenant_id, true).await?; + let job_id = job.id; + let handler = DeleteTenantJob::new(); + let state = Arc::new(app.state.clone()); + let execution = handler.handle(state, job, storage).await; + assert_job_success(&execution); + record_job_outcome(&app, job_id, &execution).await?; + assert_eq!(fetch_job_status(&app, job_id).await?, STATUS_SUCCEEDED); + + let after = tenant_snapshot(&app, tenant_id, &storage_prefix).await?; + assert_eq!(after.doc_count, 0); + assert_eq!(after.membership_count, 0); + assert!(after.storage_keys.is_empty()); + assert_storage_keys_absent(&app, &before.storage_keys).await?; + assert_eq!(storage_object_count(&app, &storage_prefix).await?, 0); + assert!(!tenant_exists(&app, tenant_id).await?); + + app.cleanup().await?; + Ok(()) +} + +#[tokio::test] +async fn delete_tenant_job_rejects_invalid_signature() -> Result<()> { + let _lock = acquire_db_lock().await; + let app = TestApp::new().await?; + + let password = "delete-invalid"; + app.insert_user("tenant-invalid", TestUserRole::Owner) + .await?; + let token = app.login_token("tenant-invalid", password).await?; + + upload_fixture(&app, &token, "invalid.pdf", b"invalid").await?; + + let tenant_id = default_tenant_id(&app)?; + set_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?; + assert_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?; + + let storage = tenant_storage(&app, tenant_id)?; + let storage_prefix = storage.root_prefix().to_string(); + let before = tenant_snapshot(&app, tenant_id, &storage_prefix).await?; + assert_eq!(before.doc_count, 1); + assert!(before.membership_count > 0); + assert!(!before.storage_keys.is_empty()); + assert_storage_keys_present(&app, &before.storage_keys).await?; + + let job = enqueue_delete_job_with_invalid_signature(&app, tenant_id, false).await?; + let job_id = job.id; + let handler = DeleteTenantJob::new(); + let state = Arc::new(app.state.clone()); + let execution = handler.handle(state, job, storage).await; + match execution { + JobExecution::Failed { ref error } => { + assert!(error.contains("signature"), "unexpected error: {error}"); + } + _ => bail!("delete job should fail when signature is invalid"), + } + record_job_outcome(&app, job_id, &execution).await?; + assert_eq!(fetch_job_status(&app, job_id).await?, STATUS_FAILED); + + let after = tenant_snapshot(&app, tenant_id, &storage_prefix).await?; + assert_eq!(after.doc_count, before.doc_count); + assert_eq!(after.membership_count, before.membership_count); + assert_storage_keys_present(&app, &before.storage_keys).await?; + assert_eq!( + storage_object_count(&app, &storage_prefix).await?, + before.storage_keys.len() + ); + assert_eq!( + fetch_tenant_status(&app, tenant_id).await?, + TenantStatus::Deleting + ); + + app.cleanup().await?; + Ok(()) +} + +#[tokio::test] +async fn delete_tenant_job_rejects_invalid_final_status() -> Result<()> { + let _lock = acquire_db_lock().await; + let app = TestApp::new().await?; + + let password = "delete-invalid-status"; + app.insert_user("tenant-invalid-status", TestUserRole::Owner) + .await?; + let token = app.login_token("tenant-invalid-status", password).await?; + + upload_fixture(&app, &token, "invalid-status.pdf", b"payload").await?; + + let tenant_id = default_tenant_id(&app)?; + set_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?; + assert_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?; + + let storage = tenant_storage(&app, tenant_id)?; + let storage_prefix = storage.root_prefix().to_string(); + let before = tenant_snapshot(&app, tenant_id, &storage_prefix).await?; + assert_eq!(before.doc_count, 1); + assert!(before.membership_count > 0); + + let job = enqueue_delete_job_with_overrides( + &app, + tenant_id, + false, + None, + PayloadOverrides { + final_status: Some("weird"), + ..PayloadOverrides::default() + }, + ) + .await?; + let job_id = job.id; + let handler = DeleteTenantJob::new(); + let state = Arc::new(app.state.clone()); + let execution = handler.handle(state, job, storage).await; + match execution { + JobExecution::Failed { ref error } => { + assert!(error.contains("payload"), "unexpected error: {error}"); + } + _ => bail!("delete job should fail when final_status is invalid"), + } + record_job_outcome(&app, job_id, &execution).await?; + assert_eq!(fetch_job_status(&app, job_id).await?, STATUS_FAILED); + + let after = tenant_snapshot(&app, tenant_id, &storage_prefix).await?; + assert_eq!(after.doc_count, before.doc_count); + assert_eq!(after.membership_count, before.membership_count); + assert_storage_keys_present(&app, &before.storage_keys).await?; + assert_eq!( + fetch_tenant_status(&app, tenant_id).await?, + TenantStatus::Deleting + ); + + app.cleanup().await?; + Ok(()) +} + +#[tokio::test] +async fn delete_tenant_job_rejects_malformed_issued_at() -> Result<()> { + let _lock = acquire_db_lock().await; + let app = TestApp::new().await?; + + let password = "delete-bad-issued-at"; + app.insert_user("tenant-issued", TestUserRole::Owner) + .await?; + let token = app.login_token("tenant-issued", password).await?; + + upload_fixture(&app, &token, "issued.pdf", b"issued").await?; + + let tenant_id = default_tenant_id(&app)?; + set_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?; + assert_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?; + + let storage = tenant_storage(&app, tenant_id)?; + let job = enqueue_delete_job_with_overrides( + &app, + tenant_id, + false, + None, + PayloadOverrides { + issued_at: Some("definitely-not-time"), + ..PayloadOverrides::default() + }, + ) + .await?; + let job_id = job.id; + let handler = DeleteTenantJob::new(); + let state = Arc::new(app.state.clone()); + let execution = handler.handle(state, job, storage).await; + match execution { + JobExecution::Failed { ref error } => { + assert!(error.contains("issued_at"), "unexpected error: {error}"); + } + _ => bail!("delete job should fail when issued_at is malformed"), + } + record_job_outcome(&app, job_id, &execution).await?; + assert_eq!(fetch_job_status(&app, job_id).await?, STATUS_FAILED); + + app.cleanup().await?; + Ok(()) +} + +#[tokio::test] +async fn delete_tenant_job_rejects_stale_confirmation() -> Result<()> { + let _lock = acquire_db_lock().await; + let app = TestApp::new().await?; + + let password = "delete-stale"; + app.insert_user("tenant-stale", TestUserRole::Owner).await?; + let token = app.login_token("tenant-stale", password).await?; + + upload_fixture(&app, &token, "stale.pdf", b"stale").await?; + + let tenant_id = default_tenant_id(&app)?; + set_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?; + assert_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?; + + let storage = tenant_storage(&app, tenant_id)?; + let stale_time = (Utc::now() - ChronoDuration::minutes(10)).to_rfc3339(); + let job = enqueue_delete_job_with_overrides( + &app, + tenant_id, + false, + None, + PayloadOverrides { + issued_at: Some(&stale_time), + ..PayloadOverrides::default() + }, + ) + .await?; + let job_id = job.id; + let handler = DeleteTenantJob::new(); + let state = Arc::new(app.state.clone()); + let execution = handler.handle(state, job, storage).await; + match execution { + JobExecution::Failed { ref error } => { + assert!(error.contains("expired"), "unexpected error: {error}"); + } + _ => bail!("delete job should fail when confirmation is stale"), + } + record_job_outcome(&app, job_id, &execution).await?; + assert_eq!(fetch_job_status(&app, job_id).await?, STATUS_FAILED); + + app.cleanup().await?; + Ok(()) +} + +async fn upload_fixture(app: &TestApp, token: &str, filename: &str, contents: &[u8]) -> Result<()> { + let response = app .upload_document( "/api/documents", - "remove.pdf", + filename, "application/pdf", - b"remove", + contents, None, - &token, + token, ) .await?; - assert_eq!(upload.status(), StatusCode::CREATED); - body_to_vec(upload.into_body()).await?; + assert_eq!(response.status(), StatusCode::CREATED); + body_to_vec(response.into_body()).await?; + Ok(()) +} - let tenant_id = app +fn default_tenant_id(app: &TestApp) -> Result { + Ok(app .state .tenants .get_by_name("test_tenant") .map_err(|err| anyhow!("tenant lookup failed: {err:?}"))? - .id; - set_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?; - - let job = enqueue_delete_job(&app, tenant_id, true).await?; - let handler = DeleteTenantJob::new(); - let state = Arc::new(app.state.clone()); - let storage = app - .state - .storage_for_tenant(tenant_id) - .map_err(|err| anyhow!("storage unavailable: {err:?}"))?; - - let execution = handler.handle(state, job, storage).await; - assert!(matches!( - execution, - papercrate::workers::JobExecution::Success - )); - - assert_eq!(app.storage().object_count().await, 0); - - let tenant_id_for_check = tenant_id; - app.with_conn(move |conn| { - use diesel::dsl::count_star; - - let doc_count: i64 = documents::table.select(count_star()).get_result(conn)?; - assert_eq!(doc_count, 0); - - let membership_count: i64 = user_memberships::table - .select(count_star()) - .get_result(conn)?; - assert_eq!(membership_count, 0, "memberships should be removed"); - - let exists: Option = tenants::table - .find(tenant_id_for_check) - .select(tenants::status) - .first(conn) - .optional()?; - assert!(exists.is_none(), "tenant row should be deleted"); - Ok(()) - }) - .await?; - - app.cleanup().await?; - Ok(()) + .id) } async fn set_tenant_status(app: &TestApp, tenant_id: Uuid, status: TenantStatus) -> Result<()> { @@ -236,6 +395,131 @@ async fn set_tenant_status(app: &TestApp, tenant_id: Uuid, status: TenantStatus) .await } +async fn assert_tenant_status( + app: &TestApp, + tenant_id: Uuid, + expected: TenantStatus, +) -> Result<()> { + let status = fetch_tenant_status(app, tenant_id).await?; + assert_eq!(status, expected); + Ok(()) +} + +async fn tenant_snapshot( + app: &TestApp, + tenant_id: Uuid, + storage_prefix: &str, +) -> Result { + let (doc_count, membership_count) = app + .with_conn(move |conn| { + let doc_count: i64 = documents::table + .filter(documents::tenant_id.eq(tenant_id)) + .select(count_star()) + .get_result(conn)?; + let membership_count: i64 = user_memberships::table + .filter(user_memberships::tenant_id.eq(tenant_id)) + .select(count_star()) + .get_result(conn)?; + Ok::<_, anyhow::Error>((doc_count, membership_count)) + }) + .await?; + + let storage_keys = app.storage().keys_with_prefix(storage_prefix).await; + + Ok(TenantSnapshot { + doc_count, + membership_count, + storage_keys, + }) +} + +async fn tenant_exists(app: &TestApp, tenant_id: Uuid) -> Result { + app.with_conn(move |conn| { + let exists_value: bool = + select(exists(tenants::table.filter(tenants::id.eq(tenant_id)))).get_result(conn)?; + Ok(exists_value) + }) + .await +} + +async fn fetch_tenant_status(app: &TestApp, tenant_id: Uuid) -> Result { + app.with_conn(move |conn| { + tenants::table + .find(tenant_id) + .select(tenants::status) + .first(conn) + .map_err(Into::into) + }) + .await +} + +async fn record_job_outcome(app: &TestApp, job_id: Uuid, execution: &JobExecution) -> Result<()> { + match execution { + JobExecution::Success => { + app.with_conn(move |conn| { + mark_job_succeeded(conn, job_id) + .map_err(|err| anyhow!("mark succeeded failed: {err}")) + }) + .await? + } + JobExecution::Failed { error } => { + let error = error.clone(); + app.with_conn(move |conn| { + mark_job_failed(conn, job_id, &error) + .map_err(|err| anyhow!("mark failed failed: {err}")) + }) + .await? + } + JobExecution::Retry { .. } => bail!("retry outcome not expected in tenant deletion tests"), + } + Ok(()) +} + +async fn fetch_job_status(app: &TestApp, job_id: Uuid) -> Result { + app.with_conn(move |conn| { + jobs::table + .find(job_id) + .select(jobs::status) + .first::(conn) + .map_err(Into::into) + }) + .await +} + +fn tenant_storage(app: &TestApp, tenant_id: Uuid) -> Result { + app.state + .storage_for_tenant(tenant_id) + .map_err(|err| anyhow!("storage unavailable: {err:?}")) +} + +async fn storage_object_count(app: &TestApp, prefix: &str) -> Result { + Ok(app.storage().object_count_with_prefix(prefix).await) +} + +async fn assert_storage_keys_present(app: &TestApp, keys: &[String]) -> Result<()> { + let storage = app.storage(); + for key in keys { + assert!( + storage.contains_key(key).await, + "expected storage object '{}' to exist", + key + ); + } + Ok(()) +} + +async fn assert_storage_keys_absent(app: &TestApp, keys: &[String]) -> Result<()> { + let storage = app.storage(); + for key in keys { + assert!( + !storage.contains_key(key).await, + "expected storage object '{}' to be deleted", + key + ); + } + Ok(()) +} + async fn enqueue_delete_job( app: &TestApp, tenant_id: Uuid, @@ -250,6 +534,58 @@ async fn enqueue_delete_job_with_status( remove_tenant: bool, final_status: Option<&'static str>, ) -> Result { + enqueue_delete_job_with_overrides( + app, + tenant_id, + remove_tenant, + final_status, + PayloadOverrides::default(), + ) + .await +} + +async fn enqueue_delete_job_with_invalid_signature( + app: &TestApp, + tenant_id: Uuid, + remove_tenant: bool, +) -> Result { + let mut job = enqueue_delete_job_with_overrides( + app, + tenant_id, + remove_tenant, + None, + PayloadOverrides::default(), + ) + .await?; + + let job_id = job.id; + let mut payload = job.payload.clone(); + payload["signature"] = json!("deadbeefdeadbeefdeadbeefdeadbeef"); + let payload_for_db = payload.clone(); + + app.with_conn(move |conn| { + diesel::update(jobs::table.find(job_id)) + .set(jobs::payload.eq(payload_for_db)) + .execute(conn)?; + Ok(()) + }) + .await?; + + job.payload = payload; + Ok(job) +} + +async fn enqueue_delete_job_with_overrides( + app: &TestApp, + tenant_id: Uuid, + remove_tenant: bool, + final_status: Option<&str>, + overrides: PayloadOverrides<'_>, +) -> Result { + let secret = app.state.config.jwt_secret.clone(); + let overrides_owned = PayloadOverridesOwned::from(overrides); + let requested_final_status = final_status.map(|value| value.to_string()); + app.with_conn(move |conn| { let tenant_name: String = tenants::table .find(tenant_id) @@ -257,16 +593,123 @@ async fn enqueue_delete_job_with_status( .first(conn) .map_err(|err| anyhow!("tenant lookup failed: {err}"))?; - let mut payload = json!({ - "remove_tenant": remove_tenant, - "tenant_name": tenant_name, - }); - if let Some(status) = final_status { - payload["final_status"] = json!(status); - } + let payload = build_signed_delete_payload( + tenant_id, + &tenant_name, + remove_tenant, + requested_final_status.as_deref(), + &secret, + overrides_owned.as_borrowed(), + )?; enqueue_job(conn, tenant_id, JOB_DELETE_TENANT, payload, None) .map_err(|err| anyhow!("enqueue failed: {err}")) }) .await } + +fn build_signed_delete_payload( + tenant_id: Uuid, + tenant_name: &str, + remove_tenant: bool, + requested_final_status: Option<&str>, + secret: &str, + overrides: PayloadOverrides<'_>, +) -> Result { + let action = if remove_tenant { + DeleteAction::Delete + } else { + DeleteAction::Reset + }; + + let nonce = overrides + .nonce + .map(|value| value.to_string()) + .unwrap_or_else(|| format!("test-delete-nonce-{}", Uuid::new_v4())); + let issued_at = overrides + .issued_at + .map(|value| value.to_string()) + .unwrap_or_else(|| Utc::now().to_rfc3339()); + let payload_final_status = if remove_tenant { + None + } else if let Some(value) = overrides.final_status { + Some(value.to_string()) + } else { + Some(requested_final_status.unwrap_or("suspended").to_string()) + }; + + let message = build_delete_proof_message( + tenant_id, + tenant_name, + action, + &nonce, + &issued_at, + payload_final_status.as_deref(), + ); + let signature = sign_delete_proof(secret, &message) + .map_err(|err| anyhow!("failed to sign delete proof: {err}"))?; + + let mut payload = json!({ + "remove_tenant": remove_tenant, + "tenant_name": tenant_name, + "action": action.as_str(), + "nonce": nonce, + "issued_at": issued_at, + "signature": signature, + }); + if let Some(status) = payload_final_status { + payload["final_status"] = json!(status); + } + + Ok(payload) +} + +struct TenantSnapshot { + doc_count: i64, + membership_count: i64, + storage_keys: Vec, +} + +#[derive(Default)] +struct PayloadOverrides<'a> { + final_status: Option<&'a str>, + issued_at: Option<&'a str>, + nonce: Option<&'a str>, +} + +fn assert_job_success(execution: &JobExecution) { + match execution { + JobExecution::Success => {} + JobExecution::Failed { error } => panic!("delete job failed unexpectedly: {error}"), + JobExecution::Retry { error, .. } => { + panic!("delete job asked for retry unexpectedly: {error}") + } + } +} + +#[derive(Default, Clone)] +struct PayloadOverridesOwned { + final_status: Option, + issued_at: Option, + nonce: Option, +} + +impl<'a> From> for PayloadOverridesOwned { + fn from(value: PayloadOverrides<'a>) -> Self { + Self { + final_status: value.final_status.map(|s| s.to_string()), + issued_at: value.issued_at.map(|s| s.to_string()), + nonce: value.nonce.map(|s| s.to_string()), + } + } +} + +impl PayloadOverridesOwned { + fn as_borrowed(&self) -> PayloadOverrides<'_> { + PayloadOverrides { + final_status: self.final_status.as_deref(), + issued_at: self.issued_at.as_deref(), + nonce: self.nonce.as_deref(), + } + } +}