reduce to 1 object per asset

This commit is contained in:
2025-11-20 16:54:12 +01:00
parent 42f3917881
commit 3ce6b0ce93
24 changed files with 366 additions and 607 deletions
@@ -0,0 +1,24 @@
CREATE TABLE tenant.document_asset_objects (
id UUID PRIMARY KEY,
asset_id UUID NOT NULL REFERENCES tenant.document_assets(id) ON DELETE CASCADE,
ordinal INT NOT NULL,
s3_key TEXT NOT NULL,
metadata JSONB NOT NULL DEFAULT '{}'::jsonb,
tenant_id UUID NOT NULL REFERENCES shared.tenants(id),
CONSTRAINT document_asset_objects_ordinal_positive CHECK (ordinal >= 1),
CONSTRAINT document_asset_objects_asset_ordinal_unique UNIQUE (asset_id, ordinal)
);
CREATE INDEX idx_document_asset_objects_asset_ordinal
ON tenant.document_asset_objects(asset_id, ordinal);
CREATE INDEX document_asset_objects_tenant_id_idx ON tenant.document_asset_objects(tenant_id);
ALTER TABLE tenant.document_assets ADD COLUMN cardinality INT;
UPDATE tenant.document_assets SET cardinality = 1;
INSERT INTO tenant.document_asset_objects (id, asset_id, ordinal, s3_key, metadata, tenant_id)
SELECT gen_random_uuid(), id, 1, s3_key, metadata, tenant_id
FROM tenant.document_assets;
ALTER TABLE tenant.document_assets DROP COLUMN s3_key;
@@ -0,0 +1,27 @@
-- Prevent concurrent inserts/updates during backfill.
LOCK TABLE tenant.document_asset_objects IN ACCESS EXCLUSIVE MODE;
LOCK TABLE tenant.document_assets IN ACCESS EXCLUSIVE MODE;
ALTER TABLE tenant.document_assets ADD COLUMN s3_key TEXT;
UPDATE tenant.document_assets AS da
SET s3_key = o.s3_key,
metadata = COALESCE(da.metadata, '{}'::jsonb) || COALESCE(o.metadata, '{}'::jsonb)
FROM tenant.document_asset_objects AS o
WHERE o.asset_id = da.id
AND o.ordinal = 1;
DELETE FROM tenant.document_asset_objects WHERE ordinal <> 1;
DO $$
BEGIN
IF EXISTS (SELECT 1 FROM tenant.document_assets WHERE s3_key IS NULL) THEN
RAISE EXCEPTION 'cannot drop document_asset_objects: some assets are missing a populated ordinal 1 object (s3_key null)';
END IF;
END
$$;
ALTER TABLE tenant.document_assets ALTER COLUMN s3_key SET NOT NULL;
ALTER TABLE tenant.document_assets DROP COLUMN cardinality;
DROP TABLE tenant.document_asset_objects;
+2 -6
View File
@@ -115,17 +115,13 @@ impl JwtService {
pub fn generate_asset_download_token( pub fn generate_asset_download_token(
&self, &self,
asset_id: Uuid, asset_id: Uuid,
object_id: Uuid,
user_id: Uuid, user_id: Uuid,
tenant_id: Uuid, tenant_id: Uuid,
) -> Result<String> { ) -> Result<String> {
let now = Utc::now(); let now = Utc::now();
let exp = now + self.download_expiry; let exp = now + self.download_expiry;
let claims = DownloadClaims { let claims = DownloadClaims {
subject: DownloadSubject::AssetObject { subject: DownloadSubject::Asset { asset_id },
asset_id,
object_id,
},
user_id, user_id,
tenant_id, tenant_id,
iss: self.issuer.clone(), iss: self.issuer.clone(),
@@ -216,7 +212,7 @@ pub struct Claims {
#[serde(tag = "scope", rename_all = "snake_case")] #[serde(tag = "scope", rename_all = "snake_case")]
pub enum DownloadSubject { pub enum DownloadSubject {
Document { doc_id: Uuid, version_id: Uuid }, Document { doc_id: Uuid, version_id: Uuid },
AssetObject { asset_id: Uuid, object_id: Uuid }, Asset { asset_id: Uuid },
} }
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
+6 -23
View File
@@ -19,14 +19,11 @@ use papercrate::{
jobs::{enqueue_job, JOB_ANALYZE_DOCUMENT, JOB_DELETE_TENANT}, jobs::{enqueue_job, JOB_ANALYZE_DOCUMENT, JOB_DELETE_TENANT},
migrations::MIGRATIONS, migrations::MIGRATIONS,
models::{ models::{
DocumentAsset, DocumentAssetObject, MagicToken, MagicTokenKind, NewUser, NewUserMembership, DocumentAsset, MagicToken, MagicTokenKind, NewUser, NewUserMembership, Tenant,
Tenant, TenantStatus, User, TenantStatus, User,
}, },
s3, s3,
schema::{ schema::{document_assets, documents, magic_tokens, tenants, user_memberships, users},
document_asset_objects, document_assets, documents, magic_tokens, tenants,
user_memberships, users,
},
storage::{ObjectStorage, S3Storage, TenantStorage}, storage::{ObjectStorage, S3Storage, TenantStorage},
tenants::{apply_tenant_guc, clear_tenant_context, TenantService}, tenants::{apply_tenant_guc, clear_tenant_context, TenantService},
utils::{text::normalize_identifier, tracing::init_tracing}, utils::{text::normalize_identifier, tracing::init_tracing},
@@ -788,29 +785,15 @@ async fn delete_assets_for_tenant(
let asset_ids: Vec<Uuid> = assets.iter().map(|asset| asset.id).collect(); let asset_ids: Vec<Uuid> = assets.iter().map(|asset| asset.id).collect();
let objects: Vec<DocumentAssetObject> = document_asset_objects::table for asset in &assets {
.filter(document_asset_objects::tenant_id.eq(tenant.id)) if let Err(err) = tenant_storage.delete_object(&asset.s3_key).await {
.filter(document_asset_objects::asset_id.eq_any(&asset_ids))
.load(&mut conn)
.with_context(|| format!("failed to load asset objects for tenant {}", tenant.name))?;
for object in &objects {
if let Err(err) = tenant_storage.delete_object(&object.s3_key).await {
eprintln!( eprintln!(
"Failed to delete object {} (tenant {}): {err}", "Failed to delete object {} (tenant {}): {err}",
object.s3_key, tenant.name asset.s3_key, tenant.name
); );
} }
} }
diesel::delete(
document_asset_objects::table
.filter(document_asset_objects::tenant_id.eq(tenant.id))
.filter(document_asset_objects::asset_id.eq_any(&asset_ids)),
)
.execute(&mut conn)
.with_context(|| format!("failed to remove asset objects for tenant {}", tenant.name))?;
diesel::delete( diesel::delete(
document_assets::table document_assets::table
.filter(document_assets::tenant_id.eq(tenant.id)) .filter(document_assets::tenant_id.eq(tenant.id))
+16 -62
View File
@@ -8,8 +8,8 @@ use utoipa::ToSchema;
use uuid::Uuid; use uuid::Uuid;
use crate::error::{AppError, AppResult}; use crate::error::{AppError, AppResult};
use crate::models::{Document, DocumentAsset, DocumentAssetObject, DocumentVersion}; use crate::models::{Document, DocumentAsset, DocumentVersion};
use crate::schema::{document_asset_objects, document_assets, document_versions}; use crate::schema::{document_assets, document_versions};
use crate::state::{AppState, PgPooledConnection}; use crate::state::{AppState, PgPooledConnection};
use crate::utils::{http::inline_content_disposition, time::to_iso}; use crate::utils::{http::inline_content_disposition, time::to_iso};
@@ -22,17 +22,6 @@ pub struct DocumentAssetResponse {
pub metadata: Value, pub metadata: Value,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
#[schema(nullable)] #[schema(nullable)]
pub cardinality: Option<i32>,
}
#[derive(Serialize, Clone, ToSchema)]
pub struct DocumentAssetObjectResponse {
pub id: Uuid,
pub ordinal: i32,
#[schema(value_type = Object)]
pub metadata: Value,
#[serde(skip_serializing_if = "Option::is_none")]
#[schema(nullable)]
pub url: Option<String>, pub url: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
#[schema(nullable)] #[schema(nullable)]
@@ -49,9 +38,10 @@ pub struct DocumentAssetDetailResponse {
pub created_at: String, pub created_at: String,
#[serde(skip_serializing_if = "Option::is_none")] #[serde(skip_serializing_if = "Option::is_none")]
#[schema(nullable)] #[schema(nullable)]
pub cardinality: Option<i32>, pub url: Option<String>,
#[serde(default, skip_serializing_if = "Vec::is_empty")] #[serde(skip_serializing_if = "Option::is_none")]
pub objects: Vec<DocumentAssetObjectResponse>, #[schema(nullable)]
pub expires_at: Option<i64>,
} }
#[derive(Serialize, Clone, ToSchema)] #[derive(Serialize, Clone, ToSchema)]
@@ -107,13 +97,15 @@ pub fn to_asset_summary(asset: DocumentAsset) -> DocumentAssetResponse {
asset_type: asset.asset_type, asset_type: asset.asset_type,
mime_type: asset.mime_type, mime_type: asset.mime_type,
metadata: asset.metadata, metadata: asset.metadata,
cardinality: asset.cardinality, url: None,
expires_at: None,
} }
} }
pub fn to_asset_detail_response( pub fn to_asset_detail_response(
asset: DocumentAsset, asset: DocumentAsset,
objects: Vec<DocumentAssetObjectResponse>, url: Option<String>,
expires_at: Option<i64>,
) -> DocumentAssetDetailResponse { ) -> DocumentAssetDetailResponse {
DocumentAssetDetailResponse { DocumentAssetDetailResponse {
id: asset.id, id: asset.id,
@@ -121,30 +113,13 @@ pub fn to_asset_detail_response(
mime_type: asset.mime_type, mime_type: asset.mime_type,
metadata: asset.metadata, metadata: asset.metadata,
created_at: to_iso(asset.created_at), created_at: to_iso(asset.created_at),
cardinality: asset.cardinality,
objects,
}
}
pub fn to_asset_object_response(
object: DocumentAssetObject,
url: Option<String>,
expires_at: Option<i64>,
) -> DocumentAssetObjectResponse {
DocumentAssetObjectResponse {
id: object.id,
ordinal: object.ordinal,
metadata: object.metadata,
url, url,
expires_at, expires_at,
} }
} }
pub fn asset_object_disposition( pub fn asset_disposition(asset: &DocumentAsset) -> Option<String> {
asset: &DocumentAsset, let filename = asset.asset_type.clone();
object: &DocumentAssetObject,
) -> Option<String> {
let filename = format!("{}-{}", asset.asset_type, object.ordinal);
inline_content_disposition(&filename) inline_content_disposition(&filename)
} }
@@ -168,25 +143,13 @@ pub fn load_asset_responses_with_conn(
tenant_id: Uuid, tenant_id: Uuid,
version_id: Uuid, version_id: Uuid,
) -> AppResult<Vec<DocumentAssetResponse>> { ) -> AppResult<Vec<DocumentAssetResponse>> {
let assets: Vec<(DocumentAsset, Option<DocumentAssetObject>)> = document_assets::table let assets: Vec<DocumentAsset> = document_assets::table
.left_outer_join(
document_asset_objects::table.on(document_asset_objects::asset_id
.eq(document_assets::id)
.and(document_asset_objects::ordinal.eq(1))),
)
.filter(document_assets::document_version_id.eq(version_id)) .filter(document_assets::document_version_id.eq(version_id))
.filter(document_assets::tenant_id.eq(tenant_id)) .filter(document_assets::tenant_id.eq(tenant_id))
.order(document_assets::created_at.asc()) .order(document_assets::created_at.asc())
.select((
document_assets::all_columns,
document_asset_objects::all_columns.nullable(),
))
.load(conn)?; .load(conn)?;
Ok(assets Ok(assets.into_iter().map(to_asset_summary).collect())
.into_iter()
.map(|(asset, _)| to_asset_summary(asset))
.collect())
} }
pub fn load_primary_assets( pub fn load_primary_assets(
@@ -216,25 +179,16 @@ pub fn load_primary_assets(
version_map.insert(version.id, version); version_map.insert(version.id, version);
} }
let assets: Vec<(DocumentAsset, Option<DocumentAssetObject>)> = document_assets::table let assets: Vec<DocumentAsset> = document_assets::table
.left_outer_join(
document_asset_objects::table.on(document_asset_objects::asset_id
.eq(document_assets::id)
.and(document_asset_objects::ordinal.eq(1))),
)
.filter(document_assets::document_version_id.eq_any(&version_ids)) .filter(document_assets::document_version_id.eq_any(&version_ids))
.order(( .order((
document_assets::document_version_id.asc(), document_assets::document_version_id.asc(),
document_assets::created_at.asc(), document_assets::created_at.asc(),
)) ))
.select((
document_assets::all_columns,
document_asset_objects::all_columns.nullable(),
))
.load(conn)?; .load(conn)?;
let mut assets_by_version: HashMap<Uuid, Vec<DocumentAssetResponse>> = HashMap::new(); let mut assets_by_version: HashMap<Uuid, Vec<DocumentAssetResponse>> = HashMap::new();
for (asset, _object) in assets { for asset in assets {
let version_id = asset.document_version_id; let version_id = asset.document_version_id;
let response = to_asset_summary(asset); let response = to_asset_summary(asset);
assets_by_version assets_by_version
+1 -24
View File
@@ -621,7 +621,7 @@ pub struct DocumentAsset {
pub mime_type: String, pub mime_type: String,
pub metadata: serde_json::Value, pub metadata: serde_json::Value,
pub created_at: NaiveDateTime, pub created_at: NaiveDateTime,
pub cardinality: Option<i32>, pub s3_key: String,
pub tenant_id: Uuid, pub tenant_id: Uuid,
} }
@@ -633,30 +633,7 @@ pub struct NewDocumentAsset {
pub asset_type: String, pub asset_type: String,
pub mime_type: String, pub mime_type: String,
pub metadata: serde_json::Value, pub metadata: serde_json::Value,
pub cardinality: Option<i32>,
pub tenant_id: Uuid,
}
#[derive(Debug, Clone, Queryable, Identifiable, Associations)]
#[diesel(table_name = document_asset_objects)]
#[diesel(belongs_to(DocumentAsset, foreign_key = asset_id))]
pub struct DocumentAssetObject {
pub id: Uuid,
pub asset_id: Uuid,
pub ordinal: i32,
pub s3_key: String, pub s3_key: String,
pub metadata: serde_json::Value,
pub tenant_id: Uuid,
}
#[derive(Debug, Insertable)]
#[diesel(table_name = document_asset_objects)]
pub struct NewDocumentAssetObject {
pub id: Uuid,
pub asset_id: Uuid,
pub ordinal: i32,
pub s3_key: String,
pub metadata: serde_json::Value,
pub tenant_id: Uuid, pub tenant_id: Uuid,
} }
+4 -4
View File
@@ -74,8 +74,8 @@ pub mod schemas {
}; };
pub use crate::auth::AuthenticatedUser; pub use crate::auth::AuthenticatedUser;
pub use crate::documents::asset::{ pub use crate::documents::asset::{
DocumentAssetDetailResponse, DocumentAssetObjectResponse, DocumentAssetResponse, DocumentAssetDetailResponse, DocumentAssetResponse, DocumentVersionDetailResponse,
DocumentVersionDetailResponse, DocumentVersionResponse, DocumentVersionResponse,
}; };
pub use crate::documents::correspondents::DocumentCorrespondentResponse; pub use crate::documents::correspondents::DocumentCorrespondentResponse;
pub use crate::error::ApiErrorResponse; pub use crate::error::ApiErrorResponse;
@@ -85,8 +85,8 @@ pub mod schemas {
UpdateCorrespondentRequest, UpdateCorrespondentRequest,
}; };
pub use crate::routes::documents::{ pub use crate::routes::documents::{
AssetObjectsQuery, AssetRequestQuery, DocumentCheckQuery, MoveDocumentRequest, AssetRequestQuery, DocumentCheckQuery, MoveDocumentRequest, RestoreDocumentRequest,
RestoreDocumentRequest, UploadDocumentForm, UploadDocumentForm,
}; };
pub use crate::routes::folders::FolderContentsResponse; pub use crate::routes::folders::FolderContentsResponse;
pub use crate::routes::tags::{CreateTagRequest, TagCatalogEntry, UpdateTagRequest}; pub use crate::routes::tags::{CreateTagRequest, TagCatalogEntry, UpdateTagRequest};
+25 -44
View File
@@ -16,17 +16,16 @@ use uuid::Uuid;
use crate::auth::{ensure_active_tenant_with_conn, jwt::DownloadSubject, TenantScopedConn}; use crate::auth::{ensure_active_tenant_with_conn, jwt::DownloadSubject, TenantScopedConn};
use crate::documents::asset::{ use crate::documents::asset::{
asset_object_disposition, DocumentAssetDetailResponse, DocumentAssetResponse, asset_disposition, DocumentAssetDetailResponse, DocumentAssetResponse,
DocumentVersionDetailResponse, DocumentVersionResponse, DocumentVersionDetailResponse, DocumentVersionResponse,
}; };
#[allow(unused_imports)] #[allow(unused_imports)]
use crate::error::ApiErrorResponse; use crate::error::ApiErrorResponse;
use crate::error::{AppError, AppResult}; use crate::error::{AppError, AppResult};
use crate::http::responders::{accepted_json, created_json, no_content, ok_json, JsonResponse}; use crate::http::responders::{accepted_json, created_json, no_content, ok_json, JsonResponse};
use crate::models::{Document, DocumentAsset, DocumentAssetObject, DocumentVersion}; use crate::models::{Document, DocumentAsset, DocumentVersion};
use crate::schema::{ use crate::schema::{
document_asset_objects, document_assets, document_versions, documents, document_assets, document_versions, documents, user_sessions::dsl as session_dsl,
user_sessions::dsl as session_dsl,
}; };
use crate::services::correspondents::{ use crate::services::correspondents::{
AssignCorrespondentsRequest, BulkCorrespondentAction, BulkCorrespondentResponse, AssignCorrespondentsRequest, BulkCorrespondentAction, BulkCorrespondentResponse,
@@ -91,15 +90,6 @@ pub struct RestoreDocumentRequest {
pub folder_id: Option<Uuid>, pub folder_id: Option<Uuid>,
} }
#[derive(Deserialize, Default, IntoParams, ToSchema)]
#[into_params(parameter_in = Query)]
pub struct AssetObjectsQuery {
#[serde(default)]
pub start: Option<i32>,
#[serde(default)]
pub limit: Option<i32>,
}
#[utoipa::path( #[utoipa::path(
get, get,
path = "/api/documents", path = "/api/documents",
@@ -467,12 +457,13 @@ pub async fn list_document_assets(
TenantScopedConn { TenantScopedConn {
mut conn, mut conn,
tenant_id, tenant_id,
user_id,
.. ..
}: TenantScopedConn, }: TenantScopedConn,
) -> AppResult<JsonResponse<Vec<DocumentAssetResponse>>> { ) -> AppResult<JsonResponse<Vec<DocumentAssetResponse>>> {
let service = DocumentsService::new(&state); let service = DocumentsService::new(&state);
let assets = service let assets = service
.list_document_assets(&mut conn, tenant_id, document_id) .list_document_assets(&mut conn, tenant_id, user_id, document_id)
.await?; .await?;
ok_json(assets) ok_json(assets)
} }
@@ -480,14 +471,13 @@ pub async fn list_document_assets(
#[utoipa::path( #[utoipa::path(
get, get,
path = "/api/assets/{asset_id}", path = "/api/assets/{asset_id}",
params(("asset_id" = Uuid, Path, description = "Asset ID"), AssetObjectsQuery), params(("asset_id" = Uuid, Path, description = "Asset ID")),
responses((status = 200, description = "Asset detail", body = DocumentAssetDetailResponse)), responses((status = 200, description = "Asset detail", body = DocumentAssetDetailResponse)),
tag = "Assets" tag = "Assets"
)] )]
pub async fn get_document_asset( pub async fn get_document_asset(
State(state): State<AppState>, State(state): State<AppState>,
Path(asset_id): Path<Uuid>, Path(asset_id): Path<Uuid>,
Query(query): Query<AssetObjectsQuery>,
TenantScopedConn { TenantScopedConn {
conn, conn,
tenant_id, tenant_id,
@@ -495,17 +485,9 @@ pub async fn get_document_asset(
.. ..
}: TenantScopedConn, }: TenantScopedConn,
) -> AppResult<JsonResponse<DocumentAssetDetailResponse>> { ) -> AppResult<JsonResponse<DocumentAssetDetailResponse>> {
let start = query.start.unwrap_or(1);
let limit = query.limit.unwrap_or(1);
if start < 1 {
return Err(AppError::bad_request("start must be at least 1"));
}
if limit < 1 {
return Err(AppError::bad_request("limit must be at least 1"));
}
let service = DocumentsService::new(&state); let service = DocumentsService::new(&state);
let detail = service let detail = service
.get_document_asset(conn, tenant_id, user_id, asset_id, start, limit) .get_document_asset(conn, tenant_id, user_id, asset_id)
.await?; .await?;
ok_json(detail) ok_json(detail)
} }
@@ -637,20 +619,7 @@ pub async fn download_with_token(
) )
.await .await
} }
DownloadSubject::AssetObject { DownloadSubject::Asset { asset_id } => {
asset_id,
object_id,
} => {
if !state.config.proxy_downloads {
return Err(AppError::not_found());
}
let object: DocumentAssetObject = document_asset_objects::table
.find(*object_id)
.filter(document_asset_objects::asset_id.eq(*asset_id))
.filter(document_asset_objects::tenant_id.eq(claims.tenant_id))
.first(&mut conn)?;
let asset: DocumentAsset = document_assets::table let asset: DocumentAsset = document_assets::table
.find(*asset_id) .find(*asset_id)
.filter(document_assets::tenant_id.eq(claims.tenant_id)) .filter(document_assets::tenant_id.eq(claims.tenant_id))
@@ -659,14 +628,28 @@ pub async fn download_with_token(
drop(conn); drop(conn);
let storage = state.storage_for_tenant(claims.tenant_id)?; let storage = state.storage_for_tenant(claims.tenant_id)?;
let disposition = asset_object_disposition(&asset, &object); let disposition = asset_disposition(&asset);
if !state.config.proxy_downloads {
let presigned_url = storage
.presign_get_object(
&asset.s3_key,
Duration::from_secs(PRESIGNED_URL_EXPIRY_SECONDS),
disposition.as_deref(),
)
.await
.storage_context("failed to generate download URL")?;
return Ok(axum::response::Redirect::temporary(&presigned_url).into_response());
}
proxy_storage_object( proxy_storage_object(
storage, storage,
&object.s3_key, &asset.s3_key,
disposition.as_deref(), disposition.as_deref(),
headers.get(header::RANGE).cloned(), headers.get(header::RANGE).cloned(),
Some(asset.mime_type.as_str()), Some(asset.mime_type.as_str()),
Some(object.id.to_string()), Some(asset.id.to_string()),
) )
.await .await
} }
@@ -1078,13 +1061,11 @@ pub async fn remove_tag(
crate::routes::documents::MoveDocumentRequest, crate::routes::documents::MoveDocumentRequest,
crate::routes::documents::BulkReanalyzeSelectionRequest, crate::routes::documents::BulkReanalyzeSelectionRequest,
crate::routes::documents::BulkReanalyzeResponse, crate::routes::documents::BulkReanalyzeResponse,
crate::routes::documents::AssetObjectsQuery,
crate::routes::documents::UploadDocumentForm, crate::routes::documents::UploadDocumentForm,
crate::documents::asset::DocumentVersionResponse, crate::documents::asset::DocumentVersionResponse,
crate::documents::asset::DocumentVersionDetailResponse, crate::documents::asset::DocumentVersionDetailResponse,
crate::documents::asset::DocumentAssetResponse, crate::documents::asset::DocumentAssetResponse,
crate::documents::asset::DocumentAssetDetailResponse, crate::documents::asset::DocumentAssetDetailResponse,
crate::documents::asset::DocumentAssetObjectResponse,
crate::documents::correspondents::DocumentCorrespondentResponse, crate::documents::correspondents::DocumentCorrespondentResponse,
crate::error::ApiErrorResponse, crate::error::ApiErrorResponse,
)) ))
+1 -1
View File
@@ -150,7 +150,7 @@ pub async fn list_folder_contents(
)?; )?;
let documents = if include_documents { let documents = if include_documents {
service.hydrate_documents(&mut conn, user_id, documents)? service.hydrate_documents(&mut conn, tenant_id, user_id, documents)?
} else { } else {
Vec::new() Vec::new()
}; };
+1 -15
View File
@@ -26,17 +26,6 @@ diesel::table! {
} }
} }
diesel::table! {
document_asset_objects (id) {
id -> Uuid,
asset_id -> Uuid,
ordinal -> Int4,
s3_key -> Text,
metadata -> Jsonb,
tenant_id -> Uuid,
}
}
diesel::table! { diesel::table! {
document_assets (id) { document_assets (id) {
id -> Uuid, id -> Uuid,
@@ -45,7 +34,7 @@ diesel::table! {
mime_type -> Text, mime_type -> Text,
metadata -> Jsonb, metadata -> Jsonb,
created_at -> Timestamptz, created_at -> Timestamptz,
cardinality -> Nullable<Int4>, s3_key -> Text,
tenant_id -> Uuid, tenant_id -> Uuid,
} }
} }
@@ -294,8 +283,6 @@ diesel::table! {
diesel::joinable!(correspondents -> tenants (tenant_id)); diesel::joinable!(correspondents -> tenants (tenant_id));
diesel::joinable!(capability_set_capabilities -> capability_sets (capability_set_id)); diesel::joinable!(capability_set_capabilities -> capability_sets (capability_set_id));
diesel::joinable!(capability_sets -> tenants (tenant_id)); diesel::joinable!(capability_sets -> tenants (tenant_id));
diesel::joinable!(document_asset_objects -> document_assets (asset_id));
diesel::joinable!(document_asset_objects -> tenants (tenant_id));
diesel::joinable!(document_assets -> document_versions (document_version_id)); diesel::joinable!(document_assets -> document_versions (document_version_id));
diesel::joinable!(document_assets -> tenants (tenant_id)); diesel::joinable!(document_assets -> tenants (tenant_id));
diesel::joinable!(document_correspondents -> correspondents (correspondent_id)); diesel::joinable!(document_correspondents -> correspondents (correspondent_id));
@@ -329,7 +316,6 @@ diesel::allow_tables_to_appear_in_same_query!(
correspondents, correspondents,
capability_set_capabilities, capability_set_capabilities,
capability_sets, capability_sets,
document_asset_objects,
document_assets, document_assets,
document_correspondents, document_correspondents,
document_tags, document_tags,
+138 -101
View File
@@ -1,9 +1,6 @@
use std::{ use std::collections::{HashMap, HashSet};
collections::{HashMap, HashSet},
time::Duration,
};
use chrono::{DateTime, NaiveDateTime, Utc}; use chrono::{DateTime, Duration as ChronoDuration, NaiveDateTime, Utc};
use diesel::{ use diesel::{
dsl::{exists, not, sql}, dsl::{exists, not, sql},
prelude::*, prelude::*,
@@ -20,11 +17,9 @@ use uuid::Uuid;
use crate::documents::{ use crate::documents::{
asset::{ asset::{
asset_object_disposition, build_download_path, derive_document_title, build_download_path, derive_document_title, filename_with_retained_extension,
filename_with_retained_extension, load_asset_responses_with_conn, load_primary_assets, to_asset_detail_response, to_version_response, DocumentAssetDetailResponse,
to_asset_detail_response, to_asset_object_response, to_version_response, DocumentAssetResponse, DocumentVersionDetailResponse, DocumentVersionResponse,
DocumentAssetDetailResponse, DocumentAssetResponse, DocumentVersionDetailResponse,
DocumentVersionResponse,
}, },
correspondents::{ correspondents::{
insert_document_correspondents, normalize_correspondent_ids, DocumentCorrespondentResponse, insert_document_correspondents, normalize_correspondent_ids, DocumentCorrespondentResponse,
@@ -39,12 +34,10 @@ use crate::documents::{
use crate::error::{AppError, AppResult}; use crate::error::{AppError, AppResult};
use crate::jobs::{enqueue_job, JobQueueError, JOB_ANALYZE_DOCUMENT, JOB_PURGE_DOCUMENT}; use crate::jobs::{enqueue_job, JobQueueError, JOB_ANALYZE_DOCUMENT, JOB_PURGE_DOCUMENT};
use crate::models::{ use crate::models::{
Document, DocumentAsset, DocumentAssetObject, DocumentVersion, NewDocument, NewDocumentVersion, Document, DocumentAsset, DocumentVersion, NewDocument, NewDocumentVersion, Tag,
Tag,
}; };
use crate::schema::{ use crate::schema::{
document_asset_objects, document_assets, document_correspondents, document_tags, document_assets, document_correspondents, document_tags, document_versions, documents, folders,
document_versions, documents, folders,
}; };
use crate::services::{ use crate::services::{
correspondents::CorrespondentAssignmentInput, folders::gather_descendant_folder_ids, correspondents::CorrespondentAssignmentInput, folders::gather_descendant_folder_ids,
@@ -52,17 +45,11 @@ use crate::services::{
}; };
use crate::state::{AppState, PgPooledConnection}; use crate::state::{AppState, PgPooledConnection};
use crate::utils::{ use crate::utils::{
db::validate_bulk_ids, db::validate_bulk_ids, error::StorageResultExt, http::inline_content_disposition,
error::StorageResultExt, json::classify_nullable, json::NullableValue, setops::intersect_option_sets,
http::inline_content_disposition, setops::load_linked_doc_ids, storage_paths::document_version_object_key, time::to_iso,
json::{classify_nullable, NullableValue},
setops::{intersect_option_sets, load_linked_doc_ids},
storage_paths::document_version_object_key,
time::to_iso,
}; };
const PRESIGNED_URL_EXPIRY_SECONDS: u64 = 300;
#[derive(Deserialize, IntoParams, ToSchema, Clone)] #[derive(Deserialize, IntoParams, ToSchema, Clone)]
#[into_params(parameter_in = Query)] #[into_params(parameter_in = Query)]
pub struct DocumentListQuery { pub struct DocumentListQuery {
@@ -371,7 +358,7 @@ impl<'a> DocumentsService<'a> {
.cloned() .cloned()
.unwrap_or_else(|| (Vec::new(), Vec::new())); .unwrap_or_else(|| (Vec::new(), Vec::new()));
let assets = load_asset_responses_with_conn(conn, tenant_id, current_version.id)?; let assets = self.load_asset_responses(conn, tenant_id, current_version.id, user_id)?;
let current_version_data = Some((to_version_response(current_version), assets)); let current_version_data = Some((to_version_response(current_version), assets));
let response = let response =
@@ -601,7 +588,7 @@ impl<'a> DocumentsService<'a> {
} }
let docs: Vec<Document> = docs_query.load(conn)?; let docs: Vec<Document> = docs_query.load(conn)?;
let mut responses = self.hydrate_documents(conn, user_id, docs)?; let mut responses = self.hydrate_documents(conn, tenant_id, user_id, docs)?;
if let Some(order) = quickwit_order { if let Some(order) = quickwit_order {
let order_map: HashMap<Uuid, usize> = order let order_map: HashMap<Uuid, usize> = order
@@ -618,6 +605,7 @@ impl<'a> DocumentsService<'a> {
pub fn hydrate_documents( pub fn hydrate_documents(
&self, &self,
conn: &mut PgPooledConnection, conn: &mut PgPooledConnection,
tenant_id: Uuid,
user_id: Uuid, user_id: Uuid,
docs: Vec<Document>, docs: Vec<Document>,
) -> AppResult<Vec<DocumentResponse>> { ) -> AppResult<Vec<DocumentResponse>> {
@@ -627,14 +615,57 @@ impl<'a> DocumentsService<'a> {
let doc_ids: Vec<Uuid> = docs.iter().map(|doc| doc.id).collect(); let doc_ids: Vec<Uuid> = docs.iter().map(|doc| doc.id).collect();
let mut relations = load_tags_and_correspondents(conn, &doc_ids)?; let mut relations = load_tags_and_correspondents(conn, &doc_ids)?;
let primary_versions = load_primary_assets(conn, &docs)?; let mut doc_to_version: HashMap<Uuid, Uuid> = HashMap::with_capacity(doc_ids.len());
let mut version_ids: Vec<Uuid> = Vec::with_capacity(doc_ids.len());
for doc in &docs {
doc_to_version.insert(doc.id, doc.current_version_id);
version_ids.push(doc.current_version_id);
}
version_ids.sort();
version_ids.dedup();
let versions: Vec<DocumentVersion> = document_versions::table
.filter(document_versions::id.eq_any(&version_ids))
.filter(document_versions::tenant_id.eq(tenant_id))
.load(conn)?;
let mut version_map: HashMap<Uuid, DocumentVersion> = HashMap::new();
for version in versions {
version_map.insert(version.id, version);
}
let assets: Vec<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq_any(&version_ids))
.filter(document_assets::tenant_id.eq(tenant_id))
.order((
document_assets::document_version_id.asc(),
document_assets::created_at.asc(),
))
.load(conn)?;
let mut assets_by_version: HashMap<Uuid, Vec<DocumentAssetResponse>> = HashMap::new();
for asset in assets {
let version_id = asset.document_version_id;
let response = self.asset_response(asset, tenant_id, user_id)?;
assets_by_version
.entry(version_id)
.or_default()
.push(response);
}
docs.into_iter() docs.into_iter()
.map(|doc| { .map(|doc| {
let (tags, correspondents) = relations let (tags, correspondents) = relations
.remove(&doc.id) .remove(&doc.id)
.unwrap_or_else(|| (Vec::new(), Vec::new())); .unwrap_or_else(|| (Vec::new(), Vec::new()));
let current_version = primary_versions.get(&doc.id).cloned(); let current_version = doc_to_version
.get(&doc.id)
.and_then(|version_id| version_map.remove(version_id))
.map(|version| {
let assets = assets_by_version.remove(&version.id).unwrap_or_default();
(to_version_response(version), assets)
});
self.to_document_response(user_id, doc, tags, correspondents, current_version) self.to_document_response(user_id, doc, tags, correspondents, current_version)
}) })
.collect() .collect()
@@ -892,12 +923,24 @@ impl<'a> DocumentsService<'a> {
&self, &self,
conn: &mut PgPooledConnection, conn: &mut PgPooledConnection,
tenant_id: Uuid, tenant_id: Uuid,
user_id: Uuid,
document_id: Uuid, document_id: Uuid,
) -> AppResult<Vec<DocumentAssetResponse>> { ) -> AppResult<Vec<DocumentAssetResponse>> {
let document = load_active_document(conn, tenant_id, document_id)?; let document = load_active_document(conn, tenant_id, document_id)?;
let version_id = document.current_version_id; let version_id = document.current_version_id;
Ok(load_asset_responses_with_conn(conn, tenant_id, version_id)?) let assets = document_assets::table
.filter(document_assets::document_version_id.eq(version_id))
.filter(document_assets::tenant_id.eq(tenant_id))
.order(document_assets::created_at.asc())
.load::<DocumentAsset>(conn)?;
let mut responses = Vec::with_capacity(assets.len());
for asset in assets {
responses.push(self.asset_response(asset, tenant_id, user_id)?);
}
Ok(responses)
} }
pub async fn get_document_asset( pub async fn get_document_asset(
@@ -906,8 +949,6 @@ impl<'a> DocumentsService<'a> {
tenant_id: Uuid, tenant_id: Uuid,
user_id: Uuid, user_id: Uuid,
asset_id: Uuid, asset_id: Uuid,
start: i32,
limit: i32,
) -> AppResult<DocumentAssetDetailResponse> { ) -> AppResult<DocumentAssetDetailResponse> {
let asset: DocumentAsset = match document_assets::table let asset: DocumentAsset = match document_assets::table
.find(asset_id) .find(asset_id)
@@ -919,77 +960,11 @@ impl<'a> DocumentsService<'a> {
None => return Err(AppError::not_found()), None => return Err(AppError::not_found()),
}; };
if start < 1 {
return Err(AppError::bad_request("start must be at least 1"));
}
if limit < 1 {
return Err(AppError::bad_request("limit must be at least 1"));
}
let end = start
.checked_add(limit - 1)
.ok_or_else(|| AppError::bad_request("requested range is too large"))?;
let objects: Vec<DocumentAssetObject> = document_asset_objects::table
.filter(document_asset_objects::asset_id.eq(asset_id))
.filter(document_asset_objects::tenant_id.eq(tenant_id))
.filter(document_asset_objects::ordinal.ge(start))
.filter(document_asset_objects::ordinal.le(end))
.order(document_asset_objects::ordinal.asc())
.load(&mut conn)?;
drop(conn); drop(conn);
let expires_at = Utc::now() let (url, expires_at) = self.asset_download_url(asset.id, tenant_id, user_id)?;
.timestamp_millis()
.checked_add((PRESIGNED_URL_EXPIRY_SECONDS as i64) * 1000)
.ok_or_else(|| AppError::internal("failed to compute expiry timestamp"))?;
let storage = (!self.state.config.proxy_downloads) Ok(to_asset_detail_response(asset, Some(url), Some(expires_at)))
.then(|| self.state.storage_for_tenant(tenant_id))
.transpose()?;
let mut object_responses = Vec::with_capacity(objects.len());
for object in objects {
let response_disposition = asset_object_disposition(&asset, &object);
let (url, object_expires_at) = if self.state.config.proxy_downloads {
let token = self
.state
.jwt
.generate_asset_download_token(asset.id, object.id, user_id, tenant_id)
.map_err(|err| {
error!(error = ?err, "failed to issue asset download token");
AppError::internal("failed to issue asset download token")
})?;
(format!("/api/download/{token}"), None)
} else {
let storage = storage
.as_ref()
.expect("storage preloaded when proxy disabled");
let url = storage
.presign_get_object(
&object.s3_key,
Duration::from_secs(PRESIGNED_URL_EXPIRY_SECONDS),
response_disposition.as_deref(),
)
.await
.storage_context("failed to generate asset URL")?;
(url, Some(expires_at))
};
object_responses.push(to_asset_object_response(
object,
Some(url),
object_expires_at,
));
}
if object_responses.is_empty() {
return Err(AppError::not_found());
}
Ok(to_asset_detail_response(asset, object_responses))
} }
pub fn list_document_versions( pub fn list_document_versions(
@@ -1025,7 +1000,7 @@ impl<'a> DocumentsService<'a> {
.filter(document_versions::tenant_id.eq(tenant_id)) .filter(document_versions::tenant_id.eq(tenant_id))
.first(conn)?; .first(conn)?;
let assets = load_asset_responses_with_conn(conn, tenant_id, version.id)?; let assets = self.load_asset_responses(conn, tenant_id, version.id, user_id)?;
let download_path = build_download_path(self.state, &document, version.id, user_id)?; let download_path = build_download_path(self.state, &document, version.id, user_id)?;
let version_core = to_version_response(version); let version_core = to_version_response(version);
@@ -1180,7 +1155,7 @@ impl<'a> DocumentsService<'a> {
let tags_and_correspondents = load_tags_and_correspondents(conn, &[document_id])?; let tags_and_correspondents = load_tags_and_correspondents(conn, &[document_id])?;
let version_id = current_version.id; let version_id = current_version.id;
let assets = load_asset_responses_with_conn(conn, tenant_id, version_id)?; let assets = self.load_asset_responses(conn, tenant_id, version_id, user_id)?;
let version_response = to_version_response(current_version); let version_response = to_version_response(current_version);
let (tags, correspondents) = tags_and_correspondents let (tags, correspondents) = tags_and_correspondents
.get(&document_id) .get(&document_id)
@@ -1493,7 +1468,7 @@ impl<'a> DocumentsService<'a> {
.cloned() .cloned()
.unwrap_or_else(|| (Vec::new(), Vec::new())); .unwrap_or_else(|| (Vec::new(), Vec::new()));
let assets = load_asset_responses_with_conn(conn, tenant_id, version.id)?; let assets = self.load_asset_responses(conn, tenant_id, version.id, user_id)?;
let version_response = to_version_response(version.clone()); let version_response = to_version_response(version.clone());
info!( info!(
@@ -1514,4 +1489,66 @@ impl<'a> DocumentsService<'a> {
Ok(Some(detail)) Ok(Some(detail))
} }
fn asset_download_url(
&self,
asset_id: Uuid,
tenant_id: Uuid,
user_id: Uuid,
) -> AppResult<(String, i64)> {
let token = self
.state
.jwt
.generate_asset_download_token(asset_id, user_id, tenant_id)
.map_err(|err| {
error!(error = ?err, "failed to issue asset download token");
AppError::internal("failed to issue asset download token")
})?;
let expires_at = Utc::now()
.checked_add_signed(ChronoDuration::minutes(
self.state.config.download_token_expiry_minutes,
))
.ok_or_else(|| AppError::internal("failed to compute download expiry"))?
.timestamp_millis();
Ok((format!("/api/download/{token}"), expires_at))
}
fn asset_response(
&self,
asset: DocumentAsset,
tenant_id: Uuid,
user_id: Uuid,
) -> AppResult<DocumentAssetResponse> {
let (url, expires_at) = self.asset_download_url(asset.id, tenant_id, user_id)?;
Ok(DocumentAssetResponse {
id: asset.id,
asset_type: asset.asset_type,
mime_type: asset.mime_type,
metadata: asset.metadata,
url: Some(url),
expires_at: Some(expires_at),
})
}
fn load_asset_responses(
&self,
conn: &mut PgPooledConnection,
tenant_id: Uuid,
version_id: Uuid,
user_id: Uuid,
) -> AppResult<Vec<DocumentAssetResponse>> {
let assets: Vec<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq(version_id))
.filter(document_assets::tenant_id.eq(tenant_id))
.order(document_assets::created_at.asc())
.load(conn)?;
assets
.into_iter()
.map(|asset| self.asset_response(asset, tenant_id, user_id))
.collect()
}
} }
+2 -1
View File
@@ -555,10 +555,11 @@ impl<'a> FolderService<'a> {
pub fn hydrate_documents( pub fn hydrate_documents(
&self, &self,
conn: &mut PgPooledConnection, conn: &mut PgPooledConnection,
tenant_id: Uuid,
user_id: Uuid, user_id: Uuid,
docs: Vec<Document>, docs: Vec<Document>,
) -> AppResult<Vec<DocumentResponse>> { ) -> AppResult<Vec<DocumentResponse>> {
DocumentsService::new(self.state).hydrate_documents(conn, user_id, docs) DocumentsService::new(self.state).hydrate_documents(conn, tenant_id, user_id, docs)
} }
} }
-1
View File
@@ -877,7 +877,6 @@ async fn prepare_database(pool: &PgPool) -> Result<()> {
fn truncate_all(conn: &mut PgConnection) -> Result<()> { fn truncate_all(conn: &mut PgConnection) -> Result<()> {
conn.batch_execute( conn.batch_execute(
"TRUNCATE TABLE \ "TRUNCATE TABLE \
tenant.document_asset_objects, \
tenant.document_assets, \ tenant.document_assets, \
tenant.document_correspondents, \ tenant.document_correspondents, \
tenant.correspondents, \ tenant.correspondents, \
+3 -23
View File
@@ -48,8 +48,8 @@ fn document_asset_type_prefix(document_id: Uuid, version_number: i32, asset_type
) )
} }
/// Returns the storage prefix under which the asset objects for a type/id pair live. /// Returns the storage key for an asset (single object).
pub fn document_asset_object_prefix( pub fn document_asset_key(
document_id: Uuid, document_id: Uuid,
version_number: i32, version_number: i32,
asset_type: &str, asset_type: &str,
@@ -62,21 +62,6 @@ pub fn document_asset_object_prefix(
) )
} }
/// Returns the full storage key for a specific asset object (ordinal).
pub fn document_asset_object_key(
document_id: Uuid,
version_number: i32,
asset_type: &str,
asset_id: Uuid,
ordinal: i32,
) -> String {
format!(
"{}/{}",
document_asset_object_prefix(document_id, version_number, asset_type, asset_id),
ordinal
)
}
#[cfg(test)] #[cfg(test)]
mod tests { mod tests {
use super::*; use super::*;
@@ -103,13 +88,8 @@ mod tests {
); );
assert_eq!( assert_eq!(
document_asset_object_prefix(document_id, 3, "thumbnail", asset_id), document_asset_key(document_id, 3, "thumbnail", asset_id),
format!("documents/{document_id}/v3/assets/thumbnail/{asset_id}") format!("documents/{document_id}/v3/assets/thumbnail/{asset_id}")
); );
assert_eq!(
document_asset_object_key(document_id, 3, "thumbnail", asset_id, 2),
format!("documents/{document_id}/v3/assets/thumbnail/{asset_id}/2")
);
} }
} }
+3 -21
View File
@@ -3,8 +3,8 @@ use std::collections::HashMap;
use diesel::{prelude::*, PgConnection}; use diesel::{prelude::*, PgConnection};
use uuid::Uuid; use uuid::Uuid;
use crate::models::{Document, DocumentAsset, DocumentAssetObject, DocumentVersion}; use crate::models::{Document, DocumentAsset, DocumentVersion};
use crate::schema::{document_asset_objects, document_assets, document_versions, documents}; use crate::schema::{document_assets, document_versions, documents};
use crate::state::AppState; use crate::state::AppState;
pub(crate) struct LoadedDocumentVersion { pub(crate) struct LoadedDocumentVersion {
@@ -41,7 +41,6 @@ pub(crate) fn load_document_version(
pub struct LoadedAsset { pub struct LoadedAsset {
pub asset: DocumentAsset, pub asset: DocumentAsset,
pub objects: Vec<DocumentAssetObject>,
} }
pub(crate) fn load_version_assets( pub(crate) fn load_version_assets(
@@ -65,26 +64,9 @@ pub(crate) fn load_version_assets(
.load(conn) .load(conn)
.map_err(|err| format!("{err:?}"))?; .map_err(|err| format!("{err:?}"))?;
let asset_ids: Vec<Uuid> = assets.iter().map(|asset| asset.id).collect();
let mut object_map: HashMap<Uuid, Vec<DocumentAssetObject>> = HashMap::new();
if !asset_ids.is_empty() {
let objects: Vec<DocumentAssetObject> = document_asset_objects::table
.filter(document_asset_objects::asset_id.eq_any(&asset_ids))
.filter(document_asset_objects::tenant_id.eq(tenant_id))
.order(document_asset_objects::ordinal.asc())
.load(conn)
.map_err(|err| format!("{err:?}"))?;
for object in objects {
object_map.entry(object.asset_id).or_default().push(object);
}
}
let mut result = HashMap::with_capacity(assets.len()); let mut result = HashMap::with_capacity(assets.len());
for asset in assets { for asset in assets {
let objects = object_map.remove(&asset.id).unwrap_or_default(); result.insert(asset.asset_type.clone(), LoadedAsset { asset });
result.insert(asset.asset_type.clone(), LoadedAsset { asset, objects });
} }
Ok(result) Ok(result)
+1 -5
View File
@@ -48,12 +48,8 @@ impl Task<DocumentVersionTaskContext> for IndexDocumentTask {
.asset(OCR_TEXT_ASSET_TYPE) .asset(OCR_TEXT_ASSET_TYPE)
.await? .await?
.ok_or_else(|| TaskError::fail("missing OCR text asset"))?; .ok_or_else(|| TaskError::fail("missing OCR text asset"))?;
let object = asset
.objects
.first()
.ok_or_else(|| TaskError::fail("missing OCR text object"))?;
let s3_key = object.s3_key.clone(); let s3_key = asset.asset.s3_key.clone();
let bytes = ctx.storage().get_object(&s3_key).await.map_err(|err| { let bytes = ctx.storage().get_object(&s3_key).await.map_err(|err| {
TaskError::retry( TaskError::retry(
Duration::from_secs(30), Duration::from_secs(30),
+1 -2
View File
@@ -430,8 +430,7 @@ async fn load_document_text(ctx: &mut DocumentVersionTaskContext) -> TaskResult<
let assets = ctx.assets().await?; let assets = ctx.assets().await?;
assets assets
.get(OCR_TEXT_ASSET_TYPE) .get(OCR_TEXT_ASSET_TYPE)
.and_then(|asset| asset.objects.first()) .map(|asset| asset.asset.s3_key.clone())
.map(|object| object.s3_key.clone())
}; };
let Some(key) = object_key else { let Some(key) = object_key else {
+9 -41
View File
@@ -19,13 +19,10 @@ use uuid::Uuid;
use crate::{ use crate::{
documents::asset::delete_asset, documents::asset::delete_asset,
error::AppResult, error::AppResult,
models::{ models::{Document, DocumentAsset, DocumentVersion, NewDocumentAsset},
Document, DocumentAsset, DocumentAssetObject, DocumentVersion, NewDocumentAsset, schema::document_assets,
NewDocumentAssetObject,
},
schema::{document_asset_objects, document_assets},
state::AppState, state::AppState,
utils::storage_paths::document_asset_object_prefix, utils::storage_paths::document_asset_key,
}; };
use super::taskflow::{ use super::taskflow::{
@@ -83,7 +80,7 @@ impl Task<DocumentVersionTaskContext> for GenerateOcrTask {
remove_existing_ocr_asset(ctx, &context).await; remove_existing_ocr_asset(ctx, &context).await;
let asset_id = Uuid::new_v4(); let asset_id = Uuid::new_v4();
let s3_key = document_asset_object_prefix( let s3_key = document_asset_key(
context.document.id, context.document.id,
context.version.version_number, context.version.version_number,
OCR_TEXT_ASSET_TYPE, OCR_TEXT_ASSET_TYPE,
@@ -123,7 +120,6 @@ struct OcrContext {
document: Document, document: Document,
version: DocumentVersion, version: DocumentVersion,
existing_asset: Option<DocumentAsset>, existing_asset: Option<DocumentAsset>,
existing_objects: Vec<DocumentAssetObject>,
skip: bool, skip: bool,
} }
@@ -135,16 +131,13 @@ async fn build_ocr_context(
let version = ctx.version().await?.clone(); let version = ctx.version().await?.clone();
let asset = ctx.asset(OCR_TEXT_ASSET_TYPE).await?; let asset = ctx.asset(OCR_TEXT_ASSET_TYPE).await?;
let (existing_asset, existing_objects) = asset let existing_asset = asset.map(|asset| asset.asset.clone());
.map(|asset| (Some(asset.asset.clone()), asset.objects.clone()))
.unwrap_or((None, Vec::new()));
if !document_is_pdf(&document) { if !document_is_pdf(&document) {
return Ok(OcrContext { return Ok(OcrContext {
document, document,
version, version,
existing_asset, existing_asset,
existing_objects,
skip: true, skip: true,
}); });
} }
@@ -155,23 +148,20 @@ async fn build_ocr_context(
document, document,
version, version,
existing_asset, existing_asset,
existing_objects,
skip, skip,
}) })
} }
async fn remove_existing_ocr_asset(ctx: &DocumentVersionTaskContext, context: &OcrContext) { async fn remove_existing_ocr_asset(ctx: &DocumentVersionTaskContext, context: &OcrContext) {
if let Some(existing_asset) = &context.existing_asset { if let Some(existing_asset) = &context.existing_asset {
for object in &context.existing_objects { if let Err(err) = ctx.storage().delete_object(&existing_asset.s3_key).await {
if let Err(err) = ctx.storage().delete_object(&object.s3_key).await {
warn!( warn!(
job_id = %ctx.job_id(), job_id = %ctx.job_id(),
error = %err, error = %err,
s3_key = %object.s3_key, s3_key = %existing_asset.s3_key,
"failed to delete existing ocr asset object" "failed to delete existing ocr asset object"
); );
} }
}
let tenant_id = context.document.tenant_id; let tenant_id = context.document.tenant_id;
let asset_id = existing_asset.id; let asset_id = existing_asset.id;
@@ -239,7 +229,7 @@ fn persist_ocr_metadata(
asset_type: OCR_TEXT_ASSET_TYPE.to_string(), asset_type: OCR_TEXT_ASSET_TYPE.to_string(),
mime_type: "text/plain".to_string(), mime_type: "text/plain".to_string(),
metadata, metadata,
cardinality: Some(1), s3_key: s3_key.to_string(),
tenant_id, tenant_id,
}; };
@@ -253,34 +243,12 @@ fn persist_ocr_metadata(
.set(( .set((
document_assets::mime_type.eq(excluded(document_assets::mime_type)), document_assets::mime_type.eq(excluded(document_assets::mime_type)),
document_assets::metadata.eq(excluded(document_assets::metadata)), document_assets::metadata.eq(excluded(document_assets::metadata)),
document_assets::cardinality.eq(excluded(document_assets::cardinality)), document_assets::s3_key.eq(excluded(document_assets::s3_key)),
document_assets::id.eq(excluded(document_assets::id)), document_assets::id.eq(excluded(document_assets::id)),
)) ))
.execute(&mut conn) .execute(&mut conn)
.map_err(|err| format!("{err:?}"))?; .map_err(|err| format!("{err:?}"))?;
diesel::delete(
document_asset_objects::table
.filter(document_asset_objects::asset_id.eq(asset_id))
.filter(document_asset_objects::tenant_id.eq(tenant_id)),
)
.execute(&mut conn)
.map_err(|err| format!("{err:?}"))?;
let object = NewDocumentAssetObject {
id: Uuid::new_v4(),
asset_id,
ordinal: 1,
s3_key: s3_key.to_string(),
metadata: json!({}),
tenant_id,
};
diesel::insert_into(document_asset_objects::table)
.values(&object)
.execute(&mut conn)
.map_err(|err| format!("{err:?}"))?;
Ok(()) Ok(())
} }
+3 -13
View File
@@ -11,7 +11,7 @@ use uuid::Uuid;
use crate::auth::ensure_active_tenant; use crate::auth::ensure_active_tenant;
use crate::jobs::JOB_PURGE_DOCUMENT; use crate::jobs::JOB_PURGE_DOCUMENT;
use crate::models::{Document, DocumentVersion}; use crate::models::{Document, DocumentVersion};
use crate::schema::{document_asset_objects, document_assets, document_versions}; use crate::schema::{document_assets, document_versions};
use crate::state::AppState; use crate::state::AppState;
use crate::storage::TenantStorage; use crate::storage::TenantStorage;
@@ -235,21 +235,11 @@ fn prepare_purge_context(
let asset_keys = if version_ids.is_empty() { let asset_keys = if version_ids.is_empty() {
Vec::new() Vec::new()
} else { } else {
let asset_ids: Vec<Uuid> = document_assets::table document_assets::table
.filter(document_assets::document_version_id.eq_any(&version_ids)) .filter(document_assets::document_version_id.eq_any(&version_ids))
.filter(document_assets::tenant_id.eq(tenant_id)) .filter(document_assets::tenant_id.eq(tenant_id))
.select(document_assets::id) .select(document_assets::s3_key)
.load(conn)?;
if asset_ids.is_empty() {
Vec::new()
} else {
document_asset_objects::table
.filter(document_asset_objects::asset_id.eq_any(&asset_ids))
.filter(document_asset_objects::tenant_id.eq(tenant_id))
.select(document_asset_objects::s3_key)
.load(conn)? .load(conn)?
}
}; };
Ok(Some(PurgeContext { Ok(Some(PurgeContext {
+4 -9
View File
@@ -22,9 +22,8 @@ use crate::documents::search::{delete_quickwit_index, ensure_quickwit_index};
use crate::jobs::{JOB_DELETE_TENANT, JOB_PROVISION_TENANT}; use crate::jobs::{JOB_DELETE_TENANT, JOB_PROVISION_TENANT};
use crate::models::{NewUserMembership, Tenant, TenantStatus}; use crate::models::{NewUserMembership, Tenant, TenantStatus};
use crate::schema::{ use crate::schema::{
api_tokens, correspondents, document_asset_objects, document_assets, document_correspondents, api_tokens, correspondents, document_assets, document_correspondents, document_tags,
document_tags, document_versions, documents, folders, tags, tenants, user_memberships, document_versions, documents, folders, tags, tenants, user_memberships, user_sessions,
user_sessions,
}; };
use crate::state::AppState; use crate::state::AppState;
use crate::tenants::TenantRepository; use crate::tenants::TenantRepository;
@@ -401,8 +400,8 @@ fn collect_object_keys(conn: &mut PgConnection) -> Result<TenantObjectKeys, dies
let version_keys = document_versions::table let version_keys = document_versions::table
.select(document_versions::s3_key) .select(document_versions::s3_key)
.load::<String>(conn)?; .load::<String>(conn)?;
let asset_keys = document_asset_objects::table let asset_keys = document_assets::table
.select(document_asset_objects::s3_key) .select(document_assets::s3_key)
.load::<String>(conn)?; .load::<String>(conn)?;
Ok(TenantObjectKeys { Ok(TenantObjectKeys {
@@ -459,10 +458,6 @@ fn delete_tenant_rows(
remove_memberships: bool, remove_memberships: bool,
) -> Result<(), diesel::result::Error> { ) -> Result<(), diesel::result::Error> {
conn.transaction(|conn| { conn.transaction(|conn| {
diesel::delete(
document_asset_objects::table.filter(document_asset_objects::tenant_id.eq(tenant_id)),
)
.execute(conn)?;
diesel::delete(document_assets::table.filter(document_assets::tenant_id.eq(tenant_id))) diesel::delete(document_assets::table.filter(document_assets::tenant_id.eq(tenant_id)))
.execute(conn)?; .execute(conn)?;
diesel::delete( diesel::delete(
+47 -148
View File
@@ -5,7 +5,7 @@ use chrono::Utc;
use diesel::{pg::upsert::excluded, prelude::*}; use diesel::{pg::upsert::excluded, prelude::*};
use image::{GenericImageView, ImageFormat, ImageReader}; use image::{GenericImageView, ImageFormat, ImageReader};
use pdfium_render::prelude::*; use pdfium_render::prelude::*;
use serde_json::{json, Map, Value}; use serde_json::{Map, Value};
use tokio::task; use tokio::task;
use tracing::{info, warn}; use tracing::{info, warn};
use uuid::Uuid; use uuid::Uuid;
@@ -13,13 +13,10 @@ use uuid::Uuid;
use crate::{ use crate::{
documents::asset::delete_asset, documents::asset::delete_asset,
error::AppResult, error::AppResult,
models::{ models::{Document, DocumentAsset, DocumentVersion, NewDocumentAsset},
Document, DocumentAsset, DocumentAssetObject, DocumentVersion, NewDocumentAsset, schema::{document_assets, document_versions},
NewDocumentAssetObject,
},
schema::{document_asset_objects, document_assets, document_versions},
state::AppState, state::AppState,
utils::storage_paths::document_asset_object_key, utils::storage_paths::document_asset_key,
}; };
use super::{ use super::{
@@ -82,7 +79,7 @@ impl Task<DocumentVersionTaskContext> for GenerateThumbnailsTask {
let thumbnail_asset_id = Uuid::new_v4(); let thumbnail_asset_id = Uuid::new_v4();
let thumbnail_objects = upload_generated_objects( let thumbnail_persistence = upload_generated_asset(
ctx, ctx,
&context, &context,
THUMBNAIL_ASSET_TYPE, THUMBNAIL_ASSET_TYPE,
@@ -91,11 +88,7 @@ impl Task<DocumentVersionTaskContext> for GenerateThumbnailsTask {
) )
.await?; .await?;
let asset_persistences = vec![AssetPersistence { let asset_persistences = vec![thumbnail_persistence];
asset_type: THUMBNAIL_ASSET_TYPE,
asset_id: thumbnail_asset_id,
objects: thumbnail_objects,
}];
let state = ctx.state().clone(); let state = ctx.state().clone();
let tenant_id = context.document.tenant_id; let tenant_id = context.document.tenant_id;
@@ -131,32 +124,22 @@ async fn build_thumbnail_context(
document, document,
version, version,
existing_thumbnail: None, existing_thumbnail: None,
existing_thumbnail_objects: Vec::new(),
skip: true, skip: true,
tenant_id, tenant_id,
}); });
} }
let assets = ctx.assets().await?; let assets = ctx.assets().await?;
let (existing_thumbnail, existing_thumbnail_objects) = assets let existing_thumbnail = assets
.get(THUMBNAIL_ASSET_TYPE) .get(THUMBNAIL_ASSET_TYPE)
.map(|entry| (Some(entry.asset.clone()), entry.objects.clone())) .map(|entry| entry.asset.clone());
.unwrap_or((None, Vec::new()));
let expected_cardinality = expected_asset_cardinality(&document, &version);
let thumbnail_cardinality = existing_thumbnail
.as_ref()
.and_then(|asset| asset.cardinality)
.unwrap_or_else(|| existing_thumbnail_objects.len() as i32);
let needs_regeneration = thumbnail_cardinality < expected_cardinality
|| (existing_thumbnail_objects.len() as i32) < expected_cardinality;
let skip = existing_thumbnail.is_some() && !force && !needs_regeneration; let skip = existing_thumbnail.is_some() && !force;
Ok(ThumbnailContext { Ok(ThumbnailContext {
document, document,
version, version,
existing_thumbnail, existing_thumbnail,
existing_thumbnail_objects,
skip, skip,
tenant_id, tenant_id,
}) })
@@ -167,26 +150,19 @@ async fn remove_existing_thumbnail_assets(
context: &ThumbnailContext, context: &ThumbnailContext,
) { ) {
if let Some(existing_thumbnail) = &context.existing_thumbnail { if let Some(existing_thumbnail) = &context.existing_thumbnail {
delete_asset_with_objects(ctx, existing_thumbnail, &context.existing_thumbnail_objects) delete_asset_object(ctx, existing_thumbnail).await;
.await;
} }
} }
async fn delete_asset_with_objects( async fn delete_asset_object(ctx: &DocumentVersionTaskContext, asset: &DocumentAsset) {
ctx: &DocumentVersionTaskContext, if let Err(err) = ctx.storage().delete_object(&asset.s3_key).await {
asset: &DocumentAsset,
objects: &[DocumentAssetObject],
) {
for object in objects {
if let Err(err) = ctx.storage().delete_object(&object.s3_key).await {
warn!( warn!(
job_id = %ctx.job_id(), job_id = %ctx.job_id(),
error = %err, error = %err,
s3_key = %object.s3_key, s3_key = %asset.s3_key,
"failed to delete existing asset object" "failed to delete existing asset object"
); );
} }
}
let tenant_id = ctx.tenant_id(); let tenant_id = ctx.tenant_id();
let asset_id = asset.id; let asset_id = asset.id;
@@ -217,26 +193,20 @@ async fn delete_asset_with_objects(
} }
} }
async fn upload_generated_objects( async fn upload_generated_asset(
ctx: &DocumentVersionTaskContext, ctx: &DocumentVersionTaskContext,
context: &ThumbnailContext, context: &ThumbnailContext,
asset_type: &str, asset_type: &str,
asset_id: Uuid, asset_id: Uuid,
asset: &GeneratedAsset, asset: &GeneratedAsset,
) -> TaskResult<Vec<AssetObjectPersistence>> { ) -> TaskResult<AssetPersistence> {
let mut objects = Vec::with_capacity(asset.objects.len()); let image = &asset.image;
for (index, image) in asset.objects.iter().enumerate() { let s3_key = document_asset_key(
if index + 1 > i32::MAX as usize {
return Err(TaskError::fail("too many generated asset objects"));
}
let ordinal = (index + 1) as i32;
let s3_key = document_asset_object_key(
context.document.id, context.document.id,
context.version.version_number, context.version.version_number,
asset_type, asset_type,
asset_id, asset_id,
ordinal,
); );
ctx.storage() ctx.storage()
@@ -249,22 +219,19 @@ async fn upload_generated_objects(
.await .await
.map_err(|err| TaskError::retry(Duration::from_secs(30), err.to_string()))?; .map_err(|err| TaskError::retry(Duration::from_secs(30), err.to_string()))?;
objects.push(AssetObjectPersistence { Ok(AssetPersistence {
ordinal, asset_type: asset_type.to_string(),
asset_id,
s3_key, s3_key,
width: image.width, width: image.width,
height: image.height, height: image.height,
}); })
}
Ok(objects)
} }
struct ThumbnailContext { struct ThumbnailContext {
document: Document, document: Document,
version: DocumentVersion, version: DocumentVersion,
existing_thumbnail: Option<DocumentAsset>, existing_thumbnail: Option<DocumentAsset>,
existing_thumbnail_objects: Vec<DocumentAssetObject>,
skip: bool, skip: bool,
tenant_id: Uuid, tenant_id: Uuid,
} }
@@ -276,7 +243,7 @@ struct GeneratedImage {
} }
struct GeneratedAsset { struct GeneratedAsset {
objects: Vec<GeneratedImage>, image: GeneratedImage,
} }
struct GeneratedAssets { struct GeneratedAssets {
@@ -284,19 +251,14 @@ struct GeneratedAssets {
page_count: Option<u32>, page_count: Option<u32>,
} }
struct AssetObjectPersistence { struct AssetPersistence {
ordinal: i32, asset_type: String,
asset_id: Uuid,
s3_key: String, s3_key: String,
width: Option<i32>, width: Option<i32>,
height: Option<i32>, height: Option<i32>,
} }
struct AssetPersistence {
asset_type: &'static str,
asset_id: Uuid,
objects: Vec<AssetObjectPersistence>,
}
fn generate_thumbnails(document: &Document, bytes: &[u8]) -> Result<GeneratedAssets, String> { fn generate_thumbnails(document: &Document, bytes: &[u8]) -> Result<GeneratedAssets, String> {
if document_is_pdf(document) { if document_is_pdf(document) {
let pdf_assets = generate_pdf_assets(bytes)?; let pdf_assets = generate_pdf_assets(bytes)?;
@@ -334,9 +296,7 @@ fn generate_image_assets(bytes: &[u8]) -> Result<GeneratedAsset, String> {
let thumbnail = encode_dynamic_image(thumbnail_image)?; let thumbnail = encode_dynamic_image(thumbnail_image)?;
Ok(GeneratedAsset { Ok(GeneratedAsset { image: thumbnail })
objects: vec![thumbnail],
})
} }
struct PdfGeneratedAssets { struct PdfGeneratedAssets {
@@ -354,6 +314,9 @@ fn generate_pdf_assets(bytes: &[u8]) -> Result<PdfGeneratedAssets, String> {
let pages = document.pages(); let pages = document.pages();
let total_pages = pages.len() as usize; let total_pages = pages.len() as usize;
if total_pages == 0 {
return Err("pdf has no pages".to_string());
}
let render_config = PdfRenderConfig::new() let render_config = PdfRenderConfig::new()
.set_target_width(RENDER_WIDTH as i32) .set_target_width(RENDER_WIDTH as i32)
@@ -361,16 +324,11 @@ fn generate_pdf_assets(bytes: &[u8]) -> Result<PdfGeneratedAssets, String> {
.render_form_data(true) .render_form_data(true)
.rotate_if_landscape(PdfPageRenderRotation::None, true); .rotate_if_landscape(PdfPageRenderRotation::None, true);
let mut thumbnail_objects: Vec<GeneratedImage> = Vec::with_capacity(total_pages); let first_page = pages.get(0).map_err(|err| format!("load page 0: {err}"))?;
for page_index in 0..total_pages { let bitmap = first_page
let page = pages
.get(u16::try_from(page_index).map_err(|_| "page index overflow".to_string())?)
.map_err(|err| format!("load page {page_index}: {err}"))?;
let bitmap = page
.render_with_config(&render_config) .render_with_config(&render_config)
.map_err(|err| format!("render pdf page {page_index}: {err}"))?; .map_err(|err| format!("render pdf page 0: {err}"))?;
let render_buffer = bitmap.as_image().to_rgb8(); let render_buffer = bitmap.as_image().to_rgb8();
let render_image = image::DynamicImage::ImageRgb8(render_buffer); let render_image = image::DynamicImage::ImageRgb8(render_buffer);
@@ -382,16 +340,13 @@ fn generate_pdf_assets(bytes: &[u8]) -> Result<PdfGeneratedAssets, String> {
render_image.clone() render_image.clone()
}; };
thumbnail_objects.push(encode_dynamic_image(thumbnail_image)?);
}
let page_count: u32 = total_pages let page_count: u32 = total_pages
.try_into() .try_into()
.map_err(|_| "page count exceeds supported range".to_string())?; .map_err(|_| "page count exceeds supported range".to_string())?;
Ok(PdfGeneratedAssets { Ok(PdfGeneratedAssets {
thumbnail: GeneratedAsset { thumbnail: GeneratedAsset {
objects: thumbnail_objects, image: encode_dynamic_image(thumbnail_image)?,
}, },
page_count, page_count,
}) })
@@ -421,28 +376,25 @@ fn persist_assets_metadata(
.map_err(|err| format!("{err:?}"))?; .map_err(|err| format!("{err:?}"))?;
for asset in assets { for asset in assets {
if asset.objects.is_empty() { let mut metadata_map = Map::new();
return Err(format!( if let Some(width) = asset.width {
"asset {} has no generated objects", metadata_map.insert("width".to_string(), Value::from(width));
asset.asset_type
));
} }
if let Some(height) = asset.height {
let object_count: i32 = asset metadata_map.insert("height".to_string(), Value::from(height));
.objects }
.len() metadata_map.insert(
.try_into() "generated_at".to_string(),
.map_err(|_| "asset contains too many objects".to_string())?; Value::from(Utc::now().to_rfc3339()),
);
let new_asset = NewDocumentAsset { let new_asset = NewDocumentAsset {
id: asset.asset_id, id: asset.asset_id,
document_version_id: version_id, document_version_id: version_id,
asset_type: asset.asset_type.to_string(), asset_type: asset.asset_type.clone(),
mime_type: "image/webp".to_string(), mime_type: "image/webp".to_string(),
metadata: json!({ metadata: Value::Object(metadata_map),
"generated_at": Utc::now().to_rfc3339(), s3_key: asset.s3_key.clone(),
}),
cardinality: Some(object_count),
tenant_id, tenant_id,
}; };
@@ -456,44 +408,10 @@ fn persist_assets_metadata(
.set(( .set((
document_assets::mime_type.eq(excluded(document_assets::mime_type)), document_assets::mime_type.eq(excluded(document_assets::mime_type)),
document_assets::metadata.eq(excluded(document_assets::metadata)), document_assets::metadata.eq(excluded(document_assets::metadata)),
document_assets::cardinality.eq(excluded(document_assets::cardinality)), document_assets::s3_key.eq(excluded(document_assets::s3_key)),
)) ))
.execute(&mut conn) .execute(&mut conn)
.map_err(|err| format!("{err:?}"))?; .map_err(|err| format!("{err:?}"))?;
diesel::delete(
document_asset_objects::table
.filter(document_asset_objects::asset_id.eq(new_asset.id))
.filter(document_asset_objects::tenant_id.eq(tenant_id)),
)
.execute(&mut conn)
.map_err(|err| format!("{err:?}"))?;
for object in &asset.objects {
let mut metadata_map = Map::new();
if let Some(width) = object.width {
metadata_map.insert("width".to_string(), Value::from(width));
}
if let Some(height) = object.height {
metadata_map.insert("height".to_string(), Value::from(height));
}
let object_metadata = Value::Object(metadata_map);
let new_object = NewDocumentAssetObject {
id: Uuid::new_v4(),
asset_id: new_asset.id,
ordinal: object.ordinal,
s3_key: object.s3_key.clone(),
metadata: object_metadata,
tenant_id,
};
diesel::insert_into(document_asset_objects::table)
.values(&new_object)
.execute(&mut conn)
.map_err(|err| format!("{err:?}"))?;
}
} }
Ok(()) Ok(())
@@ -557,22 +475,3 @@ fn document_is_pdf(document: &Document) -> bool {
.unwrap_or(false) .unwrap_or(false)
}) })
} }
fn expected_asset_cardinality(document: &Document, version: &DocumentVersion) -> i32 {
if let Value::Object(map) = &version.metadata {
if let Some(count) = map.get("page_count").and_then(|v| v.as_i64()) {
if count > 0 {
return count
.min(i64::from(i32::MAX))
.try_into()
.unwrap_or(i32::MAX);
}
}
}
if document_is_pdf(document) {
1
} else {
1
}
}
+6 -26
View File
@@ -7,8 +7,8 @@ use serde_json::{json, Value};
use uuid::Uuid; use uuid::Uuid;
use papercrate::jobs::{mark_job_succeeded, JOB_PURGE_DOCUMENT}; use papercrate::jobs::{mark_job_succeeded, JOB_PURGE_DOCUMENT};
use papercrate::models::{Job, NewDocumentAsset, NewDocumentAssetObject}; use papercrate::models::{Job, NewDocumentAsset};
use papercrate::schema::{document_asset_objects, document_assets}; use papercrate::schema::document_assets;
use papercrate::workers::{purge::PurgeDocumentJob, JobExecution, JobHandler}; use papercrate::workers::{purge::PurgeDocumentJob, JobExecution, JobHandler};
use std::sync::Arc; use std::sync::Arc;
#[derive(Deserialize)] #[derive(Deserialize)]
@@ -66,12 +66,6 @@ struct DocumentAssetInfo {
#[derive(Deserialize)] #[derive(Deserialize)]
struct AssetProxyDetail { struct AssetProxyDetail {
id: Uuid,
objects: Vec<AssetProxyObject>,
}
#[derive(Deserialize)]
struct AssetProxyObject {
id: Uuid, id: Uuid,
url: Option<String>, url: Option<String>,
expires_at: Option<i64>, expires_at: Option<i64>,
@@ -326,7 +320,7 @@ async fn asset_detail_uses_proxy_urls_when_configured() -> Result<()> {
.map_err(|err| anyhow!("tenant connection: {err:?}"))?; .map_err(|err| anyhow!("tenant connection: {err:?}"))?;
let asset_id = Uuid::new_v4(); let asset_id = Uuid::new_v4();
let object_id = Uuid::new_v4(); let s3_key = "objects/preview.png".to_string();
diesel::insert_into(document_assets::table) diesel::insert_into(document_assets::table)
.values(&NewDocumentAsset { .values(&NewDocumentAsset {
@@ -335,18 +329,7 @@ async fn asset_detail_uses_proxy_urls_when_configured() -> Result<()> {
asset_type: "preview".to_string(), asset_type: "preview".to_string(),
mime_type: "image/png".to_string(), mime_type: "image/png".to_string(),
metadata: json!({}), metadata: json!({}),
cardinality: Some(1), s3_key: s3_key.clone(),
tenant_id,
})
.execute(&mut conn)?;
diesel::insert_into(document_asset_objects::table)
.values(&NewDocumentAssetObject {
id: object_id,
asset_id,
ordinal: 1,
s3_key: "objects/preview.png".to_string(),
metadata: json!({}),
tenant_id, tenant_id,
}) })
.execute(&mut conn)?; .execute(&mut conn)?;
@@ -360,15 +343,12 @@ async fn asset_detail_uses_proxy_urls_when_configured() -> Result<()> {
let body = body_to_vec(response.into_body()).await?; let body = body_to_vec(response.into_body()).await?;
let asset_detail: AssetProxyDetail = serde_json::from_slice(&body)?; let asset_detail: AssetProxyDetail = serde_json::from_slice(&body)?;
assert_eq!(asset_detail.id, asset_id); assert_eq!(asset_detail.id, asset_id);
assert_eq!(asset_detail.objects.len(), 1); let url = asset_detail
let object = &asset_detail.objects[0];
assert_eq!(object.id, object_id);
let url = object
.url .url
.as_deref() .as_deref()
.ok_or_else(|| anyhow!("missing url"))?; .ok_or_else(|| anyhow!("missing url"))?;
assert!(url.starts_with("/api/download/")); assert!(url.starts_with("/api/download/"));
assert!(object.expires_at.is_none()); assert!(asset_detail.expires_at.is_some());
app.cleanup().await?; app.cleanup().await?;
Ok(()) Ok(())
@@ -65,6 +65,11 @@ spec:
secretKeyRef: secretKeyRef:
name: {{ $migrateSecretName }} name: {{ $migrateSecretName }}
key: {{ $migrateSecretKey }} key: {{ $migrateSecretKey }}
- name: DATABASE_URL
valueFrom:
secretKeyRef:
name: {{ $migrateSecretName }}
key: {{ $migrateSecretKey }}
{{- range .Values.migrateJob.extraEnv }} {{- range .Values.migrateJob.extraEnv }}
- name: {{ .name }} - name: {{ .name }}
value: {{ .value | quote }} value: {{ .value | quote }}