nils/papercrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

reduce to 1 object per asset

Browse Source
This commit is contained in:
nils
2025-11-20 16:54:12 +01:00
parent 42f3917881
commit 3ce6b0ce93
24 changed files with 366 additions and 607 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/services/documents.rs
+138 -101
View File
@@ -1,9 +1,6 @@
use std::{
collections::{HashMap, HashSet},
time::Duration,
};
use std::collections::{HashMap, HashSet};
use chrono::{DateTime, NaiveDateTime, Utc};
use chrono::{DateTime, Duration as ChronoDuration, NaiveDateTime, Utc};
use diesel::{
dsl::{exists, not, sql},
prelude::*,
@@ -20,11 +17,9 @@ use uuid::Uuid;
use crate::documents::{
asset::{
asset_object_disposition, build_download_path, derive_document_title,
filename_with_retained_extension, load_asset_responses_with_conn, load_primary_assets,
to_asset_detail_response, to_asset_object_response, to_version_response,
DocumentAssetDetailResponse, DocumentAssetResponse, DocumentVersionDetailResponse,
DocumentVersionResponse,
build_download_path, derive_document_title, filename_with_retained_extension,
to_asset_detail_response, to_version_response, DocumentAssetDetailResponse,
DocumentAssetResponse, DocumentVersionDetailResponse, DocumentVersionResponse,
},
correspondents::{
insert_document_correspondents, normalize_correspondent_ids, DocumentCorrespondentResponse,
@@ -39,12 +34,10 @@ use crate::documents::{
use crate::error::{AppError, AppResult};
use crate::jobs::{enqueue_job, JobQueueError, JOB_ANALYZE_DOCUMENT, JOB_PURGE_DOCUMENT};
use crate::models::{
Document, DocumentAsset, DocumentAssetObject, DocumentVersion, NewDocument, NewDocumentVersion,
Tag,
Document, DocumentAsset, DocumentVersion, NewDocument, NewDocumentVersion, Tag,
};
use crate::schema::{
document_asset_objects, document_assets, document_correspondents, document_tags,
document_versions, documents, folders,
document_assets, document_correspondents, document_tags, document_versions, documents, folders,
};
use crate::services::{
correspondents::CorrespondentAssignmentInput, folders::gather_descendant_folder_ids,
@@ -52,17 +45,11 @@ use crate::services::{
};
use crate::state::{AppState, PgPooledConnection};
use crate::utils::{
db::validate_bulk_ids,
error::StorageResultExt,
http::inline_content_disposition,
json::{classify_nullable, NullableValue},
setops::{intersect_option_sets, load_linked_doc_ids},
storage_paths::document_version_object_key,
time::to_iso,
db::validate_bulk_ids, error::StorageResultExt, http::inline_content_disposition,
json::classify_nullable, json::NullableValue, setops::intersect_option_sets,
setops::load_linked_doc_ids, storage_paths::document_version_object_key, time::to_iso,
};
const PRESIGNED_URL_EXPIRY_SECONDS: u64 = 300;
#[derive(Deserialize, IntoParams, ToSchema, Clone)]
#[into_params(parameter_in = Query)]
pub struct DocumentListQuery {
@@ -371,7 +358,7 @@ impl<'a> DocumentsService<'a> {
.cloned()
.unwrap_or_else(|| (Vec::new(), Vec::new()));
let assets = load_asset_responses_with_conn(conn, tenant_id, current_version.id)?;
let assets = self.load_asset_responses(conn, tenant_id, current_version.id, user_id)?;
let current_version_data = Some((to_version_response(current_version), assets));
let response =
@@ -601,7 +588,7 @@ impl<'a> DocumentsService<'a> {
}
let docs: Vec<Document> = docs_query.load(conn)?;
let mut responses = self.hydrate_documents(conn, user_id, docs)?;
let mut responses = self.hydrate_documents(conn, tenant_id, user_id, docs)?;
if let Some(order) = quickwit_order {
let order_map: HashMap<Uuid, usize> = order
@@ -618,6 +605,7 @@ impl<'a> DocumentsService<'a> {
pub fn hydrate_documents(
&self,
conn: &mut PgPooledConnection,
tenant_id: Uuid,
user_id: Uuid,
docs: Vec<Document>,
) -> AppResult<Vec<DocumentResponse>> {
@@ -627,14 +615,57 @@ impl<'a> DocumentsService<'a> {
let doc_ids: Vec<Uuid> = docs.iter().map(|doc| doc.id).collect();
let mut relations = load_tags_and_correspondents(conn, &doc_ids)?;
let primary_versions = load_primary_assets(conn, &docs)?;
let mut doc_to_version: HashMap<Uuid, Uuid> = HashMap::with_capacity(doc_ids.len());
let mut version_ids: Vec<Uuid> = Vec::with_capacity(doc_ids.len());
for doc in &docs {
doc_to_version.insert(doc.id, doc.current_version_id);
version_ids.push(doc.current_version_id);
}
version_ids.sort();
version_ids.dedup();
let versions: Vec<DocumentVersion> = document_versions::table
.filter(document_versions::id.eq_any(&version_ids))
.filter(document_versions::tenant_id.eq(tenant_id))
.load(conn)?;
let mut version_map: HashMap<Uuid, DocumentVersion> = HashMap::new();
for version in versions {
version_map.insert(version.id, version);
}
let assets: Vec<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq_any(&version_ids))
.filter(document_assets::tenant_id.eq(tenant_id))
.order((
document_assets::document_version_id.asc(),
document_assets::created_at.asc(),
))
.load(conn)?;
let mut assets_by_version: HashMap<Uuid, Vec<DocumentAssetResponse>> = HashMap::new();
for asset in assets {
let version_id = asset.document_version_id;
let response = self.asset_response(asset, tenant_id, user_id)?;
assets_by_version
.entry(version_id)
.or_default()
.push(response);
}
docs.into_iter()
.map(|doc| {
let (tags, correspondents) = relations
.remove(&doc.id)
.unwrap_or_else(|| (Vec::new(), Vec::new()));
let current_version = primary_versions.get(&doc.id).cloned();
let current_version = doc_to_version
.get(&doc.id)
.and_then(|version_id| version_map.remove(version_id))
.map(|version| {
let assets = assets_by_version.remove(&version.id).unwrap_or_default();
(to_version_response(version), assets)
});
self.to_document_response(user_id, doc, tags, correspondents, current_version)
})
.collect()
@@ -892,12 +923,24 @@ impl<'a> DocumentsService<'a> {
&self,
conn: &mut PgPooledConnection,
tenant_id: Uuid,
user_id: Uuid,
document_id: Uuid,
) -> AppResult<Vec<DocumentAssetResponse>> {
let document = load_active_document(conn, tenant_id, document_id)?;
let version_id = document.current_version_id;
Ok(load_asset_responses_with_conn(conn, tenant_id, version_id)?)
let assets = document_assets::table
.filter(document_assets::document_version_id.eq(version_id))
.filter(document_assets::tenant_id.eq(tenant_id))
.order(document_assets::created_at.asc())
.load::<DocumentAsset>(conn)?;
let mut responses = Vec::with_capacity(assets.len());
for asset in assets {
responses.push(self.asset_response(asset, tenant_id, user_id)?);
}
Ok(responses)
}
pub async fn get_document_asset(
@@ -906,8 +949,6 @@ impl<'a> DocumentsService<'a> {
tenant_id: Uuid,
user_id: Uuid,
asset_id: Uuid,
start: i32,
limit: i32,
) -> AppResult<DocumentAssetDetailResponse> {
let asset: DocumentAsset = match document_assets::table
.find(asset_id)
@@ -919,77 +960,11 @@ impl<'a> DocumentsService<'a> {
None => return Err(AppError::not_found()),
};
if start < 1 {
return Err(AppError::bad_request("start must be at least 1"));
}
if limit < 1 {
return Err(AppError::bad_request("limit must be at least 1"));
}
let end = start
.checked_add(limit - 1)
.ok_or_else(|| AppError::bad_request("requested range is too large"))?;
let objects: Vec<DocumentAssetObject> = document_asset_objects::table
.filter(document_asset_objects::asset_id.eq(asset_id))
.filter(document_asset_objects::tenant_id.eq(tenant_id))
.filter(document_asset_objects::ordinal.ge(start))
.filter(document_asset_objects::ordinal.le(end))
.order(document_asset_objects::ordinal.asc())
.load(&mut conn)?;
drop(conn);
let expires_at = Utc::now()
.timestamp_millis()
.checked_add((PRESIGNED_URL_EXPIRY_SECONDS as i64) * 1000)
.ok_or_else(|| AppError::internal("failed to compute expiry timestamp"))?;
let (url, expires_at) = self.asset_download_url(asset.id, tenant_id, user_id)?;
let storage = (!self.state.config.proxy_downloads)
.then(|| self.state.storage_for_tenant(tenant_id))
.transpose()?;
let mut object_responses = Vec::with_capacity(objects.len());
for object in objects {
let response_disposition = asset_object_disposition(&asset, &object);
let (url, object_expires_at) = if self.state.config.proxy_downloads {
let token = self
.state
.jwt
.generate_asset_download_token(asset.id, object.id, user_id, tenant_id)
.map_err(|err| {
error!(error = ?err, "failed to issue asset download token");
AppError::internal("failed to issue asset download token")
})?;
(format!("/api/download/{token}"), None)
} else {
let storage = storage
.as_ref()
.expect("storage preloaded when proxy disabled");
let url = storage
.presign_get_object(
&object.s3_key,
Duration::from_secs(PRESIGNED_URL_EXPIRY_SECONDS),
response_disposition.as_deref(),
)
.await
.storage_context("failed to generate asset URL")?;
(url, Some(expires_at))
};
object_responses.push(to_asset_object_response(
object,
Some(url),
object_expires_at,
));
}
if object_responses.is_empty() {
return Err(AppError::not_found());
}
Ok(to_asset_detail_response(asset, object_responses))
Ok(to_asset_detail_response(asset, Some(url), Some(expires_at)))
}
pub fn list_document_versions(
@@ -1025,7 +1000,7 @@ impl<'a> DocumentsService<'a> {
.filter(document_versions::tenant_id.eq(tenant_id))
.first(conn)?;
let assets = load_asset_responses_with_conn(conn, tenant_id, version.id)?;
let assets = self.load_asset_responses(conn, tenant_id, version.id, user_id)?;
let download_path = build_download_path(self.state, &document, version.id, user_id)?;
let version_core = to_version_response(version);
@@ -1180,7 +1155,7 @@ impl<'a> DocumentsService<'a> {
let tags_and_correspondents = load_tags_and_correspondents(conn, &[document_id])?;
let version_id = current_version.id;
let assets = load_asset_responses_with_conn(conn, tenant_id, version_id)?;
let assets = self.load_asset_responses(conn, tenant_id, version_id, user_id)?;
let version_response = to_version_response(current_version);
let (tags, correspondents) = tags_and_correspondents
.get(&document_id)
@@ -1493,7 +1468,7 @@ impl<'a> DocumentsService<'a> {
.cloned()
.unwrap_or_else(|| (Vec::new(), Vec::new()));
let assets = load_asset_responses_with_conn(conn, tenant_id, version.id)?;
let assets = self.load_asset_responses(conn, tenant_id, version.id, user_id)?;
let version_response = to_version_response(version.clone());
info!(
@@ -1514,4 +1489,66 @@ impl<'a> DocumentsService<'a> {
Ok(Some(detail))
}
fn asset_download_url(
&self,
asset_id: Uuid,
tenant_id: Uuid,
user_id: Uuid,
) -> AppResult<(String, i64)> {
let token = self
.state
.jwt
.generate_asset_download_token(asset_id, user_id, tenant_id)
.map_err(|err| {
error!(error = ?err, "failed to issue asset download token");
AppError::internal("failed to issue asset download token")
})?;
let expires_at = Utc::now()
.checked_add_signed(ChronoDuration::minutes(
self.state.config.download_token_expiry_minutes,
))
.ok_or_else(|| AppError::internal("failed to compute download expiry"))?
.timestamp_millis();
Ok((format!("/api/download/{token}"), expires_at))
}
fn asset_response(
&self,
asset: DocumentAsset,
tenant_id: Uuid,
user_id: Uuid,
) -> AppResult<DocumentAssetResponse> {
let (url, expires_at) = self.asset_download_url(asset.id, tenant_id, user_id)?;
Ok(DocumentAssetResponse {
id: asset.id,
asset_type: asset.asset_type,
mime_type: asset.mime_type,
metadata: asset.metadata,
url: Some(url),
expires_at: Some(expires_at),
})
}
fn load_asset_responses(
&self,
conn: &mut PgPooledConnection,
tenant_id: Uuid,
version_id: Uuid,
user_id: Uuid,
) -> AppResult<Vec<DocumentAssetResponse>> {
let assets: Vec<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq(version_id))
.filter(document_assets::tenant_id.eq(tenant_id))
.order(document_assets::created_at.asc())
.load(conn)?;
assets
.into_iter()
.map(|asset| self.asset_response(asset, tenant_id, user_id))
.collect()
}
}