nils/papercrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

2

Browse Source
This commit is contained in:
nils
2025-10-10 09:39:58 +02:00
parent 4a428b9af6
commit ddce0e39b3
36 changed files with 6410 additions and 1419 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/routes/documents.rs
+451 -120
View File
@@ -1,5 +1,4 @@
use std::collections::HashMap;
use std::time::Duration;
use std::{collections::HashMap, time::Duration};
use axum::extract::{Json, Multipart, Path, Query, State};
use axum::http::StatusCode;
@@ -8,16 +7,18 @@ use chrono::{DateTime, NaiveDateTime, Utc};
use diesel::dsl::exists;
use diesel::{prelude::*, PgConnection};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use serde_json::{json, Value};
use sha2::{Digest, Sha256};
use tracing::{error, info, warn};
use uuid::Uuid;
use crate::auth::AuthenticatedUser;
use crate::error::{AppError, AppResult};
use crate::jobs::{enqueue_job, JOB_ANALYZE_DOCUMENT};
use crate::models::{
Document, DocumentVersion, NewDocument, NewDocumentTag, NewDocumentVersion, Tag,
Document, DocumentAsset, DocumentVersion, NewDocument, NewDocumentTag, NewDocumentVersion, Tag,
};
use crate::schema::{document_tags, document_versions, documents, folders, tags};
use crate::schema::{document_assets, document_tags, document_versions, documents, folders, tags};
use crate::state::AppState;
const PRESIGNED_URL_EXPIRY_SECONDS: u64 = 300;
@@ -29,6 +30,12 @@ pub struct DocumentListQuery {
pub include_deleted: bool,
}
#[derive(Deserialize)]
pub struct AssetRequestQuery {
#[serde(default)]
pub force: bool,
}
#[derive(Serialize)]
pub struct TagResponse {
pub id: Uuid,
@@ -59,9 +66,10 @@ pub struct DocumentResponse {
pub deleted_at: Option<String>,
pub metadata: Value,
pub tags: Vec<TagResponse>,
pub thumbnail: Option<DocumentAssetResponse>,
}
#[derive(Serialize)]
#[derive(Serialize, Clone)]
pub struct DocumentVersionResponse {
pub id: Uuid,
pub version_number: i32,
@@ -72,10 +80,22 @@ pub struct DocumentVersionResponse {
pub operations_summary: Value,
}
#[derive(Serialize, Clone)]
pub struct DocumentAssetResponse {
pub id: Uuid,
pub asset_type: String,
pub mime_type: String,
pub width: Option<i32>,
pub height: Option<i32>,
pub url: String,
pub created_at: String,
}
#[derive(Serialize)]
pub struct DocumentDetailResponse {
pub document: DocumentResponse,
pub current_version: DocumentVersionResponse,
pub assets: Vec<DocumentAssetResponse>,
}
#[derive(Serialize)]
@@ -87,6 +107,24 @@ pub struct DocumentDownloadResponse {
pub size_bytes: i64,
}
#[derive(Serialize)]
pub struct BulkReanalyzeResponse {
pub queued: usize,
}
struct UploadRequest {
bytes: Vec<u8>,
original_name: String,
content_type: Option<String>,
folder_id: Option<Uuid>,
metadata: Value,
}
struct UploadOutcome {
detail: DocumentDetailResponse,
created: bool,
}
#[derive(Deserialize)]
pub struct MoveDocumentRequest {
pub folder_id: Option<Uuid>,
@@ -124,12 +162,16 @@ pub async fn list_documents(
let doc_ids: Vec<Uuid> = docs.iter().map(|doc| doc.id).collect();
let tags_map = load_tags_for_documents(&mut conn, &doc_ids)?;
drop(conn);
let thumbnails = load_primary_thumbnails(&state, &docs).await?;
let response = docs
.into_iter()
.map(|doc| {
let tags = tags_map.get(&doc.id).cloned();
to_document_response(doc, tags)
let thumbnail = thumbnails.get(&doc.id).cloned();
to_document_response(doc, tags, thumbnail)
})
.collect();
@@ -153,10 +195,19 @@ pub async fn get_document(
.first(&mut conn)?;
let tags_map = load_tags_for_documents(&mut conn, &[document_id])?;
let version_id = current_version.id;
drop(conn);
let assets = load_asset_responses(&state, version_id).await?;
let thumbnail = assets
.iter()
.find(|asset| asset.asset_type == "thumbnail")
.cloned();
Ok(Json(DocumentDetailResponse {
document: to_document_response(doc, tags_map.get(&document_id).cloned()),
document: to_document_response(doc, tags_map.get(&document_id).cloned(), thumbnail),
current_version: to_version_response(current_version),
assets,
}))
}
@@ -164,18 +215,18 @@ pub async fn upload_document(
State(state): State<AppState>,
_user: AuthenticatedUser,
mut multipart: Multipart,
) -> AppResult<Json<DocumentDetailResponse>> {
) -> AppResult<(StatusCode, Json<DocumentDetailResponse>)> {
let mut file_bytes: Option<Vec<u8>> = None;
let mut original_name: Option<String> = None;
let mut content_type: Option<String> = None;
let mut folder_id: Option<Uuid> = None;
let mut metadata: Value = Value::Object(Default::default());
while let Some(field) = multipart
.next_field()
.await
.map_err(|err| AppError::bad_request(format!("invalid multipart data: {err}")))?
{
while let Some(field) = multipart.next_field().await.map_err(|err| {
let msg = format!("invalid multipart data: {err}");
warn!(error = %err, "invalid multipart data");
AppError::bad_request(msg)
})? {
let name = field.name().map(|n| n.to_string());
match name.as_deref() {
Some("file") => {
@@ -183,15 +234,18 @@ pub async fn upload_document(
original_name = file_name.clone();
content_type = field.content_type().map(|mime| mime.to_string());
let data = field.bytes().await.map_err(|err| {
AppError::bad_request(format!("failed to read file bytes: {err}"))
let msg = format!("failed to read file bytes: {err}");
warn!(error = %err, "failed to read file bytes");
AppError::bad_request(msg)
})?;
file_bytes = Some(data.to_vec());
}
Some("folder_id") => {
let value = field
.text()
.await
.map_err(|err| AppError::bad_request(format!("invalid folder id: {err}")))?;
let value = field.text().await.map_err(|err| {
let msg = format!("invalid folder id: {err}");
warn!(error = %err, "invalid folder id");
AppError::bad_request(msg)
})?;
if !value.trim().is_empty() {
let parsed = Uuid::parse_str(value.trim())
.map_err(|_| AppError::bad_request("folder_id must be a valid UUID"))?;
@@ -199,127 +253,138 @@ pub async fn upload_document(
}
}
Some("metadata") => {
let value = field
.text()
.await
.map_err(|err| AppError::bad_request(format!("invalid metadata: {err}")))?;
let value = field.text().await.map_err(|err| {
let msg = format!("invalid metadata: {err}");
warn!(error = %err, "invalid metadata payload");
AppError::bad_request(msg)
})?;
metadata = serde_json::from_str(&value).map_err(|err| {
AppError::bad_request(format!("metadata must be valid JSON: {err}"))
let msg = format!("metadata must be valid JSON: {err}");
warn!(error = %err, "metadata parse failure");
AppError::bad_request(msg)
})?;
}
_ => {}
}
}
let file_bytes = file_bytes.ok_or_else(|| AppError::bad_request("file field is required"))?;
let file_bytes = file_bytes.ok_or_else(|| {
warn!("upload rejected: missing file field");
AppError::bad_request("file field is required")
})?;
let original_name = original_name.unwrap_or_else(|| "upload.bin".to_string());
if let Some(folder_id) = folder_id {
ensure_folder_exists(&state, folder_id)?;
}
let request = UploadRequest {
bytes: file_bytes,
original_name,
content_type,
folder_id,
metadata,
};
let doc_id = Uuid::new_v4();
let version_id = Uuid::new_v4();
let version_number = 1;
let stored_filename = original_name.clone();
let checksum = Sha256::digest(&file_bytes);
let checksum_hex = hex::encode(checksum);
let size_bytes = file_bytes.len() as i64;
let s3_key = format!("documents/{doc_id}/v{version_number}/{version_id}");
{
let mut conn = state.db()?;
let existing = documents::table
.inner_join(
document_versions::table.on(document_versions::document_id
.eq(documents::id)
.and(document_versions::version_number.eq(documents::current_version))),
)
.filter(document_versions::checksum.eq(&checksum_hex))
.select((documents::all_columns, document_versions::all_columns))
.first::<(Document, DocumentVersion)>(&mut conn)
.optional()?;
if let Some((mut document, version)) = existing {
if document.deleted_at.is_some() {
let now = Utc::now().naive_utc();
diesel::update(documents::table.find(document.id))
.set((
documents::deleted_at.eq(None::<NaiveDateTime>),
documents::updated_at.eq(now),
))
.execute(&mut conn)?;
document.deleted_at = None;
document.updated_at = now;
}
let tags_map = load_tags_for_documents(&mut conn, &[document.id])?;
let tags = tags_map.get(&document.id).cloned();
return Ok(Json(DocumentDetailResponse {
document: to_document_response(document, tags),
current_version: to_version_response(version),
}));
let outcome = match process_upload(&state, request).await {
Ok(outcome) => outcome,
Err(err) => {
error!(error = ?err, "document upload failed");
return Err(err);
}
};
info!(
document_id = %outcome.detail.document.id,
original_name = %outcome.detail.document.original_name,
created = outcome.created,
"document upload succeeded"
);
let status = if outcome.created {
StatusCode::CREATED
} else {
StatusCode::OK
};
Ok((status, Json(outcome.detail)))
}
pub async fn request_document_assets(
State(state): State<AppState>,
Path(document_id): Path<Uuid>,
Query(query): Query<AssetRequestQuery>,
) -> AppResult<StatusCode> {
let mut conn = state.db()?;
let document: Document = documents::table.find(document_id).first(&mut conn)?;
if document.deleted_at.is_some() {
return Err(AppError::not_found());
}
state
.storage
.put_object(&s3_key, file_bytes.clone(), content_type.clone())
.await
.map_err(|err| AppError::internal(format!("failed to store document: {err}")))?;
let version: DocumentVersion = document_versions::table
.filter(document_versions::document_id.eq(document_id))
.filter(document_versions::version_number.eq(document.current_version))
.first(&mut conn)?;
let metadata_value = if metadata.is_null() {
Value::Object(Default::default())
} else {
metadata
};
enqueue_job(
&mut conn,
JOB_ANALYZE_DOCUMENT,
json!({
"document_id": document_id,
"document_version_id": version.id,
"force": query.force,
}),
None,
)
.map_err(|err| AppError::internal(format!("failed to enqueue analyze job: {err}")))?;
let (document, version) = {
let mut conn = state.db()?;
conn.transaction(|conn| {
let new_document = NewDocument {
id: doc_id,
filename: stored_filename.clone(),
original_name: original_name.clone(),
content_type: content_type.clone(),
folder_id,
current_version: version_number,
metadata: metadata_value.clone(),
};
diesel::insert_into(documents::table)
.values(&new_document)
.execute(conn)?;
Ok(StatusCode::ACCEPTED)
}
let new_version = NewDocumentVersion {
id: version_id,
document_id: doc_id,
version_number,
s3_key: s3_key.clone(),
size_bytes,
checksum: checksum_hex.clone(),
operations_summary: Value::Object(Default::default()),
};
pub async fn reanalyze_all_documents(
State(state): State<AppState>,
_user: AuthenticatedUser,
) -> AppResult<(StatusCode, Json<BulkReanalyzeResponse>)> {
let mut conn = state.db()?;
diesel::insert_into(document_versions::table)
.values(&new_version)
.execute(conn)?;
let targets: Vec<(Uuid, Uuid)> = document_versions::table
.inner_join(documents::table.on(document_versions::document_id.eq(documents::id)))
.filter(documents::deleted_at.is_null())
.filter(document_versions::version_number.eq(documents::current_version))
.select((documents::id, document_versions::id))
.load(&mut conn)?;
let document: Document = documents::table.find(doc_id).first(conn)?;
let version: DocumentVersion = document_versions::table.find(version_id).first(conn)?;
let mut queued = 0usize;
for (document_id, version_id) in targets {
enqueue_job(
&mut conn,
JOB_ANALYZE_DOCUMENT,
json!({
"document_id": document_id,
"document_version_id": version_id,
"force": true,
}),
None,
)
.map_err(|err| AppError::internal(format!("failed to enqueue analyze job: {err}")))?;
queued += 1;
}
Ok::<_, diesel::result::Error>((document, version))
})?
};
Ok((StatusCode::ACCEPTED, Json(BulkReanalyzeResponse { queued })))
}
let response = DocumentDetailResponse {
document: to_document_response(document, None),
current_version: to_version_response(version),
};
pub async fn list_document_assets(
State(state): State<AppState>,
Path(document_id): Path<Uuid>,
) -> AppResult<Json<Vec<DocumentAssetResponse>>> {
let mut conn = state.db()?;
let document: Document = documents::table.find(document_id).first(&mut conn)?;
if document.deleted_at.is_some() {
return Err(AppError::not_found());
}
Ok(Json(response))
let version: DocumentVersion = document_versions::table
.filter(document_versions::document_id.eq(document_id))
.filter(document_versions::version_number.eq(document.current_version))
.first(&mut conn)?;
drop(conn);
let assets = load_asset_responses(&state, version.id).await?;
Ok(Json(assets))
}
pub async fn download_document(
@@ -449,6 +514,157 @@ pub async fn remove_tag(
Ok(StatusCode::NO_CONTENT)
}
async fn process_upload(state: &AppState, request: UploadRequest) -> AppResult<UploadOutcome> {
let UploadRequest {
bytes,
original_name,
content_type,
folder_id,
metadata,
} = request;
if let Some(folder) = folder_id {
ensure_folder_exists(state, folder)?;
}
let doc_id = Uuid::new_v4();
let version_id = Uuid::new_v4();
let version_number = 1;
let stored_filename = original_name.clone();
let checksum = Sha256::digest(&bytes);
let checksum_hex = hex::encode(checksum);
let size_bytes = bytes.len() as i64;
let s3_key = format!("documents/{doc_id}/v{version_number}/{version_id}");
{
let mut conn = state.db()?;
let existing = documents::table
.inner_join(
document_versions::table.on(document_versions::document_id
.eq(documents::id)
.and(document_versions::version_number.eq(documents::current_version))),
)
.filter(document_versions::checksum.eq(&checksum_hex))
.select((documents::all_columns, document_versions::all_columns))
.first::<(Document, DocumentVersion)>(&mut conn)
.optional()?;
if let Some((mut document, version)) = existing {
if document.deleted_at.is_some() {
let now = Utc::now().naive_utc();
diesel::update(documents::table.find(document.id))
.set((
documents::deleted_at.eq(None::<NaiveDateTime>),
documents::updated_at.eq(now),
))
.execute(&mut conn)?;
document.deleted_at = None;
document.updated_at = now;
}
let tags_map = load_tags_for_documents(&mut conn, &[document.id])?;
let tags = tags_map.get(&document.id).cloned();
drop(conn);
let assets = load_asset_responses(state, version.id).await?;
let thumbnail = assets
.iter()
.find(|asset| asset.asset_type == "thumbnail")
.cloned();
return Ok(UploadOutcome {
detail: DocumentDetailResponse {
document: to_document_response(document, tags, thumbnail),
current_version: to_version_response(version),
assets,
},
created: false,
});
}
}
state
.storage
.put_object(&s3_key, bytes.clone(), content_type.clone())
.await
.map_err(|err| {
error!(error = %err, key = %s3_key, "failed to store document");
AppError::internal(format!("failed to store document: {err}"))
})?;
let metadata_value = if metadata.is_null() {
Value::Object(Default::default())
} else {
metadata
};
let (document, version) = {
let mut conn = state.db()?;
conn.transaction(|conn| {
let new_document = NewDocument {
id: doc_id,
filename: stored_filename.clone(),
original_name: original_name.clone(),
content_type: content_type.clone(),
folder_id,
current_version: version_number,
metadata: metadata_value.clone(),
};
diesel::insert_into(documents::table)
.values(&new_document)
.execute(conn)?;
let new_version = NewDocumentVersion {
id: version_id,
document_id: doc_id,
version_number,
s3_key: s3_key.clone(),
size_bytes,
checksum: checksum_hex.clone(),
operations_summary: Value::Object(Default::default()),
};
diesel::insert_into(document_versions::table)
.values(&new_version)
.execute(conn)?;
let document: Document = documents::table.find(doc_id).first(conn)?;
let version: DocumentVersion = document_versions::table.find(version_id).first(conn)?;
Ok::<_, diesel::result::Error>((document, version))
})?
};
let detail = DocumentDetailResponse {
document: to_document_response(document, None, None),
current_version: to_version_response(version.clone()),
assets: Vec::new(),
};
if let Ok(mut conn) = state.db() {
if let Err(err) = enqueue_job(
&mut conn,
JOB_ANALYZE_DOCUMENT,
json!({
"document_id": doc_id,
"document_version_id": version.id,
"force": false,
}),
None,
) {
warn!(document_id = %doc_id, error = %err, "failed to enqueue analyze job");
}
} else {
warn!(document_id = %doc_id, "failed to enqueue analyze job due to pool error");
}
Ok(UploadOutcome {
detail,
created: true,
})
}
fn ensure_folder_exists(state: &AppState, folder_id: Uuid) -> AppResult<()> {
let mut conn = state.db()?;
let exists: bool = diesel::select(exists(folders::table.filter(folders::id.eq(folder_id))))
@@ -480,7 +696,82 @@ pub(crate) fn load_tags_for_documents(
Ok(map)
}
pub(crate) fn to_document_response(doc: Document, tags: Option<Vec<Tag>>) -> DocumentResponse {
pub(crate) async fn load_primary_thumbnails(
state: &AppState,
documents: &[Document],
) -> AppResult<HashMap<Uuid, DocumentAssetResponse>> {
if documents.is_empty() {
return Ok(HashMap::new());
}
let mut current_versions: HashMap<Uuid, i32> = HashMap::new();
let mut doc_ids = Vec::with_capacity(documents.len());
for doc in documents {
current_versions.insert(doc.id, doc.current_version);
doc_ids.push(doc.id);
}
let mut conn = state.db()?;
let versions: Vec<DocumentVersion> = document_versions::table
.filter(document_versions::document_id.eq_any(&doc_ids))
.load(&mut conn)?;
let mut version_id_by_doc: HashMap<Uuid, Uuid> = HashMap::new();
let mut doc_id_by_version: HashMap<Uuid, Uuid> = HashMap::new();
for version in versions {
if let Some(current_number) = current_versions.get(&version.document_id) {
if *current_number == version.version_number {
version_id_by_doc.insert(version.document_id, version.id);
doc_id_by_version.insert(version.id, version.document_id);
}
}
}
if version_id_by_doc.is_empty() {
return Ok(HashMap::new());
}
let version_ids: Vec<Uuid> = version_id_by_doc.values().copied().collect();
let assets: Vec<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq_any(&version_ids))
.filter(document_assets::asset_type.eq("thumbnail"))
.order((
document_assets::document_version_id.asc(),
document_assets::created_at.asc(),
))
.load(&mut conn)?;
drop(conn);
let mut first_assets: HashMap<Uuid, DocumentAsset> = HashMap::new();
for asset in assets {
if let Some(doc_id) = doc_id_by_version.get(&asset.document_version_id) {
first_assets.entry(*doc_id).or_insert(asset);
}
}
let mut responses = HashMap::with_capacity(first_assets.len());
for (doc_id, asset) in first_assets {
let url = state
.storage
.presign_get_object(
&asset.s3_key,
Duration::from_secs(PRESIGNED_URL_EXPIRY_SECONDS),
)
.await
.map_err(|err| AppError::internal(format!("failed to sign asset URL: {err}")))?;
responses.insert(doc_id, to_asset_response(asset, url));
}
Ok(responses)
}
pub(crate) fn to_document_response(
doc: Document,
tags: Option<Vec<Tag>>,
thumbnail: Option<DocumentAssetResponse>,
) -> DocumentResponse {
DocumentResponse {
id: doc.id,
filename: doc.filename,
@@ -497,6 +788,7 @@ pub(crate) fn to_document_response(doc: Document, tags: Option<Vec<Tag>>) -> Doc
.into_iter()
.map(TagResponse::from)
.collect(),
thumbnail,
}
}
@@ -512,6 +804,45 @@ fn to_version_response(version: DocumentVersion) -> DocumentVersionResponse {
}
}
fn to_asset_response(asset: DocumentAsset, url: String) -> DocumentAssetResponse {
DocumentAssetResponse {
id: asset.id,
asset_type: asset.asset_type,
mime_type: asset.mime_type,
width: asset.width,
height: asset.height,
url,
created_at: to_iso(asset.created_at),
}
}
async fn load_asset_responses(
state: &AppState,
version_id: Uuid,
) -> AppResult<Vec<DocumentAssetResponse>> {
let mut conn = state.db()?;
let assets: Vec<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq(version_id))
.order(document_assets::created_at.asc())
.load(&mut conn)?;
drop(conn);
let mut responses = Vec::with_capacity(assets.len());
for asset in assets {
let url = state
.storage
.presign_get_object(
&asset.s3_key,
Duration::from_secs(PRESIGNED_URL_EXPIRY_SECONDS),
)
.await
.map_err(|err| AppError::internal(format!("failed to sign asset URL: {err}")))?;
responses.push(to_asset_response(asset, url));
}
Ok(responses)
}
pub(crate) fn to_iso(dt: NaiveDateTime) -> String {
DateTime::<Utc>::from_naive_utc_and_offset(dt, Utc).to_rfc3339()
}