This commit is contained in:
2025-10-10 09:39:58 +02:00
parent 4a428b9af6
commit ddce0e39b3
36 changed files with 6410 additions and 1419 deletions
+42
View File
@@ -0,0 +1,42 @@
use std::{sync::Arc, time::Duration};
use tokio::signal;
use tracing_subscriber::EnvFilter;
use paperless_backend::{
auth::jwt::JwtService, config::AppConfig, db, default_handlers, s3::build_client,
state::AppState, storage::S3Storage, Worker,
};
#[tokio::main]
async fn main() -> anyhow::Result<()> {
dotenv::dotenv().ok();
init_tracing();
let config = AppConfig::from_env()?;
let pool = db::init_pool(&config.database_url)?;
let s3_client = build_client(&config).await?;
let storage = Arc::new(S3Storage::new(s3_client, config.s3_bucket.clone()));
let jwt = JwtService::from_config(&config)?;
let state = Arc::new(AppState::new(pool, config, storage, jwt));
let worker = Worker::new(state, default_handlers(), Duration::from_secs(2));
tokio::select! {
_ = worker.run() => {}
_ = signal::ctrl_c() => {
tracing::info!("worker received shutdown signal");
}
}
Ok(())
}
fn init_tracing() {
let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
tracing_subscriber::fmt()
.with_env_filter(filter)
.with_target(false)
.compact()
.init();
}
+127
View File
@@ -0,0 +1,127 @@
use std::time::Duration;
use chrono::{Duration as ChronoDuration, NaiveDateTime, Utc};
use diesel::pg::PgConnection;
use diesel::prelude::*;
use serde_json::Value;
use thiserror::Error;
use uuid::Uuid;
use crate::models::{Job, NewJob};
use crate::schema::jobs;
pub const STATUS_QUEUED: &str = "queued";
pub const STATUS_PROCESSING: &str = "processing";
pub const STATUS_SUCCEEDED: &str = "succeeded";
pub const STATUS_FAILED: &str = "failed";
pub const JOB_ANALYZE_DOCUMENT: &str = "analyze-document";
pub const JOB_GENERATE_THUMBNAILS: &str = "generate-thumbnails";
#[derive(Debug, Error)]
pub enum JobQueueError {
#[error("database error: {0}")]
Database(#[from] diesel::result::Error),
}
pub type JobQueueResult<T> = Result<T, JobQueueError>;
pub fn enqueue_job(
conn: &mut PgConnection,
job_type: &str,
payload: Value,
run_after: Option<NaiveDateTime>,
) -> JobQueueResult<Job> {
let new_job = NewJob {
id: Uuid::new_v4(),
job_type: job_type.to_string(),
payload,
status: STATUS_QUEUED.to_string(),
run_after: run_after.unwrap_or_else(|| Utc::now().naive_utc()),
};
diesel::insert_into(jobs::table)
.values(&new_job)
.execute(conn)?;
let job = jobs::table.find(new_job.id).first(conn)?;
Ok(job)
}
pub fn reserve_job(conn: &mut PgConnection, job_types: &[&str]) -> JobQueueResult<Option<Job>> {
let now = Utc::now().naive_utc();
conn.transaction(|conn| {
let job_opt = jobs::table
.filter(jobs::status.eq(STATUS_QUEUED))
.filter(jobs::run_after.le(now))
.filter(jobs::job_type.eq_any(job_types))
.order(jobs::run_after.asc())
.for_update()
.skip_locked()
.first::<Job>(conn)
.optional()?;
if let Some(job) = job_opt {
diesel::update(jobs::table.find(job.id))
.set((
jobs::status.eq(STATUS_PROCESSING),
jobs::attempts.eq(job.attempts + 1),
jobs::updated_at.eq(now),
))
.execute(conn)?;
let refreshed = jobs::table.find(job.id).first(conn)?;
Ok::<Option<Job>, diesel::result::Error>(Some(refreshed))
} else {
Ok::<Option<Job>, diesel::result::Error>(None)
}
})
.map_err(JobQueueError::from)
}
pub fn mark_job_succeeded(conn: &mut PgConnection, job_id: Uuid) -> JobQueueResult<()> {
diesel::update(jobs::table.find(job_id))
.set((
jobs::status.eq(STATUS_SUCCEEDED),
jobs::last_error.eq::<Option<String>>(None),
jobs::updated_at.eq(Utc::now().naive_utc()),
))
.execute(conn)?;
Ok(())
}
pub fn retry_job_after(
conn: &mut PgConnection,
job_id: Uuid,
delay: Duration,
error_message: &str,
) -> JobQueueResult<()> {
let next_run = Utc::now()
+ ChronoDuration::from_std(delay).unwrap_or_else(|_| ChronoDuration::seconds(30));
diesel::update(jobs::table.find(job_id))
.set((
jobs::status.eq(STATUS_QUEUED),
jobs::run_after.eq(next_run.naive_utc()),
jobs::last_error.eq(Some(error_message.to_string())),
jobs::updated_at.eq(Utc::now().naive_utc()),
))
.execute(conn)?;
Ok(())
}
pub fn mark_job_failed(
conn: &mut PgConnection,
job_id: Uuid,
error_message: &str,
) -> JobQueueResult<()> {
diesel::update(jobs::table.find(job_id))
.set((
jobs::status.eq(STATUS_FAILED),
jobs::last_error.eq(Some(error_message.to_string())),
jobs::updated_at.eq(Utc::now().naive_utc()),
))
.execute(conn)?;
Ok(())
}
+3
View File
@@ -2,9 +2,12 @@ pub mod auth;
pub mod config;
pub mod db;
pub mod error;
pub mod jobs;
pub mod models;
pub mod routes;
pub mod s3;
pub mod schema;
pub mod state;
pub mod storage;
pub mod workers;
pub use workers::{default_handlers, Worker};
+52
View File
@@ -98,6 +98,58 @@ pub struct NewDocumentVersion {
pub operations_summary: serde_json::Value,
}
#[derive(Debug, Clone, Queryable, Identifiable, Associations)]
#[diesel(table_name = document_assets)]
#[diesel(belongs_to(DocumentVersion, foreign_key = document_version_id))]
pub struct DocumentAsset {
pub id: Uuid,
pub document_version_id: Uuid,
pub asset_type: String,
pub s3_key: String,
pub mime_type: String,
pub width: Option<i32>,
pub height: Option<i32>,
pub metadata: serde_json::Value,
pub created_at: NaiveDateTime,
}
#[derive(Debug, Insertable)]
#[diesel(table_name = document_assets)]
pub struct NewDocumentAsset {
pub id: Uuid,
pub document_version_id: Uuid,
pub asset_type: String,
pub s3_key: String,
pub mime_type: String,
pub width: Option<i32>,
pub height: Option<i32>,
pub metadata: serde_json::Value,
}
#[derive(Debug, Clone, Queryable, Identifiable)]
#[diesel(table_name = jobs)]
pub struct Job {
pub id: Uuid,
pub job_type: String,
pub payload: serde_json::Value,
pub status: String,
pub attempts: i32,
pub run_after: NaiveDateTime,
pub last_error: Option<String>,
pub created_at: NaiveDateTime,
pub updated_at: NaiveDateTime,
}
#[derive(Debug, Insertable)]
#[diesel(table_name = jobs)]
pub struct NewJob {
pub id: Uuid,
pub job_type: String,
pub payload: serde_json::Value,
pub status: String,
pub run_after: NaiveDateTime,
}
#[derive(Debug, Clone, Queryable, Identifiable)]
#[diesel(table_name = tags)]
pub struct Tag {
+451 -120
View File
@@ -1,5 +1,4 @@
use std::collections::HashMap;
use std::time::Duration;
use std::{collections::HashMap, time::Duration};
use axum::extract::{Json, Multipart, Path, Query, State};
use axum::http::StatusCode;
@@ -8,16 +7,18 @@ use chrono::{DateTime, NaiveDateTime, Utc};
use diesel::dsl::exists;
use diesel::{prelude::*, PgConnection};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use serde_json::{json, Value};
use sha2::{Digest, Sha256};
use tracing::{error, info, warn};
use uuid::Uuid;
use crate::auth::AuthenticatedUser;
use crate::error::{AppError, AppResult};
use crate::jobs::{enqueue_job, JOB_ANALYZE_DOCUMENT};
use crate::models::{
Document, DocumentVersion, NewDocument, NewDocumentTag, NewDocumentVersion, Tag,
Document, DocumentAsset, DocumentVersion, NewDocument, NewDocumentTag, NewDocumentVersion, Tag,
};
use crate::schema::{document_tags, document_versions, documents, folders, tags};
use crate::schema::{document_assets, document_tags, document_versions, documents, folders, tags};
use crate::state::AppState;
const PRESIGNED_URL_EXPIRY_SECONDS: u64 = 300;
@@ -29,6 +30,12 @@ pub struct DocumentListQuery {
pub include_deleted: bool,
}
#[derive(Deserialize)]
pub struct AssetRequestQuery {
#[serde(default)]
pub force: bool,
}
#[derive(Serialize)]
pub struct TagResponse {
pub id: Uuid,
@@ -59,9 +66,10 @@ pub struct DocumentResponse {
pub deleted_at: Option<String>,
pub metadata: Value,
pub tags: Vec<TagResponse>,
pub thumbnail: Option<DocumentAssetResponse>,
}
#[derive(Serialize)]
#[derive(Serialize, Clone)]
pub struct DocumentVersionResponse {
pub id: Uuid,
pub version_number: i32,
@@ -72,10 +80,22 @@ pub struct DocumentVersionResponse {
pub operations_summary: Value,
}
#[derive(Serialize, Clone)]
pub struct DocumentAssetResponse {
pub id: Uuid,
pub asset_type: String,
pub mime_type: String,
pub width: Option<i32>,
pub height: Option<i32>,
pub url: String,
pub created_at: String,
}
#[derive(Serialize)]
pub struct DocumentDetailResponse {
pub document: DocumentResponse,
pub current_version: DocumentVersionResponse,
pub assets: Vec<DocumentAssetResponse>,
}
#[derive(Serialize)]
@@ -87,6 +107,24 @@ pub struct DocumentDownloadResponse {
pub size_bytes: i64,
}
#[derive(Serialize)]
pub struct BulkReanalyzeResponse {
pub queued: usize,
}
struct UploadRequest {
bytes: Vec<u8>,
original_name: String,
content_type: Option<String>,
folder_id: Option<Uuid>,
metadata: Value,
}
struct UploadOutcome {
detail: DocumentDetailResponse,
created: bool,
}
#[derive(Deserialize)]
pub struct MoveDocumentRequest {
pub folder_id: Option<Uuid>,
@@ -124,12 +162,16 @@ pub async fn list_documents(
let doc_ids: Vec<Uuid> = docs.iter().map(|doc| doc.id).collect();
let tags_map = load_tags_for_documents(&mut conn, &doc_ids)?;
drop(conn);
let thumbnails = load_primary_thumbnails(&state, &docs).await?;
let response = docs
.into_iter()
.map(|doc| {
let tags = tags_map.get(&doc.id).cloned();
to_document_response(doc, tags)
let thumbnail = thumbnails.get(&doc.id).cloned();
to_document_response(doc, tags, thumbnail)
})
.collect();
@@ -153,10 +195,19 @@ pub async fn get_document(
.first(&mut conn)?;
let tags_map = load_tags_for_documents(&mut conn, &[document_id])?;
let version_id = current_version.id;
drop(conn);
let assets = load_asset_responses(&state, version_id).await?;
let thumbnail = assets
.iter()
.find(|asset| asset.asset_type == "thumbnail")
.cloned();
Ok(Json(DocumentDetailResponse {
document: to_document_response(doc, tags_map.get(&document_id).cloned()),
document: to_document_response(doc, tags_map.get(&document_id).cloned(), thumbnail),
current_version: to_version_response(current_version),
assets,
}))
}
@@ -164,18 +215,18 @@ pub async fn upload_document(
State(state): State<AppState>,
_user: AuthenticatedUser,
mut multipart: Multipart,
) -> AppResult<Json<DocumentDetailResponse>> {
) -> AppResult<(StatusCode, Json<DocumentDetailResponse>)> {
let mut file_bytes: Option<Vec<u8>> = None;
let mut original_name: Option<String> = None;
let mut content_type: Option<String> = None;
let mut folder_id: Option<Uuid> = None;
let mut metadata: Value = Value::Object(Default::default());
while let Some(field) = multipart
.next_field()
.await
.map_err(|err| AppError::bad_request(format!("invalid multipart data: {err}")))?
{
while let Some(field) = multipart.next_field().await.map_err(|err| {
let msg = format!("invalid multipart data: {err}");
warn!(error = %err, "invalid multipart data");
AppError::bad_request(msg)
})? {
let name = field.name().map(|n| n.to_string());
match name.as_deref() {
Some("file") => {
@@ -183,15 +234,18 @@ pub async fn upload_document(
original_name = file_name.clone();
content_type = field.content_type().map(|mime| mime.to_string());
let data = field.bytes().await.map_err(|err| {
AppError::bad_request(format!("failed to read file bytes: {err}"))
let msg = format!("failed to read file bytes: {err}");
warn!(error = %err, "failed to read file bytes");
AppError::bad_request(msg)
})?;
file_bytes = Some(data.to_vec());
}
Some("folder_id") => {
let value = field
.text()
.await
.map_err(|err| AppError::bad_request(format!("invalid folder id: {err}")))?;
let value = field.text().await.map_err(|err| {
let msg = format!("invalid folder id: {err}");
warn!(error = %err, "invalid folder id");
AppError::bad_request(msg)
})?;
if !value.trim().is_empty() {
let parsed = Uuid::parse_str(value.trim())
.map_err(|_| AppError::bad_request("folder_id must be a valid UUID"))?;
@@ -199,127 +253,138 @@ pub async fn upload_document(
}
}
Some("metadata") => {
let value = field
.text()
.await
.map_err(|err| AppError::bad_request(format!("invalid metadata: {err}")))?;
let value = field.text().await.map_err(|err| {
let msg = format!("invalid metadata: {err}");
warn!(error = %err, "invalid metadata payload");
AppError::bad_request(msg)
})?;
metadata = serde_json::from_str(&value).map_err(|err| {
AppError::bad_request(format!("metadata must be valid JSON: {err}"))
let msg = format!("metadata must be valid JSON: {err}");
warn!(error = %err, "metadata parse failure");
AppError::bad_request(msg)
})?;
}
_ => {}
}
}
let file_bytes = file_bytes.ok_or_else(|| AppError::bad_request("file field is required"))?;
let file_bytes = file_bytes.ok_or_else(|| {
warn!("upload rejected: missing file field");
AppError::bad_request("file field is required")
})?;
let original_name = original_name.unwrap_or_else(|| "upload.bin".to_string());
if let Some(folder_id) = folder_id {
ensure_folder_exists(&state, folder_id)?;
}
let request = UploadRequest {
bytes: file_bytes,
original_name,
content_type,
folder_id,
metadata,
};
let doc_id = Uuid::new_v4();
let version_id = Uuid::new_v4();
let version_number = 1;
let stored_filename = original_name.clone();
let checksum = Sha256::digest(&file_bytes);
let checksum_hex = hex::encode(checksum);
let size_bytes = file_bytes.len() as i64;
let s3_key = format!("documents/{doc_id}/v{version_number}/{version_id}");
{
let mut conn = state.db()?;
let existing = documents::table
.inner_join(
document_versions::table.on(document_versions::document_id
.eq(documents::id)
.and(document_versions::version_number.eq(documents::current_version))),
)
.filter(document_versions::checksum.eq(&checksum_hex))
.select((documents::all_columns, document_versions::all_columns))
.first::<(Document, DocumentVersion)>(&mut conn)
.optional()?;
if let Some((mut document, version)) = existing {
if document.deleted_at.is_some() {
let now = Utc::now().naive_utc();
diesel::update(documents::table.find(document.id))
.set((
documents::deleted_at.eq(None::<NaiveDateTime>),
documents::updated_at.eq(now),
))
.execute(&mut conn)?;
document.deleted_at = None;
document.updated_at = now;
}
let tags_map = load_tags_for_documents(&mut conn, &[document.id])?;
let tags = tags_map.get(&document.id).cloned();
return Ok(Json(DocumentDetailResponse {
document: to_document_response(document, tags),
current_version: to_version_response(version),
}));
let outcome = match process_upload(&state, request).await {
Ok(outcome) => outcome,
Err(err) => {
error!(error = ?err, "document upload failed");
return Err(err);
}
};
info!(
document_id = %outcome.detail.document.id,
original_name = %outcome.detail.document.original_name,
created = outcome.created,
"document upload succeeded"
);
let status = if outcome.created {
StatusCode::CREATED
} else {
StatusCode::OK
};
Ok((status, Json(outcome.detail)))
}
pub async fn request_document_assets(
State(state): State<AppState>,
Path(document_id): Path<Uuid>,
Query(query): Query<AssetRequestQuery>,
) -> AppResult<StatusCode> {
let mut conn = state.db()?;
let document: Document = documents::table.find(document_id).first(&mut conn)?;
if document.deleted_at.is_some() {
return Err(AppError::not_found());
}
state
.storage
.put_object(&s3_key, file_bytes.clone(), content_type.clone())
.await
.map_err(|err| AppError::internal(format!("failed to store document: {err}")))?;
let version: DocumentVersion = document_versions::table
.filter(document_versions::document_id.eq(document_id))
.filter(document_versions::version_number.eq(document.current_version))
.first(&mut conn)?;
let metadata_value = if metadata.is_null() {
Value::Object(Default::default())
} else {
metadata
};
enqueue_job(
&mut conn,
JOB_ANALYZE_DOCUMENT,
json!({
"document_id": document_id,
"document_version_id": version.id,
"force": query.force,
}),
None,
)
.map_err(|err| AppError::internal(format!("failed to enqueue analyze job: {err}")))?;
let (document, version) = {
let mut conn = state.db()?;
conn.transaction(|conn| {
let new_document = NewDocument {
id: doc_id,
filename: stored_filename.clone(),
original_name: original_name.clone(),
content_type: content_type.clone(),
folder_id,
current_version: version_number,
metadata: metadata_value.clone(),
};
diesel::insert_into(documents::table)
.values(&new_document)
.execute(conn)?;
Ok(StatusCode::ACCEPTED)
}
let new_version = NewDocumentVersion {
id: version_id,
document_id: doc_id,
version_number,
s3_key: s3_key.clone(),
size_bytes,
checksum: checksum_hex.clone(),
operations_summary: Value::Object(Default::default()),
};
pub async fn reanalyze_all_documents(
State(state): State<AppState>,
_user: AuthenticatedUser,
) -> AppResult<(StatusCode, Json<BulkReanalyzeResponse>)> {
let mut conn = state.db()?;
diesel::insert_into(document_versions::table)
.values(&new_version)
.execute(conn)?;
let targets: Vec<(Uuid, Uuid)> = document_versions::table
.inner_join(documents::table.on(document_versions::document_id.eq(documents::id)))
.filter(documents::deleted_at.is_null())
.filter(document_versions::version_number.eq(documents::current_version))
.select((documents::id, document_versions::id))
.load(&mut conn)?;
let document: Document = documents::table.find(doc_id).first(conn)?;
let version: DocumentVersion = document_versions::table.find(version_id).first(conn)?;
let mut queued = 0usize;
for (document_id, version_id) in targets {
enqueue_job(
&mut conn,
JOB_ANALYZE_DOCUMENT,
json!({
"document_id": document_id,
"document_version_id": version_id,
"force": true,
}),
None,
)
.map_err(|err| AppError::internal(format!("failed to enqueue analyze job: {err}")))?;
queued += 1;
}
Ok::<_, diesel::result::Error>((document, version))
})?
};
Ok((StatusCode::ACCEPTED, Json(BulkReanalyzeResponse { queued })))
}
let response = DocumentDetailResponse {
document: to_document_response(document, None),
current_version: to_version_response(version),
};
pub async fn list_document_assets(
State(state): State<AppState>,
Path(document_id): Path<Uuid>,
) -> AppResult<Json<Vec<DocumentAssetResponse>>> {
let mut conn = state.db()?;
let document: Document = documents::table.find(document_id).first(&mut conn)?;
if document.deleted_at.is_some() {
return Err(AppError::not_found());
}
Ok(Json(response))
let version: DocumentVersion = document_versions::table
.filter(document_versions::document_id.eq(document_id))
.filter(document_versions::version_number.eq(document.current_version))
.first(&mut conn)?;
drop(conn);
let assets = load_asset_responses(&state, version.id).await?;
Ok(Json(assets))
}
pub async fn download_document(
@@ -449,6 +514,157 @@ pub async fn remove_tag(
Ok(StatusCode::NO_CONTENT)
}
async fn process_upload(state: &AppState, request: UploadRequest) -> AppResult<UploadOutcome> {
let UploadRequest {
bytes,
original_name,
content_type,
folder_id,
metadata,
} = request;
if let Some(folder) = folder_id {
ensure_folder_exists(state, folder)?;
}
let doc_id = Uuid::new_v4();
let version_id = Uuid::new_v4();
let version_number = 1;
let stored_filename = original_name.clone();
let checksum = Sha256::digest(&bytes);
let checksum_hex = hex::encode(checksum);
let size_bytes = bytes.len() as i64;
let s3_key = format!("documents/{doc_id}/v{version_number}/{version_id}");
{
let mut conn = state.db()?;
let existing = documents::table
.inner_join(
document_versions::table.on(document_versions::document_id
.eq(documents::id)
.and(document_versions::version_number.eq(documents::current_version))),
)
.filter(document_versions::checksum.eq(&checksum_hex))
.select((documents::all_columns, document_versions::all_columns))
.first::<(Document, DocumentVersion)>(&mut conn)
.optional()?;
if let Some((mut document, version)) = existing {
if document.deleted_at.is_some() {
let now = Utc::now().naive_utc();
diesel::update(documents::table.find(document.id))
.set((
documents::deleted_at.eq(None::<NaiveDateTime>),
documents::updated_at.eq(now),
))
.execute(&mut conn)?;
document.deleted_at = None;
document.updated_at = now;
}
let tags_map = load_tags_for_documents(&mut conn, &[document.id])?;
let tags = tags_map.get(&document.id).cloned();
drop(conn);
let assets = load_asset_responses(state, version.id).await?;
let thumbnail = assets
.iter()
.find(|asset| asset.asset_type == "thumbnail")
.cloned();
return Ok(UploadOutcome {
detail: DocumentDetailResponse {
document: to_document_response(document, tags, thumbnail),
current_version: to_version_response(version),
assets,
},
created: false,
});
}
}
state
.storage
.put_object(&s3_key, bytes.clone(), content_type.clone())
.await
.map_err(|err| {
error!(error = %err, key = %s3_key, "failed to store document");
AppError::internal(format!("failed to store document: {err}"))
})?;
let metadata_value = if metadata.is_null() {
Value::Object(Default::default())
} else {
metadata
};
let (document, version) = {
let mut conn = state.db()?;
conn.transaction(|conn| {
let new_document = NewDocument {
id: doc_id,
filename: stored_filename.clone(),
original_name: original_name.clone(),
content_type: content_type.clone(),
folder_id,
current_version: version_number,
metadata: metadata_value.clone(),
};
diesel::insert_into(documents::table)
.values(&new_document)
.execute(conn)?;
let new_version = NewDocumentVersion {
id: version_id,
document_id: doc_id,
version_number,
s3_key: s3_key.clone(),
size_bytes,
checksum: checksum_hex.clone(),
operations_summary: Value::Object(Default::default()),
};
diesel::insert_into(document_versions::table)
.values(&new_version)
.execute(conn)?;
let document: Document = documents::table.find(doc_id).first(conn)?;
let version: DocumentVersion = document_versions::table.find(version_id).first(conn)?;
Ok::<_, diesel::result::Error>((document, version))
})?
};
let detail = DocumentDetailResponse {
document: to_document_response(document, None, None),
current_version: to_version_response(version.clone()),
assets: Vec::new(),
};
if let Ok(mut conn) = state.db() {
if let Err(err) = enqueue_job(
&mut conn,
JOB_ANALYZE_DOCUMENT,
json!({
"document_id": doc_id,
"document_version_id": version.id,
"force": false,
}),
None,
) {
warn!(document_id = %doc_id, error = %err, "failed to enqueue analyze job");
}
} else {
warn!(document_id = %doc_id, "failed to enqueue analyze job due to pool error");
}
Ok(UploadOutcome {
detail,
created: true,
})
}
fn ensure_folder_exists(state: &AppState, folder_id: Uuid) -> AppResult<()> {
let mut conn = state.db()?;
let exists: bool = diesel::select(exists(folders::table.filter(folders::id.eq(folder_id))))
@@ -480,7 +696,82 @@ pub(crate) fn load_tags_for_documents(
Ok(map)
}
pub(crate) fn to_document_response(doc: Document, tags: Option<Vec<Tag>>) -> DocumentResponse {
pub(crate) async fn load_primary_thumbnails(
state: &AppState,
documents: &[Document],
) -> AppResult<HashMap<Uuid, DocumentAssetResponse>> {
if documents.is_empty() {
return Ok(HashMap::new());
}
let mut current_versions: HashMap<Uuid, i32> = HashMap::new();
let mut doc_ids = Vec::with_capacity(documents.len());
for doc in documents {
current_versions.insert(doc.id, doc.current_version);
doc_ids.push(doc.id);
}
let mut conn = state.db()?;
let versions: Vec<DocumentVersion> = document_versions::table
.filter(document_versions::document_id.eq_any(&doc_ids))
.load(&mut conn)?;
let mut version_id_by_doc: HashMap<Uuid, Uuid> = HashMap::new();
let mut doc_id_by_version: HashMap<Uuid, Uuid> = HashMap::new();
for version in versions {
if let Some(current_number) = current_versions.get(&version.document_id) {
if *current_number == version.version_number {
version_id_by_doc.insert(version.document_id, version.id);
doc_id_by_version.insert(version.id, version.document_id);
}
}
}
if version_id_by_doc.is_empty() {
return Ok(HashMap::new());
}
let version_ids: Vec<Uuid> = version_id_by_doc.values().copied().collect();
let assets: Vec<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq_any(&version_ids))
.filter(document_assets::asset_type.eq("thumbnail"))
.order((
document_assets::document_version_id.asc(),
document_assets::created_at.asc(),
))
.load(&mut conn)?;
drop(conn);
let mut first_assets: HashMap<Uuid, DocumentAsset> = HashMap::new();
for asset in assets {
if let Some(doc_id) = doc_id_by_version.get(&asset.document_version_id) {
first_assets.entry(*doc_id).or_insert(asset);
}
}
let mut responses = HashMap::with_capacity(first_assets.len());
for (doc_id, asset) in first_assets {
let url = state
.storage
.presign_get_object(
&asset.s3_key,
Duration::from_secs(PRESIGNED_URL_EXPIRY_SECONDS),
)
.await
.map_err(|err| AppError::internal(format!("failed to sign asset URL: {err}")))?;
responses.insert(doc_id, to_asset_response(asset, url));
}
Ok(responses)
}
pub(crate) fn to_document_response(
doc: Document,
tags: Option<Vec<Tag>>,
thumbnail: Option<DocumentAssetResponse>,
) -> DocumentResponse {
DocumentResponse {
id: doc.id,
filename: doc.filename,
@@ -497,6 +788,7 @@ pub(crate) fn to_document_response(doc: Document, tags: Option<Vec<Tag>>) -> Doc
.into_iter()
.map(TagResponse::from)
.collect(),
thumbnail,
}
}
@@ -512,6 +804,45 @@ fn to_version_response(version: DocumentVersion) -> DocumentVersionResponse {
}
}
fn to_asset_response(asset: DocumentAsset, url: String) -> DocumentAssetResponse {
DocumentAssetResponse {
id: asset.id,
asset_type: asset.asset_type,
mime_type: asset.mime_type,
width: asset.width,
height: asset.height,
url,
created_at: to_iso(asset.created_at),
}
}
async fn load_asset_responses(
state: &AppState,
version_id: Uuid,
) -> AppResult<Vec<DocumentAssetResponse>> {
let mut conn = state.db()?;
let assets: Vec<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq(version_id))
.order(document_assets::created_at.asc())
.load(&mut conn)?;
drop(conn);
let mut responses = Vec::with_capacity(assets.len());
for asset in assets {
let url = state
.storage
.presign_get_object(
&asset.s3_key,
Duration::from_secs(PRESIGNED_URL_EXPIRY_SECONDS),
)
.await
.map_err(|err| AppError::internal(format!("failed to sign asset URL: {err}")))?;
responses.push(to_asset_response(asset, url));
}
Ok(responses)
}
pub(crate) fn to_iso(dt: NaiveDateTime) -> String {
DateTime::<Utc>::from_naive_utc_and_offset(dt, Utc).to_rfc3339()
}
+84 -3
View File
@@ -12,7 +12,10 @@ use crate::models::{Document, Folder, NewFolder};
use crate::schema::{document_tags, documents, folders};
use crate::state::AppState;
use super::documents::{load_tags_for_documents, to_document_response, to_iso, DocumentResponse};
use super::documents::{
load_primary_thumbnails, load_tags_for_documents, to_document_response, to_iso,
DocumentResponse,
};
#[derive(Deserialize)]
pub struct CreateFolderRequest {
@@ -20,6 +23,12 @@ pub struct CreateFolderRequest {
pub parent_id: Option<Uuid>,
}
#[derive(Deserialize)]
pub struct EnsureFolderPathRequest {
pub parent_id: Option<Uuid>,
pub segments: Vec<String>,
}
#[derive(Serialize)]
pub struct FolderResponse {
pub folder: FolderInfo,
@@ -48,6 +57,70 @@ pub struct FolderInfo {
pub updated_at: String,
}
pub async fn ensure_folder_path(
State(state): State<AppState>,
Json(payload): Json<EnsureFolderPathRequest>,
) -> AppResult<Json<FolderResponse>> {
if payload.segments.is_empty() {
return Err(AppError::bad_request("segments must not be empty"));
}
let mut conn = state.db()?;
let target_folder = conn.transaction::<Folder, AppError, _>(|conn| {
let mut current_parent = payload.parent_id;
let mut last_folder: Option<Folder> = None;
for raw_name in &payload.segments {
let name = raw_name.trim();
if name.is_empty() {
return Err(AppError::bad_request("folder names must not be empty"));
}
let existing: Option<Folder> = if let Some(parent_id) = current_parent {
folders::table
.filter(folders::parent_id.eq(Some(parent_id)))
.filter(folders::name.eq(name))
.first(conn)
.optional()?
} else {
folders::table
.filter(folders::parent_id.is_null())
.filter(folders::name.eq(name))
.first(conn)
.optional()?
};
let folder = if let Some(folder) = existing {
folder
} else {
let path_cache = build_path_cache(conn, current_parent, name)?;
let new_folder = NewFolder {
id: Uuid::new_v4(),
name: name.to_string(),
parent_id: current_parent,
path_cache,
};
diesel::insert_into(folders::table)
.values(&new_folder)
.execute(conn)?;
folders::table.find(new_folder.id).first(conn)?
};
current_parent = Some(folder.id);
last_folder = Some(folder);
}
last_folder.ok_or_else(|| AppError::internal("failed to resolve folder path".to_string()))
})?;
Ok(Json(FolderResponse {
folder: folder_to_info(target_folder),
}))
}
pub async fn create_folder(
State(state): State<AppState>,
Json(payload): Json<CreateFolderRequest>,
@@ -127,12 +200,16 @@ pub async fn list_folder_contents(
let doc_ids: Vec<Uuid> = docs.iter().map(|doc| doc.id).collect();
let tags_map = load_tags_for_documents(&mut conn, &doc_ids)?;
drop(conn);
let thumbnails = load_primary_thumbnails(&state, &docs).await?;
let documents = docs
.into_iter()
.map(|doc| {
let tags = tags_map.get(&doc.id).cloned();
to_document_response(doc, tags)
let thumbnail = thumbnails.get(&doc.id).cloned();
to_document_response(doc, tags, thumbnail)
})
.collect();
@@ -228,11 +305,15 @@ pub async fn search_documents(
let doc_ids: Vec<Uuid> = docs.iter().map(|doc| doc.id).collect();
let tags_map = load_tags_for_documents(&mut conn, &doc_ids)?;
drop(conn);
let thumbnails = load_primary_thumbnails(&state, &docs).await?;
let response = docs
.into_iter()
.map(|doc| {
let tags = tags_map.get(&doc.id).cloned();
to_document_response(doc, tags)
let thumbnail = thumbnails.get(&doc.id).cloned();
to_document_response(doc, tags, thumbnail)
})
.collect();
+6
View File
@@ -30,17 +30,23 @@ pub fn create_router(state: AppState) -> Router<()> {
"/",
get(documents::list_documents).post(documents::upload_document),
)
.route("/reanalyze", post(documents::reanalyze_all_documents))
.route(
"/:id",
get(documents::get_document).delete(documents::delete_document),
)
.route("/:id/download", get(documents::download_document))
.route(
"/:id/assets",
get(documents::list_document_assets).post(documents::request_document_assets),
)
.route("/:id/folder", patch(documents::move_document))
.route("/:id/tags", post(documents::assign_tags))
.route("/:id/tags/:tag_id", delete(documents::remove_tag));
let folders_routes = Router::new()
.route("/", post(folders::create_folder))
.route("/path", post(folders::ensure_folder_path))
.route("/:id", delete(folders::delete_folder))
.route("/:id/contents", get(folders::list_folder_contents))
.route("/:id/documents", get(folders::search_documents));
+31
View File
@@ -1,5 +1,19 @@
// @generated automatically by Diesel CLI.
diesel::table! {
document_assets (id) {
id -> Uuid,
document_version_id -> Uuid,
asset_type -> Text,
s3_key -> Text,
mime_type -> Text,
width -> Nullable<Int4>,
height -> Nullable<Int4>,
metadata -> Jsonb,
created_at -> Timestamptz,
}
}
diesel::table! {
document_tags (document_id, tag_id) {
document_id -> Uuid,
@@ -55,6 +69,20 @@ diesel::table! {
}
}
diesel::table! {
jobs (id) {
id -> Uuid,
job_type -> Text,
payload -> Jsonb,
status -> Text,
attempts -> Int4,
run_after -> Timestamptz,
last_error -> Nullable<Text>,
created_at -> Timestamptz,
updated_at -> Timestamptz,
}
}
diesel::table! {
tags (id) {
id -> Uuid,
@@ -80,6 +108,7 @@ diesel::table! {
}
}
diesel::joinable!(document_assets -> document_versions (document_version_id));
diesel::joinable!(document_tags -> documents (document_id));
diesel::joinable!(document_tags -> tags (tag_id));
diesel::joinable!(document_tags -> users (assigned_by));
@@ -87,10 +116,12 @@ diesel::joinable!(document_versions -> documents (document_id));
diesel::joinable!(documents -> folders (folder_id));
diesel::allow_tables_to_appear_in_same_query!(
document_assets,
document_tags,
document_versions,
documents,
folders,
jobs,
tags,
users,
);
+23
View File
@@ -16,6 +16,8 @@ pub trait ObjectStorage: Send + Sync + 'static {
) -> Result<()>;
async fn presign_get_object(&self, key: &str, expires_in: Duration) -> Result<String>;
async fn get_object(&self, key: &str) -> Result<Vec<u8>>;
}
pub struct S3Storage {
@@ -76,4 +78,25 @@ impl ObjectStorage for S3Storage {
Ok(presigned.uri().to_string())
}
async fn get_object(&self, key: &str) -> Result<Vec<u8>> {
let response = self
.client
.get_object()
.bucket(&self.bucket)
.key(key)
.send()
.await
.context("failed to download object from S3")?;
let bytes = response
.body
.collect()
.await
.context("failed to read object stream")?
.into_bytes()
.to_vec();
Ok(bytes)
}
}
+165
View File
@@ -0,0 +1,165 @@
use std::{collections::HashSet, sync::Arc, time::Duration};
use async_trait::async_trait;
use diesel::prelude::*;
use serde::Deserialize;
use serde_json::{json, Map, Value};
use tokio::task;
use tracing::{error, warn};
use uuid::Uuid;
use crate::{
jobs::{enqueue_job, JOB_ANALYZE_DOCUMENT, JOB_GENERATE_THUMBNAILS},
models::{Document, DocumentVersion},
schema::{document_versions, documents},
state::AppState,
};
use super::{JobExecution, JobHandler};
#[derive(Debug, Deserialize)]
struct AnalyzePayload {
document_id: Uuid,
document_version_id: Uuid,
#[serde(default)]
force: bool,
}
pub struct AnalyzeDocumentJob;
impl AnalyzeDocumentJob {
pub fn new() -> Self {
Self
}
}
#[async_trait]
impl JobHandler for AnalyzeDocumentJob {
fn job_type(&self) -> &'static str {
JOB_ANALYZE_DOCUMENT
}
async fn handle(&self, state: Arc<AppState>, job: crate::models::Job) -> JobExecution {
let payload: AnalyzePayload = match serde_json::from_value(job.payload.clone()) {
Ok(payload) => payload,
Err(err) => {
return JobExecution::Failed {
error: format!("invalid analyze payload: {err}"),
}
}
};
let state_clone = state.clone();
match task::spawn_blocking(move || analyze_document(state_clone, payload)).await {
Ok(Ok(execution)) => execution,
Ok(Err(err)) => {
warn!(job_id = %job.id, error = %err, "analyze job will retry");
JobExecution::Retry {
delay: Duration::from_secs(30),
error: err,
}
}
Err(join_err) => {
error!(job_id = %job.id, error = %join_err, "analyze task panicked");
JobExecution::Retry {
delay: Duration::from_secs(60),
error: format!("worker panicked: {join_err}"),
}
}
}
}
}
fn analyze_document(state: Arc<AppState>, payload: AnalyzePayload) -> Result<JobExecution, String> {
let mut conn = state.db().map_err(|err| format!("{err:?}"))?;
let version: DocumentVersion = document_versions::table
.find(payload.document_version_id)
.first(&mut conn)
.map_err(|err| format!("{err:?}"))?;
if version.document_id != payload.document_id {
return Err("document/version mismatch".into());
}
let document: Document = documents::table
.find(payload.document_id)
.first(&mut conn)
.map_err(|err| format!("{err:?}"))?;
let (supported, reason) = determine_thumbnail_support(&document);
let mut summary_map = match version.operations_summary {
Value::Object(map) => map,
_ => Map::new(),
};
summary_map.insert("thumbnail_supported".to_string(), Value::Bool(supported));
if let Some(reason) = reason {
summary_map.insert("thumbnail_reason".to_string(), Value::String(reason));
} else {
summary_map.remove("thumbnail_reason");
}
diesel::update(document_versions::table.find(version.id))
.set(document_versions::operations_summary.eq(Value::Object(summary_map)))
.execute(&mut conn)
.map_err(|err| format!("{err:?}"))?;
if supported {
let enqueue_result = enqueue_job(
&mut conn,
JOB_GENERATE_THUMBNAILS,
json!({
"document_id": payload.document_id,
"document_version_id": payload.document_version_id,
"force": payload.force,
}),
None,
);
if let Err(err) = enqueue_result {
return Err(err.to_string());
}
}
Ok(JobExecution::Success)
}
pub(crate) fn determine_thumbnail_support(document: &Document) -> (bool, Option<String>) {
let supported_mimes: HashSet<&'static str> = [
"image/jpeg",
"image/png",
"image/gif",
"image/tiff",
"image/bmp",
"image/webp",
"application/pdf",
]
.into_iter()
.collect();
if let Some(ref content_type) = document.content_type {
if supported_mimes.contains(content_type.as_str()) {
return (true, None);
}
}
if let Some(ext) = document
.original_name
.rsplit('.')
.next()
.map(|ext| ext.to_ascii_lowercase())
{
let supported_exts = [
"jpg", "jpeg", "png", "gif", "tif", "tiff", "bmp", "webp", "pdf",
];
if supported_exts.contains(&ext.as_str()) {
return (true, None);
}
}
(
false,
Some("content type not supported for thumbnails".into()),
)
}
+131
View File
@@ -0,0 +1,131 @@
use std::{collections::HashMap, sync::Arc, time::Duration};
use async_trait::async_trait;
use tokio::time::sleep;
use tracing::{error, info, warn};
use crate::{
jobs::{mark_job_failed, mark_job_succeeded, reserve_job, retry_job_after, JobQueueError},
models::Job,
state::AppState,
};
pub mod analyze;
pub mod thumbnails;
#[derive(Debug)]
pub enum JobExecution {
Success,
Retry { delay: Duration, error: String },
Failed { error: String },
}
#[async_trait]
pub trait JobHandler: Send + Sync {
fn job_type(&self) -> &'static str;
async fn handle(&self, state: Arc<AppState>, job: Job) -> JobExecution;
}
pub struct Worker {
state: Arc<AppState>,
handlers: HashMap<&'static str, Arc<dyn JobHandler>>,
poll_interval: Duration,
}
impl Worker {
pub fn new(
state: Arc<AppState>,
handlers: Vec<Arc<dyn JobHandler>>,
poll_interval: Duration,
) -> Self {
let map = handlers
.into_iter()
.map(|handler| (handler.job_type(), handler))
.collect();
Self {
state,
handlers: map,
poll_interval,
}
}
pub async fn run(&self) {
info!("worker started");
loop {
match self.tick().await {
Ok(true) => {}
Ok(false) => sleep(self.poll_interval).await,
Err(err) => {
error!(error = %err, "worker tick failed");
sleep(self.poll_interval).await;
}
}
}
}
async fn tick(&self) -> Result<bool, JobQueueError> {
let job_types: Vec<&str> = self.handlers.keys().copied().collect();
if job_types.is_empty() {
return Ok(false);
}
let mut conn = match self.state.db() {
Ok(conn) => conn,
Err(err) => {
error!(?err, "failed to obtain database connection in worker");
return Ok(false);
}
};
let job_opt = reserve_job(&mut conn, &job_types)?;
drop(conn);
if let Some(job) = job_opt {
if let Some(handler) = self.handlers.get(job.job_type.as_str()) {
let result = handler.handle(self.state.clone(), job.clone()).await;
match result {
JobExecution::Success => {
if let Ok(mut conn) = self.state.db() {
mark_job_succeeded(&mut conn, job.id)?;
} else {
error!("failed to mark job succeeded due to pool error");
}
}
JobExecution::Retry { delay, error } => {
warn!(job_id = %job.id, job_type = %job.job_type, %error, "job will retry");
if let Ok(mut conn) = self.state.db() {
retry_job_after(&mut conn, job.id, delay, &error)?;
} else {
error!("failed to requeue job for retry due to pool error");
}
}
JobExecution::Failed { error } => {
error!(job_id = %job.id, job_type = %job.job_type, %error, "job failed");
if let Ok(mut conn) = self.state.db() {
mark_job_failed(&mut conn, job.id, &error)?;
} else {
error!("failed to mark job failed due to pool error");
}
}
}
} else {
error!(job_type = %job.job_type, "no handler registered for job type");
if let Ok(mut conn) = self.state.db() {
mark_job_failed(&mut conn, job.id, "no handler registered")?;
} else {
error!("failed to mark job failed for missing handler due to pool error");
}
}
Ok(true)
} else {
Ok(false)
}
}
}
pub fn default_handlers() -> Vec<Arc<dyn JobHandler>> {
vec![
Arc::new(analyze::AnalyzeDocumentJob::new()),
Arc::new(thumbnails::GenerateThumbnailsJob::new()),
]
}
+321
View File
@@ -0,0 +1,321 @@
use std::{io::Cursor, panic, sync::Arc, time::Duration};
use async_trait::async_trait;
use chrono::Utc;
use diesel::{pg::upsert::excluded, prelude::*};
use image::{
codecs::png::PngEncoder, ColorType, GenericImageView, ImageEncoder, ImageFormat, ImageReader,
};
use pdfium_render::prelude::*;
use serde::Deserialize;
use serde_json::json;
use tokio::task;
use tracing::{error, info, warn};
use uuid::Uuid;
use crate::{
jobs::JOB_GENERATE_THUMBNAILS,
models::{Document, DocumentAsset, DocumentVersion, NewDocumentAsset},
schema::{document_assets, document_versions, documents},
state::AppState,
};
use super::{analyze::determine_thumbnail_support, JobExecution, JobHandler};
const THUMBNAIL_WIDTH: u32 = 512;
const THUMBNAIL_HEIGHT: u32 = 512;
const THUMBNAIL_ASSET_TYPE: &str = "thumbnail";
#[derive(Debug, Deserialize)]
struct ThumbnailPayload {
document_id: Uuid,
document_version_id: Uuid,
#[serde(default)]
force: bool,
}
pub struct GenerateThumbnailsJob;
impl GenerateThumbnailsJob {
pub fn new() -> Self {
Self
}
}
#[async_trait]
impl JobHandler for GenerateThumbnailsJob {
fn job_type(&self) -> &'static str {
JOB_GENERATE_THUMBNAILS
}
async fn handle(&self, state: Arc<AppState>, job: crate::models::Job) -> JobExecution {
let payload: ThumbnailPayload = match serde_json::from_value(job.payload.clone()) {
Ok(p) => p,
Err(err) => {
return JobExecution::Failed {
error: format!("invalid thumbnail payload: {err}"),
}
}
};
let state_clone = state.clone();
let initial =
match task::spawn_blocking(move || load_thumbnail_context(state_clone, &payload)).await
{
Ok(Ok(ctx)) => ctx,
Ok(Err(err)) => {
warn!(job_id = %job.id, error = %err, "thumbnail job will retry");
return JobExecution::Retry {
delay: Duration::from_secs(30),
error: err,
};
}
Err(join_err) => {
error!(job_id = %job.id, error = %join_err, "thumbnail task panicked");
return JobExecution::Retry {
delay: Duration::from_secs(60),
error: format!("worker panicked: {join_err}"),
};
}
};
if initial.skip {
info!(job_id = %job.id, "thumbnails already exist; skipping");
return JobExecution::Success;
}
let bytes = match state.storage.get_object(&initial.version.s3_key).await {
Ok(bytes) => bytes,
Err(err) => {
warn!(job_id = %job.id, error = %err, "thumbnail fetch failed; will retry");
return JobExecution::Retry {
delay: Duration::from_secs(30),
error: err.to_string(),
};
}
};
let generation = match generate_thumbnail(&initial.document, &bytes) {
Ok(result) => result,
Err(err) => {
return JobExecution::Failed { error: err };
}
};
if let Err(err) = state
.storage
.put_object(
&generation.s3_key,
generation.image_bytes.clone(),
Some("image/png".into()),
)
.await
{
warn!(job_id = %job.id, error = %err, "failed to upload thumbnail; retrying");
return JobExecution::Retry {
delay: Duration::from_secs(30),
error: err.to_string(),
};
}
let state_clone = state.clone();
match task::spawn_blocking(move || {
persist_thumbnail_metadata(state_clone, &initial, &generation)
})
.await
{
Ok(Ok(())) => {}
Ok(Err(err)) => {
warn!(job_id = %job.id, error = %err, "failed to persist thumbnail metadata; retrying");
return JobExecution::Retry {
delay: Duration::from_secs(30),
error: err,
};
}
Err(join_err) => {
error!(job_id = %job.id, error = %join_err, "thumbnail metadata update panicked");
return JobExecution::Retry {
delay: Duration::from_secs(30),
error: format!("metadata update panic: {join_err}"),
};
}
}
JobExecution::Success
}
}
struct ThumbnailContext {
document: Document,
version: DocumentVersion,
skip: bool,
}
struct GeneratedThumbnail {
image_bytes: Vec<u8>,
width: Option<i32>,
height: Option<i32>,
s3_key: String,
}
fn load_thumbnail_context(
state: Arc<AppState>,
payload: &ThumbnailPayload,
) -> Result<ThumbnailContext, String> {
let mut conn = state.db().map_err(|err| format!("{err:?}"))?;
let version: DocumentVersion = document_versions::table
.find(payload.document_version_id)
.first(&mut conn)
.map_err(|err| format!("{err:?}"))?;
if version.document_id != payload.document_id {
return Err("document/version mismatch".into());
}
let document: Document = documents::table
.find(payload.document_id)
.first(&mut conn)
.map_err(|err| format!("{err:?}"))?;
let existing: Option<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq(payload.document_version_id))
.filter(document_assets::asset_type.eq(THUMBNAIL_ASSET_TYPE))
.first(&mut conn)
.optional()
.map_err(|err| format!("{err:?}"))?;
let (supported, _) = determine_thumbnail_support(&document);
if !supported {
return Err("thumbnail generation not supported for this document".into());
}
let skip = existing.is_some() && !payload.force;
Ok(ThumbnailContext {
document,
version,
skip,
})
}
fn generate_thumbnail(document: &Document, bytes: &[u8]) -> Result<GeneratedThumbnail, String> {
let is_pdf = document
.content_type
.as_deref()
.map(|mime| mime == "application/pdf")
.unwrap_or_else(|| {
document
.original_name
.rsplit('.')
.next()
.map(|ext| ext.eq_ignore_ascii_case("pdf"))
.unwrap_or(false)
});
let (png_bytes, width, height) = if is_pdf {
generate_pdf_thumbnail(bytes)?
} else {
generate_image_thumbnail(bytes)?
};
let s3_key = format!("thumbnails/{}/{}.png", document.id, Uuid::new_v4());
Ok(GeneratedThumbnail {
image_bytes: png_bytes,
width,
height,
s3_key,
})
}
fn generate_image_thumbnail(bytes: &[u8]) -> Result<(Vec<u8>, Option<i32>, Option<i32>), String> {
let reader = ImageReader::new(Cursor::new(bytes))
.with_guessed_format()
.map_err(|err| err.to_string())?;
let mut image = reader.decode().map_err(|err| err.to_string())?;
if image.width() > THUMBNAIL_WIDTH || image.height() > THUMBNAIL_HEIGHT {
image = image.thumbnail(THUMBNAIL_WIDTH, THUMBNAIL_HEIGHT);
}
let (width, height) = image.dimensions();
let mut cursor = Cursor::new(Vec::new());
image
.write_to(&mut cursor, ImageFormat::Png)
.map_err(|err| err.to_string())?;
let buffer = cursor.into_inner();
Ok((buffer, Some(width as i32), Some(height as i32)))
}
fn generate_pdf_thumbnail(bytes: &[u8]) -> Result<(Vec<u8>, Option<i32>, Option<i32>), String> {
let pdfium = panic::catch_unwind(|| Pdfium::default())
.map_err(|_| "failed to initialize PDFium".to_string())?;
let document = pdfium
.load_pdf_from_byte_slice(bytes, None)
.map_err(|err| format!("load pdf: {err}"))?;
let page = document
.pages()
.get(0)
.map_err(|err| format!("load first page: {err}"))?;
let render_config = PdfRenderConfig::new()
.set_target_width(THUMBNAIL_WIDTH as i32)
.set_maximum_height(THUMBNAIL_HEIGHT as i32)
.render_form_data(true)
.rotate_if_landscape(PdfPageRenderRotation::None, true);
let bitmap = page
.render_with_config(&render_config)
.map_err(|err| format!("render pdf page: {err}"))?;
let image = bitmap.as_image().to_rgb8();
let (width, height) = image.dimensions();
let mut cursor = Cursor::new(Vec::new());
PngEncoder::new(&mut cursor)
.write_image(image.as_raw(), width, height, ColorType::Rgb8.into())
.map_err(|err| format!("encode pdf thumbnail: {err}"))?;
Ok((cursor.into_inner(), Some(width as i32), Some(height as i32)))
}
fn persist_thumbnail_metadata(
state: Arc<AppState>,
context: &ThumbnailContext,
generated: &GeneratedThumbnail,
) -> Result<(), String> {
let mut conn = state.db().map_err(|err| format!("{err:?}"))?;
let new_asset = NewDocumentAsset {
id: Uuid::new_v4(),
document_version_id: context.version.id,
asset_type: THUMBNAIL_ASSET_TYPE.to_string(),
s3_key: generated.s3_key.clone(),
mime_type: "image/png".to_string(),
width: generated.width,
height: generated.height,
metadata: json!({
"generated_at": Utc::now().to_rfc3339(),
}),
};
diesel::insert_into(document_assets::table)
.values(&new_asset)
.on_conflict((
document_assets::document_version_id,
document_assets::asset_type,
))
.do_update()
.set((
document_assets::s3_key.eq(excluded(document_assets::s3_key)),
document_assets::mime_type.eq(excluded(document_assets::mime_type)),
document_assets::width.eq(excluded(document_assets::width)),
document_assets::height.eq(excluded(document_assets::height)),
document_assets::metadata.eq(excluded(document_assets::metadata)),
))
.execute(&mut conn)
.map_err(|err| format!("{err:?}"))?;
Ok(())
}