proxy downloads

This commit is contained in:
2025-11-09 12:01:19 +01:00
parent d24d2c9249
commit 81571e4c64
9 changed files with 383 additions and 71 deletions
+3
View File
@@ -66,6 +66,9 @@ The backend reads its settings from environment variables. In particular:
- `DATABASE_URL` connection string for the primary Postgres database (required).
- `DATABASE_MAX_POOL_SIZE` optional override for the r2d2 connection pool size.
Defaults to `2`; increase it in staging/production to match expected concurrency.
- `PROXY_DOWNLOADS` set to `true` when the object store is only reachable from
the backend network. When enabled, `/download/{token}` and asset-object fetches
stream bytes through the API instead of redirecting clients to S3/Hetzner.
On startup each binary logs the effective configuration with secrets redacted
(for example, the database password is masked). This makes it easier to confirm
+39 -2
View File
@@ -87,13 +87,42 @@ impl JwtService {
pub fn generate_download_token(
&self,
document_id: Uuid,
version_id: Uuid,
user_id: Uuid,
tenant_id: Uuid,
) -> Result<String> {
let now = Utc::now();
let exp = now + self.download_expiry;
let claims = DownloadClaims {
doc_id: document_id,
subject: DownloadSubject::Document {
doc_id: document_id,
version_id,
},
user_id,
tenant_id,
iss: self.issuer.clone(),
aud: self.download_audience.clone(),
iat: now.timestamp() as usize,
exp: exp.timestamp() as usize,
};
Ok(encode(&Header::default(), &claims, &self.encoding)?)
}
pub fn generate_asset_download_token(
&self,
asset_id: Uuid,
object_id: Uuid,
user_id: Uuid,
tenant_id: Uuid,
) -> Result<String> {
let now = Utc::now();
let exp = now + self.download_expiry;
let claims = DownloadClaims {
subject: DownloadSubject::AssetObject {
asset_id,
object_id,
},
user_id,
tenant_id,
iss: self.issuer.clone(),
@@ -180,9 +209,17 @@ pub struct Claims {
pub exp: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "scope", rename_all = "snake_case")]
pub enum DownloadSubject {
Document { doc_id: Uuid, version_id: Uuid },
AssetObject { asset_id: Uuid, object_id: Uuid },
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DownloadClaims {
pub doc_id: Uuid,
#[serde(flatten)]
pub subject: DownloadSubject,
pub user_id: Uuid,
pub tenant_id: Uuid,
pub iss: String,
+3
View File
@@ -41,6 +41,8 @@ pub struct AppConfig {
pub refresh_cookie_domain: Option<String>,
#[serde(default)]
pub cors_allowed_origin: Option<String>,
#[serde(default, deserialize_with = "deserialize_bool_from_anything")]
pub proxy_downloads: bool,
#[serde(default)]
pub aws_endpoint_url: Option<String>,
#[serde(default)]
@@ -79,6 +81,7 @@ impl AppConfig {
s3_bucket = %config.s3_bucket,
worker_max_document_bytes = config.worker_max_document_bytes,
upload_body_limit_bytes = config.upload_body_limit_bytes,
proxy_downloads = config.proxy_downloads,
"loaded backend configuration"
);
Ok(config)
+11 -2
View File
@@ -11,7 +11,7 @@ use crate::error::{AppError, AppResult};
use crate::models::{Document, DocumentAsset, DocumentAssetObject, DocumentVersion};
use crate::schema::{document_asset_objects, document_assets, document_versions};
use crate::state::{AppState, PgPooledConnection};
use crate::utils::time::to_iso;
use crate::utils::{http::inline_content_disposition, time::to_iso};
#[derive(Serialize, Clone, ToSchema)]
pub struct DocumentAssetResponse {
@@ -77,11 +77,12 @@ pub struct DocumentVersionDetailResponse {
pub fn build_download_path(
state: &AppState,
document: &Document,
version_id: Uuid,
user_id: Uuid,
) -> AppResult<String> {
state
.jwt
.generate_download_token(document.id, user_id, document.tenant_id)
.generate_download_token(document.id, version_id, user_id, document.tenant_id)
.map(|token| format!("/download/{token}"))
.map_err(|err| {
tracing::error!(error = ?err, "failed to generate download token");
@@ -139,6 +140,14 @@ pub fn to_asset_object_response(
}
}
pub fn asset_object_disposition(
asset: &DocumentAsset,
object: &DocumentAssetObject,
) -> Option<String> {
let filename = format!("{}-{}", asset.asset_type, object.ordinal);
inline_content_disposition(&filename)
}
pub fn delete_asset(
conn: &mut PgPooledConnection,
tenant_id: Uuid,
+165 -34
View File
@@ -1,26 +1,31 @@
use std::{collections::HashSet, time::Duration};
use axum::body::Body;
use axum::extract::{Json, Multipart, Path, Query, State};
use axum::http::StatusCode;
use axum::response::IntoResponse;
use axum::http::{header, HeaderMap, HeaderValue, StatusCode};
use axum::response::{IntoResponse, Response};
use chrono::{DateTime, NaiveDateTime, Utc};
use diesel::dsl::exists;
use diesel::{prelude::*, select};
use futures_util::StreamExt;
use serde::Deserialize;
use serde_json::Value;
use tracing::{error, info};
use utoipa::{IntoParams, ToSchema};
use uuid::Uuid;
use crate::auth::TenantScopedConn;
use crate::auth::{jwt::DownloadSubject, TenantScopedConn};
use crate::documents::asset::{
DocumentAssetDetailResponse, DocumentAssetResponse, DocumentVersionDetailResponse,
DocumentVersionResponse,
asset_object_disposition, DocumentAssetDetailResponse, DocumentAssetResponse,
DocumentVersionDetailResponse, DocumentVersionResponse,
};
use crate::error::{AppError, AppResult};
use crate::http::responders::{accepted_json, created_json, no_content, ok_json, JsonResponse};
use crate::models::{Document, DocumentVersion};
use crate::schema::{document_versions, documents, user_sessions::dsl as session_dsl};
use crate::models::{Document, DocumentAsset, DocumentAssetObject, DocumentVersion};
use crate::schema::{
document_asset_objects, document_assets, document_versions, documents,
user_sessions::dsl as session_dsl,
};
use crate::services::correspondents::{
AssignCorrespondentsRequest, BulkCorrespondentAction, BulkCorrespondentResponse,
BulkCorrespondentsRequest, CorrespondentAssignmentInput, CorrespondentsService,
@@ -34,6 +39,7 @@ use crate::services::tags::{
AssignTagsRequest, BulkTagAction, BulkTagRequest, BulkTagResponse, TagsService,
};
use crate::state::AppState;
use crate::storage::TenantStorage;
use crate::utils::{error::StorageResultExt, http::inline_content_disposition};
const PRESIGNED_URL_EXPIRY_SECONDS: u64 = 300;
@@ -479,6 +485,7 @@ pub async fn get_document_asset(
TenantScopedConn {
conn,
tenant_id,
user_id,
..
}: TenantScopedConn,
) -> AppResult<JsonResponse<DocumentAssetDetailResponse>> {
@@ -492,7 +499,7 @@ pub async fn get_document_asset(
}
let service = DocumentsService::new(&state);
let detail = service
.get_document_asset(conn, tenant_id, asset_id, start, limit)
.get_document_asset(conn, tenant_id, user_id, asset_id, start, limit)
.await?;
ok_json(detail)
}
@@ -549,13 +556,14 @@ pub async fn get_document_version(
get,
path = "/download/{token}",
params(("token" = String, Path, description = "Download token")),
responses((status = 302, description = "Redirect to pre-signed URL")),
responses((status = 200, description = "Proxied download stream or redirect")),
tag = "Documents"
)]
pub async fn download_with_token(
State(state): State<AppState>,
Path(token): Path<String>,
) -> AppResult<impl IntoResponse> {
headers: HeaderMap,
) -> AppResult<Response> {
let claims = state
.jwt
.verify_download_token(&token)
@@ -563,18 +571,6 @@ pub async fn download_with_token(
let mut conn = state.db_for_tenant(claims.tenant_id)?;
let doc: Document = documents::table
.find(claims.doc_id)
.filter(documents::tenant_id.eq(claims.tenant_id))
.first(&mut conn)?;
if doc.deleted_at.is_some() {
return Err(AppError::not_found());
}
let version: DocumentVersion = document_versions::table
.find(doc.current_version_id)
.first(&mut conn)?;
let now = Utc::now().naive_utc();
let has_active_refresh: bool = select(exists(
session_dsl::user_sessions
@@ -589,22 +585,85 @@ pub async fn download_with_token(
return Err(AppError::unauthorized());
}
drop(conn);
match &claims.subject {
DownloadSubject::Document { doc_id, version_id } => {
let doc_id = *doc_id;
let version_id = *version_id;
let doc: Document = documents::table
.find(doc_id)
.filter(documents::tenant_id.eq(claims.tenant_id))
.first(&mut conn)?;
if doc.deleted_at.is_some() {
return Err(AppError::not_found());
}
let storage = state.storage_for_tenant(claims.tenant_id)?;
let version: DocumentVersion = document_versions::table
.find(version_id)
.filter(document_versions::document_id.eq(doc_id))
.first(&mut conn)?;
let disposition = inline_content_disposition(&doc.filename);
drop(conn);
let presigned_url = storage
.presign_get_object(
&version.s3_key,
Duration::from_secs(PRESIGNED_URL_EXPIRY_SECONDS),
disposition.as_deref(),
)
.await
.storage_context("failed to generate download URL")?;
let storage = state.storage_for_tenant(claims.tenant_id)?;
let disposition = inline_content_disposition(&doc.filename);
Ok(axum::response::Redirect::temporary(&presigned_url))
if !state.config.proxy_downloads {
let presigned_url = storage
.presign_get_object(
&version.s3_key,
Duration::from_secs(PRESIGNED_URL_EXPIRY_SECONDS),
disposition.as_deref(),
)
.await
.storage_context("failed to generate download URL")?;
return Ok(axum::response::Redirect::temporary(&presigned_url).into_response());
}
proxy_storage_object(
storage,
&version.s3_key,
disposition.as_deref(),
headers.get(header::RANGE).cloned(),
doc.content_type.as_deref(),
Some(version.id.to_string()),
)
.await
}
DownloadSubject::AssetObject {
asset_id,
object_id,
} => {
if !state.config.proxy_downloads {
return Err(AppError::not_found());
}
let object: DocumentAssetObject = document_asset_objects::table
.find(*object_id)
.filter(document_asset_objects::asset_id.eq(*asset_id))
.filter(document_asset_objects::tenant_id.eq(claims.tenant_id))
.first(&mut conn)?;
let asset: DocumentAsset = document_assets::table
.find(*asset_id)
.filter(document_assets::tenant_id.eq(claims.tenant_id))
.first(&mut conn)?;
drop(conn);
let storage = state.storage_for_tenant(claims.tenant_id)?;
let disposition = asset_object_disposition(&asset, &object);
proxy_storage_object(
storage,
&object.s3_key,
disposition.as_deref(),
headers.get(header::RANGE).cloned(),
Some(asset.mime_type.as_str()),
Some(object.id.to_string()),
)
.await
}
}
}
#[utoipa::path(
@@ -628,6 +687,78 @@ pub async fn trash_document(
Ok(StatusCode::NO_CONTENT)
}
async fn proxy_storage_object(
storage: TenantStorage,
key: &str,
response_disposition: Option<&str>,
range_header: Option<HeaderValue>,
fallback_content_type: Option<&str>,
etag: Option<String>,
) -> AppResult<Response> {
let url = storage
.presign_get_object(
key,
Duration::from_secs(PRESIGNED_URL_EXPIRY_SECONDS),
response_disposition,
)
.await
.storage_context("failed to generate download URL")?;
let client = reqwest::Client::new();
let mut request = client.get(url.clone());
if let Some(range) = range_header {
request = request.header(header::RANGE, range);
}
let upstream = request.send().await.map_err(|err| {
tracing::error!(error = ?err, "failed to fetch document stream");
AppError::internal("failed to fetch document stream")
})?;
let status =
StatusCode::from_u16(upstream.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY);
if !(status.is_success() || status == StatusCode::PARTIAL_CONTENT) {
tracing::error!(status = %status, "upstream download returned error status");
return Err(AppError::internal("failed to fetch document stream"));
}
let mut builder = Response::builder().status(status);
if let Some(content_type) = upstream.headers().get(header::CONTENT_TYPE) {
builder = builder.header(header::CONTENT_TYPE, content_type);
} else if let Some(fallback) = fallback_content_type {
builder = builder.header(header::CONTENT_TYPE, fallback);
}
if let Some(content_length) = upstream.headers().get(header::CONTENT_LENGTH) {
builder = builder.header(header::CONTENT_LENGTH, content_length);
}
if let Some(range) = upstream.headers().get(header::CONTENT_RANGE) {
builder = builder.header(header::CONTENT_RANGE, range);
}
builder = builder.header("Accept-Ranges", "bytes");
if let Some(disposition) = response_disposition {
builder = builder.header(header::CONTENT_DISPOSITION, disposition);
}
if let Some(etag_value) = etag {
builder = builder.header(header::ETAG, format!("\"{}\"", etag_value));
}
let stream = upstream
.bytes_stream()
.map(|chunk| chunk.map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err)));
let body = Body::from_stream(stream);
builder.body(body).map_err(|err| {
tracing::error!(error = ?err, "failed to build proxied response");
AppError::internal("failed to build proxied response")
})
}
#[utoipa::path(
delete,
path = "/api/documents/{id}",
+44 -29
View File
@@ -20,10 +20,11 @@ use uuid::Uuid;
use crate::documents::{
asset::{
build_download_path, derive_document_title, filename_with_retained_extension,
load_asset_responses_with_conn, load_primary_assets, to_asset_detail_response,
to_asset_object_response, to_version_response, DocumentAssetDetailResponse,
DocumentAssetResponse, DocumentVersionDetailResponse, DocumentVersionResponse,
asset_object_disposition, build_download_path, derive_document_title,
filename_with_retained_extension, load_asset_responses_with_conn, load_primary_assets,
to_asset_detail_response, to_asset_object_response, to_version_response,
DocumentAssetDetailResponse, DocumentAssetResponse, DocumentVersionDetailResponse,
DocumentVersionResponse,
},
correspondents::{
insert_document_correspondents, normalize_correspondent_ids, DocumentCorrespondentResponse,
@@ -905,6 +906,7 @@ impl<'a> DocumentsService<'a> {
&self,
mut conn: PgPooledConnection,
tenant_id: Uuid,
user_id: Uuid,
asset_id: Uuid,
start: i32,
limit: i32,
@@ -945,25 +947,43 @@ impl<'a> DocumentsService<'a> {
.checked_add((PRESIGNED_URL_EXPIRY_SECONDS as i64) * 1000)
.ok_or_else(|| AppError::internal("failed to compute expiry timestamp"))?;
let storage = self.state.storage_for_tenant(tenant_id)?;
let storage = (!self.state.config.proxy_downloads)
.then(|| self.state.storage_for_tenant(tenant_id))
.transpose()?;
let mut object_responses = Vec::with_capacity(objects.len());
for object in objects {
let response_disposition = presign_disposition_for_asset(&asset, &object);
let response_disposition = asset_object_disposition(&asset, &object);
let url = storage
.presign_get_object(
&object.s3_key,
Duration::from_secs(PRESIGNED_URL_EXPIRY_SECONDS),
response_disposition.as_deref(),
)
.await
.storage_context("failed to generate asset URL")?;
let (url, object_expires_at) = if self.state.config.proxy_downloads {
let token = self
.state
.jwt
.generate_asset_download_token(asset.id, object.id, user_id, tenant_id)
.map_err(|err| {
error!(error = ?err, "failed to issue asset download token");
AppError::internal("failed to issue asset download token")
})?;
(format!("/download/{token}"), None)
} else {
let storage = storage
.as_ref()
.expect("storage preloaded when proxy disabled");
let url = storage
.presign_get_object(
&object.s3_key,
Duration::from_secs(PRESIGNED_URL_EXPIRY_SECONDS),
response_disposition.as_deref(),
)
.await
.storage_context("failed to generate asset URL")?;
(url, Some(expires_at))
};
object_responses.push(to_asset_object_response(
object,
Some(url),
Some(expires_at),
object_expires_at,
));
}
@@ -1008,7 +1028,7 @@ impl<'a> DocumentsService<'a> {
.first(conn)?;
let assets = load_asset_responses_with_conn(conn, tenant_id, version.id)?;
let download_path = build_download_path(self.state, &document, user_id)?;
let download_path = build_download_path(self.state, &document, version.id, user_id)?;
let version_core = to_version_response(version);
Ok(DocumentVersionDetailResponse {
@@ -1326,11 +1346,14 @@ impl<'a> DocumentsService<'a> {
current_version: Option<(DocumentVersionResponse, Vec<DocumentAssetResponse>)>,
) -> AppResult<DocumentResponse> {
let current_version = match current_version {
Some((version, assets)) => Some(DocumentVersionDetailResponse {
version,
assets,
download_path: build_download_path(self.state, &doc, user_id)?,
}),
Some((version, assets)) => {
let download_path = build_download_path(self.state, &doc, version.id, user_id)?;
Some(DocumentVersionDetailResponse {
version,
assets,
download_path,
})
}
None => None,
};
@@ -1466,11 +1489,3 @@ impl<'a> DocumentsService<'a> {
Ok(Some(detail))
}
}
fn presign_disposition_for_asset(
asset: &DocumentAsset,
object: &DocumentAssetObject,
) -> Option<String> {
let filename = format!("{}-{}", asset.asset_type, object.ordinal);
inline_content_disposition(&filename)
}
+2 -2
View File
@@ -78,8 +78,8 @@ impl ObjectStorage for S3Storage {
expires_in: Duration,
response_content_disposition: Option<&str>,
) -> Result<String> {
let expiry_secs = u32::try_from(expires_in.as_secs())
.context("presign expiry exceeds u32 range")?;
let expiry_secs =
u32::try_from(expires_in.as_secs()).context("presign expiry exceeds u32 range")?;
let mut queries = HashMap::new();
if let Some(value) = response_content_disposition {
+16 -1
View File
@@ -136,10 +136,17 @@ pub struct TestApp {
impl TestApp {
pub async fn new() -> Result<Self> {
Self::with_config(|_| {}).await
}
pub async fn with_config<F>(configure: F) -> Result<Self>
where
F: FnOnce(&mut AppConfig),
{
let database_url = env::var("TEST_DATABASE_URL")
.context("TEST_DATABASE_URL must be set for integration tests")?;
let config = AppConfig {
let mut config = AppConfig {
database_url: database_url.clone(),
database_max_pool_size: db::DEFAULT_MAX_POOL_SIZE,
server_host: "127.0.0.1".to_string(),
@@ -156,6 +163,7 @@ impl TestApp {
refresh_cookie_secure: false,
refresh_cookie_domain: None,
cors_allowed_origin: None,
proxy_downloads: false,
aws_endpoint_url: None,
aws_access_key_id: None,
aws_secret_access_key: None,
@@ -170,6 +178,8 @@ impl TestApp {
webauthn_rp_name: "Papercrate".to_string(),
};
configure(&mut config);
let pool = db::init_pool_with_size(&config.database_url, config.database_max_pool_size)?;
prepare_database(&pool).await?;
@@ -206,6 +216,11 @@ impl TestApp {
Ok(())
}
#[allow(dead_code)]
pub async fn tenant_id(&self) -> Result<Uuid> {
self.ensure_default_tenant().await
}
#[allow(dead_code)]
pub fn storage(&self) -> Arc<FakeStorage> {
self.storage.clone()
+100 -1
View File
@@ -3,12 +3,14 @@ mod common;
use anyhow::{anyhow, Result};
use axum::http::StatusCode;
use common::{acquire_db_lock, body_to_vec, TestApp, UploadExtras};
use diesel::prelude::*;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use uuid::Uuid;
use papercrate::jobs::{mark_job_succeeded, JOB_PURGE_DOCUMENT};
use papercrate::models::Job;
use papercrate::models::{Job, NewDocumentAsset, NewDocumentAssetObject};
use papercrate::schema::{document_asset_objects, document_assets};
use papercrate::workers::{purge::PurgeDocumentJob, JobExecution, JobHandler};
use std::sync::Arc;
#[derive(Deserialize)]
@@ -64,6 +66,19 @@ struct DocumentAssetInfo {
asset_type: String,
}
#[derive(Deserialize)]
struct AssetProxyDetail {
id: Uuid,
objects: Vec<AssetProxyObject>,
}
#[derive(Deserialize)]
struct AssetProxyObject {
id: Uuid,
url: Option<String>,
expires_at: Option<i64>,
}
#[derive(Deserialize)]
struct DocumentListItem {
id: Uuid,
@@ -277,6 +292,90 @@ async fn upload_document_with_custom_title_sets_filename() -> Result<()> {
Ok(())
}
#[tokio::test]
async fn asset_detail_uses_proxy_urls_when_configured() -> Result<()> {
let _lock = acquire_db_lock().await;
let app = TestApp::with_config(|config| config.proxy_downloads = true).await?;
let tenant_id = app.tenant_id().await?;
let username = "proxy-assets";
let password = "secret";
app.insert_user(username, password, "admin").await?;
let token = app.login_token(username, password).await?;
let upload = app
.upload_document(
"/api/documents",
"proxy.pdf",
"application/pdf",
b"dummy",
None,
&token,
)
.await?;
assert_eq!(upload.status(), StatusCode::CREATED);
let body = body_to_vec(upload.into_body()).await?;
let detail: DocumentDetail = serde_json::from_slice(&body)?;
let document = detail.document;
let version = document
.current_version
.as_ref()
.ok_or_else(|| anyhow!("current version missing"))?;
let mut conn = app
.state
.db_for_tenant(tenant_id)
.map_err(|err| anyhow!("tenant connection: {err:?}"))?;
let asset_id = Uuid::new_v4();
let object_id = Uuid::new_v4();
diesel::insert_into(document_assets::table)
.values(&NewDocumentAsset {
id: asset_id,
document_version_id: version.id,
asset_type: "preview".to_string(),
mime_type: "image/png".to_string(),
metadata: json!({}),
cardinality: Some(1),
tenant_id,
})
.execute(&mut conn)?;
diesel::insert_into(document_asset_objects::table)
.values(&NewDocumentAssetObject {
id: object_id,
asset_id,
ordinal: 1,
s3_key: "objects/preview.png".to_string(),
metadata: json!({}),
tenant_id,
})
.execute(&mut conn)?;
drop(conn);
let response = app
.get(&format!("/api/assets/{asset_id}"), Some(&token))
.await?;
assert_eq!(response.status(), StatusCode::OK);
let body = body_to_vec(response.into_body()).await?;
let asset_detail: AssetProxyDetail = serde_json::from_slice(&body)?;
assert_eq!(asset_detail.id, asset_id);
assert_eq!(asset_detail.objects.len(), 1);
let object = &asset_detail.objects[0];
assert_eq!(object.id, object_id);
let url = object
.url
.as_deref()
.ok_or_else(|| anyhow!("missing url"))?;
assert!(url.starts_with("/download/"));
assert!(object.expires_at.is_none());
app.cleanup().await?;
Ok(())
}
#[tokio::test]
async fn document_list_sorting_controls() -> Result<()> {
let _lock = acquire_db_lock().await;