backend: storage_paths.rs
This commit is contained in:
@@ -31,6 +31,7 @@ use crate::schema::{
|
|||||||
document_tags, document_versions, documents, folders, refresh_tokens::dsl as refresh_dsl, tags,
|
document_tags, document_versions, documents, folders, refresh_tokens::dsl as refresh_dsl, tags,
|
||||||
};
|
};
|
||||||
use crate::state::AppState;
|
use crate::state::AppState;
|
||||||
|
use crate::utils::storage_paths::document_version_object_key;
|
||||||
|
|
||||||
const PRESIGNED_URL_EXPIRY_SECONDS: u64 = 300;
|
const PRESIGNED_URL_EXPIRY_SECONDS: u64 = 300;
|
||||||
const QUICKWIT_MAX_HITS: usize = 200;
|
const QUICKWIT_MAX_HITS: usize = 200;
|
||||||
@@ -1606,7 +1607,7 @@ async fn process_upload(
|
|||||||
let checksum = Sha256::digest(&bytes);
|
let checksum = Sha256::digest(&bytes);
|
||||||
let checksum_hex = hex::encode(checksum);
|
let checksum_hex = hex::encode(checksum);
|
||||||
let size_bytes = bytes.len() as i64;
|
let size_bytes = bytes.len() as i64;
|
||||||
let s3_key = format!("documents/{doc_id}/v{version_number}/{version_id}");
|
let s3_key = document_version_object_key(doc_id, version_number, version_id);
|
||||||
|
|
||||||
{
|
{
|
||||||
let mut conn = state.db()?;
|
let mut conn = state.db()?;
|
||||||
|
|||||||
@@ -1 +1,2 @@
|
|||||||
pub mod json;
|
pub mod json;
|
||||||
|
pub mod storage_paths;
|
||||||
|
|||||||
@@ -0,0 +1,115 @@
|
|||||||
|
//! Document storage path helpers.
|
||||||
|
//!
|
||||||
|
//! NOTE: The path layout produced here is part of the durable storage contract.
|
||||||
|
//! External systems (presigned URLs, lifecycle jobs, migrations) expect the
|
||||||
|
//! `documents/{document_id}/...` structure to remain stable. Coordinate before
|
||||||
|
//! changing any of these helpers to avoid breaking compatibility with existing
|
||||||
|
//! objects.
|
||||||
|
|
||||||
|
use uuid::Uuid;
|
||||||
|
|
||||||
|
const DOCUMENTS_PREFIX: &str = "documents";
|
||||||
|
|
||||||
|
/// Returns the root prefix for all objects belonging to a document.
|
||||||
|
pub fn document_prefix(document_id: Uuid) -> String {
|
||||||
|
format!("{DOCUMENTS_PREFIX}/{document_id}")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the prefix for a specific document version (without the object id).
|
||||||
|
pub fn document_version_prefix(document_id: Uuid, version_number: i32) -> String {
|
||||||
|
format!("{}/v{}", document_prefix(document_id), version_number)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the storage key for a stored document version blob.
|
||||||
|
pub fn document_version_object_key(
|
||||||
|
document_id: Uuid,
|
||||||
|
version_number: i32,
|
||||||
|
version_id: Uuid,
|
||||||
|
) -> String {
|
||||||
|
format!(
|
||||||
|
"{}/{}",
|
||||||
|
document_version_prefix(document_id, version_number),
|
||||||
|
version_id
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn document_asset_prefix(document_id: Uuid, version_number: i32) -> String {
|
||||||
|
format!(
|
||||||
|
"{}/assets",
|
||||||
|
document_version_prefix(document_id, version_number)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn document_asset_type_prefix(document_id: Uuid, version_number: i32, asset_type: &str) -> String {
|
||||||
|
format!(
|
||||||
|
"{}/{}",
|
||||||
|
document_asset_prefix(document_id, version_number),
|
||||||
|
asset_type
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the storage prefix under which the asset objects for a type/id pair live.
|
||||||
|
pub fn document_asset_object_prefix(
|
||||||
|
document_id: Uuid,
|
||||||
|
version_number: i32,
|
||||||
|
asset_type: &str,
|
||||||
|
asset_id: Uuid,
|
||||||
|
) -> String {
|
||||||
|
format!(
|
||||||
|
"{}/{}",
|
||||||
|
document_asset_type_prefix(document_id, version_number, asset_type),
|
||||||
|
asset_id
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Returns the full storage key for a specific asset object (ordinal).
|
||||||
|
pub fn document_asset_object_key(
|
||||||
|
document_id: Uuid,
|
||||||
|
version_number: i32,
|
||||||
|
asset_type: &str,
|
||||||
|
asset_id: Uuid,
|
||||||
|
ordinal: i32,
|
||||||
|
) -> String {
|
||||||
|
format!(
|
||||||
|
"{}/{}",
|
||||||
|
document_asset_object_prefix(document_id, version_number, asset_type, asset_id),
|
||||||
|
ordinal
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn generates_expected_paths() {
|
||||||
|
let document_id = Uuid::nil();
|
||||||
|
let version_id = Uuid::nil();
|
||||||
|
let asset_id = Uuid::nil();
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
document_prefix(document_id),
|
||||||
|
format!("documents/{document_id}")
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
document_version_prefix(document_id, 3),
|
||||||
|
format!("documents/{document_id}/v3")
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
document_version_object_key(document_id, 3, version_id),
|
||||||
|
format!("documents/{document_id}/v3/{version_id}")
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
document_asset_object_prefix(document_id, 3, "preview", asset_id),
|
||||||
|
format!("documents/{document_id}/v3/assets/preview/{asset_id}")
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
document_asset_object_key(document_id, 3, "preview", asset_id, 2),
|
||||||
|
format!("documents/{document_id}/v3/assets/preview/{asset_id}/2")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -25,6 +25,7 @@ use crate::{
|
|||||||
},
|
},
|
||||||
schema::{document_asset_objects, document_assets, document_versions, documents},
|
schema::{document_asset_objects, document_assets, document_versions, documents},
|
||||||
state::AppState,
|
state::AppState,
|
||||||
|
utils::storage_paths::document_asset_object_prefix,
|
||||||
};
|
};
|
||||||
|
|
||||||
use super::{JobExecution, JobHandler};
|
use super::{JobExecution, JobHandler};
|
||||||
@@ -136,9 +137,11 @@ impl JobHandler for GenerateOcrTextJob {
|
|||||||
|
|
||||||
let asset_id = Uuid::new_v4();
|
let asset_id = Uuid::new_v4();
|
||||||
|
|
||||||
let s3_key = format!(
|
let s3_key = document_asset_object_prefix(
|
||||||
"documents/{}/v{}/assets/{}/{}",
|
context.document.id,
|
||||||
context.document.id, context.version.version_number, OCR_TEXT_ASSET_TYPE, asset_id
|
context.version.version_number,
|
||||||
|
OCR_TEXT_ASSET_TYPE,
|
||||||
|
asset_id,
|
||||||
);
|
);
|
||||||
|
|
||||||
if let Err(err) = state
|
if let Err(err) = state
|
||||||
|
|||||||
@@ -19,6 +19,7 @@ use crate::{
|
|||||||
},
|
},
|
||||||
schema::{document_asset_objects, document_assets, document_versions, documents},
|
schema::{document_asset_objects, document_assets, document_versions, documents},
|
||||||
state::AppState,
|
state::AppState,
|
||||||
|
utils::storage_paths::document_asset_object_key,
|
||||||
};
|
};
|
||||||
|
|
||||||
use super::{analyze::determine_thumbnail_support, JobExecution, JobHandler};
|
use super::{analyze::determine_thumbnail_support, JobExecution, JobHandler};
|
||||||
@@ -172,22 +173,8 @@ impl JobHandler for GenerateThumbnailsJob {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let preview_asset_id = Uuid::new_v4();
|
let preview_asset_id = Uuid::new_v4();
|
||||||
let preview_base = format!(
|
|
||||||
"documents/{}/v{}/assets/{}/{}",
|
|
||||||
initial.document.id,
|
|
||||||
initial.version.version_number,
|
|
||||||
PREVIEW_ASSET_TYPE,
|
|
||||||
preview_asset_id
|
|
||||||
);
|
|
||||||
|
|
||||||
let thumbnail_asset_id = Uuid::new_v4();
|
let thumbnail_asset_id = Uuid::new_v4();
|
||||||
let thumbnail_base = format!(
|
|
||||||
"documents/{}/v{}/assets/{}/{}",
|
|
||||||
initial.document.id,
|
|
||||||
initial.version.version_number,
|
|
||||||
THUMBNAIL_ASSET_TYPE,
|
|
||||||
thumbnail_asset_id
|
|
||||||
);
|
|
||||||
|
|
||||||
let mut preview_objects: Vec<AssetObjectPersistence> =
|
let mut preview_objects: Vec<AssetObjectPersistence> =
|
||||||
Vec::with_capacity(generation.preview.objects.len());
|
Vec::with_capacity(generation.preview.objects.len());
|
||||||
@@ -198,7 +185,13 @@ impl JobHandler for GenerateThumbnailsJob {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
let ordinal = (index + 1) as i32;
|
let ordinal = (index + 1) as i32;
|
||||||
let s3_key = format!("{preview_base}/{ordinal}");
|
let s3_key = document_asset_object_key(
|
||||||
|
initial.document.id,
|
||||||
|
initial.version.version_number,
|
||||||
|
PREVIEW_ASSET_TYPE,
|
||||||
|
preview_asset_id,
|
||||||
|
ordinal,
|
||||||
|
);
|
||||||
|
|
||||||
if let Err(err) = state
|
if let Err(err) = state
|
||||||
.storage
|
.storage
|
||||||
@@ -234,7 +227,13 @@ impl JobHandler for GenerateThumbnailsJob {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
let ordinal = (index + 1) as i32;
|
let ordinal = (index + 1) as i32;
|
||||||
let s3_key = format!("{thumbnail_base}/{ordinal}");
|
let s3_key = document_asset_object_key(
|
||||||
|
initial.document.id,
|
||||||
|
initial.version.version_number,
|
||||||
|
THUMBNAIL_ASSET_TYPE,
|
||||||
|
thumbnail_asset_id,
|
||||||
|
ordinal,
|
||||||
|
);
|
||||||
|
|
||||||
if let Err(err) = state
|
if let Err(err) = state
|
||||||
.storage
|
.storage
|
||||||
|
|||||||
Reference in New Issue
Block a user