10 Commits
Author SHA1 Message Date
nils d9a36a99a7 dockerfiles, TagsWorkspace 2025-10-12 01:59:26 +02:00
nils f30e455c2d patch tags 2025-10-12 01:52:30 +02:00
nils 1b950a8f9a fix breadcrumbs 2025-10-11 22:47:31 +02:00
nils 3a5d7607e3 debouncing delay 2025-10-11 22:17:26 +02:00
nils 7233e3a534 basic search 2025-10-11 17:05:12 +02:00
nils 0a669064dd quickwit 2025-10-11 16:42:25 +02:00
nils cd27f36e3d ocr 2025-10-11 14:39:58 +02:00
nils 6f6abef5d2 logout 2025-10-11 13:36:27 +02:00
nils a21853c874 preview stack, refresh token, design 2025-10-11 13:12:05 +02:00
nils 6850918e3d 6 2025-10-11 11:54:57 +02:00
36 changed files with 3454 additions and 414 deletions
+36 -1
View File
@@ -2,11 +2,14 @@
## Backend Integration Tests ## Backend Integration Tests
Integration tests require a running Postgres instance. The repository includes a lightweight compose file for local runs: Integration tests require a running Postgres instance (and, optionally, Quickwit for OCR indexing). The repository includes a lightweight compose file for local runs:
```bash ```bash
docker compose -f docker-compose.test.yml up -d docker compose -f docker-compose.test.yml up -d
export TEST_DATABASE_URL=postgres://paperless:paperless_test@localhost:5433/paperless_test export TEST_DATABASE_URL=postgres://paperless:paperless_test@localhost:5433/paperless_test
# optional, enables Quickwit indexing jobs
export QUICKWIT_ENDPOINT=http://localhost:7280
export QUICKWIT_INDEX=documents
cargo test cargo test
``` ```
@@ -17,3 +20,35 @@ docker compose -f docker-compose.test.yml down
``` ```
The compose service uses tmpfs storage, giving each test run a clean database. The compose service uses tmpfs storage, giving each test run a clean database.
## Runtime Dependencies
- `ocrmypdf` (optional but recommended): Used by the OCR worker to extract text from PDFs when no embedded text layer is available. Ensure it is installed and available on the worker hosts if OCR is desired.
- Quickwit (optional): The Quickwit indexer is used to ingest extracted text for search. Set `QUICKWIT_ENDPOINT` and `QUICKWIT_INDEX` in the environment when running workers if you want indexing jobs to run. The local compose file starts a Quickwit instance on `http://localhost:7280` and seeds the `documents` index automatically.
## Running Migrations in Kubernetes
The backend container image ships the `diesel` CLI, so schema migrations can be executed as a short-lived Job (or Helm hook) before rolling out new pods. Example manifest:
```yaml
apiVersion: batch/v1
kind: Job
metadata:
name: paperless-migrate
spec:
template:
spec:
restartPolicy: OnFailure
containers:
- name: migrate
image: ghcr.io/example/paperless-backend:<TAG>
command: ["/usr/local/bin/diesel", "migration", "run"]
env:
- name: DATABASE_URL
valueFrom:
secretKeyRef:
name: paperless-db
key: DATABASE_URL
```
Run the Job manually (`kubectl apply -f migrate-job.yaml`) or configure it as a Helm pre-install/pre-upgrade hook so migrations run automatically on each deployment. Once the Job succeeds, deploy/update the backend `Deployment` as usual.
+221 -1
View File
@@ -753,6 +753,12 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9" checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9"
[[package]]
name = "cfg_aliases"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]] [[package]]
name = "chrono" name = "chrono"
version = "0.4.42" version = "0.4.42"
@@ -1129,6 +1135,16 @@ version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
[[package]]
name = "errno"
version = "0.3.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.61.2",
]
[[package]] [[package]]
name = "fastrand" name = "fastrand"
version = "2.3.0" version = "2.3.0"
@@ -1266,9 +1282,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"js-sys",
"libc", "libc",
"r-efi", "r-efi",
"wasi 0.14.7+wasi-0.2.4", "wasi 0.14.7+wasi-0.2.4",
"wasm-bindgen",
] ]
[[package]] [[package]]
@@ -1540,6 +1558,7 @@ dependencies = [
"tokio", "tokio",
"tokio-rustls 0.26.4", "tokio-rustls 0.26.4",
"tower-service", "tower-service",
"webpki-roots",
] ]
[[package]] [[package]]
@@ -1745,6 +1764,16 @@ version = "2.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130" checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130"
[[package]]
name = "iri-string"
version = "0.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2"
dependencies = [
"memchr",
"serde",
]
[[package]] [[package]]
name = "itertools" name = "itertools"
version = "0.13.0" version = "0.13.0"
@@ -1817,6 +1846,12 @@ dependencies = [
"windows-targets 0.53.5", "windows-targets 0.53.5",
] ]
[[package]]
name = "linux-raw-sys"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039"
[[package]] [[package]]
name = "litemap" name = "litemap"
version = "0.8.0" version = "0.8.0"
@@ -1847,6 +1882,12 @@ dependencies = [
"hashbrown 0.15.5", "hashbrown 0.15.5",
] ]
[[package]]
name = "lru-slab"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
[[package]] [[package]]
name = "matchers" name = "matchers"
version = "0.2.0" version = "0.2.0"
@@ -2091,14 +2132,17 @@ dependencies = [
"mime_guess", "mime_guess",
"once_cell", "once_cell",
"pdfium-render", "pdfium-render",
"percent-encoding",
"rand 0.8.5", "rand 0.8.5",
"reqwest",
"serde", "serde",
"serde_json", "serde_json",
"sha2", "sha2",
"tempfile",
"thiserror 1.0.69", "thiserror 1.0.69",
"tokio", "tokio",
"tower 0.4.13", "tower 0.4.13",
"tower-http", "tower-http 0.5.2",
"tracing", "tracing",
"tracing-subscriber", "tracing-subscriber",
"uuid", "uuid",
@@ -2310,6 +2354,61 @@ dependencies = [
"num-traits", "num-traits",
] ]
[[package]]
name = "quinn"
version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20"
dependencies = [
"bytes",
"cfg_aliases",
"pin-project-lite",
"quinn-proto",
"quinn-udp",
"rustc-hash",
"rustls 0.23.32",
"socket2 0.6.0",
"thiserror 2.0.17",
"tokio",
"tracing",
"web-time",
]
[[package]]
name = "quinn-proto"
version = "0.11.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31"
dependencies = [
"bytes",
"getrandom 0.3.3",
"lru-slab",
"rand 0.9.2",
"ring",
"rustc-hash",
"rustls 0.23.32",
"rustls-pki-types",
"slab",
"thiserror 2.0.17",
"tinyvec",
"tracing",
"web-time",
]
[[package]]
name = "quinn-udp"
version = "0.5.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd"
dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2 0.6.0",
"tracing",
"windows-sys 0.59.0",
]
[[package]] [[package]]
name = "quote" name = "quote"
version = "1.0.41" version = "1.0.41"
@@ -2439,6 +2538,44 @@ version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001"
[[package]]
name = "reqwest"
version = "0.12.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb"
dependencies = [
"base64 0.22.1",
"bytes",
"futures-core",
"http 1.3.1",
"http-body 1.0.1",
"http-body-util",
"hyper 1.7.0",
"hyper-rustls 0.27.7",
"hyper-util",
"js-sys",
"log",
"percent-encoding",
"pin-project-lite",
"quinn",
"rustls 0.23.32",
"rustls-pki-types",
"serde",
"serde_json",
"serde_urlencoded",
"sync_wrapper",
"tokio",
"tokio-rustls 0.26.4",
"tower 0.5.2",
"tower-http 0.6.6",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
"webpki-roots",
]
[[package]] [[package]]
name = "rfc6979" name = "rfc6979"
version = "0.3.1" version = "0.3.1"
@@ -2485,6 +2622,19 @@ dependencies = [
"semver", "semver",
] ]
[[package]]
name = "rustix"
version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e"
dependencies = [
"bitflags",
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.61.2",
]
[[package]] [[package]]
name = "rustls" name = "rustls"
version = "0.21.12" version = "0.21.12"
@@ -2505,6 +2655,7 @@ checksum = "cd3c25631629d034ce7cd9940adc9d45762d46de2b0f57193c4443b92c6d4d40"
dependencies = [ dependencies = [
"aws-lc-rs", "aws-lc-rs",
"once_cell", "once_cell",
"ring",
"rustls-pki-types", "rustls-pki-types",
"rustls-webpki 0.103.7", "rustls-webpki 0.103.7",
"subtle", "subtle",
@@ -2550,6 +2701,7 @@ version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79"
dependencies = [ dependencies = [
"web-time",
"zeroize", "zeroize",
] ]
@@ -2908,6 +3060,9 @@ name = "sync_wrapper"
version = "1.0.2" version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
dependencies = [
"futures-core",
]
[[package]] [[package]]
name = "synstructure" name = "synstructure"
@@ -2920,6 +3075,19 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "tempfile"
version = "3.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16"
dependencies = [
"fastrand",
"getrandom 0.3.3",
"once_cell",
"rustix",
"windows-sys 0.61.2",
]
[[package]] [[package]]
name = "thiserror" name = "thiserror"
version = "1.0.69" version = "1.0.69"
@@ -3010,6 +3178,21 @@ dependencies = [
"zerovec", "zerovec",
] ]
[[package]]
name = "tinyvec"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa"
dependencies = [
"tinyvec_macros",
]
[[package]]
name = "tinyvec_macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]] [[package]]
name = "tokio" name = "tokio"
version = "1.47.1" version = "1.47.1"
@@ -3154,6 +3337,24 @@ dependencies = [
"tracing", "tracing",
] ]
[[package]]
name = "tower-http"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2"
dependencies = [
"bitflags",
"bytes",
"futures-util",
"http 1.3.1",
"http-body 1.0.1",
"iri-string",
"pin-project-lite",
"tower 0.5.2",
"tower-layer",
"tower-service",
]
[[package]] [[package]]
name = "tower-layer" name = "tower-layer"
version = "0.3.3" version = "0.3.3"
@@ -3451,6 +3652,25 @@ dependencies = [
"wasm-bindgen", "wasm-bindgen",
] ]
[[package]]
name = "web-time"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "webpki-roots"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32b130c0d2d49f8b6889abc456e795e82525204f27c42cf767cf0d7734e089b8"
dependencies = [
"rustls-pki-types",
]
[[package]] [[package]]
name = "windows-core" name = "windows-core"
version = "0.62.2" version = "0.62.2"
+3
View File
@@ -37,6 +37,9 @@ async-trait = "0.1"
image = { version = "0.25", default-features = false, features = ["png", "jpeg"] } image = { version = "0.25", default-features = false, features = ["png", "jpeg"] }
pdfium-render = "0.8" pdfium-render = "0.8"
mime_guess = "2.0" mime_guess = "2.0"
tempfile = "3.10"
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
percent-encoding = "2.3"
# Error handling # Error handling
thiserror = "1.0" thiserror = "1.0"
+48
View File
@@ -0,0 +1,48 @@
# syntax=docker/dockerfile:1
FROM rust:1-slim AS builder
WORKDIR /app
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
build-essential \
pkg-config \
libssl-dev \
libpq-dev \
libjpeg-dev \
libpng-dev \
&& rm -rf /var/lib/apt/lists/*
COPY Cargo.toml Cargo.lock ./
COPY src ./src
COPY migrations ./migrations
COPY tests ./tests
COPY diesel.toml ./
RUN cargo build --release --bin paperless-backend --bin worker
RUN cargo install diesel_cli --no-default-features --features postgres
FROM debian:bookworm-slim AS runtime
WORKDIR /app
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
ca-certificates \
libssl3 \
libpq5 \
libjpeg62-turbo \
libpng16-16 \
&& rm -rf /var/lib/apt/lists/* \
&& useradd --system --create-home --uid 10001 appuser
COPY --from=builder /app/target/release/paperless-backend /usr/local/bin/paperless-backend
COPY --from=builder /app/target/release/worker /usr/local/bin/paperless-worker
COPY --from=builder /usr/local/cargo/bin/diesel /usr/local/bin/diesel
COPY migrations ./migrations
COPY diesel.toml ./
ENV RUST_LOG=info
USER appuser
EXPOSE 3000
ENTRYPOINT ["/usr/local/bin/paperless-backend"]
@@ -0,0 +1 @@
DROP TABLE refresh_tokens;
@@ -0,0 +1,13 @@
CREATE TABLE refresh_tokens (
id UUID PRIMARY KEY,
user_id UUID NOT NULL REFERENCES users(id) ON DELETE CASCADE,
token_hash TEXT NOT NULL,
issued_at TIMESTAMPTZ NOT NULL,
expires_at TIMESTAMPTZ NOT NULL,
revoked_at TIMESTAMPTZ,
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
CREATE INDEX idx_refresh_tokens_user_id ON refresh_tokens(user_id);
CREATE INDEX idx_refresh_tokens_token_hash ON refresh_tokens(token_hash);
+37
View File
@@ -13,6 +13,8 @@ pub struct JwtService {
issuer: String, issuer: String,
audience: String, audience: String,
expiry: Duration, expiry: Duration,
download_audience: String,
download_expiry: Duration,
} }
impl JwtService { impl JwtService {
@@ -23,6 +25,8 @@ impl JwtService {
issuer: config.jwt_issuer.clone(), issuer: config.jwt_issuer.clone(),
audience: config.jwt_audience.clone(), audience: config.jwt_audience.clone(),
expiry: Duration::minutes(config.jwt_expiry_minutes), expiry: Duration::minutes(config.jwt_expiry_minutes),
download_audience: config.download_token_audience.clone(),
download_expiry: Duration::minutes(config.download_token_expiry_minutes),
}) })
} }
@@ -49,6 +53,29 @@ impl JwtService {
let data = decode::<Claims>(token, &self.decoding, &validation)?; let data = decode::<Claims>(token, &self.decoding, &validation)?;
Ok(data.claims) Ok(data.claims)
} }
pub fn generate_download_token(&self, document_id: Uuid, user_id: Uuid) -> Result<String> {
let now = Utc::now();
let exp = now + self.download_expiry;
let claims = DownloadClaims {
doc_id: document_id,
user_id,
iss: self.issuer.clone(),
aud: self.download_audience.clone(),
iat: now.timestamp() as usize,
exp: exp.timestamp() as usize,
};
Ok(encode(&Header::default(), &claims, &self.encoding)?)
}
pub fn verify_download_token(&self, token: &str) -> Result<DownloadClaims> {
let mut validation = Validation::default();
validation.set_audience(&[self.download_audience.clone()]);
validation.set_issuer(&[self.issuer.clone()]);
let data = decode::<DownloadClaims>(token, &self.decoding, &validation)?;
Ok(data.claims)
}
} }
#[derive(Debug, Clone, Serialize, Deserialize)] #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -61,3 +88,13 @@ pub struct Claims {
pub iat: usize, pub iat: usize,
pub exp: usize, pub exp: usize,
} }
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DownloadClaims {
pub doc_id: Uuid,
pub user_id: Uuid,
pub iss: String,
pub aud: String,
pub iat: usize,
pub exp: usize,
}
+68
View File
@@ -0,0 +1,68 @@
use std::env;
use anyhow::{Context, Result};
use diesel::prelude::*;
use paperless_backend::{
config::AppConfig,
db,
models::DocumentAsset,
s3,
schema::document_assets,
storage::{ObjectStorage, S3Storage},
};
#[tokio::main]
async fn main() -> Result<()> {
let mut args = env::args().skip(1);
match args.next().as_deref() {
Some("delete-assets") => delete_all_assets().await?,
Some(cmd) => {
eprintln!("Unknown command: {cmd}\nUsage: maintenance delete-assets");
std::process::exit(1);
}
None => {
eprintln!("Usage: maintenance delete-assets");
std::process::exit(1);
}
}
Ok(())
}
async fn delete_all_assets() -> Result<()> {
let config = AppConfig::from_env()?;
let pool = db::init_pool(&config.database_url)?;
let s3_client = s3::build_client(&config).await?;
let storage = S3Storage::new(s3_client, config.s3_bucket.clone());
let mut conn = pool.get().context("failed to get database connection")?;
let assets: Vec<DocumentAsset> = document_assets::table
.load(&mut conn)
.context("failed to load document assets")?;
if assets.is_empty() {
println!("No assets found.");
return Ok(());
}
println!("Deleting {} assets…", assets.len());
for asset in &assets {
if let Err(err) = storage.delete_object(&asset.s3_key).await {
eprintln!(
"Failed to delete object {} from storage: {err}",
asset.s3_key
);
}
}
diesel::delete(document_assets::table)
.execute(&mut conn)
.context("failed to remove asset records")?;
println!("Asset records deleted.");
Ok(())
}
+33
View File
@@ -11,11 +11,19 @@ pub struct AppConfig {
pub jwt_issuer: String, pub jwt_issuer: String,
pub jwt_audience: String, pub jwt_audience: String,
pub jwt_expiry_minutes: i64, pub jwt_expiry_minutes: i64,
pub download_token_audience: String,
pub download_token_expiry_minutes: i64,
pub refresh_token_expiry_days: i64,
pub refresh_cookie_secure: bool,
pub refresh_cookie_domain: Option<String>,
pub cors_allowed_origin: Option<String>,
pub aws_endpoint_url: Option<String>, pub aws_endpoint_url: Option<String>,
pub aws_access_key_id: Option<String>, pub aws_access_key_id: Option<String>,
pub aws_secret_access_key: Option<String>, pub aws_secret_access_key: Option<String>,
pub aws_region: String, pub aws_region: String,
pub s3_bucket: String, pub s3_bucket: String,
pub quickwit_endpoint: Option<String>,
pub quickwit_index: Option<String>,
} }
impl AppConfig { impl AppConfig {
@@ -34,11 +42,28 @@ impl AppConfig {
.unwrap_or_else(|_| "60".to_string()) .unwrap_or_else(|_| "60".to_string())
.parse() .parse()
.context("JWT_EXPIRY_MINUTES must be an integer")?; .context("JWT_EXPIRY_MINUTES must be an integer")?;
let download_token_audience = env::var("DOWNLOAD_TOKEN_AUDIENCE")
.unwrap_or_else(|_| "paperless-neo-download".to_string());
let download_token_expiry_minutes = env::var("DOWNLOAD_TOKEN_EXPIRY_MINUTES")
.unwrap_or_else(|_| "60".to_string())
.parse()
.context("DOWNLOAD_TOKEN_EXPIRY_MINUTES must be an integer")?;
let refresh_token_expiry_days = env::var("REFRESH_TOKEN_EXPIRY_DAYS")
.unwrap_or_else(|_| "30".to_string())
.parse()
.context("REFRESH_TOKEN_EXPIRY_DAYS must be an integer")?;
let refresh_cookie_secure = env::var("REFRESH_COOKIE_SECURE")
.map(|v| v == "1" || v.eq_ignore_ascii_case("true"))
.unwrap_or(false);
let refresh_cookie_domain = env::var("REFRESH_COOKIE_DOMAIN").ok();
let cors_allowed_origin = env::var("CORS_ALLOWED_ORIGIN").ok();
let aws_endpoint_url = env::var("AWS_ENDPOINT_URL").ok(); let aws_endpoint_url = env::var("AWS_ENDPOINT_URL").ok();
let aws_access_key_id = env::var("AWS_ACCESS_KEY_ID").ok(); let aws_access_key_id = env::var("AWS_ACCESS_KEY_ID").ok();
let aws_secret_access_key = env::var("AWS_SECRET_ACCESS_KEY").ok(); let aws_secret_access_key = env::var("AWS_SECRET_ACCESS_KEY").ok();
let aws_region = env::var("AWS_REGION").unwrap_or_else(|_| "us-east-1".to_string()); let aws_region = env::var("AWS_REGION").unwrap_or_else(|_| "us-east-1".to_string());
let s3_bucket = env::var("S3_BUCKET").context("S3_BUCKET must be set")?; let s3_bucket = env::var("S3_BUCKET").context("S3_BUCKET must be set")?;
let quickwit_endpoint = env::var("QUICKWIT_ENDPOINT").ok();
let quickwit_index = env::var("QUICKWIT_INDEX").ok();
Ok(Self { Ok(Self {
database_url, database_url,
@@ -48,11 +73,19 @@ impl AppConfig {
jwt_issuer, jwt_issuer,
jwt_audience, jwt_audience,
jwt_expiry_minutes, jwt_expiry_minutes,
download_token_audience,
download_token_expiry_minutes,
refresh_token_expiry_days,
refresh_cookie_secure,
refresh_cookie_domain,
cors_allowed_origin,
aws_endpoint_url, aws_endpoint_url,
aws_access_key_id, aws_access_key_id,
aws_secret_access_key, aws_secret_access_key,
aws_region, aws_region,
s3_bucket, s3_bucket,
quickwit_endpoint,
quickwit_index,
}) })
} }
} }
+2
View File
@@ -17,6 +17,8 @@ pub const STATUS_FAILED: &str = "failed";
pub const JOB_ANALYZE_DOCUMENT: &str = "analyze-document"; pub const JOB_ANALYZE_DOCUMENT: &str = "analyze-document";
pub const JOB_GENERATE_THUMBNAILS: &str = "generate-thumbnails"; pub const JOB_GENERATE_THUMBNAILS: &str = "generate-thumbnails";
pub const JOB_GENERATE_OCR_TEXT: &str = "generate-ocr-text";
pub const JOB_INDEX_DOCUMENT_TEXT: &str = "index-document-text";
#[derive(Debug, Error)] #[derive(Debug, Error)]
pub enum JobQueueError { pub enum JobQueueError {
+1
View File
@@ -9,5 +9,6 @@ pub mod s3;
pub mod schema; pub mod schema;
pub mod state; pub mod state;
pub mod storage; pub mod storage;
pub mod utils;
pub mod workers; pub mod workers;
pub use workers::{default_handlers, Worker}; pub use workers::{default_handlers, Worker};
+24
View File
@@ -191,3 +191,27 @@ pub struct NewDocumentTag {
pub tag_id: Uuid, pub tag_id: Uuid,
pub assigned_by: Option<Uuid>, pub assigned_by: Option<Uuid>,
} }
#[derive(Debug, Clone, Queryable, Identifiable, Associations)]
#[diesel(table_name = refresh_tokens)]
#[diesel(belongs_to(User))]
pub struct RefreshToken {
pub id: Uuid,
pub user_id: Uuid,
pub token_hash: String,
pub issued_at: NaiveDateTime,
pub expires_at: NaiveDateTime,
pub revoked_at: Option<NaiveDateTime>,
pub created_at: NaiveDateTime,
pub updated_at: NaiveDateTime,
}
#[derive(Debug, Insertable)]
#[diesel(table_name = refresh_tokens)]
pub struct NewRefreshToken {
pub id: Uuid,
pub user_id: Uuid,
pub token_hash: String,
pub issued_at: NaiveDateTime,
pub expires_at: NaiveDateTime,
}
+218 -12
View File
@@ -1,15 +1,28 @@
use axum::{extract::State, http::StatusCode, response::IntoResponse, Json}; use axum::{
extract::State,
http::{header::SET_COOKIE, HeaderMap, HeaderValue, StatusCode},
Json,
};
use axum_extra::{headers::Cookie, typed_header::TypedHeader};
use chrono::{Duration as ChronoDuration, Utc};
use diesel::prelude::*; use diesel::prelude::*;
use rand::{rngs::OsRng, RngCore};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use uuid::Uuid;
use crate::{ use crate::{
auth::{password, AuthenticatedUser}, auth::{password, AuthenticatedUser},
error::{AppError, AppResult}, error::{AppError, AppResult},
models::User, models::{NewRefreshToken, RefreshToken, User},
schema::users::dsl, schema::{refresh_tokens, users::dsl},
state::AppState, state::AppState,
}; };
use crate::schema::refresh_tokens::dsl as refresh_dsl;
const REFRESH_COOKIE_NAME: &str = "refresh_token";
#[derive(Deserialize)] #[derive(Deserialize)]
pub struct LoginRequest { pub struct LoginRequest {
pub username: String, pub username: String,
@@ -26,7 +39,7 @@ pub struct LoginResponse {
pub async fn login( pub async fn login(
State(state): State<AppState>, State(state): State<AppState>,
Json(payload): Json<LoginRequest>, Json(payload): Json<LoginRequest>,
) -> AppResult<Json<LoginResponse>> { ) -> AppResult<(HeaderMap, Json<LoginResponse>)> {
let mut conn = state.db()?; let mut conn = state.db()?;
let user: User = dsl::users let user: User = dsl::users
@@ -40,22 +53,215 @@ pub async fn login(
return Err(AppError::unauthorized()); return Err(AppError::unauthorized());
} }
let token = state let access_token = state
.jwt .jwt
.generate_token(user.id, &user.username, &user.role) .generate_token(user.id, &user.username, &user.role)
.map_err(AppError::from)?; .map_err(AppError::from)?;
Ok(Json(LoginResponse { let now = Utc::now();
access_token: token, let refresh_value = generate_refresh_token();
token_type: "Bearer".to_string(), let refresh_hash = hash_refresh_token(&refresh_value);
expires_in: state.config.jwt_expiry_minutes * 60, let refresh_expires_at = now + ChronoDuration::days(state.config.refresh_token_expiry_days);
}))
let new_refresh = NewRefreshToken {
id: Uuid::new_v4(),
user_id: user.id,
token_hash: refresh_hash,
issued_at: now.naive_utc(),
expires_at: refresh_expires_at.naive_utc(),
};
diesel::insert_into(refresh_tokens::table)
.values(&new_refresh)
.execute(&mut conn)?;
let mut headers = HeaderMap::new();
headers.insert(
SET_COOKIE,
build_refresh_cookie(&state, &refresh_value, refresh_expires_at),
);
Ok((
headers,
Json(LoginResponse {
access_token,
token_type: "Bearer".to_string(),
expires_in: state.config.jwt_expiry_minutes * 60,
}),
))
} }
pub async fn logout(_user: AuthenticatedUser) -> impl IntoResponse { pub async fn refresh(
StatusCode::NO_CONTENT State(state): State<AppState>,
jar: Option<TypedHeader<Cookie>>,
) -> AppResult<(HeaderMap, Json<LoginResponse>)> {
let cookies = jar.ok_or_else(AppError::unauthorized)?;
let refresh_value = cookies
.get(REFRESH_COOKIE_NAME)
.ok_or_else(AppError::unauthorized)?;
let hashed = hash_refresh_token(refresh_value);
let mut conn = state.db()?;
let now = Utc::now();
let now_naive = now.naive_utc();
let token = match refresh_dsl::refresh_tokens
.filter(refresh_dsl::token_hash.eq(&hashed))
.filter(refresh_dsl::revoked_at.is_null())
.filter(refresh_dsl::expires_at.gt(now_naive))
.first::<RefreshToken>(&mut conn)
{
Ok(token) => token,
Err(diesel::result::Error::NotFound) => return Err(AppError::unauthorized()),
Err(err) => return Err(AppError::from(err)),
};
diesel::update(refresh_dsl::refresh_tokens.filter(refresh_dsl::id.eq(token.id)))
.set((
refresh_dsl::revoked_at.eq(now_naive),
refresh_dsl::updated_at.eq(now_naive),
))
.execute(&mut conn)?;
let user: User = dsl::users
.find(token.user_id)
.first(&mut conn)
.map_err(AppError::from)?;
let access_token = state
.jwt
.generate_token(user.id, &user.username, &user.role)
.map_err(AppError::from)?;
let new_refresh_value = generate_refresh_token();
let new_refresh_hash = hash_refresh_token(&new_refresh_value);
let new_refresh_expires = now + ChronoDuration::days(state.config.refresh_token_expiry_days);
let new_refresh = NewRefreshToken {
id: Uuid::new_v4(),
user_id: user.id,
token_hash: new_refresh_hash,
issued_at: now_naive,
expires_at: new_refresh_expires.naive_utc(),
};
diesel::insert_into(refresh_tokens::table)
.values(&new_refresh)
.execute(&mut conn)?;
let mut headers = HeaderMap::new();
headers.insert(
SET_COOKIE,
build_refresh_cookie(&state, &new_refresh_value, new_refresh_expires),
);
Ok((
headers,
Json(LoginResponse {
access_token,
token_type: "Bearer".to_string(),
expires_in: state.config.jwt_expiry_minutes * 60,
}),
))
}
pub async fn logout(
State(state): State<AppState>,
user: AuthenticatedUser,
jar: Option<TypedHeader<Cookie>>,
) -> AppResult<(HeaderMap, StatusCode)> {
let mut conn = state.db()?;
let now = Utc::now().naive_utc();
let mut rows_affected = 0;
if let Some(cookies) = jar {
if let Some(value) = cookies.get(REFRESH_COOKIE_NAME) {
let hashed = hash_refresh_token(value);
rows_affected = diesel::update(
refresh_dsl::refresh_tokens
.filter(refresh_dsl::token_hash.eq(hashed))
.filter(refresh_dsl::user_id.eq(user.user_id))
.filter(refresh_dsl::revoked_at.is_null()),
)
.set((
refresh_dsl::revoked_at.eq(now),
refresh_dsl::updated_at.eq(now),
))
.execute(&mut conn)
.unwrap_or(0);
}
}
if rows_affected == 0 {
let _ = diesel::update(
refresh_dsl::refresh_tokens
.filter(refresh_dsl::user_id.eq(user.user_id))
.filter(refresh_dsl::revoked_at.is_null()),
)
.set((
refresh_dsl::revoked_at.eq(now),
refresh_dsl::updated_at.eq(now),
))
.execute(&mut conn);
}
let mut headers = HeaderMap::new();
headers.insert(SET_COOKIE, build_clear_refresh_cookie(&state));
Ok((headers, StatusCode::NO_CONTENT))
} }
pub async fn me(user: AuthenticatedUser) -> Json<AuthenticatedUser> { pub async fn me(user: AuthenticatedUser) -> Json<AuthenticatedUser> {
Json(user) Json(user)
} }
fn hash_refresh_token(token: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(token.as_bytes());
hex::encode(hasher.finalize())
}
fn generate_refresh_token() -> String {
let mut bytes = [0u8; 32];
OsRng.fill_bytes(&mut bytes);
hex::encode(bytes)
}
fn build_refresh_cookie(
state: &AppState,
token: &str,
expires_at: chrono::DateTime<Utc>,
) -> HeaderValue {
let max_age = ChronoDuration::days(state.config.refresh_token_expiry_days).num_seconds();
let mut parts = vec![format!("{}={}", REFRESH_COOKIE_NAME, token)];
parts.push("Path=/".into());
parts.push("HttpOnly".into());
parts.push("SameSite=Strict".into());
parts.push(format!("Max-Age={}", max_age));
parts.push(format!("Expires={}", expires_at.to_rfc2822()));
if state.config.refresh_cookie_secure {
parts.push("Secure".into());
}
if let Some(domain) = &state.config.refresh_cookie_domain {
parts.push(format!("Domain={}", domain));
}
HeaderValue::from_str(&parts.join("; ")).expect("valid refresh cookie")
}
fn build_clear_refresh_cookie(state: &AppState) -> HeaderValue {
let mut parts = vec![format!("{}=", REFRESH_COOKIE_NAME)];
parts.push("Path=/".into());
parts.push("HttpOnly".into());
parts.push("SameSite=Strict".into());
parts.push("Max-Age=0".into());
parts.push("Expires=Thu, 01 Jan 1970 00:00:00 GMT".into());
if state.config.refresh_cookie_secure {
parts.push("Secure".into());
}
if let Some(domain) = &state.config.refresh_cookie_domain {
parts.push(format!("Domain={}", domain));
}
HeaderValue::from_str(&parts.join("; ")).expect("valid refresh cookie")
}
+129 -20
View File
@@ -5,7 +5,7 @@ use axum::http::StatusCode;
use axum::response::IntoResponse; use axum::response::IntoResponse;
use chrono::{DateTime, NaiveDateTime, Utc}; use chrono::{DateTime, NaiveDateTime, Utc};
use diesel::dsl::exists; use diesel::dsl::exists;
use diesel::{prelude::*, PgConnection}; use diesel::{prelude::*, select, PgConnection};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use serde_json::{json, Value}; use serde_json::{json, Value};
use sha2::{Digest, Sha256}; use sha2::{Digest, Sha256};
@@ -18,11 +18,35 @@ use crate::jobs::{enqueue_job, JOB_ANALYZE_DOCUMENT};
use crate::models::{ use crate::models::{
Document, DocumentAsset, DocumentVersion, NewDocument, NewDocumentTag, NewDocumentVersion, Tag, Document, DocumentAsset, DocumentVersion, NewDocument, NewDocumentTag, NewDocumentVersion, Tag,
}; };
use crate::schema::{document_assets, document_tags, document_versions, documents, folders, tags}; use crate::schema::{
document_assets, document_tags, document_versions, documents, folders,
refresh_tokens::dsl as refresh_dsl, tags,
};
use crate::state::AppState; use crate::state::AppState;
const PRESIGNED_URL_EXPIRY_SECONDS: u64 = 300; const PRESIGNED_URL_EXPIRY_SECONDS: u64 = 300;
fn inline_content_disposition(filename: &str) -> Option<String> {
if filename.is_empty() {
return None;
}
let sanitized: String = filename
.chars()
.map(|ch| match ch {
'"' | '\\' => '_',
_ => ch,
})
.collect();
let encoded =
percent_encoding::utf8_percent_encode(&sanitized, percent_encoding::NON_ALPHANUMERIC);
Some(format!(
"inline; filename=\"{}\"; filename*=UTF-8''{}",
sanitized, encoded
))
}
#[derive(Deserialize)] #[derive(Deserialize)]
pub struct DocumentListQuery { pub struct DocumentListQuery {
pub folder_id: Option<Uuid>, pub folder_id: Option<Uuid>,
@@ -69,6 +93,7 @@ pub struct DocumentResponse {
pub metadata: Value, pub metadata: Value,
pub tags: Vec<TagResponse>, pub tags: Vec<TagResponse>,
pub thumbnail: Option<DocumentAssetResponse>, pub thumbnail: Option<DocumentAssetResponse>,
pub download_path: String,
} }
#[derive(Serialize, Clone)] #[derive(Serialize, Clone)]
@@ -182,6 +207,7 @@ pub struct AssignTagsRequest {
pub async fn list_documents( pub async fn list_documents(
State(state): State<AppState>, State(state): State<AppState>,
Query(query): Query<DocumentListQuery>, Query(query): Query<DocumentListQuery>,
user: AuthenticatedUser,
) -> AppResult<Json<Vec<DocumentResponse>>> { ) -> AppResult<Json<Vec<DocumentResponse>>> {
let mut conn = state.db()?; let mut conn = state.db()?;
@@ -210,14 +236,18 @@ pub async fn list_documents(
let thumbnails = load_primary_thumbnails(&state, &docs).await?; let thumbnails = load_primary_thumbnails(&state, &docs).await?;
let response = docs let mut response = Vec::with_capacity(doc_ids.len());
.into_iter() for doc in docs {
.map(|doc| { let tags = tags_map.get(&doc.id).cloned();
let tags = tags_map.get(&doc.id).cloned(); let thumbnail = thumbnails.get(&doc.id).cloned();
let thumbnail = thumbnails.get(&doc.id).cloned(); response.push(to_document_response(
to_document_response(doc, tags, thumbnail) &state,
}) user.user_id,
.collect(); doc,
tags,
thumbnail,
)?);
}
Ok(Json(response)) Ok(Json(response))
} }
@@ -225,6 +255,7 @@ pub async fn list_documents(
pub async fn get_document( pub async fn get_document(
State(state): State<AppState>, State(state): State<AppState>,
Path(document_id): Path<Uuid>, Path(document_id): Path<Uuid>,
user: AuthenticatedUser,
) -> AppResult<Json<DocumentDetailResponse>> { ) -> AppResult<Json<DocumentDetailResponse>> {
let mut conn = state.db()?; let mut conn = state.db()?;
@@ -249,7 +280,13 @@ pub async fn get_document(
.cloned(); .cloned();
Ok(Json(DocumentDetailResponse { Ok(Json(DocumentDetailResponse {
document: to_document_response(doc, tags_map.get(&document_id).cloned(), thumbnail), document: to_document_response(
&state,
user.user_id,
doc,
tags_map.get(&document_id).cloned(),
thumbnail,
)?,
current_version: to_version_response(current_version), current_version: to_version_response(current_version),
assets, assets,
})) }))
@@ -257,7 +294,7 @@ pub async fn get_document(
pub async fn upload_document( pub async fn upload_document(
State(state): State<AppState>, State(state): State<AppState>,
_user: AuthenticatedUser, user: AuthenticatedUser,
mut multipart: Multipart, mut multipart: Multipart,
) -> AppResult<(StatusCode, Json<DocumentDetailResponse>)> { ) -> AppResult<(StatusCode, Json<DocumentDetailResponse>)> {
let mut file_bytes: Option<Vec<u8>> = None; let mut file_bytes: Option<Vec<u8>> = None;
@@ -335,7 +372,7 @@ pub async fn upload_document(
metadata, metadata,
}; };
let outcome = match process_upload(&state, request).await { let outcome = match process_upload(&state, request, user.user_id).await {
Ok(outcome) => { Ok(outcome) => {
info!( info!(
document_id = %outcome.detail.document.id, document_id = %outcome.detail.document.id,
@@ -527,6 +564,54 @@ pub async fn download_document(
})) }))
} }
pub async fn download_with_token(
State(state): State<AppState>,
Path(token): Path<String>,
) -> AppResult<impl IntoResponse> {
let claims = state
.jwt
.verify_download_token(&token)
.map_err(|_| AppError::unauthorized())?;
let mut conn = state.db()?;
let doc: Document = documents::table.find(claims.doc_id).first(&mut conn)?;
if doc.deleted_at.is_some() {
return Err(AppError::not_found());
}
let version: DocumentVersion = document_versions::table
.filter(document_versions::document_id.eq(claims.doc_id))
.filter(document_versions::version_number.eq(doc.current_version))
.first(&mut conn)?;
let now = Utc::now().naive_utc();
let has_active_refresh: bool = select(exists(
refresh_dsl::refresh_tokens
.filter(refresh_dsl::user_id.eq(claims.user_id))
.filter(refresh_dsl::revoked_at.is_null())
.filter(refresh_dsl::expires_at.gt(now)),
))
.get_result(&mut conn)?;
if !has_active_refresh {
return Err(AppError::unauthorized());
}
drop(conn);
let presigned_url = state
.storage
.presign_get_object(
&version.s3_key,
Duration::from_secs(PRESIGNED_URL_EXPIRY_SECONDS),
)
.await
.map_err(|err| AppError::internal(format!("failed to generate download URL: {err}")))?;
Ok(axum::response::Redirect::temporary(&presigned_url))
}
pub async fn delete_document( pub async fn delete_document(
State(state): State<AppState>, State(state): State<AppState>,
Path(document_id): Path<Uuid>, Path(document_id): Path<Uuid>,
@@ -757,7 +842,11 @@ pub async fn remove_tag(
Ok(StatusCode::NO_CONTENT) Ok(StatusCode::NO_CONTENT)
} }
async fn process_upload(state: &AppState, request: UploadRequest) -> AppResult<UploadOutcome> { async fn process_upload(
state: &AppState,
request: UploadRequest,
user_id: Uuid,
) -> AppResult<UploadOutcome> {
let UploadRequest { let UploadRequest {
bytes, bytes,
original_name, original_name,
@@ -824,7 +913,7 @@ async fn process_upload(state: &AppState, request: UploadRequest) -> AppResult<U
return Ok(UploadOutcome { return Ok(UploadOutcome {
detail: DocumentDetailResponse { detail: DocumentDetailResponse {
document: to_document_response(document, tags, thumbnail), document: to_document_response(state, user_id, document, tags, thumbnail)?,
current_version: to_version_response(version), current_version: to_version_response(version),
assets, assets,
}, },
@@ -833,9 +922,16 @@ async fn process_upload(state: &AppState, request: UploadRequest) -> AppResult<U
} }
} }
let content_disposition = inline_content_disposition(&original_name);
state state
.storage .storage
.put_object(&s3_key, bytes.clone(), content_type.clone()) .put_object(
&s3_key,
bytes.clone(),
content_type.clone(),
content_disposition.clone(),
)
.await .await
.map_err(|err| { .map_err(|err| {
error!(error = %err, key = %s3_key, "failed to store document"); error!(error = %err, key = %s3_key, "failed to store document");
@@ -888,7 +984,7 @@ async fn process_upload(state: &AppState, request: UploadRequest) -> AppResult<U
}; };
let detail = DocumentDetailResponse { let detail = DocumentDetailResponse {
document: to_document_response(document, None, None), document: to_document_response(state, user_id, document, None, None)?,
current_version: to_version_response(version.clone()), current_version: to_version_response(version.clone()),
assets: Vec::new(), assets: Vec::new(),
}; };
@@ -1019,11 +1115,15 @@ pub(crate) async fn load_primary_thumbnails(
} }
pub(crate) fn to_document_response( pub(crate) fn to_document_response(
state: &AppState,
user_id: Uuid,
doc: Document, doc: Document,
tags: Option<Vec<Tag>>, tags: Option<Vec<Tag>>,
thumbnail: Option<DocumentAssetResponse>, thumbnail: Option<DocumentAssetResponse>,
) -> DocumentResponse { ) -> AppResult<DocumentResponse> {
DocumentResponse { let download_path = build_download_path(state, doc.id, user_id)?;
Ok(DocumentResponse {
id: doc.id, id: doc.id,
filename: doc.filename, filename: doc.filename,
title: doc.title, title: doc.title,
@@ -1042,7 +1142,16 @@ pub(crate) fn to_document_response(
.map(TagResponse::from) .map(TagResponse::from)
.collect(), .collect(),
thumbnail, thumbnail,
} download_path,
})
}
fn build_download_path(state: &AppState, document_id: Uuid, user_id: Uuid) -> AppResult<String> {
state
.jwt
.generate_download_token(document_id, user_id)
.map(|token| format!("/download/{token}"))
.map_err(|err| AppError::internal(format!("failed to generate download token: {err}")))
} }
fn to_version_response(version: DocumentVersion) -> DocumentVersionResponse { fn to_version_response(version: DocumentVersion) -> DocumentVersionResponse {
+250 -25
View File
@@ -1,22 +1,30 @@
use anyhow::anyhow;
use axum::{ use axum::{
extract::{Json, Path, Query, State}, extract::{Json, Path, Query, State},
http::StatusCode, http::StatusCode,
}; };
use diesel::{dsl::exists, prelude::*, PgConnection}; use diesel::{dsl::exists, prelude::*, PgConnection};
use reqwest::Client;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::collections::HashSet; use serde_json::{json, Value};
use std::collections::{HashMap, HashSet};
use uuid::Uuid; use uuid::Uuid;
use crate::error::{AppError, AppResult};
use crate::models::{Document, Folder, NewFolder}; use crate::models::{Document, Folder, NewFolder};
use crate::schema::{document_tags, documents, folders}; use crate::schema::{document_tags, documents, folders};
use crate::state::AppState; use crate::state::AppState;
use crate::{
auth::AuthenticatedUser,
error::{AppError, AppResult},
};
use super::documents::{ use super::documents::{
load_primary_thumbnails, load_tags_for_documents, to_document_response, to_iso, load_primary_thumbnails, load_tags_for_documents, to_document_response, to_iso,
DocumentResponse, DocumentResponse,
}; };
const QUICKWIT_MAX_HITS: usize = 200;
#[derive(Deserialize)] #[derive(Deserialize)]
pub struct CreateFolderRequest { pub struct CreateFolderRequest {
pub name: String, pub name: String,
@@ -157,6 +165,7 @@ pub async fn create_folder(
pub async fn list_folder_contents( pub async fn list_folder_contents(
State(state): State<AppState>, State(state): State<AppState>,
Path(folder_identifier): Path<String>, Path(folder_identifier): Path<String>,
user: AuthenticatedUser,
) -> AppResult<Json<FolderContentsResponse>> { ) -> AppResult<Json<FolderContentsResponse>> {
let mut conn = state.db()?; let mut conn = state.db()?;
@@ -209,14 +218,18 @@ pub async fn list_folder_contents(
let thumbnails = load_primary_thumbnails(&state, &docs).await?; let thumbnails = load_primary_thumbnails(&state, &docs).await?;
let documents = docs let mut documents = Vec::with_capacity(doc_ids.len());
.into_iter() for doc in docs {
.map(|doc| { let tags = tags_map.get(&doc.id).cloned();
let tags = tags_map.get(&doc.id).cloned(); let thumbnail = thumbnails.get(&doc.id).cloned();
let thumbnail = thumbnails.get(&doc.id).cloned(); documents.push(to_document_response(
to_document_response(doc, tags, thumbnail) &state,
}) user.user_id,
.collect(); doc,
tags,
thumbnail,
)?);
}
Ok(Json(FolderContentsResponse { Ok(Json(FolderContentsResponse {
folder, folder,
@@ -229,6 +242,7 @@ pub async fn search_documents(
State(state): State<AppState>, State(state): State<AppState>,
Path(folder_identifier): Path<String>, Path(folder_identifier): Path<String>,
Query(params): Query<DocumentSearchQuery>, Query(params): Query<DocumentSearchQuery>,
user: AuthenticatedUser,
) -> AppResult<Json<Vec<DocumentResponse>>> { ) -> AppResult<Json<Vec<DocumentResponse>>> {
let mut conn = state.db()?; let mut conn = state.db()?;
@@ -250,14 +264,40 @@ pub async fn search_documents(
docs_query = docs_query.filter(documents::folder_id.eq_any(descendant_ids)); docs_query = docs_query.filter(documents::folder_id.eq_any(descendant_ids));
} }
let mut filter_ids: Option<HashSet<Uuid>> = None;
let mut quickwit_order: Option<Vec<Uuid>> = None;
if let Some(query) = params if let Some(query) = params
.query .query
.as_ref() .as_ref()
.map(|s| s.trim()) .map(|s| s.trim())
.filter(|s| !s.is_empty()) .filter(|s| !s.is_empty())
{ {
let pattern = format!("%{}%", query); let endpoint = state
docs_query = docs_query.filter(documents::original_name.ilike(pattern)); .config
.quickwit_endpoint
.as_ref()
.ok_or_else(|| AppError::internal("quickwit endpoint not configured"))?;
let index = state
.config
.quickwit_index
.as_ref()
.ok_or_else(|| AppError::internal("quickwit index not configured"))?;
let ids = quickwit_search(endpoint, index, query)
.await
.map_err(|err| AppError::internal(format!("quickwit search failed: {err}")))?;
if ids.is_empty() {
return Ok(Json(vec![]));
}
quickwit_order = Some(ids.clone());
let set: HashSet<Uuid> = ids.into_iter().collect();
filter_ids = Some(match &filter_ids {
Some(existing) => existing.intersection(&set).copied().collect(),
None => set,
});
} }
if let Some(tags_param) = params if let Some(tags_param) = params
@@ -292,37 +332,222 @@ pub async fn search_documents(
} }
} }
let matching_doc_ids: Vec<Uuid> = let matching_doc_ids: HashSet<Uuid> = doc_id_set.unwrap_or_default();
doc_id_set.unwrap_or_default().into_iter().collect();
if matching_doc_ids.is_empty() { if matching_doc_ids.is_empty() {
return Ok(Json(vec![])); return Ok(Json(vec![]));
} }
docs_query = docs_query.filter(documents::id.eq_any(matching_doc_ids)); let new_filter = match &filter_ids {
Some(existing) => existing.intersection(&matching_doc_ids).copied().collect(),
None => matching_doc_ids.clone(),
};
filter_ids = Some(new_filter);
} }
} }
} }
let docs: Vec<Document> = docs_query if let Some(ref set) = filter_ids {
.order(documents::uploaded_at.desc()) if set.is_empty() {
.load(&mut conn)?; return Ok(Json(vec![]));
}
let ids_vec: Vec<Uuid> = set.iter().copied().collect();
docs_query = docs_query.filter(documents::id.eq_any(ids_vec));
}
let docs: Vec<Document> = if let Some(order_ids) = quickwit_order.as_ref() {
let relevant_ids: Vec<Uuid> = if let Some(filter_set) = filter_ids.as_ref() {
order_ids
.iter()
.copied()
.filter(|id| filter_set.contains(id))
.collect()
} else {
order_ids.clone()
};
if relevant_ids.is_empty() {
return Ok(Json(vec![]));
}
let fetched: Vec<Document> = docs_query.load(&mut conn)?;
let mut by_id: HashMap<Uuid, Document> =
fetched.into_iter().map(|doc| (doc.id, doc)).collect();
let mut ordered = Vec::with_capacity(by_id.len());
for id in relevant_ids {
if let Some(doc) = by_id.remove(&id) {
ordered.push(doc);
}
}
if !by_id.is_empty() {
let mut remaining: Vec<Document> = by_id.into_values().collect();
remaining.sort_by(|a, b| b.uploaded_at.cmp(&a.uploaded_at));
ordered.extend(remaining);
}
ordered
} else {
docs_query
.order(documents::uploaded_at.desc())
.load(&mut conn)?
};
let doc_ids: Vec<Uuid> = docs.iter().map(|doc| doc.id).collect(); let doc_ids: Vec<Uuid> = docs.iter().map(|doc| doc.id).collect();
let tags_map = load_tags_for_documents(&mut conn, &doc_ids)?; let tags_map = load_tags_for_documents(&mut conn, &doc_ids)?;
drop(conn); drop(conn);
let thumbnails = load_primary_thumbnails(&state, &docs).await?; let thumbnails = load_primary_thumbnails(&state, &docs).await?;
let response = docs let mut response = Vec::with_capacity(doc_ids.len());
.into_iter() for doc in docs {
.map(|doc| { let tags = tags_map.get(&doc.id).cloned();
let tags = tags_map.get(&doc.id).cloned(); let thumbnail = thumbnails.get(&doc.id).cloned();
let thumbnail = thumbnails.get(&doc.id).cloned(); response.push(to_document_response(
to_document_response(doc, tags, thumbnail) &state,
user.user_id,
doc,
tags,
thumbnail,
)?);
}
Ok(Json(response))
}
async fn quickwit_search(endpoint: &str, index: &str, query: &str) -> anyhow::Result<Vec<Uuid>> {
let quickwit_query = match build_quickwit_query(query) {
Some(q) => q,
None => return Ok(vec![]),
};
let client = Client::new();
let url = format!("{}/api/v1/{}/search", endpoint.trim_end_matches('/'), index);
let payload = json!({
"query": quickwit_query,
"max_hits": QUICKWIT_MAX_HITS,
});
let response = client.post(url).json(&payload).send().await?;
if !response.status().is_success() {
let status = response.status();
let body = response.text().await.unwrap_or_default();
return Err(anyhow!(
"quickwit search failed with status {status}: {body}"
));
}
let data: QuickwitSearchResponse = response.json().await?;
let mut seen = HashSet::new();
let mut doc_ids = Vec::new();
for hit in data.hits {
if let Some(doc_id) = extract_document_id(&hit) {
if seen.insert(doc_id) {
doc_ids.push(doc_id);
}
}
}
Ok(doc_ids)
}
fn build_quickwit_query(input: &str) -> Option<String> {
let tokens: Vec<String> = input
.split_whitespace()
.filter(|token| !token.is_empty())
.map(|token| {
let normalized = token.to_lowercase();
escape_quickwit_token(&normalized)
}) })
.collect(); .collect();
Ok(Json(response)) if tokens.is_empty() {
return None;
}
let parts: Vec<String> = tokens
.into_iter()
.map(|token| format!("(title:{token} OR text:{token})"))
.collect();
Some(parts.join(" AND "))
}
fn escape_quickwit_token(token: &str) -> String {
let mut escaped = String::with_capacity(token.len());
for ch in token.chars() {
match ch {
'+' | '-' | '&' | '|' | '!' | '(' | ')' | '{' | '}' | '[' | ']' | '^' | '"' | '~'
| '*' | '?' | ':' | '\\' | '/' => {
escaped.push('\\');
escaped.push(ch);
}
_ => escaped.push(ch),
}
}
escaped
}
#[derive(Deserialize)]
struct QuickwitSearchResponse {
#[serde(default)]
hits: Vec<Value>,
}
fn extract_document_id(hit: &Value) -> Option<Uuid> {
for key in ["_source", "source", "fields", "stored_fields"] {
if let Some(value) = hit.get(key) {
if let Some(uuid) = extract_uuid_from_value(value) {
return Some(uuid);
}
}
}
if let Some(value) = hit.get("document_id") {
if let Some(uuid) = extract_uuid_from_value(value) {
return Some(uuid);
}
}
None
}
fn extract_uuid_from_value(value: &Value) -> Option<Uuid> {
if let Some(obj) = value.as_object() {
if let Some(inner) = obj.get("document_id") {
return parse_uuid_value(inner);
}
}
if let Some(arr) = value.as_array() {
for item in arr {
if let Some(uuid) = extract_uuid_from_value(item) {
return Some(uuid);
}
}
}
parse_uuid_value(value)
}
fn parse_uuid_value(value: &Value) -> Option<Uuid> {
if let Some(s) = value.as_str() {
return Uuid::parse_str(s).ok();
}
if let Some(arr) = value.as_array() {
for item in arr {
if let Some(uuid) = parse_uuid_value(item) {
return Some(uuid);
}
}
}
None
} }
pub async fn delete_folder( pub async fn delete_folder(
+37 -6
View File
@@ -1,10 +1,11 @@
use axum::http::HeaderValue;
use axum::{ use axum::{
extract::DefaultBodyLimit, extract::DefaultBodyLimit,
middleware, middleware,
routing::{delete, get, patch, post}, routing::{delete, get, patch, post},
Router, Router,
}; };
use tower_http::cors::{Any, CorsLayer}; use tower_http::cors::{AllowOrigin, CorsLayer};
use crate::{auth::AuthenticatedUser, state::AppState}; use crate::{auth::AuthenticatedUser, state::AppState};
@@ -15,13 +16,37 @@ pub mod health;
pub mod tags; pub mod tags;
pub fn create_router(state: AppState) -> Router<()> { pub fn create_router(state: AppState) -> Router<()> {
let cors = CorsLayer::new() let cors = if let Some(origins) = state.config.cors_allowed_origin.as_ref() {
.allow_origin(Any) let headers: Vec<HeaderValue> = origins
.allow_methods(Any) .split(',')
.allow_headers(Any); .filter_map(|value| {
let trimmed = value.trim();
(!trimmed.is_empty()).then(|| {
trimmed
.parse::<HeaderValue>()
.expect("invalid CORS allowed origin")
})
})
.collect();
let allow_origin = AllowOrigin::list(headers);
CorsLayer::new()
.allow_origin(allow_origin)
.allow_methods(tower_http::cors::AllowMethods::mirror_request())
.allow_headers(tower_http::cors::AllowHeaders::mirror_request())
.allow_credentials(true)
} else {
CorsLayer::new()
.allow_origin(AllowOrigin::mirror_request())
.allow_methods(tower_http::cors::AllowMethods::mirror_request())
.allow_headers(tower_http::cors::AllowHeaders::mirror_request())
.allow_credentials(true)
};
let auth_routes = Router::new() let auth_routes = Router::new()
.route("/login", post(auth::login)) .route("/login", post(auth::login))
.route("/refresh", post(auth::refresh))
.route("/logout", post(auth::logout)) .route("/logout", post(auth::logout))
.route("/me", get(auth::me)); .route("/me", get(auth::me));
@@ -50,6 +75,9 @@ pub fn create_router(state: AppState) -> Router<()> {
.route("/:id/tags", post(documents::assign_tags)) .route("/:id/tags", post(documents::assign_tags))
.route("/:id/tags/:tag_id", delete(documents::remove_tag)); .route("/:id/tags/:tag_id", delete(documents::remove_tag));
let download_routes =
Router::new().route("/download/:token", get(documents::download_with_token));
let folders_routes = Router::new() let folders_routes = Router::new()
.route("/", post(folders::create_folder)) .route("/", post(folders::create_folder))
.route("/path", post(folders::ensure_folder_path)) .route("/path", post(folders::ensure_folder_path))
@@ -60,7 +88,9 @@ pub fn create_router(state: AppState) -> Router<()> {
.route("/:id/contents", get(folders::list_folder_contents)) .route("/:id/contents", get(folders::list_folder_contents))
.route("/:id/documents", get(folders::search_documents)); .route("/:id/documents", get(folders::search_documents));
let tags_routes = Router::new().route("/", get(tags::list_tags).post(tags::create_tag)); let tags_routes = Router::new()
.route("/", get(tags::list_tags).post(tags::create_tag))
.route("/:id", patch(tags::update_tag));
let protected_state = state.clone(); let protected_state = state.clone();
let protected_routes = Router::new() let protected_routes = Router::new()
@@ -71,6 +101,7 @@ pub fn create_router(state: AppState) -> Router<()> {
.layer(middleware::from_extractor_with_state::<AuthenticatedUser, _>(protected_state)); .layer(middleware::from_extractor_with_state::<AuthenticatedUser, _>(protected_state));
Router::new() Router::new()
.merge(download_routes)
.nest("/api/auth", auth_routes) .nest("/api/auth", auth_routes)
.merge(protected_routes) .merge(protected_routes)
.with_state(state) .with_state(state)
+144 -10
View File
@@ -1,32 +1,69 @@
use axum::{extract::State, Json}; use crate::utils::json::{classify_nullable, NullableValue};
use diesel::prelude::*; use axum::{
use serde::Deserialize; extract::{Path, State},
Json,
};
use diesel::{dsl::count_star, prelude::*};
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::collections::HashMap;
use uuid::Uuid; use uuid::Uuid;
use crate::error::{AppError, AppResult}; use crate::error::{AppError, AppResult};
use crate::models::{NewTag, Tag}; use crate::models::{NewTag, Tag};
use crate::schema::tags; use crate::schema::{document_tags, tags};
use crate::state::AppState; use crate::state::AppState;
use super::documents::TagResponse;
#[derive(Deserialize)] #[derive(Deserialize)]
pub struct CreateTagRequest { pub struct CreateTagRequest {
pub label: String, pub label: String,
pub color: Option<String>, pub color: Option<String>,
} }
pub async fn list_tags(State(state): State<AppState>) -> AppResult<Json<Vec<TagResponse>>> { #[derive(AsChangeset, Default)]
#[diesel(table_name = tags)]
struct UpdateTagChangeset<'a> {
label: Option<&'a str>,
color: Option<Option<&'a str>>,
}
#[derive(Serialize)]
pub struct TagCatalogEntry {
pub id: Uuid,
pub label: String,
pub color: Option<String>,
pub usage_count: i64,
}
pub async fn list_tags(State(state): State<AppState>) -> AppResult<Json<Vec<TagCatalogEntry>>> {
let mut conn = state.db()?; let mut conn = state.db()?;
let tag_list: Vec<Tag> = tags::table.order(tags::label.asc()).load(&mut conn)?; let tag_list: Vec<Tag> = tags::table.order(tags::label.asc()).load(&mut conn)?;
let response = tag_list.into_iter().map(TagResponse::from).collect();
let usage_rows: Vec<(Uuid, i64)> = document_tags::table
.group_by(document_tags::tag_id)
.select((document_tags::tag_id, count_star()))
.load(&mut conn)?;
let usage_map: HashMap<Uuid, i64> = usage_rows.into_iter().collect();
let response = tag_list
.into_iter()
.map(|tag| TagCatalogEntry {
id: tag.id,
label: tag.label,
color: tag.color,
usage_count: *usage_map.get(&tag.id).unwrap_or(&0),
})
.collect();
Ok(Json(response)) Ok(Json(response))
} }
pub async fn create_tag( pub async fn create_tag(
State(state): State<AppState>, State(state): State<AppState>,
Json(payload): Json<CreateTagRequest>, Json(payload): Json<CreateTagRequest>,
) -> AppResult<Json<TagResponse>> { ) -> AppResult<Json<TagCatalogEntry>> {
if payload.label.trim().is_empty() { if payload.label.trim().is_empty() {
return Err(AppError::bad_request("label must not be empty")); return Err(AppError::bad_request("label must not be empty"));
} }
@@ -53,5 +90,102 @@ pub async fn create_tag(
} }
let tag: Tag = tags::table.find(new_tag.id).first(&mut conn)?; let tag: Tag = tags::table.find(new_tag.id).first(&mut conn)?;
Ok(Json(TagResponse::from(tag))) Ok(Json(TagCatalogEntry {
id: tag.id,
label: tag.label,
color: tag.color,
usage_count: 0,
}))
}
pub async fn update_tag(
State(state): State<AppState>,
Path(tag_id): Path<Uuid>,
Json(body): Json<Value>,
) -> AppResult<Json<TagCatalogEntry>> {
let mut conn = state.db()?;
let existing: Tag = tags::table.find(tag_id).first(&mut conn)?;
let label_class = classify_nullable(body.get("label")).map_err(AppError::bad_request)?;
let color_class = classify_nullable(body.get("color")).map_err(AppError::bad_request)?;
if matches!(label_class, NullableValue::Omitted)
&& matches!(color_class, NullableValue::Omitted)
{
return Err(AppError::bad_request("no changes supplied"));
}
let mut new_label: Option<String> = None;
let mut label_changed = false;
match label_class {
NullableValue::Omitted => {}
NullableValue::Null => {
return Err(AppError::bad_request("label cannot be null"));
}
NullableValue::String(value) => {
let trimmed = value.trim();
if trimmed.is_empty() {
return Err(AppError::bad_request("label must not be empty"));
}
if trimmed != existing.label {
let duplicate = tags::table
.filter(tags::label.eq(trimmed))
.filter(tags::id.ne(tag_id))
.first::<Tag>(&mut conn)
.optional()?;
if duplicate.is_some() {
return Err(AppError::bad_request("tag label already exists"));
}
new_label = Some(trimmed.to_string());
label_changed = true;
}
}
}
let mut color_change: Option<Option<String>> = None;
let mut color_changed = false;
match color_class {
NullableValue::Omitted => {}
NullableValue::Null => {
color_change = Some(None);
color_changed = true;
}
NullableValue::String(value) => {
let trimmed = value.trim();
if trimmed.is_empty() {
return Err(AppError::bad_request("color must not be empty"));
}
if existing.color.as_deref() != Some(trimmed) {
color_change = Some(Some(trimmed.to_string()));
color_changed = true;
}
}
}
if !label_changed && !color_changed {
return Err(AppError::bad_request("no changes supplied"));
}
let changeset = UpdateTagChangeset {
label: new_label.as_deref(),
color: color_change
.as_ref()
.map(|opt| opt.as_ref().map(|value| value.as_str())),
};
diesel::update(tags::table.find(tag_id))
.set(&changeset)
.execute(&mut conn)?;
let updated: Tag = tags::table.find(tag_id).first(&mut conn)?;
let usage_count: i64 = document_tags::table
.filter(document_tags::tag_id.eq(tag_id))
.select(count_star())
.first(&mut conn)?;
Ok(Json(TagCatalogEntry {
id: updated.id,
label: updated.label,
color: updated.color,
usage_count,
}))
} }
+15
View File
@@ -86,6 +86,19 @@ diesel::table! {
} }
} }
diesel::table! {
refresh_tokens (id) {
id -> Uuid,
user_id -> Uuid,
token_hash -> Text,
issued_at -> Timestamptz,
expires_at -> Timestamptz,
revoked_at -> Nullable<Timestamptz>,
created_at -> Timestamptz,
updated_at -> Timestamptz,
}
}
diesel::table! { diesel::table! {
tags (id) { tags (id) {
id -> Uuid, id -> Uuid,
@@ -117,6 +130,7 @@ diesel::joinable!(document_tags -> tags (tag_id));
diesel::joinable!(document_tags -> users (assigned_by)); diesel::joinable!(document_tags -> users (assigned_by));
diesel::joinable!(document_versions -> documents (document_id)); diesel::joinable!(document_versions -> documents (document_id));
diesel::joinable!(documents -> folders (folder_id)); diesel::joinable!(documents -> folders (folder_id));
diesel::joinable!(refresh_tokens -> users (user_id));
diesel::allow_tables_to_appear_in_same_query!( diesel::allow_tables_to_appear_in_same_query!(
document_assets, document_assets,
@@ -125,6 +139,7 @@ diesel::allow_tables_to_appear_in_same_query!(
documents, documents,
folders, folders,
jobs, jobs,
refresh_tokens,
tags, tags,
users, users,
); );
+19
View File
@@ -13,11 +13,14 @@ pub trait ObjectStorage: Send + Sync + 'static {
key: &str, key: &str,
bytes: Vec<u8>, bytes: Vec<u8>,
content_type: Option<String>, content_type: Option<String>,
content_disposition: Option<String>,
) -> Result<()>; ) -> Result<()>;
async fn presign_get_object(&self, key: &str, expires_in: Duration) -> Result<String>; async fn presign_get_object(&self, key: &str, expires_in: Duration) -> Result<String>;
async fn get_object(&self, key: &str) -> Result<Vec<u8>>; async fn get_object(&self, key: &str) -> Result<Vec<u8>>;
async fn delete_object(&self, key: &str) -> Result<()>;
} }
pub struct S3Storage { pub struct S3Storage {
@@ -41,6 +44,7 @@ impl ObjectStorage for S3Storage {
key: &str, key: &str,
bytes: Vec<u8>, bytes: Vec<u8>,
content_type: Option<String>, content_type: Option<String>,
content_disposition: Option<String>,
) -> Result<()> { ) -> Result<()> {
let mut request = self let mut request = self
.client .client
@@ -53,6 +57,10 @@ impl ObjectStorage for S3Storage {
request = request.content_type(content_type); request = request.content_type(content_type);
} }
if let Some(content_disposition) = content_disposition {
request = request.content_disposition(content_disposition);
}
request request
.send() .send()
.await .await
@@ -99,4 +107,15 @@ impl ObjectStorage for S3Storage {
Ok(bytes) Ok(bytes)
} }
async fn delete_object(&self, key: &str) -> Result<()> {
self.client
.delete_object()
.bucket(&self.bucket)
.key(key)
.send()
.await
.context("failed to delete object from S3")?;
Ok(())
}
} }
+16
View File
@@ -0,0 +1,16 @@
use serde_json::Value;
pub enum NullableValue {
Omitted,
Null,
String(String),
}
pub fn classify_nullable(optional_value: Option<&Value>) -> Result<NullableValue, String> {
match optional_value {
None => Ok(NullableValue::Omitted),
Some(Value::Null) => Ok(NullableValue::Null),
Some(Value::String(s)) => Ok(NullableValue::String(s.to_owned())),
Some(other) => Err(format!("expected string or null, got {other}")),
}
}
+1
View File
@@ -0,0 +1 @@
pub mod json;
+41 -3
View File
@@ -8,10 +8,11 @@ use tokio::task;
use tracing::{error, warn}; use tracing::{error, warn};
use uuid::Uuid; use uuid::Uuid;
use super::ocr::{document_is_pdf, OCR_TEXT_ASSET_TYPE};
use crate::{ use crate::{
jobs::{enqueue_job, JOB_ANALYZE_DOCUMENT, JOB_GENERATE_THUMBNAILS}, jobs::{enqueue_job, JOB_ANALYZE_DOCUMENT, JOB_GENERATE_OCR_TEXT, JOB_GENERATE_THUMBNAILS},
models::{Document, DocumentVersion}, models::{Document, DocumentAsset, DocumentVersion},
schema::{document_versions, documents}, schema::{document_assets, document_versions, documents},
state::AppState, state::AppState,
}; };
@@ -88,6 +89,16 @@ fn analyze_document(state: Arc<AppState>, payload: AnalyzePayload) -> Result<Job
.map_err(|err| format!("{err:?}"))?; .map_err(|err| format!("{err:?}"))?;
let (supported, reason) = determine_thumbnail_support(&document); let (supported, reason) = determine_thumbnail_support(&document);
let ocr_supported = document_is_pdf(&document);
let existing_ocr: Option<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq(payload.document_version_id))
.filter(document_assets::asset_type.eq(OCR_TEXT_ASSET_TYPE))
.first(&mut conn)
.optional()
.map_err(|err| format!("{err:?}"))?;
let skip_ocr = existing_ocr.is_some() && !payload.force;
let mut summary_map = match version.operations_summary { let mut summary_map = match version.operations_summary {
Value::Object(map) => map, Value::Object(map) => map,
@@ -100,6 +111,16 @@ fn analyze_document(state: Arc<AppState>, payload: AnalyzePayload) -> Result<Job
summary_map.remove("thumbnail_reason"); summary_map.remove("thumbnail_reason");
} }
summary_map.insert("ocr_supported".to_string(), Value::Bool(ocr_supported));
if ocr_supported {
summary_map.remove("ocr_reason");
} else {
summary_map.insert(
"ocr_reason".to_string(),
Value::String("document is not a PDF".into()),
);
}
diesel::update(document_versions::table.find(version.id)) diesel::update(document_versions::table.find(version.id))
.set(document_versions::operations_summary.eq(Value::Object(summary_map))) .set(document_versions::operations_summary.eq(Value::Object(summary_map)))
.execute(&mut conn) .execute(&mut conn)
@@ -122,6 +143,23 @@ fn analyze_document(state: Arc<AppState>, payload: AnalyzePayload) -> Result<Job
} }
} }
if ocr_supported && !skip_ocr {
let enqueue_result = enqueue_job(
&mut conn,
JOB_GENERATE_OCR_TEXT,
json!({
"document_id": payload.document_id,
"document_version_id": payload.document_version_id,
"force": payload.force,
}),
None,
);
if let Err(err) = enqueue_result {
return Err(err.to_string());
}
}
Ok(JobExecution::Success) Ok(JobExecution::Success)
} }
+204
View File
@@ -0,0 +1,204 @@
use std::sync::Arc;
use std::time::Duration;
use async_trait::async_trait;
use diesel::prelude::*;
use reqwest::Client;
use serde::Deserialize;
use serde_json::json;
use tokio::task;
use tracing::{error, warn};
use uuid::Uuid;
use crate::{
jobs::JOB_INDEX_DOCUMENT_TEXT,
models::{Document, DocumentAsset, DocumentVersion},
schema::{document_assets, document_versions, documents},
state::AppState,
};
use super::{ocr::OCR_TEXT_ASSET_TYPE, JobExecution, JobHandler};
#[derive(Debug, Deserialize)]
struct IndexPayload {
document_id: Uuid,
document_version_id: Uuid,
}
pub struct IndexDocumentTextJob;
impl IndexDocumentTextJob {
pub fn new() -> Self {
Self
}
}
#[async_trait]
impl JobHandler for IndexDocumentTextJob {
fn job_type(&self) -> &'static str {
JOB_INDEX_DOCUMENT_TEXT
}
async fn handle(&self, state: Arc<AppState>, job: crate::models::Job) -> JobExecution {
let payload: IndexPayload = match serde_json::from_value(job.payload.clone()) {
Ok(payload) => payload,
Err(err) => {
return JobExecution::Failed {
error: format!("invalid index payload: {err}"),
}
}
};
let quickwit_endpoint = match &state.config.quickwit_endpoint {
Some(endpoint) => endpoint.clone(),
None => {
warn!("quickwit endpoint missing; skipping indexing");
return JobExecution::Success;
}
};
let quickwit_index = match &state.config.quickwit_index {
Some(index) => index.clone(),
None => {
warn!("quickwit index missing; skipping indexing");
return JobExecution::Success;
}
};
let client = Client::new();
let state_clone = state.clone();
let context = match task::spawn_blocking(move || load_context(state_clone, &payload)).await
{
Ok(Ok(ctx)) => ctx,
Ok(Err(err)) => {
warn!(job_id = %job.id, error = %err, "index job will retry");
return JobExecution::Retry {
delay: Duration::from_secs(30),
error: err,
};
}
Err(join_err) => {
error!(job_id = %job.id, error = %join_err, "index task panicked");
return JobExecution::Retry {
delay: Duration::from_secs(60),
error: format!("worker panicked: {join_err}"),
};
}
};
if context.text_asset.is_none() {
warn!(job_id = %job.id, "missing OCR text asset; failing indexing job");
return JobExecution::Failed {
error: "missing OCR text asset".into(),
};
}
let asset = context.text_asset.unwrap();
let text = match state.storage.get_object(&asset.s3_key).await {
Ok(bytes) => match String::from_utf8(bytes) {
Ok(text) => text,
Err(err) => {
warn!(job_id = %job.id, error = %err, "ocr text not valid UTF-8");
return JobExecution::Failed {
error: "ocr text not valid UTF-8".into(),
};
}
},
Err(err) => {
warn!(job_id = %job.id, error = %err, "failed to download ocr text");
return JobExecution::Retry {
delay: Duration::from_secs(30),
error: err.to_string(),
};
}
};
if text.trim().is_empty() {
warn!(job_id = %job.id, "ocr text empty; skipping");
return JobExecution::Failed {
error: "ocr text empty".into(),
};
}
let client = client;
let url = format!(
"{}/api/v1/{}/ingest?commit=auto",
quickwit_endpoint, quickwit_index
);
let payload = json!({
"document_id": context.document.id,
"version_id": context.version.id,
"title": context.document.title,
"text": text,
});
let body = serde_json::to_string(&payload).unwrap();
match client
.post(&url)
.header("content-type", "application/x-ndjson")
.body(format!("{}\n", body))
.send()
.await
{
Ok(response) => {
if response.status().is_success() {
JobExecution::Success
} else {
let status = response.status();
let body = response.text().await.unwrap_or_default();
warn!(job_id = %job.id, %status, %body, "quickwit ingest failed");
JobExecution::Retry {
delay: Duration::from_secs(30),
error: format!("quickwit ingest failed with status {status}"),
}
}
}
Err(err) => {
warn!(job_id = %job.id, error = %err, "quickwit request failed");
JobExecution::Retry {
delay: Duration::from_secs(30),
error: err.to_string(),
}
}
}
}
}
struct IndexContext {
document: Document,
version: DocumentVersion,
text_asset: Option<DocumentAsset>,
}
fn load_context(state: Arc<AppState>, payload: &IndexPayload) -> Result<IndexContext, String> {
let mut conn = state.db().map_err(|err| format!("{err:?}"))?;
let version: DocumentVersion = document_versions::table
.find(payload.document_version_id)
.first(&mut conn)
.map_err(|err| format!("{err:?}"))?;
if version.document_id != payload.document_id {
return Err("document/version mismatch".into());
}
let document: Document = documents::table
.find(payload.document_id)
.first(&mut conn)
.map_err(|err| format!("{err:?}"))?;
let text_asset: Option<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq(payload.document_version_id))
.filter(document_assets::asset_type.eq(OCR_TEXT_ASSET_TYPE))
.first(&mut conn)
.optional()
.map_err(|err| format!("{err:?}"))?;
Ok(IndexContext {
document,
version,
text_asset,
})
}
+4
View File
@@ -11,6 +11,8 @@ use crate::{
}; };
pub mod analyze; pub mod analyze;
pub mod index;
pub mod ocr;
pub mod thumbnails; pub mod thumbnails;
#[derive(Debug)] #[derive(Debug)]
@@ -128,5 +130,7 @@ pub fn default_handlers() -> Vec<Arc<dyn JobHandler>> {
vec![ vec![
Arc::new(analyze::AnalyzeDocumentJob::new()), Arc::new(analyze::AnalyzeDocumentJob::new()),
Arc::new(thumbnails::GenerateThumbnailsJob::new()), Arc::new(thumbnails::GenerateThumbnailsJob::new()),
Arc::new(ocr::GenerateOcrTextJob::new()),
Arc::new(index::IndexDocumentTextJob::new()),
] ]
} }
+440
View File
@@ -0,0 +1,440 @@
use std::{
fmt, fs,
io::{ErrorKind, Write},
process::Command,
sync::Arc,
time::Duration,
};
use async_trait::async_trait;
use chrono::Utc;
use diesel::{pg::upsert::excluded, prelude::*};
use pdfium_render::prelude::*;
use serde::Deserialize;
use serde_json::json;
use tempfile::NamedTempFile;
use tokio::task;
use tracing::{error, info, warn};
use uuid::Uuid;
use crate::{
jobs::{enqueue_job, JOB_GENERATE_OCR_TEXT, JOB_INDEX_DOCUMENT_TEXT},
models::{Document, DocumentAsset, DocumentVersion, NewDocumentAsset},
schema::{document_assets, document_versions, documents},
state::AppState,
};
use super::{JobExecution, JobHandler};
pub const OCR_TEXT_ASSET_TYPE: &str = "ocr-text";
const MIN_TEXT_LENGTH: usize = 50;
#[derive(Clone, Debug, Deserialize)]
struct OcrPayload {
document_id: Uuid,
document_version_id: Uuid,
#[serde(default)]
force: bool,
}
pub struct GenerateOcrTextJob;
impl GenerateOcrTextJob {
pub fn new() -> Self {
Self
}
}
#[async_trait]
impl JobHandler for GenerateOcrTextJob {
fn job_type(&self) -> &'static str {
JOB_GENERATE_OCR_TEXT
}
async fn handle(&self, state: Arc<AppState>, job: crate::models::Job) -> JobExecution {
let payload: OcrPayload = match serde_json::from_value(job.payload.clone()) {
Ok(payload) => payload,
Err(err) => {
return JobExecution::Failed {
error: format!("invalid OCR payload: {err}"),
}
}
};
let state_clone = state.clone();
let payload_clone = payload.clone();
let context =
match task::spawn_blocking(move || load_ocr_context(state_clone, &payload_clone)).await
{
Ok(Ok(ctx)) => ctx,
Ok(Err(err)) => {
warn!(job_id = %job.id, error = %err, "ocr job will retry");
return JobExecution::Retry {
delay: Duration::from_secs(30),
error: err,
};
}
Err(join_err) => {
error!(job_id = %job.id, error = %join_err, "ocr task panicked");
return JobExecution::Retry {
delay: Duration::from_secs(60),
error: format!("worker panicked: {join_err}"),
};
}
};
if context.skip {
info!(job_id = %job.id, "ocr already present; skipping");
return JobExecution::Success;
}
let bytes = match state.storage.get_object(&context.version.s3_key).await {
Ok(bytes) => bytes,
Err(err) => {
warn!(job_id = %job.id, error = %err, "failed to fetch document for ocr");
return JobExecution::Retry {
delay: Duration::from_secs(30),
error: err.to_string(),
};
}
};
let doc_meta = PdfDocumentMeta {
content_type: context.document.content_type.clone(),
original_name: context.document.original_name.clone(),
};
let generation =
match task::spawn_blocking(move || generate_ocr_text(&doc_meta, &bytes)).await {
Ok(result) => result,
Err(join_err) => {
error!(job_id = %job.id, error = %join_err, "ocr text task panicked");
return JobExecution::Retry {
delay: Duration::from_secs(60),
error: format!("worker panicked: {join_err}"),
};
}
};
let Some(generation) = generation else {
warn!(job_id = %job.id, "no text extracted from document; failing job");
return JobExecution::Failed {
error: "no text extracted and OCR unavailable".into(),
};
};
let asset_id = context
.existing_asset
.as_ref()
.map(|asset| asset.id)
.unwrap_or_else(Uuid::new_v4);
let s3_key = format!(
"documents/{}/v{}/assets/{}/{}",
context.document.id, context.version.version_number, OCR_TEXT_ASSET_TYPE, asset_id
);
if let Err(err) = state
.storage
.put_object(
&s3_key,
generation.text.into_bytes(),
Some("text/plain".into()),
None,
)
.await
{
warn!(job_id = %job.id, error = %err, "failed to upload ocr text");
return JobExecution::Retry {
delay: Duration::from_secs(30),
error: err.to_string(),
};
}
let state_clone = state.clone();
match task::spawn_blocking(move || {
persist_ocr_metadata(state_clone, &context, asset_id, &s3_key, generation.source)
})
.await
{
Ok(Ok(())) => {
if state.config.quickwit_endpoint.is_some() && state.config.quickwit_index.is_some()
{
if let Err(err) = enqueue_index_job(&state, &payload) {
warn!(job_id = %job.id, error = %err, "failed to enqueue index job");
}
}
JobExecution::Success
}
Ok(Err(err)) => {
warn!(job_id = %job.id, error = %err, "failed to persist ocr metadata");
JobExecution::Retry {
delay: Duration::from_secs(30),
error: err,
}
}
Err(join_err) => {
error!(job_id = %job.id, error = %join_err, "ocr metadata task panicked");
JobExecution::Retry {
delay: Duration::from_secs(30),
error: format!("metadata update panic: {join_err}"),
}
}
}
}
}
struct PdfDocumentMeta {
content_type: Option<String>,
original_name: String,
}
struct OcrContext {
document: Document,
version: DocumentVersion,
existing_asset: Option<DocumentAsset>,
skip: bool,
}
struct OcrGeneration {
text: String,
source: &'static str,
}
fn load_ocr_context(state: Arc<AppState>, payload: &OcrPayload) -> Result<OcrContext, String> {
let mut conn = state.db().map_err(|err| format!("{err:?}"))?;
let version: DocumentVersion = document_versions::table
.find(payload.document_version_id)
.first(&mut conn)
.map_err(|err| format!("{err:?}"))?;
if version.document_id != payload.document_id {
return Err("document/version mismatch".into());
}
let document: Document = documents::table
.find(payload.document_id)
.first(&mut conn)
.map_err(|err| format!("{err:?}"))?;
let existing: Option<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq(payload.document_version_id))
.filter(document_assets::asset_type.eq(OCR_TEXT_ASSET_TYPE))
.first(&mut conn)
.optional()
.map_err(|err| format!("{err:?}"))?;
let is_pdf = document_is_pdf(&document);
if !is_pdf {
return Ok(OcrContext {
document,
version,
existing_asset: existing,
skip: true,
});
}
let skip = existing.is_some() && !payload.force;
Ok(OcrContext {
document,
version,
existing_asset: existing,
skip,
})
}
fn generate_ocr_text(meta: &PdfDocumentMeta, bytes: &[u8]) -> Option<OcrGeneration> {
if !document_meta_is_pdf(meta) {
return None;
}
if let Ok(text) = extract_pdf_text(bytes) {
if text.trim().chars().count() >= MIN_TEXT_LENGTH {
return Some(OcrGeneration {
text,
source: "pdf-text",
});
}
}
match run_ocr(bytes) {
Ok(Some(text)) => Some(OcrGeneration {
text,
source: "ocr",
}),
Ok(None) => None,
Err(OcrError::BinaryMissing) => {
warn!("ocrmypdf not installed; cannot perform OCR");
None
}
Err(err) => {
warn!(error = ?err, "ocr command failed");
None
}
}
}
fn extract_pdf_text(bytes: &[u8]) -> Result<String, String> {
let pdfium = Pdfium::default();
let document = pdfium
.load_pdf_from_byte_slice(bytes, None)
.map_err(|err| format!("load pdf: {err}"))?;
let mut combined = String::new();
let pages = document.pages();
for page_index in 0..pages.len() {
let page = pages
.get(page_index)
.map_err(|err| format!("load page {page_index}: {err}"))?;
if let Ok(page_text) = page.text() {
for segment in page_text.segments().iter() {
combined.push_str(&segment.text());
combined.push('\n');
}
};
}
Ok(combined)
}
#[derive(Debug)]
enum OcrError {
BinaryMissing,
Failed(String),
}
impl fmt::Display for OcrError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
OcrError::BinaryMissing => write!(f, "ocrmypdf binary not found"),
OcrError::Failed(msg) => write!(f, "ocr failed: {msg}"),
}
}
}
fn run_ocr(bytes: &[u8]) -> Result<Option<String>, OcrError> {
let mut input = NamedTempFile::new().map_err(|err| OcrError::Failed(err.to_string()))?;
input
.write_all(bytes)
.map_err(|err| OcrError::Failed(err.to_string()))?;
input
.flush()
.map_err(|err| OcrError::Failed(err.to_string()))?;
let output_pdf = NamedTempFile::new().map_err(|err| OcrError::Failed(err.to_string()))?;
let sidecar = NamedTempFile::new().map_err(|err| OcrError::Failed(err.to_string()))?;
let status = Command::new("ocrmypdf")
.arg("--sidecar")
.arg(sidecar.path())
.arg("--skip-text")
.arg(input.path())
.arg(output_pdf.path())
.output();
match status {
Ok(output) => {
if !output.status.success() {
return Err(OcrError::Failed(format!(
"ocrmypdf failed: exit={} stderr={}",
output.status,
String::from_utf8_lossy(&output.stderr)
)));
}
let text = fs::read_to_string(sidecar.path())
.map_err(|err| OcrError::Failed(err.to_string()))?;
if text.trim().chars().count() >= MIN_TEXT_LENGTH {
Ok(Some(text))
} else {
Ok(None)
}
}
Err(err) => {
if err.kind() == ErrorKind::NotFound {
Err(OcrError::BinaryMissing)
} else {
Err(OcrError::Failed(err.to_string()))
}
}
}
}
fn persist_ocr_metadata(
state: Arc<AppState>,
context: &OcrContext,
asset_id: Uuid,
s3_key: &str,
source: &'static str,
) -> Result<(), String> {
let mut conn = state.db().map_err(|err| format!("{err:?}"))?;
let new_asset = NewDocumentAsset {
id: asset_id,
document_version_id: context.version.id,
asset_type: OCR_TEXT_ASSET_TYPE.to_string(),
s3_key: s3_key.to_string(),
mime_type: "text/plain".to_string(),
width: None,
height: None,
metadata: json!({
"generated_at": Utc::now().to_rfc3339(),
"source": source,
}),
};
diesel::insert_into(document_assets::table)
.values(&new_asset)
.on_conflict((
document_assets::document_version_id,
document_assets::asset_type,
))
.do_update()
.set((
document_assets::s3_key.eq(excluded(document_assets::s3_key)),
document_assets::mime_type.eq(excluded(document_assets::mime_type)),
document_assets::metadata.eq(excluded(document_assets::metadata)),
))
.execute(&mut conn)
.map_err(|err| format!("{err:?}"))?;
Ok(())
}
fn enqueue_index_job(state: &AppState, payload: &OcrPayload) -> Result<(), String> {
let mut conn = state.db().map_err(|err| format!("{err:?}"))?;
enqueue_job(
&mut conn,
JOB_INDEX_DOCUMENT_TEXT,
json!({
"document_id": payload.document_id,
"document_version_id": payload.document_version_id,
}),
None,
)
.map(|_| ())
.map_err(|err| err.to_string())
}
pub fn document_is_pdf(document: &Document) -> bool {
document_meta_is_pdf(&PdfDocumentMeta {
content_type: document.content_type.clone(),
original_name: document.original_name.clone(),
})
}
fn document_meta_is_pdf(meta: &PdfDocumentMeta) -> bool {
if let Some(content_type) = &meta.content_type {
if content_type.eq_ignore_ascii_case("application/pdf") {
return true;
}
}
meta.original_name
.rsplit('.')
.next()
.map(|ext| ext.eq_ignore_ascii_case("pdf"))
.unwrap_or(false)
}
+19 -8
View File
@@ -102,12 +102,23 @@ impl JobHandler for GenerateThumbnailsJob {
} }
}; };
let asset_id = initial
.existing_asset
.as_ref()
.map(|asset| asset.id)
.unwrap_or_else(Uuid::new_v4);
let s3_key = format!(
"documents/{}/v{}/assets/{}/{}",
initial.document.id, initial.version.version_number, THUMBNAIL_ASSET_TYPE, asset_id
);
if let Err(err) = state if let Err(err) = state
.storage .storage
.put_object( .put_object(
&generation.s3_key, &s3_key,
generation.image_bytes.clone(), generation.image_bytes.clone(),
Some("image/png".into()), Some("image/png".into()),
None,
) )
.await .await
{ {
@@ -120,7 +131,7 @@ impl JobHandler for GenerateThumbnailsJob {
let state_clone = state.clone(); let state_clone = state.clone();
match task::spawn_blocking(move || { match task::spawn_blocking(move || {
persist_thumbnail_metadata(state_clone, &initial, &generation) persist_thumbnail_metadata(state_clone, &initial, &generation, asset_id, &s3_key)
}) })
.await .await
{ {
@@ -148,6 +159,7 @@ impl JobHandler for GenerateThumbnailsJob {
struct ThumbnailContext { struct ThumbnailContext {
document: Document, document: Document,
version: DocumentVersion, version: DocumentVersion,
existing_asset: Option<DocumentAsset>,
skip: bool, skip: bool,
} }
@@ -155,7 +167,6 @@ struct GeneratedThumbnail {
image_bytes: Vec<u8>, image_bytes: Vec<u8>,
width: Option<i32>, width: Option<i32>,
height: Option<i32>, height: Option<i32>,
s3_key: String,
} }
fn load_thumbnail_context( fn load_thumbnail_context(
@@ -195,6 +206,7 @@ fn load_thumbnail_context(
Ok(ThumbnailContext { Ok(ThumbnailContext {
document, document,
version, version,
existing_asset: existing,
skip, skip,
}) })
} }
@@ -219,13 +231,10 @@ fn generate_thumbnail(document: &Document, bytes: &[u8]) -> Result<GeneratedThum
generate_image_thumbnail(bytes)? generate_image_thumbnail(bytes)?
}; };
let s3_key = format!("thumbnails/{}/{}.png", document.id, Uuid::new_v4());
Ok(GeneratedThumbnail { Ok(GeneratedThumbnail {
image_bytes: png_bytes, image_bytes: png_bytes,
width, width,
height, height,
s3_key,
}) })
} }
@@ -284,14 +293,16 @@ fn persist_thumbnail_metadata(
state: Arc<AppState>, state: Arc<AppState>,
context: &ThumbnailContext, context: &ThumbnailContext,
generated: &GeneratedThumbnail, generated: &GeneratedThumbnail,
asset_id: Uuid,
s3_key: &str,
) -> Result<(), String> { ) -> Result<(), String> {
let mut conn = state.db().map_err(|err| format!("{err:?}"))?; let mut conn = state.db().map_err(|err| format!("{err:?}"))?;
let new_asset = NewDocumentAsset { let new_asset = NewDocumentAsset {
id: Uuid::new_v4(), id: asset_id,
document_version_id: context.version.id, document_version_id: context.version.id,
asset_type: THUMBNAIL_ASSET_TYPE.to_string(), asset_type: THUMBNAIL_ASSET_TYPE.to_string(),
s3_key: generated.s3_key.clone(), s3_key: s3_key.to_string(),
mime_type: "image/png".to_string(), mime_type: "image/png".to_string(),
width: generated.width, width: generated.width,
height: generated.height, height: generated.height,
+17
View File
@@ -37,6 +37,7 @@ pub struct StoredObject {
pub key: String, pub key: String,
pub bytes: Vec<u8>, pub bytes: Vec<u8>,
pub content_type: Option<String>, pub content_type: Option<String>,
pub content_disposition: Option<String>,
} }
#[derive(Default)] #[derive(Default)]
@@ -51,11 +52,13 @@ impl ObjectStorage for FakeStorage {
key: &str, key: &str,
bytes: Vec<u8>, bytes: Vec<u8>,
content_type: Option<String>, content_type: Option<String>,
content_disposition: Option<String>,
) -> Result<()> { ) -> Result<()> {
let stored = StoredObject { let stored = StoredObject {
key: key.to_string(), key: key.to_string(),
bytes, bytes,
content_type, content_type,
content_disposition,
}; };
let mut guard = self.objects.lock().await; let mut guard = self.objects.lock().await;
guard.insert(stored.key.clone(), stored); guard.insert(stored.key.clone(), stored);
@@ -78,6 +81,12 @@ impl ObjectStorage for FakeStorage {
.map(|obj| obj.bytes.clone()) .map(|obj| obj.bytes.clone())
.ok_or_else(|| anyhow!("object {key} missing")) .ok_or_else(|| anyhow!("object {key} missing"))
} }
async fn delete_object(&self, key: &str) -> Result<()> {
let mut guard = self.objects.lock().await;
guard.remove(key);
Ok(())
}
} }
impl FakeStorage { impl FakeStorage {
@@ -113,11 +122,19 @@ impl TestApp {
jwt_issuer: "test-issuer".to_string(), jwt_issuer: "test-issuer".to_string(),
jwt_audience: "test-audience".to_string(), jwt_audience: "test-audience".to_string(),
jwt_expiry_minutes: 60, jwt_expiry_minutes: 60,
download_token_audience: "test-download".to_string(),
download_token_expiry_minutes: 60,
refresh_token_expiry_days: 30,
refresh_cookie_secure: false,
refresh_cookie_domain: None,
cors_allowed_origin: None,
aws_endpoint_url: None, aws_endpoint_url: None,
aws_access_key_id: None, aws_access_key_id: None,
aws_secret_access_key: None, aws_secret_access_key: None,
aws_region: "us-east-1".to_string(), aws_region: "us-east-1".to_string(),
s3_bucket: "test-bucket".to_string(), s3_bucket: "test-bucket".to_string(),
quickwit_endpoint: None,
quickwit_index: None,
}; };
let pool = db::init_pool(&config.database_url)?; let pool = db::init_pool(&config.database_url)?;
+13
View File
@@ -22,6 +22,7 @@ struct DocumentInfo {
deleted_at: Option<String>, deleted_at: Option<String>,
issued_at: Option<String>, issued_at: Option<String>,
tags: Vec<TagSummary>, tags: Vec<TagSummary>,
download_path: String,
} }
#[derive(Deserialize)] #[derive(Deserialize)]
@@ -42,6 +43,7 @@ struct DocumentAssetInfo {
struct DocumentListItem { struct DocumentListItem {
id: Uuid, id: Uuid,
current_version: i32, current_version: i32,
download_path: String,
} }
#[derive(Deserialize)] #[derive(Deserialize)]
@@ -154,6 +156,7 @@ async fn upload_and_list_document() -> Result<()> {
assert_eq!(detail.document.deleted_at, None); assert_eq!(detail.document.deleted_at, None);
assert!(detail.document.issued_at.is_none()); assert!(detail.document.issued_at.is_none());
assert!(detail.document.tags.is_empty()); assert!(detail.document.tags.is_empty());
assert!(detail.document.download_path.starts_with("/download/"));
assert_eq!(detail.current_version.size_bytes, file_bytes.len() as i64); assert_eq!(detail.current_version.size_bytes, file_bytes.len() as i64);
assert!(detail.assets.is_empty()); assert!(detail.assets.is_empty());
@@ -173,6 +176,7 @@ async fn upload_and_list_document() -> Result<()> {
let item = list.pop().unwrap(); let item = list.pop().unwrap();
assert_eq!(item.id, detail.document.id); assert_eq!(item.id, detail.document.id);
assert_eq!(item.current_version, 1); assert_eq!(item.current_version, 1);
assert!(item.download_path.starts_with("/download/"));
let download = app let download = app
.get( .get(
@@ -186,6 +190,15 @@ async fn upload_and_list_document() -> Result<()> {
assert!(download_info.url.contains(&detail.current_version.s3_key)); assert!(download_info.url.contains(&detail.current_version.s3_key));
assert_eq!(download_info.filename, "doc.txt"); assert_eq!(download_info.filename, "doc.txt");
let redirect = app.get(&detail.document.download_path, None).await?;
assert_eq!(redirect.status(), StatusCode::TEMPORARY_REDIRECT);
let location = redirect
.headers()
.get("location")
.expect("redirect location header");
let location = location.to_str().expect("location header utf8");
assert!(location.contains(&detail.current_version.s3_key));
app.cleanup().await?; app.cleanup().await?;
Ok(()) Ok(())
} }
+53 -1
View File
@@ -21,11 +21,16 @@ struct DocumentInfo {
#[derive(Deserialize)] #[derive(Deserialize)]
struct TagInfo { struct TagInfo {
label: String, label: String,
#[allow(dead_code)]
color: Option<String>,
} }
#[derive(Deserialize)] #[derive(Deserialize)]
struct TagResponse { struct TagResponse {
id: Uuid, id: Uuid,
label: String,
color: Option<String>,
usage_count: i64,
} }
#[derive(Serialize)] #[derive(Serialize)]
@@ -75,6 +80,53 @@ async fn tag_assignment_flow() -> Result<()> {
assert_eq!(create_tag.status(), StatusCode::OK); assert_eq!(create_tag.status(), StatusCode::OK);
let body = body_to_vec(create_tag.into_body()).await?; let body = body_to_vec(create_tag.into_body()).await?;
let tag: TagResponse = serde_json::from_slice(&body)?; let tag: TagResponse = serde_json::from_slice(&body)?;
assert_eq!(tag.label, "Important");
assert_eq!(tag.color.as_deref(), Some("#FF0000"));
assert_eq!(tag.usage_count, 0);
let update = app
.patch_json(
&format!("/api/tags/{}", tag.id),
&serde_json::json!({
"label": "Critical",
"color": "#00FF00"
}),
Some(&token),
)
.await?;
let updated_status = update.status();
let updated_body = body_to_vec(update.into_body()).await?;
if updated_status != StatusCode::OK {
panic!(
"update tag failed: {}",
String::from_utf8_lossy(&updated_body)
);
}
let updated: TagResponse = serde_json::from_slice(&updated_body)?;
assert_eq!(updated.label, "Critical");
assert_eq!(updated.color.as_deref(), Some("#00FF00"));
assert_eq!(updated.usage_count, 0);
let clear_color = app
.patch_json(
&format!("/api/tags/{}", tag.id),
&serde_json::json!({
"color": null
}),
Some(&token),
)
.await?;
let cleared_status = clear_color.status();
let cleared_body = body_to_vec(clear_color.into_body()).await?;
if cleared_status != StatusCode::OK {
panic!(
"clear color failed: {}",
String::from_utf8_lossy(&cleared_body)
);
}
let cleared: TagResponse = serde_json::from_slice(&cleared_body)?;
assert_eq!(cleared.color, None);
assert_eq!(cleared.usage_count, 0);
let assign = app let assign = app
.post_json( .post_json(
@@ -97,7 +149,7 @@ async fn tag_assignment_flow() -> Result<()> {
let refreshed_body = body_to_vec(refreshed.into_body()).await?; let refreshed_body = body_to_vec(refreshed.into_body()).await?;
let refreshed_detail: DocumentDetail = serde_json::from_slice(&refreshed_body)?; let refreshed_detail: DocumentDetail = serde_json::from_slice(&refreshed_body)?;
assert_eq!(refreshed_detail.document.tags.len(), 1); assert_eq!(refreshed_detail.document.tags.len(), 1);
assert_eq!(refreshed_detail.document.tags[0].label, "Important"); assert_eq!(refreshed_detail.document.tags[0].label, "Critical");
let remove = app let remove = app
.delete( .delete(
+36
View File
@@ -14,3 +14,39 @@ services:
interval: 5s interval: 5s
timeout: 5s timeout: 5s
retries: 5 retries: 5
quickwit:
image: quickwit/quickwit:0.8.2
command: ["run"]
environment:
QW_ENABLE_API_AUTH: "false"
QW_DATA_DIR: /quickwit/data
ports:
- "7280:7280"
volumes:
- quickwit_test_data:/quickwit/data
healthcheck:
test: ["CMD", "curl", "-sf", "http://127.0.0.1:7280/api/v1/version"]
interval: 10s
timeout: 5s
retries: 5
quickwit-create-index:
image: quickwit/quickwit:0.8.2
depends_on:
quickwit:
condition: service_healthy
volumes:
- quickwit_test_data:/quickwit/data
- ./quickwit/documents-index.yaml:/tmp/documents-index.yaml:ro
environment:
QW_DATA_DIR: /quickwit/data
QW_NODE_URI: http://quickwit:7280
entrypoint: ["/bin/sh", "-c"]
command: >
quickwit index list --node-uri http://quickwit:7280 2>/dev/null | grep -q '"documents"' \
|| quickwit index create --index-config /tmp/documents-index.yaml --node-uri http://quickwit:7280
restart: "no"
volumes:
quickwit_test_data:
+28 -2
View File
@@ -1,5 +1,3 @@
version: '3.8'
services: services:
postgres: postgres:
image: postgres:16-alpine image: postgres:16-alpine
@@ -48,6 +46,34 @@ services:
exit 0; exit 0;
" "
quickwit:
image: quickwit/quickwit:0.8.2
command: ["run"]
environment:
QW_ENABLE_API_AUTH: "false"
QW_DATA_DIR: /quickwit/data
ports:
- "7280:7280"
volumes:
- quickwit_data:/quickwit/data
healthcheck:
test: ["CMD", "curl", "-sf", "http://127.0.0.1:7280/api/v1/version"]
interval: 10s
timeout: 5s
retries: 5
quickwit-create-index:
image: quickwit/quickwit:0.8.2
depends_on:
quickwit:
condition: service_healthy
volumes:
- quickwit_data:/quickwit/data
- ./quickwit/documents-index.yaml:/tmp/documents-index.yaml:ro
command: index create --endpoint=http://quickwit:7280 --index-config /tmp/documents-index.yaml
restart: "no"
volumes: volumes:
postgres_data: postgres_data:
minio_data: minio_data:
quickwit_data:
+19
View File
@@ -0,0 +1,19 @@
# syntax=docker/dockerfile:1
FROM node:20-alpine AS build
WORKDIR /app
COPY package.json package-lock.json ./
RUN npm ci --no-audit --no-fund
COPY . .
RUN npm run build
FROM nginx:alpine
WORKDIR /usr/share/nginx/html
COPY nginx.conf /etc/nginx/conf.d/default.conf
COPY --from=build /app/dist ./
EXPOSE 80
CMD ["nginx", "-g", "daemon off;"]
+11
View File
@@ -0,0 +1,11 @@
server {
listen 80;
server_name _;
root /usr/share/nginx/html;
index index.html;
location / {
try_files $uri /index.html;
}
}
+1015 -310
View File
File diff suppressed because it is too large Load Diff
+212 -15
View File
@@ -52,6 +52,30 @@ button:hover:not([disabled]) {
background: #2047d9; background: #2047d9;
} }
a.button-link {
display: inline-flex;
align-items: center;
gap: 0.35rem;
font: inherit;
border-radius: 2px;
border: 1px solid transparent;
padding: 0.35rem 0.85rem;
background: var(--accent);
color: #fff;
text-decoration: none;
font-weight: 500;
transition: background 0.15s ease, border-color 0.15s ease;
}
a.button-link[aria-disabled='true'] {
opacity: 0.55;
pointer-events: none;
}
a.button-link:hover:not([aria-disabled='true']) {
background: #2047d9;
}
button.secondary { button.secondary {
background: transparent; background: transparent;
color: var(--fg); color: var(--fg);
@@ -194,13 +218,21 @@ button.icon-button.ghost:hover:not([disabled]) {
overflow: hidden; overflow: hidden;
} }
.tags-main {
flex: 1;
display: grid;
grid-template-columns: 220px minmax(0, 1fr);
gap: 1.5rem;
padding: 0.75rem 1.5rem 1.25rem;
min-height: 0;
overflow: hidden;
}
.preview-main { .preview-main {
flex: 1; flex: 1;
display: flex; display: flex;
flex-direction: column; flex-direction: column;
padding: 0.75rem 1.5rem 1.25rem;
min-height: 0; min-height: 0;
overflow: hidden;
} }
.preview-workspace { .preview-workspace {
@@ -216,6 +248,7 @@ button.icon-button.ghost:hover:not([disabled]) {
align-items: center; align-items: center;
justify-content: space-between; justify-content: space-between;
gap: 1rem; gap: 1rem;
padding: 0.75rem 1.5rem 0;
} }
.preview-workspace__meta { .preview-workspace__meta {
@@ -243,9 +276,6 @@ button.icon-button.ghost:hover:not([disabled]) {
.preview-workspace__body { .preview-workspace__body {
flex: 1; flex: 1;
background: var(--surface-subtle);
border: 1px solid var(--border);
border-radius: 2px;
display: flex; display: flex;
align-items: center; align-items: center;
justify-content: center; justify-content: center;
@@ -258,6 +288,94 @@ button.icon-button.ghost:hover:not([disabled]) {
border: none; border: none;
} }
.tags-panel__body {
overflow-y: auto;
}
.tags-table {
width: 100%;
overflow: auto;
}
.tags-table table {
width: 100%;
border-collapse: collapse;
min-width: 320px;
}
.tags-table th,
.tags-table td {
padding: 0.45rem 0.6rem;
text-align: left;
border-bottom: 1px solid var(--divider);
font-size: 0.85rem;
}
.tags-table th.numeric,
.tags-table td.numeric {
text-align: right;
}
.tags-table th.actions,
.tags-table td.actions {
text-align: right;
width: 0;
}
.tags-table tbody tr:hover {
background: var(--surface-subtle);
}
.tags-table tr.editing {
background: rgba(43, 92, 255, 0.08);
}
.tags-table__label {
max-width: 24rem;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.tags-table__swatch {
display: inline-block;
width: 1rem;
height: 1rem;
border-radius: 2px;
box-shadow: inset 0 0 0 1px rgba(0, 0, 0, 0.1);
}
.tags-panel__error {
margin: 0.5rem 0;
color: var(--danger);
font-size: 0.8rem;
}
.tags-table__label-input {
width: 100%;
}
.tags-table__color-editor {
display: flex;
align-items: center;
gap: 0.4rem;
}
.tags-table__color-picker {
width: 2.25rem;
height: 2.25rem;
padding: 0;
border: none;
background: none;
cursor: pointer;
}
.tags-table__edit-controls {
display: flex;
justify-content: flex-end;
gap: 0.4rem;
}
.preview-workspace__message { .preview-workspace__message {
color: var(--muted); color: var(--muted);
font-size: 0.95rem; font-size: 0.95rem;
@@ -445,6 +563,52 @@ button.icon-button.ghost:hover:not([disabled]) {
padding: 0; padding: 0;
} }
.sidebar-section {
margin-top: 1rem;
display: flex;
flex-direction: column;
gap: 0.4rem;
}
.sidebar-section__header {
display: flex;
align-items: center;
justify-content: space-between;
font-size: 0.75rem;
color: var(--muted);
text-transform: uppercase;
letter-spacing: 0.04em;
}
.sidebar-section__header h3 {
margin: 0;
font-size: 0.75rem;
font-weight: 600;
color: inherit;
}
.sidebar-link {
border: none;
background: transparent;
padding: 0.3rem 0.3rem;
text-align: left;
font-size: 0.82rem;
color: var(--fg);
border-radius: 3px;
cursor: pointer;
transition: background 0.12s ease, color 0.12s ease;
}
.sidebar-link:hover {
background: var(--surface-subtle);
}
.sidebar-link.active {
background: rgba(43, 92, 255, 0.12);
color: var(--accent);
font-weight: 600;
}
.folder-row.is-drop-target { .folder-row.is-drop-target {
outline: 2px dashed var(--accent); outline: 2px dashed var(--accent);
outline-offset: 2px; outline-offset: 2px;
@@ -635,22 +799,22 @@ button.icon-button.ghost:hover:not([disabled]) {
.tag-filter { .tag-filter {
border: 1px solid var(--border); border: 1px solid var(--border);
background: transparent; background: var(--surface-subtle);
color: var(--fg); color: var(--fg);
padding: 0.25rem 0.6rem; padding: 0.25rem 0.6rem;
border-radius: 2px; border-radius: 2px;
cursor: pointer; cursor: pointer;
font-size: 0.85rem; font-size: 0.85rem;
transition: background 0.15s ease, color 0.15s ease, box-shadow 0.15s ease; transition: transform 0.12s ease, box-shadow 0.12s ease, filter 0.12s ease;
} }
.tag-filter:hover { .tag-filter:hover {
background: var(--surface-subtle); filter: brightness(0.97);
} }
.tag-filter.active { .tag-filter.active {
background: var(--accent); box-shadow: 0 0 0 1px currentColor inset;
color: white; transform: translateY(-1px);
} }
.filter-actions { .filter-actions {
@@ -672,7 +836,11 @@ button.icon-button.ghost:hover:not([disabled]) {
background: var(--surface-subtle); background: var(--surface-subtle);
color: var(--fg); color: var(--fg);
font-size: 0.74rem; font-size: 0.74rem;
margin-right: 0.25rem; border: 1px solid transparent;
}
.tag-chip {
gap: 0.25rem;
} }
.empty-state { .empty-state {
@@ -775,14 +943,17 @@ button.icon-button.ghost:hover:not([disabled]) {
left: 50%; left: 50%;
width: 100%; width: 100%;
height: 100%; height: 100%;
display: flex;
align-items: center;
justify-content: center;
transition: transform 120ms ease; transition: transform 120ms ease;
filter: drop-shadow(0 2px 6px rgba(0, 0, 0, 0.18)); filter: drop-shadow(0 2px 6px rgba(0, 0, 0, 0.18));
transform-origin: center; transform-origin: center;
border-radius: 6px; border-radius: 6px;
overflow: hidden; overflow: hidden;
pointer-events: none;
} }
.preview-stack .preview-stack__item.orientation-portrait { .preview-stack .preview-stack__item.orientation-portrait {
width: 80%; width: 80%;
height: 100%; height: 100%;
@@ -794,10 +965,31 @@ button.icon-button.ghost:hover:not([disabled]) {
} }
.preview-stack__image { .preview-stack__image {
width: 100%; display: block;
height: 100%; width: auto;
height: auto;
max-width: 100%;
max-height: 100%;
object-fit: contain; object-fit: contain;
background: transparent; background: transparent;
cursor: pointer;
transition:
outline-color 120ms ease,
box-shadow 120ms ease,
filter 120ms ease,
background-color 120ms ease;
outline: 2px solid transparent;
outline-offset: -2px;
pointer-events: auto;
}
.preview-stack__image:hover,
.preview-stack__image:focus-visible {
outline-color: rgba(24, 119, 242, 0.85);
box-shadow:
inset 0 0 0 999px rgba(24, 119, 242, 0.18),
0 6px 18px rgba(24, 119, 242, 0.24);
filter: saturate(118%) brightness(1.05);
} }
.preview-pane--stack { .preview-pane--stack {
@@ -846,10 +1038,15 @@ button.icon-button.ghost:hover:not([disabled]) {
.tag-pill button { .tag-pill button {
background: none; background: none;
border: none; border: none;
color: var(--muted); color: inherit;
padding: 0; padding: 0;
cursor: pointer; cursor: pointer;
font-size: 0.85rem; font-size: 0.85rem;
opacity: 0.8;
}
.tag-pill button:hover {
opacity: 1;
} }
input, input,
+26
View File
@@ -0,0 +1,26 @@
version: 0.8
index_id: documents
doc_mapping:
tokenizers:
- name: substring
type: ngram
min_gram: 2
max_gram: 20
prefix_only: false
field_mappings:
- name: document_id
type: text
stored: true
- name: version_id
type: text
stored: true
- name: title
type: text
tokenizer: substring
stored: true
- name: text
type: text
tokenizer: substring
record: position
search_settings:
default_search_fields: [title, text]