1 Commits
Author SHA1 Message Date
nils 3c9a9fc060 Initial commit
ci / docker (backend, backend/Dockerfile, backend) (push) Failing after 13s
ci / docker (frontend, frontend/Dockerfile, frontend) (push) Failing after 12s
2025-12-17 13:37:31 +01:00
349 changed files with 42793 additions and 26604 deletions
-95
View File
@@ -1,95 +0,0 @@
name: ci
on:
push:
branches:
- main
- staging
- dev
tags:
- '*'
jobs:
docker:
runs-on: ubuntu-latest
strategy:
matrix:
include:
- service: frontend
context: frontend
dockerfile: frontend/Dockerfile
- service: backend
context: backend
dockerfile: backend/Dockerfile
steps:
- name: Checkout with submodules
uses: actions/checkout@v4
with:
submodules: recursive
- name: Derive repository metadata
id: repo_meta
run: |
repo="${GITHUB_REPOSITORY:-$GITEA_REPOSITORY}"
owner="${repo%%/*}"
name="${repo##*/}"
echo "repo_owner=$owner" >> "$GITHUB_OUTPUT"
echo "repo_name=$name" >> "$GITHUB_OUTPUT"
- name: Compute base tag
id: compute_tag
run: |
sha="${GITHUB_SHA:-$GITEA_SHA}"
ref_type="${GITHUB_REF_TYPE:-$GITEA_REF_TYPE}"
ref_name="${GITHUB_REF_NAME:-$GITEA_REF_NAME}"
if [ -z "$sha" ]; then
echo "base_tag=$ref_name" >> "$GITHUB_OUTPUT"
exit 0
fi
short="${sha:0:7}"
tag="$short"
if [ "$ref_type" = "tag" ]; then
tag="$ref_name"
elif [ "$ref_name" = "dev" ]; then
tag="${tag}-dev"
fi
echo "base_tag=$tag" >> "$GITHUB_OUTPUT"
- name: Login to local registry
uses: docker/login-action@v2
with:
registry: ${{ vars.REGISTRY_URL }}
username: ${{ vars.REGISTRY_USER }}
password: ${{ secrets.REGISTRY_PASSWORD }}
- name: Login to GHCR
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ vars.GHCR_USERNAME }}
password: ${{ secrets.GHCR_PASSWORD }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
driver: remote
endpoint: ${{ env.BUILDKIT_ARM64_ENDPOINT }}
- name: Build and Push ${{ matrix.service }} Image
uses: docker/build-push-action@v6
with:
context: ${{ matrix.context }}
file: ${{ matrix.dockerfile }}
platforms: linux/arm64
push: true
provenance: false
tags: |
${{ vars.REGISTRY_URL }}/${{ steps.repo_meta.outputs.repo_name }}-${{ matrix.service }}:${{ steps.compute_tag.outputs.base_tag }}
${{ vars.REGISTRY_URL }}/${{ steps.repo_meta.outputs.repo_name }}-${{ matrix.service }}:${{ gitea.sha }}
ghcr.io/paperless-dms/${{ steps.repo_meta.outputs.repo_name }}-${{ matrix.service }}:${{ steps.compute_tag.outputs.base_tag }}
ghcr.io/paperless-dms/${{ steps.repo_meta.outputs.repo_name }}-${{ matrix.service }}:${{ gitea.sha }}
+146
View File
@@ -0,0 +1,146 @@
name: ci
on:
push:
branches:
- main
- staging
- dev
tags:
- '*'
jobs:
docker:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
strategy:
matrix:
include:
- service: frontend
context: frontend
dockerfile: frontend/Dockerfile
- service: backend
context: backend
dockerfile: backend/Dockerfile
steps:
- name: Checkout with submodules
uses: actions/checkout@v4
with:
submodules: recursive
- name: Derive repository metadata
id: repo_meta
run: |
repo="${GITHUB_REPOSITORY}"
owner="${repo%%/*}"
name="${repo##*/}"
echo "repo_owner=$owner" >> "$GITHUB_OUTPUT"
echo "repo_name=$name" >> "$GITHUB_OUTPUT"
- name: Compute base tag
id: compute_tag
env:
GITHUB_SHA: ${{ github.sha }}
GITHUB_REF_TYPE: ${{ github.ref_type }}
GITHUB_REF_NAME: ${{ github.ref_name }}
run: |
sha="${GITHUB_SHA}"
ref_type="${GITHUB_REF_TYPE}"
ref_name="${GITHUB_REF_NAME}"
short="${sha:0:7}"
tag="$short"
if [ "$ref_type" = "tag" ]; then
tag="$ref_name"
elif [ "$ref_name" = "dev" ]; then
tag="${tag}-dev"
elif [ "$ref_name" = "staging" ]; then
tag="${tag}-staging"
fi
echo "base_tag=$tag" >> "$GITHUB_OUTPUT"
- name: Login to local registry
if: ${{ vars.REGISTRY_URL != '' }}
uses: docker/login-action@v2
with:
registry: ${{ vars.REGISTRY_URL }}
username: ${{ vars.REGISTRY_USER }}
password: ${{ secrets.REGISTRY_PASSWORD }}
- name: Login to GHCR
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ github.token }}
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Determine branch alias tag
id: branch_alias
env:
REF_NAME: ${{ github.ref_name }}
run: |
alias=""
case "${REF_NAME}" in
dev) alias="latest-dev" ;;
staging) alias="latest-staging" ;;
main) alias="latest" ;;
esac
echo "alias=$alias" >> "$GITHUB_OUTPUT"
- name: Assemble image tags
id: tag_list
env:
REGISTRY_URL: ${{ vars.REGISTRY_URL }}
REPO_NAME: ${{ steps.repo_meta.outputs.repo_name }}
SERVICE: ${{ matrix.service }}
BASE_TAG: ${{ steps.compute_tag.outputs.base_tag }}
GIT_SHA: ${{ github.sha }}
BRANCH_ALIAS: ${{ steps.branch_alias.outputs.alias }}
shell: bash
run: |
tags=""
if [ -n "${REGISTRY_URL}" ]; then
repo_tag="${REGISTRY_URL}/${REPO_NAME}-${SERVICE}"
tags="${tags}${repo_tag}:${BASE_TAG}\n${repo_tag}:${GIT_SHA}"
fi
ghcr_tag="ghcr.io/papercrate-dms/${REPO_NAME}-${SERVICE}"
if [ -n "${tags}" ]; then
tags="${tags}\n"
fi
tags="${tags}${ghcr_tag}:${BASE_TAG}\n${ghcr_tag}:${GIT_SHA}"
if [ -n "${BRANCH_ALIAS}" ]; then
if [ -n "${REGISTRY_URL}" ]; then
tags="${tags}\n${repo_tag}:${BRANCH_ALIAS}"
fi
tags="${tags}\n${ghcr_tag}:${BRANCH_ALIAS}"
fi
export TAGS="${tags}"
python -c 'import os; tags=[t.strip() for t in os.environ["TAGS"].split("\\n") if t.strip()]; print("tags=" + ",".join(tags))' | tee -a "$GITHUB_OUTPUT"
- name: Build and Push ${{ matrix.service }} Image
uses: docker/build-push-action@v6
with:
context: ${{ matrix.context }}
file: ${{ matrix.dockerfile }}
platforms: linux/amd64,linux/arm64
push: true
provenance: false
tags: ${{ steps.tag_list.outputs.tags }}
cache-from: type=gha
cache-to: type=gha,mode=max
+3
View File
@@ -66,6 +66,9 @@ The backend reads its settings from environment variables. In particular:
- `DATABASE_URL` connection string for the primary Postgres database (required).
- `DATABASE_MAX_POOL_SIZE` optional override for the r2d2 connection pool size.
Defaults to `2`; increase it in staging/production to match expected concurrency.
- `PROXY_DOWNLOADS` set to `true` when the object store is only reachable from
the backend network. When enabled, `/api/download/{token}` and asset-object fetches
stream bytes through the API instead of redirecting clients to S3/Hetzner.
On startup each binary logs the effective configuration with secrets redacted
(for example, the database password is masked). This makes it easier to confirm
+985 -1119
View File
File diff suppressed because it is too large Load Diff
+26 -17
View File
@@ -5,22 +5,20 @@ edition = "2021"
[dependencies]
# Web framework
axum = { version = "0.7", features = ["multipart"] }
tokio = { version = "1", features = ["full"] }
tower = { version = "0.4", features = ["make", "util"] }
axum = { version = "0.8", features = ["multipart"] }
tokio = { version = "1.48", features = ["full"] }
tower = { version = "0.5", features = ["make", "util"] }
tower-http = { version = "0.6", features = ["cors", "trace"] }
axum-extra = { version = "0.9", features = ["typed-header"] }
axum-extra = { version = "0.12", features = ["typed-header"] }
# Database
diesel = { version = "2.1", features = ["postgres", "uuid", "chrono", "serde_json", "r2d2"] }
diesel = { version = "2.3.3", features = ["postgres", "uuid", "chrono", "serde_json", "r2d2"] }
diesel_migrations = "2.1"
uuid = { version = "1.6", features = ["v4", "serde"] }
chrono = { version = "0.4", features = ["serde"] }
# S3
aws-config = "1.1"
aws-sdk-s3 = "1.14"
aws-credential-types = "1.2"
rust-s3 = { version = "0.37", features = ["with-tokio", "tokio-rustls-tls"] }
# Serialization
serde = { version = "1.0", features = ["derive"] }
@@ -34,41 +32,52 @@ tracing-subscriber = { version = "0.3", features = ["env-filter"] }
dotenv = "0.15"
sha2 = "0.10"
hex = "0.4"
hmac = "0.12"
bytes = "1.5"
async-trait = "0.1"
image = { version = "0.25", default-features = false, features = ["png", "jpeg", "webp"] }
pdfium-render = "0.8"
pdfium-render = "0.8.36"
mime_guess = "2.0"
tempfile = "3.10"
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls", "stream"] }
reqwest = { version = "0.12.24", default-features = false, features = ["json", "rustls-tls", "stream"] }
percent-encoding = "2.3"
base64 = "0.21"
quick-xml = "0.32"
base64 = "0.22"
quick-xml = "0.38"
futures-util = "0.3"
url = "2.5"
once_cell = "1.19"
regex = "1.11"
infer = "0.19"
utoipa = { version = "4.2", default-features = false, features = ["chrono", "uuid", "preserve_order"] }
clap = { version = "4.5", features = ["derive"] }
# Error handling
thiserror = "1.0"
thiserror = "2.0"
anyhow = "1.0"
# Authentication & security
argon2 = "0.5"
jsonwebtoken = "9"
jsonwebtoken = { version = "10", features = ["rust_crypto"] }
webauthn-rs = { version = "0.5", features = ["danger-allow-state-serialisation", "danger-credential-internals"] }
serde_bytes = "0.11"
serde_cbor_2 = "0.13"
# Misc
rand = "0.8"
rand = "0.9"
hyper = "1.2"
http-body-util = "0.1"
chrono-tz = "0.8"
[dev-dependencies]
once_cell = "1.19"
hyper = "1.2"
http-body-util = "0.1"
webauthn-rs-core = "0.5"
serde_yaml = "0.9"
[build-dependencies]
serde = { version = "1.0", features = ["derive"] }
serde_yaml = "0.9"
serde_json = "1.0"
regex = "1.11"
[[bin]]
name = "backend"
+102 -24
View File
@@ -1,29 +1,114 @@
# syntax=docker/dockerfile:1
# syntax=docker/dockerfile:1.7
FROM rust:1-slim AS builder
ARG RUST_VERSION=1
FROM --platform=$BUILDPLATFORM rust:${RUST_VERSION}-slim AS builder
ARG TARGETARCH
ENV TARGETARCH=${TARGETARCH}
WORKDIR /app
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
RUN cat <<'SCRIPT' >/usr/local/bin/resolve-target.sh
#!/bin/sh
set -e
case "$1" in
amd64)
echo x86_64-unknown-linux-gnu
;;
arm64)
echo aarch64-unknown-linux-gnu
;;
*)
echo "Unsupported TARGETARCH: $1" >&2
exit 1
;;
esac
SCRIPT
RUN chmod +x /usr/local/bin/resolve-target.sh
ENV PKG_CONFIG_ALLOW_CROSS=1 \
CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc \
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc \
PKG_CONFIG_LIBDIR_aarch64_unknown_linux_gnu=/usr/lib/aarch64-linux-gnu/pkgconfig \
PKG_CONFIG_PATH_aarch64_unknown_linux_gnu=/usr/lib/aarch64-linux-gnu/pkgconfig \
PKG_CONFIG_SYSROOT_DIR_aarch64_unknown_linux_gnu=/usr/aarch64-linux-gnu \
OPENSSL_DIR_aarch64_unknown_linux_gnu=/usr/lib/aarch64-linux-gnu \
OPENSSL_LIB_DIR_aarch64_unknown_linux_gnu=/usr/lib/aarch64-linux-gnu \
OPENSSL_INCLUDE_DIR_aarch64_unknown_linux_gnu=/usr/include/aarch64-linux-gnu \
CC_x86_64_unknown_linux_gnu=x86_64-linux-gnu-gcc \
CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-linux-gnu-gcc \
PKG_CONFIG_LIBDIR_x86_64_unknown_linux_gnu=/usr/lib/x86_64-linux-gnu/pkgconfig \
PKG_CONFIG_PATH_x86_64_unknown_linux_gnu=/usr/lib/x86_64-linux-gnu/pkgconfig \
OPENSSL_DIR_x86_64_unknown_linux_gnu=/usr/lib/x86_64-linux-gnu \
OPENSSL_LIB_DIR_x86_64_unknown_linux_gnu=/usr/lib/x86_64-linux-gnu \
OPENSSL_INCLUDE_DIR_x86_64_unknown_linux_gnu=/usr/include/x86_64-linux-gnu
RUN set -eux; \
case "${TARGETARCH}" in \
amd64) TARGET_DEB_ARCH=amd64 ;; \
arm64) TARGET_DEB_ARCH=arm64 ;; \
*) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \
esac; \
BUILD_DEB_ARCH="$(dpkg --print-architecture)"; \
if [ "${TARGET_DEB_ARCH}" != "${BUILD_DEB_ARCH}" ]; then \
dpkg --add-architecture "${TARGET_DEB_ARCH}"; \
fi; \
apt-get update; \
apt-get install -y --no-install-recommends \
build-essential \
pkg-config \
curl \
libssl-dev \
libpq-dev \
libjpeg-dev \
libpng-dev \
curl \
&& rm -rf /var/lib/apt/lists/*
zlib1g-dev; \
if [ "${TARGET_DEB_ARCH}" != "${BUILD_DEB_ARCH}" ]; then \
case "${TARGETARCH}" in \
arm64) CROSS_GCC=gcc-aarch64-linux-gnu ;; \
amd64) CROSS_GCC=gcc-x86-64-linux-gnu ;; \
esac; \
apt-get install -y --no-install-recommends \
"${CROSS_GCC}" \
"libc6-dev:${TARGET_DEB_ARCH}" \
"libssl-dev:${TARGET_DEB_ARCH}" \
"libpq-dev:${TARGET_DEB_ARCH}" \
"libjpeg-dev:${TARGET_DEB_ARCH}" \
"libpng-dev:${TARGET_DEB_ARCH}" \
"zlib1g-dev:${TARGET_DEB_ARCH}"; \
fi; \
rm -rf /var/lib/apt/lists/*
COPY Cargo.toml Cargo.lock ./
COPY Cargo.toml Cargo.lock build.rs ./
COPY src ./src
COPY migrations ./migrations
COPY tests ./tests
COPY resources ./resources
COPY diesel.toml ./
RUN cargo build --release --bin backend --bin worker --bin webdav --bin admin
RUN cargo install diesel_cli --no-default-features --features postgres
RUN set -eux; \
TARGET="$(/usr/local/bin/resolve-target.sh "${TARGETARCH}")"; \
rustup target add "${TARGET}"; \
cargo build --release --target "${TARGET}" --bin backend --bin worker --bin webdav --bin admin; \
mkdir -p /artifacts; \
for bin in backend worker webdav admin; do \
cp "target/${TARGET}/release/${bin}" "/artifacts/${bin}"; \
done
FROM debian:trixie-slim AS runtime
RUN set -eux; \
case "${TARGETARCH}" in \
amd64) pdfium_package=pdfium-linux-x64.tgz ;; \
arm64) pdfium_package=pdfium-linux-arm64.tgz ;; \
*) echo "Unsupported TARGETARCH: ${TARGETARCH}" >&2; exit 1 ;; \
esac; \
curl -fsSL "https://github.com/bblanchon/pdfium-binaries/releases/latest/download/${pdfium_package}" -o /tmp/pdfium.tgz; \
mkdir -p /tmp/pdfium; \
tar -xzf /tmp/pdfium.tgz -C /tmp/pdfium --strip-components=1; \
pdfium_so="$(find /tmp/pdfium -name libpdfium.so -type f | head -n1)"; \
[ -n "${pdfium_so}" ]; \
cp "${pdfium_so}" /artifacts/libpdfium.so; \
rm -rf /tmp/pdfium.tgz /tmp/pdfium
FROM --platform=$TARGETPLATFORM debian:trixie-slim AS runtime
ARG TARGETARCH
WORKDIR /app
RUN apt-get update \
@@ -38,25 +123,18 @@ RUN apt-get update \
tesseract-ocr \
ghostscript \
qpdf \
ffmpeg \
&& rm -rf /var/lib/apt/lists/* \
&& mkdir -p /usr/local/lib \
&& curl -fsSL https://github.com/bblanchon/pdfium-binaries/releases/latest/download/pdfium-linux-arm64.tgz -o /tmp/pdfium.tgz \
&& mkdir -p /tmp/pdfium \
&& tar -xzf /tmp/pdfium.tgz -C /tmp/pdfium --strip-components=1 \
&& pdfium_so="$(find /tmp/pdfium -name libpdfium.so -type f | head -n1)" \
&& [ -n "${pdfium_so}" ] \
&& mv "${pdfium_so}" /usr/local/lib/libpdfium.so \
&& ldconfig \
&& rm -rf /tmp/pdfium.tgz /tmp/pdfium \
&& useradd --system --create-home --uid 10001 appuser
COPY --from=builder /app/target/release/backend /usr/local/bin/papercrate-backend
COPY --from=builder /app/target/release/worker /usr/local/bin/papercrate-worker
COPY --from=builder /app/target/release/webdav /usr/local/bin/papercrate-webdav
COPY --from=builder /app/target/release/admin /usr/local/bin/papercrate-admin
COPY --from=builder /usr/local/cargo/bin/diesel /usr/local/bin/diesel
COPY --from=builder /artifacts/backend /usr/local/bin/papercrate-backend
COPY --from=builder /artifacts/worker /usr/local/bin/papercrate-worker
COPY --from=builder /artifacts/webdav /usr/local/bin/papercrate-webdav
COPY --from=builder /artifacts/admin /usr/local/bin/papercrate-admin
COPY --from=builder /artifacts/libpdfium.so /usr/local/lib/libpdfium.so
RUN ldconfig
COPY migrations ./migrations
COPY diesel.toml ./
ENV RUST_LOG=info
USER appuser
+123
View File
@@ -0,0 +1,123 @@
use std::env;
use std::fs;
use std::path::PathBuf;
use regex::escape;
use serde::Deserialize;
#[derive(Deserialize)]
struct CaseSuite {
cases: Vec<CaseName>,
}
#[derive(Deserialize)]
struct CaseName {
name: String,
}
#[derive(Deserialize)]
struct MonthSuite {
months: Vec<MonthDefinition>,
}
#[derive(Deserialize)]
struct MonthDefinition {
name: String,
month: u32,
#[serde(default)]
locales: Vec<String>,
}
fn sanitize(name: &str) -> String {
let mut out = String::with_capacity(name.len());
for ch in name.chars() {
if ch.is_ascii_alphanumeric() {
out.push(ch);
} else {
out.push('_');
}
}
if out.is_empty() {
"case".to_string()
} else if out.chars().next().unwrap().is_ascii_digit() {
format!("_{}", out)
} else {
out
}
}
fn quote(value: &str) -> String {
serde_json::to_string(value).expect("string literal")
}
fn generate_tests() -> Result<String, Box<dyn std::error::Error>> {
let yaml_path = PathBuf::from("tests/data/issued_at_cases.yaml");
let contents = fs::read_to_string(&yaml_path)?;
let suite: CaseSuite = serde_yaml::from_str(&contents)?;
let mut output =
String::from("#[cfg(test)]\npub mod issued_at_generated_tests {\n use super::*;\n");
for case in suite.cases {
let ident = sanitize(&case.name);
output.push_str(&format!(
" #[test]\n fn {}() {{\n run_named_case(\"{}\");\n }}\n",
ident, case.name
));
}
output.push_str("}\n");
Ok(output)
}
fn generate_months() -> Result<String, Box<dyn std::error::Error>> {
let yaml_path = PathBuf::from("resources/issued_at_months.yaml");
let contents = fs::read_to_string(&yaml_path)?;
let suite: MonthSuite = serde_yaml::from_str(&contents)?;
let mut pattern_parts = Vec::with_capacity(suite.months.len());
let mut entries = String::new();
for entry in &suite.months {
pattern_parts.push(escape(&entry.name));
let locales_literal = if entry.locales.is_empty() {
"&[]".to_string()
} else {
let joined = entry
.locales
.iter()
.map(|loc| quote(loc))
.collect::<Vec<_>>()
.join(", ");
format!("&[{}]", joined)
};
entries.push_str(&format!(
" MonthVariant {{ name: {}, month: {}, locales: {} }},\n",
quote(&entry.name),
entry.month,
locales_literal
));
}
let pattern_literal = quote(&pattern_parts.join("|"));
let output = format!(
"pub(super) static MONTH_VARIANTS: &[MonthVariant] = &[\n{entries}];\n\n",
entries = entries
) + &format!(
"pub(super) const MONTH_PATTERN: &str = {};\n",
pattern_literal
);
Ok(output)
}
fn main() -> Result<(), Box<dyn std::error::Error>> {
println!("cargo:rerun-if-changed=tests/data/issued_at_cases.yaml");
println!("cargo:rerun-if-changed=resources/issued_at_months.yaml");
let out_dir = PathBuf::from(env::var("OUT_DIR")?);
fs::write(
out_dir.join("issued_at_generated_tests.rs"),
generate_tests()?,
)?;
fs::write(out_dir.join("issued_at_months.rs"), generate_months()?)?;
Ok(())
}
@@ -0,0 +1,13 @@
ALTER TABLE tenant.document_tags
DROP CONSTRAINT IF EXISTS document_tags_assigned_by_fkey,
ADD CONSTRAINT document_tags_assigned_by_fkey
FOREIGN KEY (assigned_by)
REFERENCES shared.users (id)
ON DELETE NO ACTION;
ALTER TABLE tenant.document_correspondents
DROP CONSTRAINT IF EXISTS document_correspondents_assigned_by_fkey,
ADD CONSTRAINT document_correspondents_assigned_by_fkey
FOREIGN KEY (assigned_by)
REFERENCES shared.users (id)
ON DELETE NO ACTION;
@@ -0,0 +1,13 @@
ALTER TABLE tenant.document_tags
DROP CONSTRAINT IF EXISTS document_tags_assigned_by_fkey,
ADD CONSTRAINT document_tags_assigned_by_fkey
FOREIGN KEY (assigned_by)
REFERENCES shared.users (id)
ON DELETE SET NULL;
ALTER TABLE tenant.document_correspondents
DROP CONSTRAINT IF EXISTS document_correspondents_assigned_by_fkey,
ADD CONSTRAINT document_correspondents_assigned_by_fkey
FOREIGN KEY (assigned_by)
REFERENCES shared.users (id)
ON DELETE SET NULL;
@@ -0,0 +1,13 @@
ALTER TABLE shared.jobs
DROP CONSTRAINT jobs_tenant_id_fkey;
ALTER TABLE shared.jobs
ALTER COLUMN tenant_id SET NOT NULL;
ALTER TABLE shared.jobs
ADD CONSTRAINT jobs_tenant_id_fkey
FOREIGN KEY (tenant_id)
REFERENCES shared.tenants(id);
ALTER TABLE shared.jobs
DROP COLUMN result;
@@ -0,0 +1,14 @@
ALTER TABLE shared.jobs
ALTER COLUMN tenant_id DROP NOT NULL;
ALTER TABLE shared.jobs
DROP CONSTRAINT jobs_tenant_id_fkey;
ALTER TABLE shared.jobs
ADD CONSTRAINT jobs_tenant_id_fkey
FOREIGN KEY (tenant_id)
REFERENCES shared.tenants(id)
ON DELETE SET NULL;
ALTER TABLE shared.jobs
ADD COLUMN result JSONB;
@@ -0,0 +1,4 @@
-- diesel:run_in_transaction = false
-- Enum values cannot be removed safely; this down migration intentionally left empty.
SELECT 1;
@@ -0,0 +1,46 @@
-- diesel:run_in_transaction = false
DO $$
BEGIN
BEGIN
EXECUTE 'ALTER TYPE tenant.api_capability ADD VALUE ''tenants:write''';
EXCEPTION
WHEN undefined_object THEN
BEGIN
EXECUTE 'ALTER TYPE api_capability ADD VALUE ''tenants:write''';
EXCEPTION
WHEN duplicate_object THEN NULL;
END;
WHEN duplicate_object THEN NULL;
END;
END $$;
DO $$
BEGIN
BEGIN
EXECUTE 'ALTER TYPE tenant.api_capability ADD VALUE ''tenants:reset''';
EXCEPTION
WHEN undefined_object THEN
BEGIN
EXECUTE 'ALTER TYPE api_capability ADD VALUE ''tenants:reset''';
EXCEPTION
WHEN duplicate_object THEN NULL;
END;
WHEN duplicate_object THEN NULL;
END;
END $$;
DO $$
BEGIN
BEGIN
EXECUTE 'ALTER TYPE tenant.api_capability ADD VALUE ''tenants:delete''';
EXCEPTION
WHEN undefined_object THEN
BEGIN
EXECUTE 'ALTER TYPE api_capability ADD VALUE ''tenants:delete''';
EXCEPTION
WHEN duplicate_object THEN NULL;
END;
WHEN duplicate_object THEN NULL;
END;
END $$;
@@ -0,0 +1,9 @@
-- diesel:run_in_transaction = false
DELETE FROM tenant.capability_set_capabilities
WHERE capability IN (
'tenants:write'::api_capability,
'tenants:reset'::api_capability,
'tenants:delete'::api_capability
)
AND capability_set_id IN (SELECT id FROM tenant.capability_sets WHERE slug = 'owner');
@@ -0,0 +1,22 @@
-- diesel:run_in_transaction = false
WITH owner_sets AS (
SELECT id FROM tenant.capability_sets WHERE slug = 'owner'
)
INSERT INTO tenant.capability_set_capabilities (capability_set_id, capability)
SELECT id, 'tenants:write'::api_capability FROM owner_sets
ON CONFLICT DO NOTHING;
WITH owner_sets AS (
SELECT id FROM tenant.capability_sets WHERE slug = 'owner'
)
INSERT INTO tenant.capability_set_capabilities (capability_set_id, capability)
SELECT id, 'tenants:reset'::api_capability FROM owner_sets
ON CONFLICT DO NOTHING;
WITH owner_sets AS (
SELECT id FROM tenant.capability_sets WHERE slug = 'owner'
)
INSERT INTO tenant.capability_set_capabilities (capability_set_id, capability)
SELECT id, 'tenants:delete'::api_capability FROM owner_sets
ON CONFLICT DO NOTHING;
@@ -0,0 +1,24 @@
CREATE TABLE tenant.document_asset_objects (
id UUID PRIMARY KEY,
asset_id UUID NOT NULL REFERENCES tenant.document_assets(id) ON DELETE CASCADE,
ordinal INT NOT NULL,
s3_key TEXT NOT NULL,
metadata JSONB NOT NULL DEFAULT '{}'::jsonb,
tenant_id UUID NOT NULL REFERENCES shared.tenants(id),
CONSTRAINT document_asset_objects_ordinal_positive CHECK (ordinal >= 1),
CONSTRAINT document_asset_objects_asset_ordinal_unique UNIQUE (asset_id, ordinal)
);
CREATE INDEX idx_document_asset_objects_asset_ordinal
ON tenant.document_asset_objects(asset_id, ordinal);
CREATE INDEX document_asset_objects_tenant_id_idx ON tenant.document_asset_objects(tenant_id);
ALTER TABLE tenant.document_assets ADD COLUMN cardinality INT;
UPDATE tenant.document_assets SET cardinality = 1;
INSERT INTO tenant.document_asset_objects (id, asset_id, ordinal, s3_key, metadata, tenant_id)
SELECT gen_random_uuid(), id, 1, s3_key, metadata, tenant_id
FROM tenant.document_assets;
ALTER TABLE tenant.document_assets DROP COLUMN s3_key;
@@ -0,0 +1,27 @@
-- Prevent concurrent inserts/updates during backfill.
LOCK TABLE tenant.document_asset_objects IN ACCESS EXCLUSIVE MODE;
LOCK TABLE tenant.document_assets IN ACCESS EXCLUSIVE MODE;
ALTER TABLE tenant.document_assets ADD COLUMN s3_key TEXT;
UPDATE tenant.document_assets AS da
SET s3_key = o.s3_key,
metadata = COALESCE(da.metadata, '{}'::jsonb) || COALESCE(o.metadata, '{}'::jsonb)
FROM tenant.document_asset_objects AS o
WHERE o.asset_id = da.id
AND o.ordinal = 1;
DELETE FROM tenant.document_asset_objects WHERE ordinal <> 1;
DO $$
BEGIN
IF EXISTS (SELECT 1 FROM tenant.document_assets WHERE s3_key IS NULL) THEN
RAISE EXCEPTION 'cannot drop document_asset_objects: some assets are missing a populated ordinal 1 object (s3_key null)';
END IF;
END
$$;
ALTER TABLE tenant.document_assets ALTER COLUMN s3_key SET NOT NULL;
ALTER TABLE tenant.document_assets DROP COLUMN cardinality;
DROP TABLE tenant.document_asset_objects;
@@ -0,0 +1,3 @@
-- Revert column rename.
ALTER TABLE tenant.documents
RENAME COLUMN mime_type TO content_type;
@@ -0,0 +1,3 @@
-- Rename document content_type column to mime_type for consistency with API.
ALTER TABLE tenant.documents
RENAME COLUMN content_type TO mime_type;
@@ -0,0 +1,3 @@
UPDATE tenant.document_assets
SET asset_type = 'ocr-text'
WHERE asset_type = 'text-content';
@@ -0,0 +1,3 @@
UPDATE tenant.document_assets
SET asset_type = 'text-content'
WHERE asset_type = 'ocr-text';
@@ -0,0 +1 @@
DROP INDEX IF EXISTS tenant.idx_documents_title_trgm;
@@ -0,0 +1,6 @@
CREATE EXTENSION IF NOT EXISTS pg_trgm;
CREATE INDEX idx_documents_title_trgm
ON tenant.documents
USING gin (title gin_trgm_ops)
WHERE deleted_at IS NULL;
+648
View File
@@ -0,0 +1,648 @@
months:
- name: "january"
month: 1
locales:
- "en"
- name: "jan"
month: 1
locales:
- "en"
- name: "janvier"
month: 1
locales:
- "fr"
- name: "janv"
month: 1
locales:
- "fr"
- name: "januar"
month: 1
locales:
- "de"
- name: "janu\u00e1r"
month: 1
locales:
- "hu"
- name: "leden"
month: 1
locales:
- "cs"
- name: "sije\u010danj"
month: 1
locales:
- "hr"
- name: "sijecanj"
month: 1
locales:
- "hr"
- name: "stycze\u0144"
month: 1
locales:
- "pl"
- name: "styczen"
month: 1
locales:
- "pl"
- name: "ocak"
month: 1
locales:
- "tr"
- name: "february"
month: 2
locales:
- "en"
- name: "feb"
month: 2
locales:
- "en"
- name: "f\u00e9vrier"
month: 2
locales:
- "fr"
- name: "fevrier"
month: 2
locales:
- "fr"
- name: "f\u00e9vr"
month: 2
locales:
- "fr"
- name: "fevr"
month: 2
locales:
- "fr"
- name: "februar"
month: 2
locales:
- "de"
- name: "\u00fanor"
month: 2
locales:
- "cs"
- name: "unor"
month: 2
locales:
- "cs"
- name: "\u00fanora"
month: 2
locales:
- "cs"
- name: "unora"
month: 2
locales:
- "cs"
- name: "velja\u010da"
month: 2
locales:
- "hr"
- name: "veljaca"
month: 2
locales:
- "hr"
- name: "velja\u010de"
month: 2
locales:
- "hr"
- name: "veljace"
month: 2
locales:
- "hr"
- name: "luty"
month: 2
locales:
- "pl"
- name: "\u015fubat"
month: 2
locales:
- "tr"
- name: "subat"
month: 2
locales:
- "tr"
- name: "march"
month: 3
locales:
- "en"
- name: "mar"
month: 3
locales:
- "en"
- name: "m\u00e4rz"
month: 3
locales:
- "de"
- name: "maerz"
month: 3
locales:
- "de"
- name: "mars"
month: 3
locales:
- "fr"
- name: "m\u00e4r"
month: 3
locales:
- "de"
- name: "marz"
month: 3
locales:
- "de"
- name: "b\u0159ezen"
month: 3
locales:
- "cs"
- name: "brezen"
month: 3
locales:
- "cs"
- name: "b\u0159ezna"
month: 3
locales:
- "cs"
- name: "brezna"
month: 3
locales:
- "cs"
- name: "o\u017eujak"
month: 3
locales:
- "hr"
- name: "ozujak"
month: 3
locales:
- "hr"
- name: "o\u017eujka"
month: 3
locales:
- "hr"
- name: "ozujka"
month: 3
locales:
- "hr"
- name: "marzec"
month: 3
locales:
- "pl"
- name: "mart"
month: 3
locales:
- "sr"
- "bs"
- name: "m\u00e1rcius"
month: 3
locales:
- "hu"
- name: "martie"
month: 3
locales:
- "ro"
- name: "april"
month: 4
locales:
- "en"
- name: "apr"
month: 4
locales:
- "en"
- name: "avril"
month: 4
locales:
- "fr"
- name: "abril"
month: 4
locales:
- "es"
- "pt"
- name: "duben"
month: 4
locales:
- "cs"
- name: "travanj"
month: 4
locales:
- "hr"
- name: "nisan"
month: 4
locales:
- "tr"
- name: "kwiecie\u0144"
month: 4
locales:
- "pl"
- name: "kwiecien"
month: 4
locales:
- "pl"
- name: "aprile"
month: 4
locales:
- "it"
- name: "may"
month: 5
locales:
- "en"
- name: "mai"
month: 5
locales:
- "fr"
- "de"
- name: "mayo"
month: 5
locales:
- "es"
- name: "kv\u011bten"
month: 5
locales:
- "cs"
- name: "kveten"
month: 5
locales:
- "cs"
- name: "kv\u011btna"
month: 5
locales:
- "cs"
- name: "kvetna"
month: 5
locales:
- "cs"
- name: "svibanj"
month: 5
locales:
- "hr"
- name: "maj"
month: 5
locales:
- "pl"
- "bs"
- "sr"
- "hr"
- name: "may\u0131s"
month: 5
locales:
- "tr"
- name: "mayis"
month: 5
locales:
- "tr"
- name: "maggio"
month: 5
locales:
- "it"
- name: "m\u00e1j"
month: 5
locales:
- "hu"
- "sk"
- name: "june"
month: 6
locales:
- "en"
- name: "jun"
month: 6
locales:
- "en"
- "de"
- name: "juin"
month: 6
locales:
- "fr"
- name: "junio"
month: 6
locales:
- "es"
- name: "juni"
month: 6
locales:
- "de"
- name: "\u010derven"
month: 6
locales:
- "cs"
- name: "cerven"
month: 6
locales:
- "cs"
- name: "lipanj"
month: 6
locales:
- "hr"
- name: "haziran"
month: 6
locales:
- "tr"
- name: "czerwiec"
month: 6
locales:
- "pl"
- name: "j\u00fanius"
month: 6
locales:
- "hu"
- name: "giugno"
month: 6
locales:
- "it"
- name: "july"
month: 7
locales:
- "en"
- name: "jul"
month: 7
locales:
- "en"
- "de"
- name: "juillet"
month: 7
locales:
- "fr"
- name: "julio"
month: 7
locales:
- "es"
- name: "temmuz"
month: 7
locales:
- "tr"
- name: "\u010dervenec"
month: 7
locales:
- "cs"
- name: "cervenec"
month: 7
locales:
- "cs"
- name: "srpanj"
month: 7
locales:
- "hr"
- name: "lipiec"
month: 7
locales:
- "pl"
- name: "luglio"
month: 7
locales:
- "it"
- name: "j\u00falius"
month: 7
locales:
- "hu"
- name: "august"
month: 8
locales:
- "en"
- name: "aug"
month: 8
locales:
- "en"
- "de"
- name: "ao\u00fbt"
month: 8
locales:
- "fr"
- name: "aout"
month: 8
locales:
- "fr"
- name: "agosto"
month: 8
locales:
- "es"
- "pt"
- "it"
- name: "a\u011fustos"
month: 8
locales:
- "tr"
- name: "agustos"
month: 8
locales:
- "tr"
- name: "kolovoz"
month: 8
locales:
- "hr"
- name: "srpen"
month: 8
locales:
- "cs"
- name: "sierpie\u0144"
month: 8
locales:
- "pl"
- name: "sierpien"
month: 8
locales:
- "pl"
- name: "augustus"
month: 8
locales:
- "nl"
- name: "agost"
month: 8
locales:
- "it"
- "ca"
- name: "september"
month: 9
locales:
- "en"
- name: "sept"
month: 9
locales:
- "en"
- "fr"
- name: "sep"
month: 9
locales:
- "en"
- name: "septembre"
month: 9
locales:
- "fr"
- name: "septiembre"
month: 9
locales:
- "es"
- name: "eyl\u00fcl"
month: 9
locales:
- "tr"
- name: "eylul"
month: 9
locales:
- "tr"
- name: "z\u00e1\u0159\u00ed"
month: 9
locales:
- "cs"
- name: "zari"
month: 9
locales:
- "cs"
- name: "rujan"
month: 9
locales:
- "hr"
- name: "wrzesie\u0144"
month: 9
locales:
- "pl"
- name: "wrzesien"
month: 9
locales:
- "pl"
- name: "septembrie"
month: 9
locales:
- "ro"
- name: "settembre"
month: 9
locales:
- "it"
- name: "october"
month: 10
locales:
- "en"
- name: "oct"
month: 10
locales:
- "en"
- name: "oktober"
month: 10
locales:
- "de"
- name: "okt\u00f3ber"
month: 10
locales:
- "hu"
- name: "octobre"
month: 10
locales:
- "fr"
- name: "octubre"
month: 10
locales:
- "es"
- name: "\u0159\u00edjen"
month: 10
locales:
- "cs"
- name: "rijen"
month: 10
locales:
- "hr"
- name: "pa\u017adziernik"
month: 10
locales:
- "pl"
- name: "pazdziernik"
month: 10
locales:
- "pl"
- name: "ekim"
month: 10
locales:
- "tr"
- name: "octombrie"
month: 10
locales:
- "ro"
- name: "november"
month: 11
locales:
- "en"
- name: "nov"
month: 11
locales:
- "en"
- name: "novembre"
month: 11
locales:
- "fr"
- name: "noviembre"
month: 11
locales:
- "es"
- name: "studeni"
month: 11
locales:
- "hr"
- name: "kas\u0131m"
month: 11
locales:
- "tr"
- name: "kasim"
month: 11
locales:
- "tr"
- name: "listopad"
month: 11
locales:
- "pl"
- "cs"
- name: "novembro"
month: 11
locales:
- "pt"
- name: "listopadu"
month: 11
locales:
- "cs"
- name: "noiembrie"
month: 11
locales:
- "ro"
- name: "december"
month: 12
locales:
- "en"
- name: "dec"
month: 12
locales:
- "en"
- name: "dezember"
month: 12
locales:
- "de"
- name: "d\u00e9cembre"
month: 12
locales:
- "fr"
- name: "decembre"
month: 12
locales:
- "fr"
- name: "prosinec"
month: 12
locales:
- "cs"
- name: "prosinac"
month: 12
locales:
- "hr"
- name: "grudzie\u0144"
month: 12
locales:
- "pl"
- name: "grudzien"
month: 12
locales:
- "pl"
- name: "grudnia"
month: 12
locales:
- "pl"
- name: "aral\u0131k"
month: 12
locales:
- "tr"
- name: "aralik"
month: 12
locales:
- "tr"
- name: "decembrie"
month: 12
locales:
- "ro"
+4 -4
View File
@@ -1,11 +1,10 @@
use argon2::{
password_hash::{PasswordHasher, SaltString},
password_hash::{rand_core::OsRng as PasswordHashOsRng, PasswordHasher, SaltString},
Argon2,
};
use chrono::{NaiveDateTime, Utc};
use diesel::prelude::*;
use rand::rngs::OsRng;
use rand::RngCore;
use rand::{rngs::OsRng, TryRngCore};
use uuid::Uuid;
use crate::{
@@ -264,7 +263,8 @@ fn generate_secret() -> Result<String, AppError> {
}
fn hash_secret(secret: &str) -> Result<String, AppError> {
let salt = SaltString::generate(&mut OsRng);
let mut salt_rng = PasswordHashOsRng;
let salt = SaltString::generate(&mut salt_rng);
let hash = Argon2::default()
.hash_password(secret.as_bytes(), &salt)
.map_err(|err| {
+4 -1
View File
@@ -12,7 +12,7 @@ use crate::{
},
};
const OWNER_CAPABILITIES: [ApiCapability; 19] = [
const OWNER_CAPABILITIES: [ApiCapability; 22] = [
ApiCapability::CorrespondentsEdit,
ApiCapability::CorrespondentsRead,
ApiCapability::CorrespondentsWrite,
@@ -32,6 +32,9 @@ const OWNER_CAPABILITIES: [ApiCapability; 19] = [
ApiCapability::WebdavWrite,
ApiCapability::CapabilitySetsRead,
ApiCapability::CapabilitySetsWrite,
ApiCapability::TenantsWrite,
ApiCapability::TenantsReset,
ApiCapability::TenantsDelete,
];
const USER_CAPABILITIES: [ApiCapability; 16] = [
+41 -5
View File
@@ -2,11 +2,12 @@ use anyhow::Result;
use chrono::{Duration, Utc};
use jsonwebtoken::{decode, encode, DecodingKey, EncodingKey, Header, Validation};
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
use uuid::Uuid;
use crate::config::AppConfig;
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, ToSchema)]
#[serde(rename_all = "snake_case")]
pub enum PrincipalKind {
UserSession,
@@ -41,16 +42,18 @@ pub struct JwtService {
impl JwtService {
pub fn from_config(config: &AppConfig) -> Result<Self> {
let access_expiry = Duration::minutes(config.jwt_expiry_minutes);
Ok(Self {
encoding: EncodingKey::from_secret(config.jwt_secret.as_bytes()),
decoding: DecodingKey::from_secret(config.jwt_secret.as_bytes()),
issuer: config.jwt_issuer.clone(),
audience: config.jwt_audience.clone(),
expiry: Duration::minutes(config.jwt_expiry_minutes),
expiry: access_expiry,
download_audience: config.download_token_audience.clone(),
download_expiry: Duration::minutes(config.download_token_expiry_minutes),
selector_audience: format!("{}:tenant-selector", config.jwt_audience),
selector_expiry: Duration::minutes(15),
selector_expiry: access_expiry,
signup_audience: format!("{}:signup", config.jwt_audience),
signup_expiry: Duration::minutes(15),
})
@@ -87,13 +90,38 @@ impl JwtService {
pub fn generate_download_token(
&self,
document_id: Uuid,
version_id: Uuid,
user_id: Uuid,
tenant_id: Uuid,
) -> Result<String> {
let now = Utc::now();
let exp = now + self.download_expiry;
let claims = DownloadClaims {
doc_id: document_id,
subject: DownloadSubject::Document {
doc_id: document_id,
version_id,
},
user_id,
tenant_id,
iss: self.issuer.clone(),
aud: self.download_audience.clone(),
iat: now.timestamp() as usize,
exp: exp.timestamp() as usize,
};
Ok(encode(&Header::default(), &claims, &self.encoding)?)
}
pub fn generate_asset_download_token(
&self,
asset_id: Uuid,
user_id: Uuid,
tenant_id: Uuid,
) -> Result<String> {
let now = Utc::now();
let exp = now + self.download_expiry;
let claims = DownloadClaims {
subject: DownloadSubject::Asset { asset_id },
user_id,
tenant_id,
iss: self.issuer.clone(),
@@ -180,9 +208,17 @@ pub struct Claims {
pub exp: usize,
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(tag = "scope", rename_all = "snake_case")]
pub enum DownloadSubject {
Document { doc_id: Uuid, version_id: Uuid },
Asset { asset_id: Uuid },
}
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct DownloadClaims {
pub doc_id: Uuid,
#[serde(flatten)]
pub subject: DownloadSubject,
pub user_id: Uuid,
pub tenant_id: Uuid,
pub iss: String,
+137 -62
View File
@@ -7,22 +7,65 @@ pub mod password;
use std::sync::{Arc, Mutex};
use axum::{async_trait, extract::FromRequestParts, http::request::Parts};
use axum::{
extract::FromRequestParts,
http::{request::Parts, StatusCode},
};
use axum_extra::headers::{authorization::Bearer, Authorization};
use axum_extra::TypedHeader;
use diesel::{pg::PgConnection, prelude::*};
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
use crate::{
auth::capability_sets::{load_capabilities_for_set, load_capability_set},
error::AppError,
models::ApiCapability,
error::{AppError, AppResult},
models::{ApiCapability, TenantStatus},
schema::tenants::dsl as tenant_dsl,
state::{AppState, PgPooledConnection},
};
use uuid::Uuid;
use crate::auth::jwt::PrincipalKind;
#[derive(Debug, Clone)]
pub struct TenantMembershipUser {
pub user_id: Uuid,
}
impl FromRequestParts<AppState> for TenantMembershipUser {
type Rejection = AppError;
#[allow(refining_impl_trait)]
fn from_request_parts<'a>(
parts: &'a mut Parts,
state: &AppState,
) -> impl std::future::Future<Output = Result<Self, Self::Rejection>> + Send + 'a {
let state = state.clone();
async move {
let TypedHeader(Authorization(bearer)) =
TypedHeader::<Authorization<Bearer>>::from_request_parts(parts, &state)
.await
.map_err(|_| AppError::unauthorized())?;
if let Ok(claims) = state.jwt.verify_token(bearer.token()) {
return Ok(Self {
user_id: claims.sub,
});
}
let selector = state
.jwt
.verify_tenant_selector_token(bearer.token())
.map_err(|_| AppError::unauthorized())?;
Ok(Self {
user_id: selector.sub,
})
}
}
}
#[derive(Clone)]
pub struct TenantConnectionHolder {
inner: Arc<Mutex<Option<PgPooledConnection>>>,
@@ -52,56 +95,59 @@ pub struct AuthenticatedUser {
pub capabilities: Vec<ApiCapability>,
}
#[async_trait]
impl FromRequestParts<AppState> for AuthenticatedUser {
type Rejection = AppError;
async fn from_request_parts(
parts: &mut Parts,
#[allow(refining_impl_trait)]
fn from_request_parts<'a>(
parts: &'a mut Parts,
state: &AppState,
) -> Result<Self, Self::Rejection> {
if let Some(user) = parts.extensions.get::<AuthenticatedUser>() {
return Ok(user.clone());
}
) -> impl std::future::Future<Output = Result<Self, Self::Rejection>> + Send + 'a {
let state = state.clone();
async move {
if let Some(user) = parts.extensions.get::<AuthenticatedUser>() {
return Ok(user.clone());
}
let TypedHeader(Authorization(bearer)) =
TypedHeader::<Authorization<Bearer>>::from_request_parts(parts, state)
.await
let TypedHeader(Authorization(bearer)) =
TypedHeader::<Authorization<Bearer>>::from_request_parts(parts, &state)
.await
.map_err(|_| AppError::unauthorized())?;
let claims = state
.jwt
.verify_token(bearer.token())
.map_err(|_| AppError::unauthorized())?;
let claims = state
.jwt
.verify_token(bearer.token())
.map_err(|_| AppError::unauthorized())?;
let mut tenant_conn = state.db_for_tenant(claims.tenant_id)?;
let capability_set = load_capability_set(&mut tenant_conn, claims.capability_set_id)
.map_err(|_| AppError::unauthorized())?;
let mut tenant_conn = state.db_for_tenant(claims.tenant_id)?;
let capability_set = load_capability_set(&mut tenant_conn, claims.capability_set_id)
.map_err(|_| AppError::unauthorized())?;
if capability_set.cap_version != claims.cap_version {
return Err(AppError::unauthorized());
}
if capability_set.cap_version != claims.cap_version {
return Err(AppError::unauthorized());
let capabilities = load_capabilities_for_set(&mut tenant_conn, capability_set.id)
.map_err(|_| AppError::unauthorized())?;
let user = AuthenticatedUser {
user_id: claims.sub,
username: claims.username,
tenant_id: claims.tenant_id,
principal_kind: claims.principal_kind,
principal_id: claims.principal_id,
capability_set_id: claims.capability_set_id,
cap_version: claims.cap_version,
capabilities,
};
parts.extensions.insert(user.clone());
parts
.extensions
.insert(TenantConnectionHolder::new(tenant_conn));
Ok(user)
}
let capabilities = load_capabilities_for_set(&mut tenant_conn, capability_set.id)
.map_err(|_| AppError::unauthorized())?;
let user = AuthenticatedUser {
user_id: claims.sub,
username: claims.username,
tenant_id: claims.tenant_id,
principal_kind: claims.principal_kind,
principal_id: claims.principal_id,
capability_set_id: claims.capability_set_id,
cap_version: claims.cap_version,
capabilities,
};
parts.extensions.insert(user.clone());
parts
.extensions
.insert(TenantConnectionHolder::new(tenant_conn));
Ok(user)
}
}
@@ -118,29 +164,58 @@ impl TenantScopedConn {
}
}
#[async_trait]
impl FromRequestParts<AppState> for TenantScopedConn {
type Rejection = AppError;
async fn from_request_parts(
parts: &mut Parts,
#[allow(refining_impl_trait)]
fn from_request_parts<'a>(
parts: &'a mut Parts,
state: &AppState,
) -> Result<Self, Self::Rejection> {
let user = AuthenticatedUser::from_request_parts(parts, state).await?;
let tenant_id = user.tenant_id;
let conn = if let Some(holder) = parts.extensions.remove::<TenantConnectionHolder>() {
holder
.into_conn()
.ok_or_else(|| AppError::internal("tenant connection unavailable"))?
} else {
state.db_for_tenant(tenant_id)?
};
) -> impl std::future::Future<Output = Result<Self, Self::Rejection>> + Send + 'a {
let state = state.clone();
async move {
let user = AuthenticatedUser::from_request_parts(parts, &state).await?;
let tenant_id = user.tenant_id;
let mut conn = if let Some(holder) = parts.extensions.remove::<TenantConnectionHolder>()
{
holder
.into_conn()
.ok_or_else(|| AppError::internal("tenant connection unavailable"))?
} else {
state.db_for_tenant(tenant_id)?
};
Ok(Self {
conn,
tenant_id,
user_id: user.user_id,
user,
})
ensure_active_tenant_with_conn(&mut conn, tenant_id)?;
Ok(Self {
conn,
tenant_id,
user_id: user.user_id,
user,
})
}
}
}
pub(crate) fn ensure_active_tenant(state: &AppState, tenant_id: Uuid) -> AppResult<()> {
let mut conn = state.db_unscoped()?;
ensure_active_tenant_with_conn(&mut conn, tenant_id)
}
pub(crate) fn ensure_active_tenant_with_conn(
conn: &mut PgConnection,
tenant_id: Uuid,
) -> AppResult<()> {
use tenant_dsl::tenants;
let status: TenantStatus = tenants
.find(tenant_id)
.select(tenant_dsl::status)
.first(conn)?;
if status != TenantStatus::Active {
return Err(AppError::new(StatusCode::FORBIDDEN, "tenant is not active"));
}
Ok(())
}
+6 -3
View File
@@ -1,9 +1,11 @@
use anyhow::{anyhow, Result};
use argon2::{
password_hash::{PasswordHash, PasswordHasher, PasswordVerifier, SaltString},
password_hash::{
rand_core::OsRng as PasswordHashOsRng, PasswordHash, PasswordHasher, PasswordVerifier,
SaltString,
},
Argon2,
};
use rand::rngs::OsRng;
pub fn verify_password(password: &str, password_hash: &str) -> Result<bool> {
let parsed_hash = PasswordHash::new(password_hash).map_err(|err| anyhow!(err))?;
@@ -13,7 +15,8 @@ pub fn verify_password(password: &str, password_hash: &str) -> Result<bool> {
}
pub fn hash_password(password: &str) -> Result<String> {
let salt = SaltString::generate(&mut OsRng);
let mut rng = PasswordHashOsRng;
let salt = SaltString::generate(&mut rng);
let hash = Argon2::default()
.hash_password(password.as_bytes(), &salt)
.map_err(|err| anyhow!(err))?;
+332 -71
View File
@@ -3,30 +3,31 @@ use std::sync::Arc;
use anyhow::{anyhow, bail, Context, Result};
use chrono::{Duration as ChronoDuration, Utc};
use clap::{Parser, Subcommand, ValueEnum};
use diesel::{dsl::exists, prelude::*, select};
use rand::{rngs::OsRng, RngCore};
use diesel::{dsl::exists, pg::PgConnection, prelude::*, select};
use diesel_migrations::MigrationHarness;
use rand::{rngs::OsRng, TryRngCore};
use reqwest::{Client, Method, StatusCode};
use sha2::{Digest, Sha256};
use tokio::task;
use uuid::Uuid;
use papercrate::{
auth::capability_sets::{ensure_capability_set, owner_capabilities},
config::AppConfig,
config::{redact_database_url, AppConfig},
db::{self, PgPool},
documents::search::ensure_quickwit_index,
jobs::{enqueue_job, JOB_ANALYZE_DOCUMENT},
jobs::{enqueue_job, JOB_ANALYZE_DOCUMENT, JOB_DELETE_TENANT},
migrations::MIGRATIONS,
models::{
DocumentAsset, DocumentAssetObject, MagicToken, MagicTokenKind, NewUser, NewUserMembership,
Tenant, TenantStatus, User,
DocumentAsset, MagicToken, MagicTokenKind, NewUser, NewUserMembership, Tenant,
TenantStatus, User,
},
s3,
schema::{
document_asset_objects, document_assets, documents, magic_tokens, tenants,
user_memberships, users,
},
schema::{document_assets, documents, magic_tokens, tenants, user_memberships, users},
storage::{ObjectStorage, S3Storage, TenantStorage},
tenants::TenantService,
tenants::{apply_tenant_guc, clear_tenant_context, TenantService},
utils::{text::normalize_identifier, tracing::init_tracing},
workers::tenants::{build_delete_proof_message, sign_delete_proof, DeleteAction},
};
#[derive(Parser)]
@@ -58,6 +59,15 @@ enum Command {
},
DeleteTenant {
tenant_id: Uuid,
#[arg(long = "tenant-name")]
tenant_name: String,
},
ResetTenant {
tenant_id: Uuid,
#[arg(long = "tenant-name")]
tenant_name: String,
#[arg(long = "final-status", value_enum, default_value_t = TenantFinalStatusArg::Active)]
final_status: TenantFinalStatusArg,
},
AddUserToTenant {
username: String,
@@ -73,6 +83,10 @@ enum Command {
ListTenants,
DeleteAssets {
tenant_id: Uuid,
#[arg(long = "asset-type")]
asset_type: Option<String>,
#[arg(long = "all", help = "Confirm deleting every asset for the tenant")]
delete_all: bool,
},
QuickwitCreate {
tenant_id: Uuid,
@@ -80,6 +94,15 @@ enum Command {
QuickwitDelete {
tenant_id: Uuid,
},
EnqueueDeleteTenant {
tenant_id: Uuid,
#[arg(long = "tenant-name")]
tenant_name: String,
#[arg(long = "remove-tenant")]
remove_tenant: bool,
#[arg(long = "final-status", value_enum)]
final_status: Option<TenantFinalStatusArg>,
},
MagicToken {
username: String,
#[arg(long = "ttl-minutes", default_value_t = 10)]
@@ -93,6 +116,14 @@ enum Command {
#[arg(long = "kind", value_enum, default_value_t = MagicTokenKindArg::EmailLogin)]
kind: MagicTokenKindArg,
},
MigrateDatabase {
#[arg(
long = "database-url",
value_name = "URL",
help = "Override the migrations database URL (defaults to MIGRATIONS_DATABASE_URL or DATABASE_URL)"
)]
database_url: Option<String>,
},
}
#[derive(Copy, Clone, Debug, ValueEnum)]
@@ -112,6 +143,21 @@ impl From<MagicTokenKindArg> for MagicTokenKind {
}
}
#[derive(Copy, Clone, Debug, ValueEnum)]
enum TenantFinalStatusArg {
Active,
Suspended,
}
impl TenantFinalStatusArg {
fn as_str(&self) -> &'static str {
match self {
TenantFinalStatusArg::Active => "active",
TenantFinalStatusArg::Suspended => "suspended",
}
}
}
#[tokio::main]
async fn main() -> Result<()> {
init_tracing("info");
@@ -128,7 +174,15 @@ async fn main() -> Result<()> {
storage_root,
quickwit_index,
} => create_tenant(&pool, &name, storage_root, quickwit_index)?,
Command::DeleteTenant { tenant_id } => delete_tenant(&pool, tenant_id)?,
Command::DeleteTenant {
tenant_id,
tenant_name,
} => delete_tenant(&config, &pool, tenant_id, &tenant_name)?,
Command::ResetTenant {
tenant_id,
tenant_name,
final_status,
} => reset_tenant(&config, &pool, tenant_id, &tenant_name, final_status)?,
Command::AddUserToTenant {
username,
tenant_id,
@@ -139,8 +193,17 @@ async fn main() -> Result<()> {
} => remove_user_from_tenant(&pool, &username, tenant_id)?,
Command::ReanalyzeDocuments { tenant_id } => reanalyze_documents(&pool, tenant_id)?,
Command::ListTenants => list_tenants(&pool)?,
Command::DeleteAssets { tenant_id } => {
delete_assets_for_tenant(&config, &pool, tenant_id).await?
Command::DeleteAssets {
tenant_id,
asset_type,
delete_all,
} => {
let asset_type = asset_type.as_deref();
if asset_type.is_none() && !delete_all {
bail!("refusing to delete all assets without --all confirmation");
}
delete_assets_for_tenant(&config, &pool, tenant_id, asset_type).await?
}
Command::QuickwitCreate { tenant_id } => {
quickwit_index(&config, &pool, tenant_id, Method::POST).await?
@@ -148,6 +211,19 @@ async fn main() -> Result<()> {
Command::QuickwitDelete { tenant_id } => {
quickwit_index(&config, &pool, tenant_id, Method::DELETE).await?
}
Command::EnqueueDeleteTenant {
tenant_id,
tenant_name,
remove_tenant,
final_status,
} => enqueue_delete_tenant_job(
&config,
&pool,
tenant_id,
&tenant_name,
remove_tenant,
final_status,
)?,
Command::MagicToken {
username,
ttl_minutes,
@@ -156,11 +232,34 @@ async fn main() -> Result<()> {
} => {
create_magic_token(&pool, &username, ttl_minutes, max_uses, kind.into())?;
}
Command::MigrateDatabase { database_url } => {
let url = database_url.unwrap_or_else(|| config.migrations_database_url().to_string());
migrate_database(url).await?;
}
}
Ok(())
}
async fn migrate_database(database_url: String) -> Result<()> {
let redacted = redact_database_url(&database_url);
tracing::info!(database_url = %redacted, "running pending migrations");
let result = task::spawn_blocking(move || -> Result<()> {
let mut conn =
PgConnection::establish(&database_url).context("failed to connect to database")?;
conn.run_pending_migrations(MIGRATIONS)
.map_err(|err| anyhow!("failed to run migrations: {err}"))?;
Ok(())
})
.await
.context("migration task panicked")?;
result?;
tracing::info!(database_url = %redacted, "migrations completed");
Ok(())
}
fn create_user(pool: &PgPool, username: &str) -> Result<()> {
let username = normalize_identifier(
username,
@@ -233,8 +332,17 @@ fn delete_user(pool: &PgPool, username: &str) -> Result<()> {
.optional()?
.ok_or_else(|| anyhow!("user '{}' not found", username))?;
diesel::delete(user_memberships::table.filter(user_memberships::user_id.eq(user.id)))
.execute(&mut conn)?;
let has_memberships: bool = select(exists(
user_memberships::table.filter(user_memberships::user_id.eq(user.id)),
))
.get_result(&mut conn)?;
if has_memberships {
bail!(
"user '{}' is still a member of one or more tenants; remove memberships first",
username
);
}
diesel::delete(users::table.filter(users::id.eq(user.id))).execute(&mut conn)?;
println!("deleted user '{}'", username);
@@ -323,7 +431,9 @@ fn create_magic_token(
fn generate_random_token() -> String {
let mut bytes = [0u8; 32];
OsRng.fill_bytes(&mut bytes);
OsRng
.try_fill_bytes(&mut bytes)
.expect("failed to read random bytes");
hex::encode(bytes)
}
@@ -333,7 +443,51 @@ fn hash_token(token: &str) -> String {
hex::encode(hasher.finalize())
}
fn delete_tenant(pool: &PgPool, tenant_id: Uuid) -> Result<()> {
fn delete_tenant(
config: &AppConfig,
pool: &PgPool,
tenant_id: Uuid,
tenant_name: &str,
) -> Result<()> {
enqueue_delete_tenant_job_internal(config, pool, tenant_id, tenant_name, true, None)?;
println!(
"delete job enqueued; tenant '{}' will be permanently removed",
tenant_name
);
Ok(())
}
fn reset_tenant(
config: &AppConfig,
pool: &PgPool,
tenant_id: Uuid,
tenant_name: &str,
final_status: TenantFinalStatusArg,
) -> Result<()> {
enqueue_delete_tenant_job_internal(
config,
pool,
tenant_id,
tenant_name,
false,
Some(final_status),
)?;
println!(
"reset job enqueued; tenant '{}' will be wiped and set to {}",
tenant_name,
final_status.as_str()
);
Ok(())
}
fn enqueue_delete_tenant_job_internal(
config: &AppConfig,
pool: &PgPool,
tenant_id: Uuid,
expected_name: &str,
remove_tenant: bool,
final_status: Option<TenantFinalStatusArg>,
) -> Result<()> {
let mut conn = pool.get().context("failed to get database connection")?;
let tenant: Tenant = tenants::table
@@ -342,19 +496,100 @@ fn delete_tenant(pool: &PgPool, tenant_id: Uuid) -> Result<()> {
.optional()?
.ok_or_else(|| anyhow!("tenant '{}' not found", tenant_id))?;
let member_exists: bool = select(exists(
user_memberships::table.filter(user_memberships::tenant_id.eq(tenant.id)),
))
.get_result(&mut conn)?;
if member_exists {
bail!("tenant '{}' still has user memberships", tenant.name);
apply_tenant_guc(&mut conn, tenant.id)
.map_err(|err| anyhow!("failed to set tenant context for {}: {err:?}", tenant.name))?;
if tenant.name != expected_name {
bail!(
"tenant name mismatch: expected '{}', database has '{}'",
expected_name,
tenant.name
);
}
diesel::delete(tenants::table.filter(tenants::id.eq(tenant.id))).execute(&mut conn)?;
println!("deleted tenant '{}'", tenant.name);
diesel::update(tenants::table.find(tenant.id))
.set(tenants::status.eq(TenantStatus::Deleting))
.execute(&mut conn)?;
let nonce = generate_random_token();
let issued_at = Utc::now();
let issued_at_str = issued_at.to_rfc3339();
let action = if remove_tenant {
DeleteAction::Delete
} else {
DeleteAction::Reset
};
let resolved_final_status = if remove_tenant {
None
} else {
Some(
final_status
.unwrap_or(TenantFinalStatusArg::Suspended)
.as_str(),
)
};
let message = build_delete_proof_message(
tenant.id,
expected_name,
action,
&nonce,
&issued_at_str,
resolved_final_status,
);
let signature = sign_delete_proof(&config.jwt_secret, &message).map_err(|err| anyhow!(err))?;
let mut payload = serde_json::json!({
"remove_tenant": remove_tenant,
"tenant_name": tenant.name.clone(),
"action": action.as_str(),
"nonce": nonce,
"issued_at": issued_at_str,
"signature": signature,
});
if let Some(status) = resolved_final_status {
payload["final_status"] = serde_json::json!(status);
}
enqueue_job(&mut conn, tenant.id, JOB_DELETE_TENANT, payload, None)?;
let status_label = if remove_tenant {
"deleted"
} else {
final_status.map(|s| s.as_str()).unwrap_or("suspended")
};
println!(
"delete-tenant job enqueued for '{}' (remove_tenant={}, final_status={})",
tenant.name, remove_tenant, status_label
);
clear_tenant_context(&mut conn).map_err(|err| {
anyhow!(
"failed to clear tenant context for {}: {err:?}",
tenant.name
)
})?;
Ok(())
}
fn enqueue_delete_tenant_job(
config: &AppConfig,
pool: &PgPool,
tenant_id: Uuid,
tenant_name: &str,
remove_tenant: bool,
final_status: Option<TenantFinalStatusArg>,
) -> Result<()> {
enqueue_delete_tenant_job_internal(
config,
pool,
tenant_id,
tenant_name,
remove_tenant,
final_status,
)
}
fn add_user_to_tenant(pool: &PgPool, username: &str, tenant_id: Uuid) -> Result<()> {
let mut conn = pool.get().context("failed to get database connection")?;
@@ -491,10 +726,10 @@ async fn delete_assets_for_tenant(
config: &AppConfig,
pool: &PgPool,
tenant_id: Uuid,
asset_type: Option<&str>,
) -> Result<()> {
let s3_client = s3::build_client(config).await?;
let storage: Arc<dyn ObjectStorage> =
Arc::new(S3Storage::new(s3_client, config.s3_bucket.clone()));
let bucket = s3::build_bucket(config)?;
let storage: Arc<dyn ObjectStorage> = Arc::new(S3Storage::new(bucket));
let mut conn = pool.get().context("failed to get database connection")?;
let tenant: Tenant = tenants::table
@@ -503,57 +738,83 @@ async fn delete_assets_for_tenant(
.optional()?
.ok_or_else(|| anyhow!("tenant '{}' not found", tenant_id))?;
let tenant_storage = TenantStorage::new(Arc::clone(&storage), &tenant)
.with_context(|| format!("missing storage root for tenant {}", tenant.name))?;
apply_tenant_guc(&mut conn, tenant.id)
.map_err(|err| anyhow!("failed to set tenant context for {}: {err:?}", tenant.name))?;
let assets: Vec<DocumentAsset> = document_assets::table
.filter(document_assets::tenant_id.eq(tenant.id))
.load(&mut conn)
.with_context(|| format!("failed to load assets for tenant {}", tenant.name))?;
let result = async {
let tenant_storage = TenantStorage::new(Arc::clone(&storage), &tenant)
.with_context(|| format!("missing storage root for tenant {}", tenant.name))?;
if assets.is_empty() {
println!("Tenant {}: no assets", tenant.name);
return Ok(());
}
let mut asset_query = document_assets::table
.filter(document_assets::tenant_id.eq(tenant.id))
.into_boxed();
println!(
"Tenant {} ({}): deleting {} assets…",
tenant.name,
tenant.id,
assets.len()
);
let asset_ids: Vec<Uuid> = assets.iter().map(|asset| asset.id).collect();
let objects: Vec<DocumentAssetObject> = document_asset_objects::table
.filter(document_asset_objects::tenant_id.eq(tenant.id))
.filter(document_asset_objects::asset_id.eq_any(&asset_ids))
.load(&mut conn)
.with_context(|| format!("failed to load asset objects for tenant {}", tenant.name))?;
for object in &objects {
if let Err(err) = tenant_storage.delete_object(&object.s3_key).await {
eprintln!(
"Failed to delete object {} (tenant {}): {err}",
object.s3_key, tenant.name
);
if let Some(asset_type) = asset_type {
asset_query = asset_query.filter(document_assets::asset_type.eq(asset_type));
}
}
diesel::delete(
document_asset_objects::table
.filter(document_asset_objects::tenant_id.eq(tenant.id))
.filter(document_asset_objects::asset_id.eq_any(&asset_ids)),
)
.execute(&mut conn)
.with_context(|| format!("failed to remove asset objects for tenant {}", tenant.name))?;
let assets: Vec<DocumentAsset> = asset_query
.load(&mut conn)
.with_context(|| format!("failed to load assets for tenant {}", tenant.name))?;
diesel::delete(document_assets::table.filter(document_assets::tenant_id.eq(tenant.id)))
if assets.is_empty() {
match asset_type {
Some(asset_type) => {
println!("Tenant {}: no assets of type '{}'", tenant.name, asset_type)
}
None => println!("Tenant {}: no assets", tenant.name),
}
return Ok(());
}
match asset_type {
Some(asset_type) => println!(
"Tenant {} ({}): deleting {} '{}' assets…",
tenant.name,
tenant.id,
assets.len(),
asset_type
),
None => println!(
"Tenant {} ({}): deleting {} assets…",
tenant.name,
tenant.id,
assets.len()
),
}
let asset_ids: Vec<Uuid> = assets.iter().map(|asset| asset.id).collect();
for asset in &assets {
if let Err(err) = tenant_storage.delete_object(&asset.s3_key).await {
eprintln!(
"Failed to delete object {} (tenant {}): {err}",
asset.s3_key, tenant.name
);
}
}
diesel::delete(
document_assets::table
.filter(document_assets::tenant_id.eq(tenant.id))
.filter(document_assets::id.eq_any(&asset_ids)),
)
.execute(&mut conn)
.with_context(|| format!("failed to remove asset records for tenant {}", tenant.name))?;
println!("Tenant {}: asset records deleted.", tenant.name);
Ok(())
println!("Tenant {}: asset records deleted.", tenant.name);
Ok(())
}
.await;
clear_tenant_context(&mut conn).map_err(|err| {
anyhow!(
"failed to clear tenant context for {}: {err:?}",
tenant.name
)
})?;
result
}
async fn quickwit_index(
+76 -1
View File
@@ -1,6 +1,7 @@
use anyhow::{Context, Result};
use url::Url;
use serde::de::Deserializer;
use serde::Deserialize;
use serde_aux::field_attributes::deserialize_bool_from_anything;
@@ -9,6 +10,8 @@ use crate::db::DEFAULT_MAX_POOL_SIZE;
#[derive(Clone, Debug, Deserialize)]
pub struct AppConfig {
pub database_url: String,
#[serde(default)]
pub migrations_database_url: Option<String>,
#[serde(default = "default_database_max_pool_size")]
pub database_max_pool_size: u32,
#[serde(default = "default_server_host")]
@@ -41,6 +44,8 @@ pub struct AppConfig {
pub refresh_cookie_domain: Option<String>,
#[serde(default)]
pub cors_allowed_origin: Option<String>,
#[serde(default, deserialize_with = "deserialize_bool_from_anything")]
pub proxy_downloads: bool,
#[serde(default)]
pub aws_endpoint_url: Option<String>,
#[serde(default)]
@@ -58,6 +63,16 @@ pub struct AppConfig {
pub worker_max_document_bytes: u64,
#[serde(default = "default_upload_body_limit_bytes")]
pub upload_body_limit_bytes: u64,
#[serde(default = "default_service_timezone")]
pub service_timezone: String,
#[serde(default = "default_issued_at_date_order")]
pub issued_at_date_order: String,
#[serde(default)]
pub issued_at_filename_date_order: Option<String>,
#[serde(default, deserialize_with = "deserialize_string_list")]
pub issued_at_date_parser_locales: Vec<String>,
#[serde(default, deserialize_with = "deserialize_string_list")]
pub issued_at_ignore_dates: Vec<String>,
#[serde(default)]
pub webauthn_rp_id: Option<String>,
#[serde(default)]
@@ -73,12 +88,14 @@ impl AppConfig {
tracing::info!(
component,
database_url = %config.redacted_database_url(),
migrations_database_url = %config.redacted_migrations_database_url(),
pool_size = config.database_max_pool_size,
quickwit_enabled = config.quickwit_endpoint.is_some(),
passkeys_enabled = config.webauthn_origin.is_some(),
s3_bucket = %config.s3_bucket,
worker_max_document_bytes = config.worker_max_document_bytes,
upload_body_limit_bytes = config.upload_body_limit_bytes,
proxy_downloads = config.proxy_downloads,
"loaded backend configuration"
);
Ok(config)
@@ -93,6 +110,18 @@ impl AppConfig {
pub fn redacted_database_url(&self) -> String {
redact_database_url(&self.database_url)
}
pub fn redacted_migrations_database_url(&self) -> String {
redact_database_url(self.migrations_database_url())
}
pub fn migrations_database_url(&self) -> &str {
if let Some(ref url) = self.migrations_database_url {
url
} else {
&self.database_url
}
}
}
impl AppConfig {
@@ -180,11 +209,57 @@ fn default_upload_body_limit_bytes() -> u64 {
128 * 1024 * 1024
}
fn default_service_timezone() -> String {
"UTC".to_string()
}
fn default_issued_at_date_order() -> String {
"DMY".to_string()
}
fn deserialize_string_list<'de, D>(deserializer: D) -> Result<Vec<String>, D::Error>
where
D: Deserializer<'de>,
{
#[derive(Deserialize)]
#[serde(untagged)]
enum Helper {
List(Vec<String>),
Single(String),
}
let helper = Option::<Helper>::deserialize(deserializer)?;
let mut values = Vec::new();
if let Some(helper) = helper {
match helper {
Helper::List(list) => {
for entry in list {
let trimmed = entry.trim();
if !trimmed.is_empty() {
values.push(trimmed.to_string());
}
}
}
Helper::Single(value) => {
for part in value.split(',') {
let trimmed = part.trim();
if !trimmed.is_empty() {
values.push(trimmed.to_string());
}
}
}
}
}
Ok(values)
}
fn default_webauthn_rp_name() -> String {
"Papercrate".to_string()
}
fn redact_database_url(raw: &str) -> String {
pub fn redact_database_url(raw: &str) -> String {
match Url::parse(raw) {
Ok(mut parsed) => {
if parsed.password().is_some() {
+40 -67
View File
@@ -1,6 +1,7 @@
use std::collections::HashMap;
use std::path::Path as FsPath;
use chrono::{Duration as ChronoDuration, Utc};
use diesel::prelude::*;
use serde::Serialize;
use serde_json::Value;
@@ -8,10 +9,16 @@ use utoipa::ToSchema;
use uuid::Uuid;
use crate::error::{AppError, AppResult};
use crate::models::{Document, DocumentAsset, DocumentAssetObject, DocumentVersion};
use crate::schema::{document_asset_objects, document_assets, document_versions};
use crate::models::{Document, DocumentAsset, DocumentVersion};
use crate::schema::{document_assets, document_versions};
use crate::state::{AppState, PgPooledConnection};
use crate::utils::time::to_iso;
use crate::utils::{http::inline_content_disposition, time::to_iso};
#[derive(Serialize, Clone, ToSchema)]
pub struct DownloadLink {
pub url: String,
pub expires_at: i64,
}
#[derive(Serialize, Clone, ToSchema)]
pub struct DocumentAssetResponse {
@@ -22,21 +29,7 @@ pub struct DocumentAssetResponse {
pub metadata: Value,
#[serde(skip_serializing_if = "Option::is_none")]
#[schema(nullable)]
pub cardinality: Option<i32>,
}
#[derive(Serialize, Clone, ToSchema)]
pub struct DocumentAssetObjectResponse {
pub id: Uuid,
pub ordinal: i32,
#[schema(value_type = Object)]
pub metadata: Value,
#[serde(skip_serializing_if = "Option::is_none")]
#[schema(nullable)]
pub url: Option<String>,
#[serde(skip_serializing_if = "Option::is_none")]
#[schema(nullable)]
pub expires_at: Option<i64>,
pub download: Option<DownloadLink>,
}
#[derive(Serialize, ToSchema)]
@@ -49,9 +42,7 @@ pub struct DocumentAssetDetailResponse {
pub created_at: String,
#[serde(skip_serializing_if = "Option::is_none")]
#[schema(nullable)]
pub cardinality: Option<i32>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub objects: Vec<DocumentAssetObjectResponse>,
pub download: Option<DownloadLink>,
}
#[derive(Serialize, Clone, ToSchema)]
@@ -71,22 +62,35 @@ pub struct DocumentVersionDetailResponse {
pub version: DocumentVersionResponse,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub assets: Vec<DocumentAssetResponse>,
pub download_path: String,
pub download: DownloadLink,
}
pub fn build_download_path(
pub fn build_download_link(
state: &AppState,
document: &Document,
version_id: Uuid,
user_id: Uuid,
) -> AppResult<String> {
) -> AppResult<DownloadLink> {
state
.jwt
.generate_download_token(document.id, user_id, document.tenant_id)
.map(|token| format!("/download/{token}"))
.generate_download_token(document.id, version_id, user_id, document.tenant_id)
.map_err(|err| {
tracing::error!(error = ?err, "failed to generate download token");
AppError::internal("failed to generate download token")
})
.and_then(|token| {
let expires_at = Utc::now()
.checked_add_signed(ChronoDuration::minutes(
state.config.download_token_expiry_minutes,
))
.ok_or_else(|| AppError::internal("failed to compute download expiry"))?
.timestamp_millis();
Ok(DownloadLink {
url: format!("/api/download/{token}"),
expires_at,
})
})
}
pub fn to_version_response(version: DocumentVersion) -> DocumentVersionResponse {
@@ -106,13 +110,13 @@ pub fn to_asset_summary(asset: DocumentAsset) -> DocumentAssetResponse {
asset_type: asset.asset_type,
mime_type: asset.mime_type,
metadata: asset.metadata,
cardinality: asset.cardinality,
download: None,
}
}
pub fn to_asset_detail_response(
asset: DocumentAsset,
objects: Vec<DocumentAssetObjectResponse>,
download: Option<DownloadLink>,
) -> DocumentAssetDetailResponse {
DocumentAssetDetailResponse {
id: asset.id,
@@ -120,23 +124,13 @@ pub fn to_asset_detail_response(
mime_type: asset.mime_type,
metadata: asset.metadata,
created_at: to_iso(asset.created_at),
cardinality: asset.cardinality,
objects,
download,
}
}
pub fn to_asset_object_response(
object: DocumentAssetObject,
url: Option<String>,
expires_at: Option<i64>,
) -> DocumentAssetObjectResponse {
DocumentAssetObjectResponse {
id: object.id,
ordinal: object.ordinal,
metadata: object.metadata,
url,
expires_at,
}
pub fn asset_disposition(asset: &DocumentAsset) -> Option<String> {
let filename = asset.asset_type.clone();
inline_content_disposition(&filename)
}
pub fn delete_asset(
@@ -159,25 +153,13 @@ pub fn load_asset_responses_with_conn(
tenant_id: Uuid,
version_id: Uuid,
) -> AppResult<Vec<DocumentAssetResponse>> {
let assets: Vec<(DocumentAsset, Option<DocumentAssetObject>)> = document_assets::table
.left_outer_join(
document_asset_objects::table.on(document_asset_objects::asset_id
.eq(document_assets::id)
.and(document_asset_objects::ordinal.eq(1))),
)
let assets: Vec<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq(version_id))
.filter(document_assets::tenant_id.eq(tenant_id))
.order(document_assets::created_at.asc())
.select((
document_assets::all_columns,
document_asset_objects::all_columns.nullable(),
))
.load(conn)?;
Ok(assets
.into_iter()
.map(|(asset, _)| to_asset_summary(asset))
.collect())
Ok(assets.into_iter().map(to_asset_summary).collect())
}
pub fn load_primary_assets(
@@ -207,25 +189,16 @@ pub fn load_primary_assets(
version_map.insert(version.id, version);
}
let assets: Vec<(DocumentAsset, Option<DocumentAssetObject>)> = document_assets::table
.left_outer_join(
document_asset_objects::table.on(document_asset_objects::asset_id
.eq(document_assets::id)
.and(document_asset_objects::ordinal.eq(1))),
)
let assets: Vec<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq_any(&version_ids))
.order((
document_assets::document_version_id.asc(),
document_assets::created_at.asc(),
))
.select((
document_assets::all_columns,
document_asset_objects::all_columns.nullable(),
))
.load(conn)?;
let mut assets_by_version: HashMap<Uuid, Vec<DocumentAssetResponse>> = HashMap::new();
for (asset, _object) in assets {
for asset in assets {
let version_id = asset.document_version_id;
let response = to_asset_summary(asset);
assets_by_version
+14
View File
@@ -163,6 +163,20 @@ pub async fn ensure_quickwit_index(client: &Client, endpoint: &str, index_id: &s
}
}
pub async fn delete_quickwit_index(client: &Client, endpoint: &str, index_id: &str) -> Result<()> {
let base = endpoint.trim_end_matches('/');
let url = format!("{}/api/v1/indexes/{}", base, index_id);
let response = client.delete(&url).send().await?;
match response.status() {
status if status.is_success() => Ok(()),
StatusCode::NOT_FOUND => Ok(()),
status => {
let body = response.text().await.unwrap_or_default();
bail!("quickwit delete index failed with status {status}: {body}");
}
}
}
pub fn extract_document_id(hit: &Value) -> Option<Uuid> {
for key in ["_source", "source", "fields", "stored_fields"] {
if let Some(value) = hit.get(key) {
+18 -6
View File
@@ -5,7 +5,8 @@ use axum::{
};
use serde::Serialize;
use serde_json::Value;
use std::fmt::Display;
use std::fmt::{self, Display};
use utoipa::ToSchema;
pub type AppResult<T> = Result<T, AppError>;
@@ -39,6 +40,10 @@ impl AppError {
Self::new(StatusCode::UNAUTHORIZED, "unauthorized")
}
pub fn forbidden(message: impl Into<String>) -> Self {
Self::new(StatusCode::FORBIDDEN, message)
}
pub fn not_found() -> Self {
Self::new(StatusCode::NOT_FOUND, "resource not found")
}
@@ -58,10 +63,16 @@ impl AppError {
}
}
impl fmt::Display for AppError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}: {}", self.status, self.message)
}
}
impl IntoResponse for AppError {
fn into_response(self) -> Response {
let status = self.status;
let body = Json(ErrorResponse {
let body = Json(ApiErrorResponse {
error: self.message,
code: self.code,
details: self.details,
@@ -70,8 +81,8 @@ impl IntoResponse for AppError {
}
}
#[derive(Serialize)]
struct ErrorResponse {
#[derive(Serialize, ToSchema)]
pub struct ApiErrorResponse {
error: String,
#[serde(skip_serializing_if = "Option::is_none")]
code: Option<String>,
@@ -84,8 +95,9 @@ impl From<diesel::result::Error> for AppError {
match value {
diesel::result::Error::NotFound => AppError::not_found(),
other => {
tracing::error!(error = ?other, "database operation failed");
AppError::internal("database operation failed")
let message = format!("database operation failed: {other}");
tracing::error!(error = ?other, message);
AppError::internal(message)
}
}
}
+8
View File
@@ -77,6 +77,14 @@ impl<T> JsonResponse<T> {
pub fn accepted(payload: T) -> Self {
Self::new(StatusCode::ACCEPTED, payload)
}
pub fn into_inner(self) -> T {
self.payload
}
pub fn as_inner(&self) -> &T {
&self.payload
}
}
impl<T> From<T> for JsonResponse<T> {
+167
View File
@@ -0,0 +1,167 @@
use std::collections::HashSet;
use chrono::{DateTime, Datelike, NaiveDate, TimeZone, Utc};
use chrono_tz::Tz;
use once_cell::sync::Lazy;
use tracing::warn;
use crate::config::AppConfig;
#[derive(Clone, Copy, Debug)]
pub enum DateOrder {
Dmy,
Mdy,
Ymd,
}
impl DateOrder {
pub fn parse(value: &str) -> Self {
Self::try_parse(value).unwrap_or(DateOrder::Dmy)
}
pub fn try_parse(value: &str) -> Option<Self> {
match value.trim().to_ascii_uppercase().as_str() {
"YMD" => Some(DateOrder::Ymd),
"MDY" => Some(DateOrder::Mdy),
"DMY" => Some(DateOrder::Dmy),
_ => None,
}
}
}
static MIN_ISSUED_AT_DATE: Lazy<NaiveDate> =
Lazy::new(|| NaiveDate::from_ymd_opt(1901, 1, 1).expect("valid minimum issued_at date"));
#[derive(Clone, Debug)]
pub struct IssuedAtSettings {
pub timezone: Tz,
pub date_order: DateOrder,
pub filename_date_order: Option<DateOrder>,
pub locales: HashSet<String>,
pub ignore_dates: HashSet<NaiveDate>,
pub min_date: NaiveDate,
}
impl IssuedAtSettings {
pub fn from_config(config: &AppConfig) -> Self {
let timezone = config.service_timezone.parse::<Tz>().unwrap_or_else(|_| {
warn!(
timezone = %config.service_timezone,
"invalid service timezone configured; falling back to UTC"
);
chrono_tz::UTC
});
let date_order = DateOrder::parse(&config.issued_at_date_order);
let filename_date_order =
config
.issued_at_filename_date_order
.as_deref()
.and_then(|value| {
DateOrder::try_parse(value).or_else(|| {
warn!(value, "invalid issued_at filename date order; ignoring");
None
})
});
let locales = config
.issued_at_date_parser_locales
.iter()
.filter_map(|value| {
let trimmed = value.trim();
if trimmed.is_empty() {
None
} else {
Some(trimmed.to_ascii_lowercase())
}
})
.collect::<HashSet<_>>();
// Ignore dates are evaluated after normalizing candidate timestamps to
// the configured service timezone, so administrators should provide
// local calendar dates rather than UTC midnights.
let ignore_dates = config
.issued_at_ignore_dates
.iter()
.filter_map(|value| {
let trimmed = value.trim();
if trimmed.is_empty() {
return None;
}
match NaiveDate::parse_from_str(trimmed, "%Y-%m-%d") {
Ok(date) => Some(date),
Err(err) => {
warn!(value = trimmed, error = %err, "invalid issued_at ignore date");
None
}
}
})
.collect();
Self {
timezone,
date_order,
filename_date_order,
locales,
ignore_dates,
min_date: *MIN_ISSUED_AT_DATE,
}
}
/// Returns the immutable (lowercase) locale allowlist supplied via config.
pub fn locales(&self) -> &HashSet<String> {
&self.locales
}
/// Returns the immutable set of local-calendar dates that should be ignored.
pub fn ignore_dates(&self) -> &HashSet<NaiveDate> {
&self.ignore_dates
}
/// Returns the configured service timezone (copy type).
pub fn timezone(&self) -> Tz {
self.timezone
}
pub fn date_order(&self) -> DateOrder {
self.date_order
}
pub fn filename_date_order(&self) -> Option<DateOrder> {
self.filename_date_order
}
pub fn normalize_naive(
&self,
date: NaiveDate,
now_utc: chrono::DateTime<Utc>,
) -> Option<DateTime<Utc>> {
if !self.is_valid_with_now(date, now_utc) {
return None;
}
self.timezone
.with_ymd_and_hms(date.year(), date.month(), date.day(), 0, 0, 0)
.earliest()
.map(|dt| dt.with_timezone(&Utc))
}
pub fn normalize_datetime(
&self,
dt: chrono::DateTime<Utc>,
now_utc: chrono::DateTime<Utc>,
) -> Option<DateTime<Utc>> {
let local_date = dt.with_timezone(&self.timezone).date_naive();
self.normalize_naive(local_date, now_utc)
}
fn is_valid_with_now(&self, date: NaiveDate, now_utc: chrono::DateTime<Utc>) -> bool {
if date < self.min_date {
return false;
}
let now_local = now_utc.with_timezone(&self.timezone).date_naive();
if date > now_local {
return false;
}
!self.ignore_dates.contains(&date)
}
}
+1 -1
View File
@@ -18,9 +18,9 @@ pub const STATUS_FAILED: &str = "failed";
pub const JOB_ANALYZE_DOCUMENT: &str = "analyze-document";
pub const JOB_GENERATE_THUMBNAILS: &str = "generate-thumbnails";
pub const JOB_GENERATE_OCR_TEXT: &str = "generate-ocr-text";
pub const JOB_INDEX_DOCUMENT_TEXT: &str = "index-document-text";
pub const JOB_PROVISION_TENANT: &str = "provision-tenant";
pub const JOB_PURGE_DOCUMENT: &str = "purge-document";
pub const JOB_DELETE_TENANT: &str = "delete-tenant";
#[derive(Debug, Error)]
pub enum JobQueueError {
+3
View File
@@ -4,6 +4,7 @@ pub mod db;
pub mod documents;
pub mod error;
pub mod http;
pub mod issued_at;
pub mod jobs;
pub mod models;
pub mod openapi;
@@ -17,3 +18,5 @@ pub mod tenants;
pub mod utils;
pub mod workers;
pub use workers::{default_handlers, Worker};
pub mod migrations;
pub mod test_support;
+3
View File
@@ -0,0 +1,3 @@
use diesel_migrations::{embed_migrations, EmbeddedMigrations};
pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations");
+23 -27
View File
@@ -111,6 +111,12 @@ pub enum ApiCapability {
CapabilitySetsRead,
#[serde(rename = "capability_sets:write")]
CapabilitySetsWrite,
#[serde(rename = "tenants:write")]
TenantsWrite,
#[serde(rename = "tenants:reset")]
TenantsReset,
#[serde(rename = "tenants:delete")]
TenantsDelete,
}
impl MagicTokenKind {
@@ -148,6 +154,9 @@ impl ApiCapability {
ApiCapability::WebdavWrite => "webdav:write",
ApiCapability::CapabilitySetsRead => "capability_sets:read",
ApiCapability::CapabilitySetsWrite => "capability_sets:write",
ApiCapability::TenantsWrite => "tenants:write",
ApiCapability::TenantsReset => "tenants:reset",
ApiCapability::TenantsDelete => "tenants:delete",
}
}
@@ -172,6 +181,9 @@ impl ApiCapability {
"webdav:write",
"capability_sets:read",
"capability_sets:write",
"tenants:write",
"tenants:reset",
"tenants:delete",
]
}
}
@@ -237,6 +249,9 @@ impl FromSql<ApiCapabilitySql, Pg> for ApiCapability {
"webdav:write" => Ok(ApiCapability::WebdavWrite),
"capability_sets:read" => Ok(ApiCapability::CapabilitySetsRead),
"capability_sets:write" => Ok(ApiCapability::CapabilitySetsWrite),
"tenants:write" => Ok(ApiCapability::TenantsWrite),
"tenants:reset" => Ok(ApiCapability::TenantsReset),
"tenants:delete" => Ok(ApiCapability::TenantsDelete),
other => Err(Box::new(std::io::Error::new(
std::io::ErrorKind::InvalidData,
format!("invalid api_capability '{other}'"),
@@ -281,6 +296,9 @@ impl str::FromStr for ApiCapability {
"webdav:write" => Ok(ApiCapability::WebdavWrite),
"capability_sets:read" => Ok(ApiCapability::CapabilitySetsRead),
"capability_sets:write" => Ok(ApiCapability::CapabilitySetsWrite),
"tenants:write" => Ok(ApiCapability::TenantsWrite),
"tenants:reset" => Ok(ApiCapability::TenantsReset),
"tenants:delete" => Ok(ApiCapability::TenantsDelete),
_ => Err("unsupported api capability"),
}
}
@@ -522,7 +540,7 @@ pub struct Document {
pub id: Uuid,
pub filename: String,
pub original_name: String,
pub content_type: Option<String>,
pub mime_type: Option<String>,
pub folder_id: Option<Uuid>,
pub created_at: NaiveDateTime,
pub updated_at: NaiveDateTime,
@@ -540,7 +558,7 @@ pub struct NewDocument {
pub id: Uuid,
pub filename: String,
pub original_name: String,
pub content_type: Option<String>,
pub mime_type: Option<String>,
pub folder_id: Option<Uuid>,
pub current_version_id: Uuid,
pub metadata: serde_json::Value,
@@ -603,7 +621,7 @@ pub struct DocumentAsset {
pub mime_type: String,
pub metadata: serde_json::Value,
pub created_at: NaiveDateTime,
pub cardinality: Option<i32>,
pub s3_key: String,
pub tenant_id: Uuid,
}
@@ -615,30 +633,7 @@ pub struct NewDocumentAsset {
pub asset_type: String,
pub mime_type: String,
pub metadata: serde_json::Value,
pub cardinality: Option<i32>,
pub tenant_id: Uuid,
}
#[derive(Debug, Clone, Queryable, Identifiable, Associations)]
#[diesel(table_name = document_asset_objects)]
#[diesel(belongs_to(DocumentAsset, foreign_key = asset_id))]
pub struct DocumentAssetObject {
pub id: Uuid,
pub asset_id: Uuid,
pub ordinal: i32,
pub s3_key: String,
pub metadata: serde_json::Value,
pub tenant_id: Uuid,
}
#[derive(Debug, Insertable)]
#[diesel(table_name = document_asset_objects)]
pub struct NewDocumentAssetObject {
pub id: Uuid,
pub asset_id: Uuid,
pub ordinal: i32,
pub s3_key: String,
pub metadata: serde_json::Value,
pub tenant_id: Uuid,
}
@@ -654,7 +649,8 @@ pub struct Job {
pub last_error: Option<String>,
pub created_at: NaiveDateTime,
pub updated_at: NaiveDateTime,
pub tenant_id: Uuid,
pub tenant_id: Option<Uuid>,
pub result: Option<serde_json::Value>,
}
#[derive(Debug, Insertable)]
+14 -5
View File
@@ -13,6 +13,7 @@ impl OpenApi for ApiDoc {
doc.merge(crate::routes::correspondents::CorrespondentsApiDoc::openapi());
doc.merge(crate::routes::profile::ProfileApiDoc::openapi());
doc.merge(crate::routes::capability_sets::CapabilitySetsApiDoc::openapi());
doc.merge(crate::routes::tenants::TenantsApiDoc::openapi());
doc.info = InfoBuilder::new()
.title("Papercrate API")
@@ -56,6 +57,10 @@ impl OpenApi for ApiDoc {
.name("Capability Sets")
.description(Some("Capability set management"))
.build(),
TagBuilder::new()
.name("Tenants")
.description(Some("Tenant catalog"))
.build(),
]);
doc
@@ -69,18 +74,19 @@ pub mod schemas {
};
pub use crate::auth::AuthenticatedUser;
pub use crate::documents::asset::{
DocumentAssetDetailResponse, DocumentAssetObjectResponse, DocumentAssetResponse,
DocumentVersionDetailResponse, DocumentVersionResponse,
DocumentAssetDetailResponse, DocumentAssetResponse, DocumentVersionDetailResponse,
DocumentVersionResponse,
};
pub use crate::documents::correspondents::DocumentCorrespondentResponse;
pub use crate::error::ApiErrorResponse;
pub use crate::models::ApiCapability;
pub use crate::routes::correspondents::{
CorrespondentSummary, CorrespondentUsage, CreateCorrespondentRequest,
CorrespondentSummary, CreateCorrespondentRequest,
UpdateCorrespondentRequest,
};
pub use crate::routes::documents::{
AssetObjectsQuery, AssetRequestQuery, DocumentCheckQuery, MoveDocumentRequest,
RestoreDocumentRequest, UploadDocumentForm,
AssetRequestQuery, DocumentCheckQuery, MoveDocumentRequest, RestoreDocumentRequest,
UploadDocumentForm,
};
pub use crate::routes::folders::FolderContentsResponse;
pub use crate::routes::tags::{CreateTagRequest, TagCatalogEntry, UpdateTagRequest};
@@ -111,6 +117,9 @@ pub mod schemas {
pub use crate::services::tags::{
AssignTagsRequest, BulkTagAction, BulkTagRequest, BulkTagResponse,
};
pub use crate::services::tenants::{
TenantUserListResponse, TenantUserSummary, UpdateTenantRequest, UpdateTenantUserRequest,
};
}
#[cfg(test)]
-14
View File
@@ -38,7 +38,6 @@ use crate::{
refresh,
logout,
me,
list_tenants,
select_tenant,
passkey_register_start,
passkey_register_finish,
@@ -201,19 +200,6 @@ pub async fn me(user: AuthenticatedUser) -> Json<AuthenticatedUser> {
Json(user)
}
#[utoipa::path(
get,
path = "/api/auth/tenants",
responses((status = 200, description = "List of tenants", body = TenantListResponse)),
tag = "Auth"
)]
pub async fn list_tenants(
State(state): State<AppState>,
user: AuthenticatedUser,
) -> AppResult<JsonResponse<TenantListResponse>> {
AuthService::new(&state).list_tenants(user)
}
#[utoipa::path(
post,
path = "/api/auth/passkeys/register/start",
+3 -9
View File
@@ -20,11 +20,6 @@ use crate::{
},
};
#[derive(Serialize, ToSchema)]
pub struct CorrespondentUsage {
pub total: i64,
}
#[derive(Serialize, ToSchema)]
pub struct CorrespondentSummary {
pub id: Uuid,
@@ -33,7 +28,7 @@ pub struct CorrespondentSummary {
pub metadata: Value,
pub created_at: String,
pub updated_at: String,
pub usage: CorrespondentUsage,
pub usage_count: i64,
}
#[derive(Deserialize, ToSchema)]
@@ -268,14 +263,14 @@ pub async fn delete_correspondent(
no_content()
}
fn build_summary(correspondent: Correspondent, total: i64) -> CorrespondentSummary {
fn build_summary(correspondent: Correspondent, usage_count: i64) -> CorrespondentSummary {
CorrespondentSummary {
id: correspondent.id,
name: correspondent.name,
metadata: correspondent.metadata,
created_at: to_iso(correspondent.created_at),
updated_at: to_iso(correspondent.updated_at),
usage: CorrespondentUsage { total },
usage_count,
}
}
@@ -310,7 +305,6 @@ fn load_usage_for_correspondent(
),
components(schemas(
crate::routes::correspondents::CorrespondentSummary,
crate::routes::correspondents::CorrespondentUsage,
crate::routes::correspondents::CreateCorrespondentRequest,
crate::routes::correspondents::UpdateCorrespondentRequest
))
+261 -62
View File
@@ -1,26 +1,32 @@
use std::{collections::HashSet, time::Duration};
use axum::body::Body;
use axum::extract::{Json, Multipart, Path, Query, State};
use axum::http::StatusCode;
use axum::response::IntoResponse;
use axum::http::{header, HeaderMap, HeaderValue, StatusCode};
use axum::response::{IntoResponse, Response};
use chrono::{DateTime, NaiveDateTime, Utc};
use diesel::dsl::exists;
use diesel::{prelude::*, select};
use futures_util::StreamExt;
use serde::Deserialize;
use serde_json::Value;
use tracing::{error, info};
use utoipa::{IntoParams, ToSchema};
use uuid::Uuid;
use crate::auth::TenantScopedConn;
use crate::auth::{ensure_active_tenant_with_conn, jwt::DownloadSubject, TenantScopedConn};
use crate::documents::asset::{
DocumentAssetDetailResponse, DocumentAssetResponse, DocumentVersionDetailResponse,
DocumentVersionResponse,
asset_disposition, DocumentAssetDetailResponse, DocumentAssetResponse,
DocumentVersionDetailResponse, DocumentVersionResponse, DownloadLink,
};
#[allow(unused_imports)]
use crate::error::ApiErrorResponse;
use crate::error::{AppError, AppResult};
use crate::http::responders::{accepted_json, created_json, no_content, ok_json, JsonResponse};
use crate::models::{Document, DocumentVersion};
use crate::schema::{document_versions, documents, user_sessions::dsl as session_dsl};
use crate::models::{Document, DocumentAsset, DocumentVersion};
use crate::schema::{
document_assets, document_versions, documents, user_sessions::dsl as session_dsl,
};
use crate::services::correspondents::{
AssignCorrespondentsRequest, BulkCorrespondentAction, BulkCorrespondentResponse,
BulkCorrespondentsRequest, CorrespondentAssignmentInput, CorrespondentsService,
@@ -34,6 +40,7 @@ use crate::services::tags::{
AssignTagsRequest, BulkTagAction, BulkTagRequest, BulkTagResponse, TagsService,
};
use crate::state::AppState;
use crate::storage::TenantStorage;
use crate::utils::{error::StorageResultExt, http::inline_content_disposition};
const PRESIGNED_URL_EXPIRY_SECONDS: u64 = 300;
@@ -83,15 +90,6 @@ pub struct RestoreDocumentRequest {
pub folder_id: Option<Uuid>,
}
#[derive(Deserialize, Default, IntoParams, ToSchema)]
#[into_params(parameter_in = Query)]
pub struct AssetObjectsQuery {
#[serde(default)]
pub start: Option<i32>,
#[serde(default)]
pub limit: Option<i32>,
}
#[utoipa::path(
get,
path = "/api/documents",
@@ -180,7 +178,11 @@ pub async fn get_document(
responses(
(status = 201, description = "Document created", body = DocumentDetailResponse),
(status = 200, description = "Existing document reused", body = DocumentDetailResponse),
(status = 409, description = "Document with identical contents already exists")
(
status = 409,
description = "Document with identical contents already exists",
body = ApiErrorResponse
)
),
tag = "Documents"
)]
@@ -198,7 +200,7 @@ pub async fn upload_document(
let user_id = user_id;
let mut file_bytes: Option<Vec<u8>> = None;
let mut original_name: Option<String> = None;
let mut content_type: Option<String> = None;
let mut mime_type: Option<String> = None;
let mut folder_id: Option<Uuid> = None;
let mut metadata: Value = Value::Object(Default::default());
let mut tag_ids: Vec<Uuid> = Vec::new();
@@ -217,7 +219,7 @@ pub async fn upload_document(
Some("file") => {
let file_name = field.file_name().map(|n| n.to_string());
original_name = file_name.clone();
content_type = field.content_type().map(|mime| mime.to_string());
mime_type = field.content_type().map(|mime| mime.to_string());
let data = field.bytes().await.map_err(|err| {
let msg = format!("failed to read file bytes: {err}");
error!(error = %err, "failed to read file bytes");
@@ -346,7 +348,7 @@ pub async fn upload_document(
let request = DocumentUploadRequest {
bytes: file_bytes,
original_name,
content_type,
mime_type,
folder_id,
metadata,
title_override,
@@ -455,12 +457,13 @@ pub async fn list_document_assets(
TenantScopedConn {
mut conn,
tenant_id,
user_id,
..
}: TenantScopedConn,
) -> AppResult<JsonResponse<Vec<DocumentAssetResponse>>> {
let service = DocumentsService::new(&state);
let assets = service
.list_document_assets(&mut conn, tenant_id, document_id)
.list_document_assets(&mut conn, tenant_id, user_id, document_id)
.await?;
ok_json(assets)
}
@@ -468,35 +471,102 @@ pub async fn list_document_assets(
#[utoipa::path(
get,
path = "/api/assets/{asset_id}",
params(("asset_id" = Uuid, Path, description = "Asset ID"), AssetObjectsQuery),
params(("asset_id" = Uuid, Path, description = "Asset ID")),
responses((status = 200, description = "Asset detail", body = DocumentAssetDetailResponse)),
tag = "Assets"
)]
pub async fn get_document_asset(
State(state): State<AppState>,
Path(asset_id): Path<Uuid>,
Query(query): Query<AssetObjectsQuery>,
TenantScopedConn {
conn,
tenant_id,
user_id,
..
}: TenantScopedConn,
) -> AppResult<JsonResponse<DocumentAssetDetailResponse>> {
let start = query.start.unwrap_or(1);
let limit = query.limit.unwrap_or(1);
if start < 1 {
return Err(AppError::bad_request("start must be at least 1"));
}
if limit < 1 {
return Err(AppError::bad_request("limit must be at least 1"));
}
let service = DocumentsService::new(&state);
let detail = service
.get_document_asset(conn, tenant_id, asset_id, start, limit)
.get_document_asset(conn, tenant_id, user_id, asset_id)
.await?;
ok_json(detail)
}
#[utoipa::path(
post,
path = "/api/documents/{id}/download",
params(("id" = Uuid, Path, description = "Document ID")),
responses((status = 200, description = "Download link for current version", body = DownloadLink)),
tag = "Documents"
)]
pub async fn refresh_document_download(
State(state): State<AppState>,
Path(document_id): Path<Uuid>,
TenantScopedConn {
mut conn,
tenant_id,
user_id,
..
}: TenantScopedConn,
) -> AppResult<JsonResponse<DownloadLink>> {
let service = DocumentsService::new(&state);
let link = service
.get_document_download_link(&mut conn, tenant_id, user_id, document_id)
.await?;
ok_json(link)
}
#[utoipa::path(
post,
path = "/api/documents/{id}/versions/{version_id}/download",
params(
("id" = Uuid, Path, description = "Document ID"),
("version_id" = Uuid, Path, description = "Version ID")
),
responses((status = 200, description = "Download link for version", body = DownloadLink)),
tag = "Documents"
)]
pub async fn refresh_document_version_download(
State(state): State<AppState>,
Path((document_id, version_id)): Path<(Uuid, Uuid)>,
TenantScopedConn {
mut conn,
tenant_id,
user_id,
..
}: TenantScopedConn,
) -> AppResult<JsonResponse<DownloadLink>> {
let service = DocumentsService::new(&state);
let link = service
.get_document_version_download_link(&mut conn, tenant_id, user_id, document_id, version_id)
.await?;
ok_json(link)
}
#[utoipa::path(
post,
path = "/api/assets/{asset_id}/download",
params(("asset_id" = Uuid, Path, description = "Asset ID")),
responses((status = 200, description = "Asset download link", body = DownloadLink)),
tag = "Assets"
)]
pub async fn refresh_asset_download(
State(state): State<AppState>,
Path(asset_id): Path<Uuid>,
TenantScopedConn {
mut conn,
tenant_id,
user_id,
..
}: TenantScopedConn,
) -> AppResult<JsonResponse<DownloadLink>> {
let service = DocumentsService::new(&state);
let link = service
.get_asset_download_link(&mut conn, tenant_id, user_id, asset_id)
.await?;
ok_json(link)
}
#[utoipa::path(
get,
path = "/api/documents/{id}/versions",
@@ -547,33 +617,23 @@ pub async fn get_document_version(
#[utoipa::path(
get,
path = "/download/{token}",
path = "/api/download/{token}",
params(("token" = String, Path, description = "Download token")),
responses((status = 302, description = "Redirect to pre-signed URL")),
responses((status = 200, description = "Proxied download stream or redirect")),
tag = "Documents"
)]
pub async fn download_with_token(
State(state): State<AppState>,
Path(token): Path<String>,
) -> AppResult<impl IntoResponse> {
headers: HeaderMap,
) -> AppResult<Response> {
let claims = state
.jwt
.verify_download_token(&token)
.map_err(|_| AppError::unauthorized())?;
let mut conn = state.db_for_tenant(claims.tenant_id)?;
let doc: Document = documents::table
.find(claims.doc_id)
.filter(documents::tenant_id.eq(claims.tenant_id))
.first(&mut conn)?;
if doc.deleted_at.is_some() {
return Err(AppError::not_found());
}
let version: DocumentVersion = document_versions::table
.find(doc.current_version_id)
.first(&mut conn)?;
ensure_active_tenant_with_conn(&mut conn, claims.tenant_id)?;
let now = Utc::now().naive_utc();
let has_active_refresh: bool = select(exists(
@@ -589,22 +649,86 @@ pub async fn download_with_token(
return Err(AppError::unauthorized());
}
drop(conn);
match &claims.subject {
DownloadSubject::Document { doc_id, version_id } => {
let doc_id = *doc_id;
let version_id = *version_id;
let doc: Document = documents::table
.find(doc_id)
.filter(documents::tenant_id.eq(claims.tenant_id))
.first(&mut conn)?;
if doc.deleted_at.is_some() {
return Err(AppError::not_found());
}
let storage = state.storage_for_tenant(claims.tenant_id)?;
let version: DocumentVersion = document_versions::table
.find(version_id)
.filter(document_versions::document_id.eq(doc_id))
.first(&mut conn)?;
let disposition = inline_content_disposition(&doc.filename);
drop(conn);
let presigned_url = storage
.presign_get_object(
&version.s3_key,
Duration::from_secs(PRESIGNED_URL_EXPIRY_SECONDS),
disposition.as_deref(),
)
.await
.storage_context("failed to generate download URL")?;
let storage = state.storage_for_tenant(claims.tenant_id)?;
let disposition = inline_content_disposition(&doc.filename);
Ok(axum::response::Redirect::temporary(&presigned_url))
if !state.config.proxy_downloads {
let presigned_url = storage
.presign_get_object(
&version.s3_key,
Duration::from_secs(PRESIGNED_URL_EXPIRY_SECONDS),
disposition.as_deref(),
)
.await
.storage_context("failed to generate download URL")?;
return Ok(axum::response::Redirect::temporary(&presigned_url).into_response());
}
proxy_storage_object(
storage,
&version.s3_key,
disposition.as_deref(),
headers.get(header::RANGE).cloned(),
doc.mime_type.as_deref(),
Some(version.id.to_string()),
)
.await
}
DownloadSubject::Asset { asset_id } => {
let asset: DocumentAsset = document_assets::table
.find(*asset_id)
.filter(document_assets::tenant_id.eq(claims.tenant_id))
.first(&mut conn)?;
drop(conn);
let storage = state.storage_for_tenant(claims.tenant_id)?;
let disposition = asset_disposition(&asset);
if !state.config.proxy_downloads {
let presigned_url = storage
.presign_get_object(
&asset.s3_key,
Duration::from_secs(PRESIGNED_URL_EXPIRY_SECONDS),
disposition.as_deref(),
)
.await
.storage_context("failed to generate download URL")?;
return Ok(axum::response::Redirect::temporary(&presigned_url).into_response());
}
proxy_storage_object(
storage,
&asset.s3_key,
disposition.as_deref(),
headers.get(header::RANGE).cloned(),
Some(asset.mime_type.as_str()),
Some(asset.id.to_string()),
)
.await
}
}
}
#[utoipa::path(
@@ -628,6 +752,78 @@ pub async fn trash_document(
Ok(StatusCode::NO_CONTENT)
}
async fn proxy_storage_object(
storage: TenantStorage,
key: &str,
response_disposition: Option<&str>,
range_header: Option<HeaderValue>,
fallback_content_type: Option<&str>,
etag: Option<String>,
) -> AppResult<Response> {
let url = storage
.presign_get_object(
key,
Duration::from_secs(PRESIGNED_URL_EXPIRY_SECONDS),
response_disposition,
)
.await
.storage_context("failed to generate download URL")?;
let client = reqwest::Client::new();
let mut request = client.get(url.clone());
if let Some(range) = range_header {
request = request.header(header::RANGE, range);
}
let upstream = request.send().await.map_err(|err| {
tracing::error!(error = ?err, "failed to fetch document stream");
AppError::internal("failed to fetch document stream")
})?;
let status =
StatusCode::from_u16(upstream.status().as_u16()).unwrap_or(StatusCode::BAD_GATEWAY);
if !(status.is_success() || status == StatusCode::PARTIAL_CONTENT) {
tracing::error!(status = %status, "upstream download returned error status");
return Err(AppError::internal("failed to fetch document stream"));
}
let mut builder = Response::builder().status(status);
if let Some(content_type) = upstream.headers().get(header::CONTENT_TYPE) {
builder = builder.header(header::CONTENT_TYPE, content_type);
} else if let Some(fallback) = fallback_content_type {
builder = builder.header(header::CONTENT_TYPE, fallback);
}
if let Some(content_length) = upstream.headers().get(header::CONTENT_LENGTH) {
builder = builder.header(header::CONTENT_LENGTH, content_length);
}
if let Some(range) = upstream.headers().get(header::CONTENT_RANGE) {
builder = builder.header(header::CONTENT_RANGE, range);
}
builder = builder.header("Accept-Ranges", "bytes");
if let Some(disposition) = response_disposition {
builder = builder.header(header::CONTENT_DISPOSITION, disposition);
}
if let Some(etag_value) = etag {
builder = builder.header(header::ETAG, format!("\"{}\"", etag_value));
}
let stream = upstream
.bytes_stream()
.map(|chunk| chunk.map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err)));
let body = Body::from_stream(stream);
builder.body(body).map_err(|err| {
tracing::error!(error = ?err, "failed to build proxied response");
AppError::internal("failed to build proxied response")
})
}
#[utoipa::path(
delete,
path = "/api/documents/{id}",
@@ -654,7 +850,7 @@ pub async fn delete_document(
patch,
path = "/api/documents/{id}",
params(("id" = Uuid, Path, description = "Document ID")),
request_body = UpdateDocumentRequest,
request_body = crate::services::documents::UpdateDocumentRequest,
responses((status = 200, description = "Updated document", body = DocumentDetailResponse)),
tag = "Documents"
)]
@@ -896,6 +1092,7 @@ pub async fn remove_tag(
crate::routes::documents::check_document,
crate::routes::documents::upload_document,
crate::routes::documents::get_document,
crate::routes::documents::refresh_document_download,
crate::routes::documents::update_document,
crate::routes::documents::trash_document,
crate::routes::documents::delete_document,
@@ -915,6 +1112,8 @@ pub async fn remove_tag(
crate::routes::documents::get_document_asset,
crate::routes::documents::list_document_versions,
crate::routes::documents::get_document_version,
crate::routes::documents::refresh_document_version_download,
crate::routes::documents::refresh_asset_download,
),
components(schemas(
crate::services::documents::DocumentListQuery,
@@ -940,14 +1139,14 @@ pub async fn remove_tag(
crate::routes::documents::MoveDocumentRequest,
crate::routes::documents::BulkReanalyzeSelectionRequest,
crate::routes::documents::BulkReanalyzeResponse,
crate::routes::documents::AssetObjectsQuery,
crate::routes::documents::UploadDocumentForm,
crate::documents::asset::DocumentVersionResponse,
crate::documents::asset::DocumentVersionDetailResponse,
crate::documents::asset::DocumentAssetResponse,
crate::documents::asset::DocumentAssetDetailResponse,
crate::documents::asset::DocumentAssetObjectResponse,
crate::documents::asset::DownloadLink,
crate::documents::correspondents::DocumentCorrespondentResponse,
crate::error::ApiErrorResponse,
))
)]
pub struct DocumentsApiDoc;
+1 -1
View File
@@ -150,7 +150,7 @@ pub async fn list_folder_contents(
)?;
let documents = if include_documents {
service.hydrate_documents(&mut conn, user_id, documents)?
service.hydrate_documents(&mut conn, tenant_id, user_id, documents)?
} else {
Vec::new()
};
+81 -36
View File
@@ -28,6 +28,7 @@ pub mod folders;
pub mod health;
pub mod profile;
pub mod tags;
pub mod tenants;
pub mod webdav;
pub fn create_router(state: AppState) -> Router<()> {
@@ -67,7 +68,6 @@ pub fn create_router(state: AppState) -> Router<()> {
.route("/refresh", post(auth::refresh))
.route("/logout", post(auth::logout))
.route("/select-tenant", post(auth::select_tenant))
.route("/tenants", get(auth::list_tenants))
.route(
"/passkeys/register/start",
post(auth::passkey_register_start),
@@ -125,92 +125,104 @@ pub fn create_router(state: AppState) -> Router<()> {
])),
)
.route(
"/:id",
"/{id}",
get(documents::get_document).layer(RequireCapabilitiesLayer::all([
ApiCapability::DocumentsRead,
])),
)
.route(
"/:id/trash",
"/{id}/download",
post(documents::refresh_document_download).layer(RequireCapabilitiesLayer::all([
ApiCapability::DocumentsRead,
])),
)
.route(
"/{id}/trash",
post(documents::trash_document).layer(RequireCapabilitiesLayer::all([
ApiCapability::DocumentsWrite,
])),
)
.route(
"/:id",
"/{id}",
delete(documents::delete_document).layer(RequireCapabilitiesLayer::all([
ApiCapability::DocumentsWrite,
])),
)
.route(
"/:id",
"/{id}",
patch(documents::update_document).layer(RequireCapabilitiesLayer::all([
ApiCapability::DocumentsEdit,
])),
)
.route(
"/:id/assets",
"/{id}/assets",
get(documents::list_document_assets).layer(RequireCapabilitiesLayer::all([
ApiCapability::DocumentsRead,
])),
)
.route(
"/:id/assets",
"/{id}/assets",
post(documents::request_document_assets).layer(RequireCapabilitiesLayer::all([
ApiCapability::DocumentsWrite,
])),
)
.route(
"/:id/folder",
"/{id}/folder",
patch(documents::move_document).layer(RequireCapabilitiesLayer::all([
ApiCapability::DocumentsEdit,
])),
)
.route(
"/:id/versions",
"/{id}/versions",
get(documents::list_document_versions).layer(RequireCapabilitiesLayer::all([
ApiCapability::DocumentsRead,
])),
)
.route(
"/:id/versions/:version_id",
"/{id}/versions/{version_id}",
get(documents::get_document_version).layer(RequireCapabilitiesLayer::all([
ApiCapability::DocumentsRead,
])),
)
.route(
"/:id/restore",
"/{id}/versions/{version_id}/download",
post(documents::refresh_document_version_download).layer(
RequireCapabilitiesLayer::all([ApiCapability::DocumentsRead]),
),
)
.route(
"/{id}/restore",
post(documents::restore_document).layer(RequireCapabilitiesLayer::all([
ApiCapability::DocumentsEdit,
])),
)
.route(
"/:id/tags",
"/{id}/tags",
post(documents::assign_tags).layer(RequireCapabilitiesLayer::all([
ApiCapability::DocumentsEdit,
])),
)
.route(
"/:id/tags/:tag_id",
"/{id}/tags/{tag_id}",
delete(documents::remove_tag).layer(RequireCapabilitiesLayer::all([
ApiCapability::DocumentsEdit,
])),
)
.route(
"/:id/correspondents",
"/{id}/correspondents",
post(documents::assign_correspondents).layer(RequireCapabilitiesLayer::all([
ApiCapability::DocumentsEdit,
])),
)
.route(
"/:id/correspondents/:correspondent_id",
"/{id}/correspondents/{correspondent_id}",
delete(documents::remove_correspondent).layer(RequireCapabilitiesLayer::all([
ApiCapability::DocumentsEdit,
])),
);
let download_routes =
Router::new().route("/download/:token", get(documents::download_with_token));
Router::new().route("/api/download/{token}", get(documents::download_with_token));
let folders_routes = Router::new()
.route(
@@ -229,22 +241,22 @@ pub fn create_router(state: AppState) -> Router<()> {
.layer(RequireCapabilitiesLayer::all([ApiCapability::FoldersRead])),
)
.route(
"/:id",
"/{id}",
get(folders::get_folder)
.layer(RequireCapabilitiesLayer::all([ApiCapability::FoldersRead])),
)
.route(
"/:id",
"/{id}",
delete(folders::delete_folder)
.layer(RequireCapabilitiesLayer::all([ApiCapability::FoldersWrite])),
)
.route(
"/:id",
"/{id}",
patch(folders::update_folder)
.layer(RequireCapabilitiesLayer::all([ApiCapability::FoldersEdit])),
)
.route(
"/:id/contents",
"/{id}/contents",
get(folders::list_folder_contents)
.layer(RequireCapabilitiesLayer::all([ApiCapability::FoldersRead])),
);
@@ -259,11 +271,11 @@ pub fn create_router(state: AppState) -> Router<()> {
post(tags::create_tag).layer(RequireCapabilitiesLayer::all([ApiCapability::TagsWrite])),
)
.route(
"/:id",
"/{id}",
patch(tags::update_tag).layer(RequireCapabilitiesLayer::all([ApiCapability::TagsEdit])),
)
.route(
"/:id",
"/{id}",
delete(tags::delete_tag)
.layer(RequireCapabilitiesLayer::all([ApiCapability::TagsWrite])),
);
@@ -282,13 +294,13 @@ pub fn create_router(state: AppState) -> Router<()> {
])),
)
.route(
"/:id",
"/{id}",
patch(correspondents::update_correspondent).layer(RequireCapabilitiesLayer::all([
ApiCapability::CorrespondentsEdit,
])),
)
.route(
"/:id",
"/{id}",
delete(correspondents::delete_correspondent).layer(RequireCapabilitiesLayer::all([
ApiCapability::CorrespondentsWrite,
])),
@@ -306,12 +318,12 @@ pub fn create_router(state: AppState) -> Router<()> {
.layer(RequireCapabilitiesLayer::all([ApiCapability::ProfileWrite])),
)
.route(
"/api-tokens/:id/regenerate",
"/api-tokens/{id}/regenerate",
post(profile::regenerate_api_token)
.layer(RequireCapabilitiesLayer::all([ApiCapability::ProfileWrite])),
)
.route(
"/api-tokens/:id",
"/api-tokens/{id}",
delete(profile::delete_api_token)
.layer(RequireCapabilitiesLayer::all([ApiCapability::ProfileWrite])),
)
@@ -321,7 +333,7 @@ pub fn create_router(state: AppState) -> Router<()> {
.layer(RequireCapabilitiesLayer::all([ApiCapability::ProfileRead])),
)
.route(
"/passkeys/:id",
"/passkeys/{id}",
delete(profile::delete_passkey)
.layer(RequireCapabilitiesLayer::all([ApiCapability::ProfileWrite])),
);
@@ -340,19 +352,19 @@ pub fn create_router(state: AppState) -> Router<()> {
])),
)
.route(
"/:id",
"/{id}",
get(capability_sets::get_capability_set).layer(RequireCapabilitiesLayer::all([
ApiCapability::CapabilitySetsRead,
])),
)
.route(
"/:id",
"/{id}",
patch(capability_sets::update_capability_set).layer(RequireCapabilitiesLayer::all([
ApiCapability::CapabilitySetsWrite,
])),
)
.route(
"/:id",
"/{id}",
delete(capability_sets::delete_capability_set).layer(RequireCapabilitiesLayer::all([
ApiCapability::CapabilitySetsWrite,
])),
@@ -366,12 +378,44 @@ pub fn create_router(state: AppState) -> Router<()> {
);
let protected_state = state.clone();
let assets_routes = Router::new().route(
"/:asset_id",
get(documents::get_document_asset).layer(RequireCapabilitiesLayer::all([
ApiCapability::DocumentsRead,
])),
);
let assets_routes = Router::new()
.route(
"/{asset_id}",
get(documents::get_document_asset).layer(RequireCapabilitiesLayer::all([
ApiCapability::DocumentsRead,
])),
)
.route(
"/{asset_id}/download",
post(documents::refresh_asset_download).layer(RequireCapabilitiesLayer::all([
ApiCapability::DocumentsRead,
])),
);
let manage_tenants_layer = RequireCapabilitiesLayer::all([ApiCapability::TenantsWrite]);
let tenants_routes = Router::new()
.route("/", get(tenants::list_tenants))
.route("/{tenant_id}", get(tenants::get_tenant))
.route(
"/{tenant_id}",
patch(tenants::update_tenant).layer(manage_tenants_layer.clone()),
)
.route(
"/{tenant_id}/users",
get(tenants::list_tenant_users).layer(manage_tenants_layer.clone()),
)
.route(
"/{tenant_id}/users/{user_id}",
get(tenants::get_tenant_user).layer(manage_tenants_layer.clone()),
)
.route(
"/{tenant_id}/users/{user_id}",
patch(tenants::update_tenant_user).layer(manage_tenants_layer.clone()),
)
.route(
"/{tenant_id}/users/{user_id}",
delete(tenants::delete_tenant_user).layer(manage_tenants_layer.clone()),
);
let protected_routes = Router::new()
.nest("/api/documents", documents_routes)
@@ -382,6 +426,7 @@ pub fn create_router(state: AppState) -> Router<()> {
.nest("/api/capability-sets", capability_sets_routes)
.nest("/api/capabilities", capabilities_routes)
.nest("/api/assets", assets_routes)
.nest("/api/tenants", tenants_routes)
.layer(middleware::from_extractor_with_state::<AuthenticatedUser, _>(protected_state));
let upload_limit = state.config.upload_body_limit_bytes;
+154
View File
@@ -0,0 +1,154 @@
use axum::extract::{Path, State};
use axum::{http::StatusCode, Json};
use uuid::Uuid;
use crate::auth::{AuthenticatedUser, TenantMembershipUser};
use crate::error::AppResult;
use crate::http::responders::JsonResponse;
use crate::services::auth::{AuthService, TenantSnippet};
use crate::services::tenants::{
TenantApiService, TenantUserListResponse, TenantUserSummary, UpdateTenantRequest,
UpdateTenantUserRequest,
};
use crate::state::AppState;
#[utoipa::path(
get,
path = "/api/tenants",
responses((status = 200, body = [TenantSnippet], description = "Tenant memberships for the current user")),
tag = "Tenants"
)]
pub async fn list_tenants(
State(state): State<AppState>,
user: TenantMembershipUser,
) -> AppResult<Json<Vec<TenantSnippet>>> {
let response = AuthService::new(&state).list_tenants(user.user_id)?;
Ok(Json(response.into_inner().tenants))
}
#[utoipa::path(
get,
path = "/api/tenants/{tenant_id}",
params(("tenant_id" = Uuid, Path, description = "Tenant identifier")),
responses((status = 200, body = TenantSnippet, description = "Tenant details")),
tag = "Tenants"
)]
pub async fn get_tenant(
State(state): State<AppState>,
Path(tenant_id): Path<Uuid>,
user: TenantMembershipUser,
) -> AppResult<JsonResponse<TenantSnippet>> {
AuthService::new(&state).get_tenant(user.user_id, tenant_id)
}
#[utoipa::path(
patch,
path = "/api/tenants/{tenant_id}",
params(("tenant_id" = Uuid, Path, description = "Tenant identifier")),
request_body = UpdateTenantRequest,
responses((status = 200, body = TenantSnippet, description = "Updated tenant")),
tag = "Tenants"
)]
pub async fn update_tenant(
State(state): State<AppState>,
Path(tenant_id): Path<Uuid>,
user: AuthenticatedUser,
Json(payload): Json<UpdateTenantRequest>,
) -> AppResult<JsonResponse<TenantSnippet>> {
TenantApiService::new(&state).update_name(user, tenant_id, payload)
}
#[utoipa::path(
get,
path = "/api/tenants/{tenant_id}/users",
params(("tenant_id" = Uuid, Path, description = "Tenant ID")),
responses((status = 200, body = [TenantUserSummary], description = "All users for the tenant")),
tag = "Tenants"
)]
pub async fn list_tenant_users(
State(state): State<AppState>,
Path(tenant_id): Path<Uuid>,
user: AuthenticatedUser,
) -> AppResult<Json<Vec<TenantUserSummary>>> {
let response = TenantApiService::new(&state).list_users(&user, tenant_id)?;
Ok(Json(response.into_inner().users))
}
#[utoipa::path(
get,
path = "/api/tenants/{tenant_id}/users/{user_id}",
params(
("tenant_id" = Uuid, Path, description = "Tenant ID"),
("user_id" = Uuid, Path, description = "User ID")
),
responses((status = 200, body = TenantUserSummary, description = "Tenant user details")),
tag = "Tenants"
)]
pub async fn get_tenant_user(
State(state): State<AppState>,
Path((tenant_id, target_user_id)): Path<(Uuid, Uuid)>,
user: AuthenticatedUser,
) -> AppResult<JsonResponse<TenantUserSummary>> {
TenantApiService::new(&state).get_user(&user, tenant_id, target_user_id)
}
#[utoipa::path(
patch,
path = "/api/tenants/{tenant_id}/users/{user_id}",
params(
("tenant_id" = Uuid, Path, description = "Tenant ID"),
("user_id" = Uuid, Path, description = "User ID")
),
request_body = UpdateTenantUserRequest,
responses((status = 200, body = TenantUserSummary, description = "Updated tenant user")),
tag = "Tenants"
)]
pub async fn update_tenant_user(
State(state): State<AppState>,
Path((tenant_id, target_user_id)): Path<(Uuid, Uuid)>,
user: AuthenticatedUser,
Json(payload): Json<UpdateTenantUserRequest>,
) -> AppResult<JsonResponse<TenantUserSummary>> {
TenantApiService::new(&state).update_user(&user, tenant_id, target_user_id, payload)
}
#[utoipa::path(
delete,
path = "/api/tenants/{tenant_id}/users/{user_id}",
params(
("tenant_id" = Uuid, Path, description = "Tenant ID"),
("user_id" = Uuid, Path, description = "User ID")
),
responses((status = 204, description = "Membership removed")),
tag = "Tenants"
)]
pub async fn delete_tenant_user(
State(state): State<AppState>,
Path((tenant_id, target_user_id)): Path<(Uuid, Uuid)>,
user: AuthenticatedUser,
) -> AppResult<StatusCode> {
TenantApiService::new(&state).remove_user(&user, tenant_id, target_user_id)?;
Ok(StatusCode::NO_CONTENT)
}
#[derive(utoipa::OpenApi)]
#[openapi(
paths(
list_tenants,
get_tenant,
update_tenant,
list_tenant_users,
get_tenant_user,
update_tenant_user,
delete_tenant_user,
),
components(schemas(
crate::services::auth::TenantListResponse,
crate::services::auth::TenantSnippet,
UpdateTenantRequest,
UpdateTenantUserRequest,
TenantUserListResponse,
TenantUserSummary,
))
)]
pub struct TenantsApiDoc;
+15 -7
View File
@@ -16,7 +16,10 @@ use quick_xml::events::{BytesDecl, BytesEnd, BytesStart, BytesText, Event};
use quick_xml::Writer;
use uuid::Uuid;
use crate::auth::api_tokens::{find_active_token_by_secret, touch_api_token};
use crate::auth::{
api_tokens::{find_active_token_by_secret, touch_api_token},
ensure_active_tenant_with_conn,
};
use crate::error::{AppError, AppResult};
use crate::models::{ApiCapability, Document, DocumentVersion, Folder, User};
use crate::schema::{
@@ -348,7 +351,7 @@ async fn stream_document(
if let Some(content_type) = upstream.headers().get(header::CONTENT_TYPE) {
builder = builder.header(header::CONTENT_TYPE, content_type);
} else if let Some(ref typ) = document.content_type {
} else if let Some(ref typ) = document.mime_type {
builder = builder.header(header::CONTENT_TYPE, typ);
}
@@ -483,6 +486,11 @@ fn authenticate(state: &AppState, headers: &HeaderMap) -> Result<Option<WebDavCo
}
};
if let Err(err) = ensure_active_tenant_with_conn(&mut conn, tenant_id) {
tracing::warn!(tenant_id = %tenant_id, error = ?err, "webdav tenant not active");
return Ok(None);
}
apply_tenant_guc(&mut conn, tenant_id)?;
touch_api_token(&mut conn, token.id)?;
@@ -521,7 +529,7 @@ fn build_resources_for_folder(
display_name,
is_collection: true,
content_length: None,
content_type: None,
mime_type: None,
last_modified,
});
@@ -537,7 +545,7 @@ fn build_resources_for_folder(
display_name: subfolder.name.clone(),
is_collection: true,
content_length: None,
content_type: None,
mime_type: None,
last_modified: Some(to_http_date(subfolder.updated_at)),
});
}
@@ -575,7 +583,7 @@ fn document_to_resource(
display_name: document.title.clone(),
is_collection: false,
content_length: Some(version.size_bytes),
content_type: document.content_type.clone(),
mime_type: document.mime_type.clone(),
last_modified: Some(to_http_date(document.updated_at)),
}
}
@@ -631,7 +639,7 @@ fn render_multistatus(resources: &[DavResource]) -> Result<Vec<u8>, quick_xml::E
writer.write_event(Event::End(BytesEnd::new("D:getcontentlength")))?;
}
if let Some(content_type) = &resource.content_type {
if let Some(content_type) = &resource.mime_type {
writer.write_event(Event::Start(BytesStart::new("D:getcontenttype")))?;
writer.write_event(Event::Text(BytesText::new(content_type)))?;
writer.write_event(Event::End(BytesEnd::new("D:getcontenttype")))?;
@@ -673,7 +681,7 @@ struct DavResource {
display_name: String,
is_collection: bool,
content_length: Option<i64>,
content_type: Option<String>,
mime_type: Option<String>,
last_modified: Option<String>,
}
enum ResolvedPath {
+26 -30
View File
@@ -1,38 +1,34 @@
use anyhow::Result;
use aws_config::meta::region::RegionProviderChain;
use aws_credential_types::Credentials;
use aws_sdk_s3::{
config::{Builder as S3ConfigBuilder, Region},
Client as S3Client,
};
use anyhow::{anyhow, Context, Result};
use s3::{bucket::Bucket, creds::Credentials, region::Region};
use crate::config::AppConfig;
pub async fn build_client(config: &AppConfig) -> Result<S3Client> {
let region = Region::new(config.aws_region.clone());
let region_provider = RegionProviderChain::first_try(Some(region))
.or_default_provider()
.or_else("us-east-1");
pub fn build_bucket(config: &AppConfig) -> Result<Bucket> {
let region = if let Some(endpoint) = &config.aws_endpoint_url {
Region::Custom {
region: config.aws_region.clone(),
endpoint: endpoint.clone(),
}
} else {
config
.aws_region
.parse::<Region>()
.context("invalid AWS region")?
};
#[allow(deprecated)]
let mut loader = aws_config::from_env().region(region_provider);
if let Some(endpoint) = &config.aws_endpoint_url {
loader = loader.endpoint_url(endpoint);
}
if let (Some(access_key), Some(secret_key)) = (
config.aws_access_key_id.clone(),
config.aws_secret_access_key.clone(),
let credentials = if let (Some(access_key), Some(secret_key)) = (
config.aws_access_key_id.as_deref(),
config.aws_secret_access_key.as_deref(),
) {
let credentials = Credentials::new(access_key, secret_key, None, None, "static");
loader = loader.credentials_provider(credentials);
}
Credentials::new(Some(access_key), Some(secret_key), None, None, None)
.context("failed to create static AWS credentials")?
} else {
Credentials::default().context("failed to load AWS credentials")?
};
let base_config = loader.load().await;
let s3_config = S3ConfigBuilder::from(&base_config)
.force_path_style(true)
.build();
let bucket = Bucket::new(&config.s3_bucket, region, credentials)
.map_err(|err| anyhow!("failed to create S3 bucket client: {err}"))?;
let bucket = bucket.with_path_style();
Ok(S3Client::from_conf(s3_config))
Ok(*bucket)
}
+6 -18
View File
@@ -26,17 +26,6 @@ diesel::table! {
}
}
diesel::table! {
document_asset_objects (id) {
id -> Uuid,
asset_id -> Uuid,
ordinal -> Int4,
s3_key -> Text,
metadata -> Jsonb,
tenant_id -> Uuid,
}
}
diesel::table! {
document_assets (id) {
id -> Uuid,
@@ -45,7 +34,7 @@ diesel::table! {
mime_type -> Text,
metadata -> Jsonb,
created_at -> Timestamptz,
cardinality -> Nullable<Int4>,
s3_key -> Text,
tenant_id -> Uuid,
}
}
@@ -94,7 +83,7 @@ diesel::table! {
#[max_length = 255]
original_name -> Varchar,
#[max_length = 100]
content_type -> Nullable<Varchar>,
mime_type -> Nullable<Varchar>,
folder_id -> Nullable<Uuid>,
created_at -> Timestamptz,
updated_at -> Timestamptz,
@@ -131,7 +120,8 @@ diesel::table! {
last_error -> Nullable<Text>,
created_at -> Timestamptz,
updated_at -> Timestamptz,
tenant_id -> Uuid,
tenant_id -> Nullable<Uuid>,
result -> Nullable<Jsonb>,
}
}
@@ -293,8 +283,6 @@ diesel::table! {
diesel::joinable!(correspondents -> tenants (tenant_id));
diesel::joinable!(capability_set_capabilities -> capability_sets (capability_set_id));
diesel::joinable!(capability_sets -> tenants (tenant_id));
diesel::joinable!(document_asset_objects -> document_assets (asset_id));
diesel::joinable!(document_asset_objects -> tenants (tenant_id));
diesel::joinable!(document_assets -> document_versions (document_version_id));
diesel::joinable!(document_assets -> tenants (tenant_id));
diesel::joinable!(document_correspondents -> correspondents (correspondent_id));
@@ -310,6 +298,7 @@ diesel::joinable!(documents -> folders (folder_id));
diesel::joinable!(documents -> tenants (tenant_id));
diesel::joinable!(folders -> tenants (tenant_id));
diesel::joinable!(jobs -> tenants (tenant_id));
diesel::joinable!(magic_tokens -> users (user_id));
diesel::joinable!(user_sessions -> tenants (tenant_id));
diesel::joinable!(user_sessions -> users (user_id));
diesel::joinable!(tags -> tenants (tenant_id));
@@ -323,10 +312,10 @@ diesel::joinable!(api_tokens -> capability_sets (capability_set_id));
diesel::joinable!(api_tokens -> users (user_id));
diesel::allow_tables_to_appear_in_same_query!(
api_tokens,
correspondents,
capability_set_capabilities,
capability_sets,
document_asset_objects,
document_assets,
document_correspondents,
document_tags,
@@ -342,5 +331,4 @@ diesel::allow_tables_to_appear_in_same_query!(
user_passkeys,
users,
webauthn_challenges,
api_tokens,
);
+76 -27
View File
@@ -2,7 +2,7 @@ use axum::http::{header::SET_COOKIE, HeaderMap, HeaderValue, StatusCode};
use axum::response::{IntoResponse, Response};
use chrono::{Duration as ChronoDuration, TimeZone, Utc};
use diesel::{pg::PgConnection, prelude::*, Connection, OptionalExtension};
use rand::{rngs::OsRng, RngCore};
use rand::{rngs::OsRng, TryRngCore};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use utoipa::ToSchema;
@@ -442,15 +442,12 @@ impl<'a> AuthService<'a> {
Ok((headers, StatusCode::NO_CONTENT))
}
pub fn list_tenants(
&self,
user: AuthenticatedUser,
) -> AppResult<JsonResponse<TenantListResponse>> {
pub fn list_tenants(&self, user_id: Uuid) -> AppResult<JsonResponse<TenantListResponse>> {
let mut conn = self.state.db_unscoped()?;
apply_user_guc(&mut conn, user.user_id)?;
apply_user_guc(&mut conn, user_id)?;
let tenant_ids: Vec<Uuid> = memberships_dsl::user_memberships
.filter(memberships_dsl::user_id.eq(user.user_id))
.filter(memberships_dsl::user_id.eq(user_id))
.select(memberships_dsl::tenant_id)
.load(&mut conn)?;
@@ -458,11 +455,14 @@ impl<'a> AuthService<'a> {
let mut tenants = Vec::with_capacity(tenant_ids.len());
for tenant_id in tenant_ids {
let name: String = tenant_dsl::tenants
let (name, status): (String, TenantStatus) = tenant_dsl::tenants
.find(tenant_id)
.select(tenant_dsl::name)
.select((tenant_dsl::name, tenant_dsl::status))
.first(&mut conn)
.map_err(AppError::from)?;
if status != TenantStatus::Active {
continue;
}
tenants.push(TenantSnippet {
id: tenant_id,
name,
@@ -472,6 +472,43 @@ impl<'a> AuthService<'a> {
ok_json(TenantListResponse { tenants })
}
pub fn get_tenant(
&self,
user_id: Uuid,
tenant_id: Uuid,
) -> AppResult<JsonResponse<TenantSnippet>> {
let mut conn = self.state.db_unscoped()?;
apply_user_guc(&mut conn, user_id)?;
let is_member: bool = diesel::select(diesel::dsl::exists(
memberships_dsl::user_memberships
.filter(memberships_dsl::user_id.eq(user_id))
.filter(memberships_dsl::tenant_id.eq(tenant_id)),
))
.get_result(&mut conn)?;
clear_user_guc(&mut conn)?;
if !is_member {
return Err(AppError::not_found());
}
let (name, status): (String, TenantStatus) = tenant_dsl::tenants
.find(tenant_id)
.select((tenant_dsl::name, tenant_dsl::status))
.first(&mut conn)
.map_err(AppError::from)?;
if status != TenantStatus::Active {
return Err(AppError::not_found());
}
ok_json(TenantSnippet {
id: tenant_id,
name,
})
}
pub fn passkey_register_start(
&self,
user: AuthenticatedUser,
@@ -575,14 +612,33 @@ impl<'a> AuthService<'a> {
return Err(AppError::unauthorized());
}
let mut active_tenants = Vec::new();
for tenant_id in tenant_ids {
let (name, status): (String, TenantStatus) = tenant_dsl::tenants
.find(tenant_id)
.select((tenant_dsl::name, tenant_dsl::status))
.first(conn)
.map_err(AppError::from)?;
if status == TenantStatus::Active {
active_tenants.push((tenant_id, name));
}
}
if active_tenants.is_empty() {
return Err(AppError::new(
StatusCode::FORBIDDEN,
"no active tenants available",
));
}
if let Some(preferred_id) = preferred_tenant_id {
if tenant_ids.iter().any(|id| *id == preferred_id) {
if active_tenants.iter().any(|(id, _)| *id == preferred_id) {
return self.issue_session(conn, user, preferred_id);
}
}
if tenant_ids.len() == 1 {
return self.issue_session(conn, user, tenant_ids[0]);
if active_tenants.len() == 1 {
return self.issue_session(conn, user, active_tenants[0].0);
}
let selection_token = self
@@ -591,20 +647,10 @@ impl<'a> AuthService<'a> {
.generate_tenant_selector_token(user.id)
.map_err(AppError::from)?;
let mut tenants = Vec::with_capacity(tenant_ids.len());
for tenant_id in tenant_ids {
apply_tenant_guc(conn, tenant_id)?;
clear_user_guc(conn)?;
let name: String = tenant_dsl::tenants
.find(tenant_id)
.select(tenant_dsl::name)
.first(conn)
.map_err(AppError::from)?;
tenants.push(TenantSnippet {
id: tenant_id,
name,
});
}
let tenants = active_tenants
.into_iter()
.map(|(id, name)| TenantSnippet { id, name })
.collect();
let response = ok_json(LoginResponseVariants::Selection(TenantSelectionResponse {
access_token: selection_token,
@@ -677,6 +723,7 @@ impl<'a> AuthService<'a> {
user: &User,
tenant_id: Uuid,
) -> AppResult<Response> {
crate::auth::ensure_active_tenant_with_conn(conn, tenant_id)?;
apply_tenant_guc(conn, tenant_id)?;
clear_user_guc(conn)?;
clear_user_session_hash(conn)?;
@@ -781,7 +828,9 @@ fn hash_magic_token(token: &str) -> String {
fn generate_session_token() -> String {
let mut bytes = [0u8; 32];
OsRng.fill_bytes(&mut bytes);
OsRng
.try_fill_bytes(&mut bytes)
.expect("failed to read random bytes");
hex::encode(bytes)
}
+295 -153
View File
@@ -1,9 +1,6 @@
use std::{
collections::{HashMap, HashSet},
time::Duration,
};
use std::collections::{HashMap, HashSet};
use chrono::{DateTime, NaiveDateTime, Utc};
use chrono::{DateTime, Duration as ChronoDuration, NaiveDateTime, Utc};
use diesel::{
dsl::{exists, not, sql},
prelude::*,
@@ -20,10 +17,10 @@ use uuid::Uuid;
use crate::documents::{
asset::{
build_download_path, derive_document_title, filename_with_retained_extension,
load_asset_responses_with_conn, load_primary_assets, to_asset_detail_response,
to_asset_object_response, to_version_response, DocumentAssetDetailResponse,
build_download_link, derive_document_title, filename_with_retained_extension,
to_asset_detail_response, to_version_response, DocumentAssetDetailResponse,
DocumentAssetResponse, DocumentVersionDetailResponse, DocumentVersionResponse,
DownloadLink,
},
correspondents::{
insert_document_correspondents, normalize_correspondent_ids, DocumentCorrespondentResponse,
@@ -36,16 +33,12 @@ use crate::documents::{
tags::assign_tags as assign_tags_to_document,
};
use crate::error::{AppError, AppResult};
use crate::jobs::{
enqueue_job, JobQueueError, JOB_ANALYZE_DOCUMENT, JOB_INDEX_DOCUMENT_TEXT, JOB_PURGE_DOCUMENT,
};
use crate::jobs::{enqueue_job, JobQueueError, JOB_ANALYZE_DOCUMENT, JOB_PURGE_DOCUMENT};
use crate::models::{
Document, DocumentAsset, DocumentAssetObject, DocumentVersion, NewDocument, NewDocumentVersion,
Tag,
Document, DocumentAsset, DocumentVersion, NewDocument, NewDocumentVersion, Tag,
};
use crate::schema::{
document_asset_objects, document_assets, document_correspondents, document_tags,
document_versions, documents, folders,
document_assets, document_correspondents, document_tags, document_versions, documents, folders,
};
use crate::services::{
correspondents::CorrespondentAssignmentInput, folders::gather_descendant_folder_ids,
@@ -53,17 +46,11 @@ use crate::services::{
};
use crate::state::{AppState, PgPooledConnection};
use crate::utils::{
db::validate_bulk_ids,
error::StorageResultExt,
http::inline_content_disposition,
json::{classify_nullable, NullableValue},
setops::{intersect_option_sets, load_linked_doc_ids},
storage_paths::document_version_object_key,
time::to_iso,
db::validate_bulk_ids, error::StorageResultExt, http::inline_content_disposition,
json::classify_nullable, json::NullableValue, setops::intersect_option_sets,
setops::load_linked_doc_ids, storage_paths::document_version_object_key, time::to_iso,
};
const PRESIGNED_URL_EXPIRY_SECONDS: u64 = 300;
#[derive(Deserialize, IntoParams, ToSchema, Clone)]
#[into_params(parameter_in = Query)]
pub struct DocumentListQuery {
@@ -146,7 +133,7 @@ pub struct DocumentResponse {
pub title: String,
pub original_name: String,
#[schema(nullable)]
pub content_type: Option<String>,
pub mime_type: Option<String>,
#[schema(nullable)]
pub folder_id: Option<Uuid>,
pub created_at: String,
@@ -191,7 +178,6 @@ struct DocumentUpdateChangeset {
struct DocumentUpdatePlan {
changeset: DocumentUpdateChangeset,
title_changed: bool,
}
#[derive(Deserialize, ToSchema)]
@@ -238,7 +224,7 @@ fn default_true() -> bool {
pub struct DocumentUploadRequest {
pub bytes: Vec<u8>,
pub original_name: String,
pub content_type: Option<String>,
pub mime_type: Option<String>,
pub folder_id: Option<Uuid>,
pub metadata: Value,
pub title_override: Option<String>,
@@ -286,7 +272,6 @@ impl<'a> DocumentsService<'a> {
let mut changes = DocumentUpdateChangeset::default();
let mut has_changes = false;
let mut title_changed = false;
if let Some(ref candidate) = title {
let trimmed = candidate.trim();
@@ -301,7 +286,6 @@ impl<'a> DocumentsService<'a> {
changes.filename = Some(new_filename);
}
has_changes = true;
title_changed = true;
}
}
@@ -349,7 +333,6 @@ impl<'a> DocumentsService<'a> {
Ok(DocumentUpdatePlan {
changeset: changes,
title_changed,
})
}
@@ -372,8 +355,9 @@ impl<'a> DocumentsService<'a> {
.cloned()
.unwrap_or_else(|| (Vec::new(), Vec::new()));
let assets = load_asset_responses_with_conn(conn, tenant_id, current_version.id)?;
let current_version_data = Some((to_version_response(current_version), assets));
let assets = self.load_asset_responses(conn, tenant_id, current_version.id, user_id)?;
let download = build_download_link(self.state, &doc, current_version.id, user_id)?;
let current_version_data = Some((to_version_response(current_version), assets, download));
let response =
self.to_document_response(user_id, doc, tags, correspondents, current_version_data)?;
@@ -488,7 +472,9 @@ impl<'a> DocumentsService<'a> {
let mut quickwit_order: Option<Vec<Uuid>> = None;
if let Some(query_str) = search_text.as_ref() {
debug!(query = %query_str, "performing quickwit document search");
debug!(query = %query_str, "performing hybrid document search");
// 1. Quickwit Search
let endpoint = self
.state
.config
@@ -501,19 +487,37 @@ impl<'a> DocumentsService<'a> {
.as_ref()
.ok_or_else(|| AppError::internal("quickwit index not configured for tenant"))?;
let ids = quickwit_search(endpoint, index, tenant_id, query_str)
let quickwit_ids = quickwit_search(endpoint, index, tenant_id, query_str)
.await
.map_err(|err| {
error!(error = ?err, "quickwit search failed");
AppError::internal("quickwit search failed")
})?;
if ids.is_empty() {
// 2. Postgres Title Search
let postgres_ids: Vec<Uuid> = documents::table
.filter(documents::tenant_id.eq(tenant_id))
.filter(documents::deleted_at.is_null())
.filter(documents::title.ilike(format!("%{}%", query_str)))
.select(documents::id)
.load(conn)?;
// 3. Combine Results
let mut combined_ids = quickwit_ids.clone();
let quickwit_set: HashSet<Uuid> = quickwit_ids.iter().cloned().collect();
for id in postgres_ids {
if !quickwit_set.contains(&id) {
combined_ids.push(id);
}
}
if combined_ids.is_empty() {
return Ok(Vec::new());
}
quickwit_order = Some(ids.clone());
let set: HashSet<Uuid> = ids.into_iter().collect();
quickwit_order = Some(combined_ids.clone());
let set: HashSet<Uuid> = combined_ids.into_iter().collect();
filter_ids = intersect_option_sets(filter_ids, set);
}
@@ -602,10 +606,10 @@ impl<'a> DocumentsService<'a> {
}
let docs: Vec<Document> = docs_query.load(conn)?;
let mut responses = self.hydrate_documents(conn, user_id, docs)?;
let mut responses = self.hydrate_documents(conn, tenant_id, user_id, docs)?;
if let Some(order) = quickwit_order {
let order_map: HashMap<Uuid, usize> = order
let order_map: HashMap<Uuid, usize> = order
.into_iter()
.enumerate()
.map(|(idx, id)| (id, idx))
@@ -619,6 +623,7 @@ impl<'a> DocumentsService<'a> {
pub fn hydrate_documents(
&self,
conn: &mut PgPooledConnection,
tenant_id: Uuid,
user_id: Uuid,
docs: Vec<Document>,
) -> AppResult<Vec<DocumentResponse>> {
@@ -628,14 +633,59 @@ impl<'a> DocumentsService<'a> {
let doc_ids: Vec<Uuid> = docs.iter().map(|doc| doc.id).collect();
let mut relations = load_tags_and_correspondents(conn, &doc_ids)?;
let primary_versions = load_primary_assets(conn, &docs)?;
let mut doc_to_version: HashMap<Uuid, Uuid> = HashMap::with_capacity(doc_ids.len());
let mut version_ids: Vec<Uuid> = Vec::with_capacity(doc_ids.len());
for doc in &docs {
doc_to_version.insert(doc.id, doc.current_version_id);
version_ids.push(doc.current_version_id);
}
version_ids.sort();
version_ids.dedup();
let versions: Vec<DocumentVersion> = document_versions::table
.filter(document_versions::id.eq_any(&version_ids))
.filter(document_versions::tenant_id.eq(tenant_id))
.load(conn)?;
let mut version_map: HashMap<Uuid, DocumentVersion> = HashMap::new();
for version in versions {
version_map.insert(version.id, version);
}
let assets: Vec<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq_any(&version_ids))
.filter(document_assets::tenant_id.eq(tenant_id))
.order((
document_assets::document_version_id.asc(),
document_assets::created_at.asc(),
))
.load(conn)?;
let mut assets_by_version: HashMap<Uuid, Vec<DocumentAssetResponse>> = HashMap::new();
for asset in assets {
let version_id = asset.document_version_id;
let response = self.asset_response(asset, tenant_id, user_id)?;
assets_by_version
.entry(version_id)
.or_default()
.push(response);
}
docs.into_iter()
.map(|doc| {
let (tags, correspondents) = relations
.remove(&doc.id)
.unwrap_or_else(|| (Vec::new(), Vec::new()));
let current_version = primary_versions.get(&doc.id).cloned();
let current_version = doc_to_version
.get(&doc.id)
.and_then(|version_id| version_map.remove(version_id))
.map(|version| -> AppResult<_> {
let assets = assets_by_version.remove(&version.id).unwrap_or_default();
let download = build_download_link(self.state, &doc, version.id, user_id)?;
Ok((to_version_response(version), assets, download))
})
.transpose()?;
self.to_document_response(user_id, doc, tags, correspondents, current_version)
})
.collect()
@@ -651,7 +701,7 @@ impl<'a> DocumentsService<'a> {
let DocumentUploadRequest {
bytes,
original_name,
content_type,
mime_type,
folder_id,
metadata,
title_override,
@@ -705,7 +755,7 @@ impl<'a> DocumentsService<'a> {
.put_object(
&s3_key,
bytes.clone(),
content_type.clone(),
mime_type.clone(),
content_disposition.clone(),
)
.await
@@ -722,7 +772,7 @@ impl<'a> DocumentsService<'a> {
id: doc_id,
filename: stored_filename.clone(),
original_name: original_name.clone(),
content_type: content_type.clone(),
mime_type: mime_type.clone(),
folder_id,
current_version_id: version_id,
metadata: metadata_value.clone(),
@@ -788,13 +838,15 @@ impl<'a> DocumentsService<'a> {
.cloned()
.unwrap_or_else(|| (Vec::new(), Vec::new()));
let download = build_download_link(self.state, &document, version.id, user_id)?;
DocumentDetailResponse {
document: self.to_document_response(
user_id,
document,
tags,
correspondents,
Some((to_version_response(version.clone()), Vec::new())),
Some((to_version_response(version.clone()), Vec::new(), download)),
)?,
}
};
@@ -893,21 +945,32 @@ impl<'a> DocumentsService<'a> {
&self,
conn: &mut PgPooledConnection,
tenant_id: Uuid,
user_id: Uuid,
document_id: Uuid,
) -> AppResult<Vec<DocumentAssetResponse>> {
let document = load_active_document(conn, tenant_id, document_id)?;
let version_id = document.current_version_id;
Ok(load_asset_responses_with_conn(conn, tenant_id, version_id)?)
let assets = document_assets::table
.filter(document_assets::document_version_id.eq(version_id))
.filter(document_assets::tenant_id.eq(tenant_id))
.order(document_assets::created_at.asc())
.load::<DocumentAsset>(conn)?;
let mut responses = Vec::with_capacity(assets.len());
for asset in assets {
responses.push(self.asset_response(asset, tenant_id, user_id)?);
}
Ok(responses)
}
pub async fn get_document_asset(
&self,
mut conn: PgPooledConnection,
tenant_id: Uuid,
user_id: Uuid,
asset_id: Uuid,
start: i32,
limit: i32,
) -> AppResult<DocumentAssetDetailResponse> {
let asset: DocumentAsset = match document_assets::table
.find(asset_id)
@@ -919,59 +982,71 @@ impl<'a> DocumentsService<'a> {
None => return Err(AppError::not_found()),
};
if start < 1 {
return Err(AppError::bad_request("start must be at least 1"));
}
if limit < 1 {
return Err(AppError::bad_request("limit must be at least 1"));
}
let end = start
.checked_add(limit - 1)
.ok_or_else(|| AppError::bad_request("requested range is too large"))?;
let objects: Vec<DocumentAssetObject> = document_asset_objects::table
.filter(document_asset_objects::asset_id.eq(asset_id))
.filter(document_asset_objects::tenant_id.eq(tenant_id))
.filter(document_asset_objects::ordinal.ge(start))
.filter(document_asset_objects::ordinal.le(end))
.order(document_asset_objects::ordinal.asc())
.load(&mut conn)?;
drop(conn);
let expires_at = Utc::now()
.timestamp_millis()
.checked_add((PRESIGNED_URL_EXPIRY_SECONDS as i64) * 1000)
.ok_or_else(|| AppError::internal("failed to compute expiry timestamp"))?;
let download = self.asset_download_link(asset.id, tenant_id, user_id)?;
let storage = self.state.storage_for_tenant(tenant_id)?;
Ok(to_asset_detail_response(asset, Some(download)))
}
let mut object_responses = Vec::with_capacity(objects.len());
for object in objects {
let response_disposition = presign_disposition_for_asset(&asset, &object);
pub async fn get_document_download_link(
&self,
conn: &mut PgPooledConnection,
tenant_id: Uuid,
user_id: Uuid,
document_id: Uuid,
) -> AppResult<DownloadLink> {
let document = load_active_document(conn, tenant_id, document_id)?;
let version: DocumentVersion = document_versions::table
.find(document.current_version_id)
.filter(document_versions::tenant_id.eq(tenant_id))
.first(conn)?;
let url = storage
.presign_get_object(
&object.s3_key,
Duration::from_secs(PRESIGNED_URL_EXPIRY_SECONDS),
response_disposition.as_deref(),
)
.await
.storage_context("failed to generate asset URL")?;
build_download_link(self.state, &document, version.id, user_id)
}
object_responses.push(to_asset_object_response(
object,
Some(url),
Some(expires_at),
));
}
pub async fn get_document_version_download_link(
&self,
conn: &mut PgPooledConnection,
tenant_id: Uuid,
user_id: Uuid,
document_id: Uuid,
version_id: Uuid,
) -> AppResult<DownloadLink> {
let document = load_active_document(conn, tenant_id, document_id)?;
if object_responses.is_empty() {
let version: Option<DocumentVersion> = document_versions::table
.find(version_id)
.filter(document_versions::document_id.eq(document_id))
.filter(document_versions::tenant_id.eq(tenant_id))
.first(conn)
.optional()?;
let Some(version) = version else {
return Err(AppError::not_found());
}
};
Ok(to_asset_detail_response(asset, object_responses))
build_download_link(self.state, &document, version.id, user_id)
}
pub async fn get_asset_download_link(
&self,
conn: &mut PgPooledConnection,
tenant_id: Uuid,
user_id: Uuid,
asset_id: Uuid,
) -> AppResult<DownloadLink> {
let asset: Option<DocumentAsset> = document_assets::table
.find(asset_id)
.filter(document_assets::tenant_id.eq(tenant_id))
.first(conn)
.optional()?;
let Some(asset) = asset else {
return Err(AppError::not_found());
};
self.asset_download_link(asset.id, tenant_id, user_id)
}
pub fn list_document_versions(
@@ -1007,14 +1082,14 @@ impl<'a> DocumentsService<'a> {
.filter(document_versions::tenant_id.eq(tenant_id))
.first(conn)?;
let assets = load_asset_responses_with_conn(conn, tenant_id, version.id)?;
let download_path = build_download_path(self.state, &document, user_id)?;
let assets = self.load_asset_responses(conn, tenant_id, version.id, user_id)?;
let download = build_download_link(self.state, &document, version.id, user_id)?;
let version_core = to_version_response(version);
Ok(DocumentVersionDetailResponse {
version: version_core,
assets,
download_path,
download,
})
}
@@ -1024,19 +1099,33 @@ impl<'a> DocumentsService<'a> {
tenant_id: Uuid,
document_id: Uuid,
) -> AppResult<()> {
let now = Utc::now().naive_utc();
diesel::update(
documents::table
conn.transaction::<_, AppError, _>(|conn| {
let document: Document = documents::table
.find(document_id)
.filter(documents::tenant_id.eq(tenant_id)),
)
.set((
documents::deleted_at.eq(Some(now)),
documents::updated_at.eq(now),
))
.execute(conn)?;
.filter(documents::tenant_id.eq(tenant_id))
.for_update()
.first(conn)
.optional()?
.ok_or_else(AppError::not_found)?;
Ok(())
if document.deleted_at.is_some() {
return Err(AppError::conflict("document already trashed"));
}
let now = Utc::now().naive_utc();
diesel::update(
documents::table
.find(document_id)
.filter(documents::tenant_id.eq(tenant_id)),
)
.set((
documents::deleted_at.eq(Some(now)),
documents::updated_at.eq(now),
))
.execute(conn)?;
Ok(())
})
}
pub fn delete_document(
@@ -1091,10 +1180,8 @@ impl<'a> DocumentsService<'a> {
let payload_obj = payload
.as_object()
.ok_or_else(|| AppError::bad_request("request body must be a JSON object"))?;
let DocumentUpdatePlan {
mut changeset,
title_changed,
} = Self::build_document_update_plan(&document, payload_obj)?;
let DocumentUpdatePlan { mut changeset } =
Self::build_document_update_plan(&document, payload_obj)?;
let now = Utc::now().naive_utc();
changeset.updated_at = Some(now);
@@ -1125,30 +1212,11 @@ impl<'a> DocumentsService<'a> {
.find(document.current_version_id)
.first(conn)?;
if title_changed {
if let Err(err) = enqueue_job(
conn,
tenant_id,
JOB_INDEX_DOCUMENT_TEXT,
json!({
"document_id": document.id,
"document_version_id": current_version.id,
}),
None,
) {
warn!(
document_id = %document.id,
version_id = %current_version.id,
error = %err,
"failed to enqueue reindex job after title change"
);
}
}
let tags_and_correspondents = load_tags_and_correspondents(conn, &[document_id])?;
let version_id = current_version.id;
let assets = load_asset_responses_with_conn(conn, tenant_id, version_id)?;
let assets = self.load_asset_responses(conn, tenant_id, version_id, user_id)?;
let version_response = to_version_response(current_version);
let download = build_download_link(self.state, &document, version_id, user_id)?;
let (tags, correspondents) = tags_and_correspondents
.get(&document_id)
.cloned()
@@ -1159,7 +1227,7 @@ impl<'a> DocumentsService<'a> {
document,
tags,
correspondents,
Some((version_response, assets)),
Some((version_response, assets, download)),
)?;
Ok(DocumentDetailResponse {
@@ -1319,17 +1387,21 @@ impl<'a> DocumentsService<'a> {
fn to_document_response(
&self,
user_id: Uuid,
_user_id: Uuid,
doc: Document,
tags: Vec<Tag>,
correspondents: Vec<DocumentCorrespondentResponse>,
current_version: Option<(DocumentVersionResponse, Vec<DocumentAssetResponse>)>,
current_version: Option<(
DocumentVersionResponse,
Vec<DocumentAssetResponse>,
DownloadLink,
)>,
) -> AppResult<DocumentResponse> {
let current_version = match current_version {
Some((version, assets)) => Some(DocumentVersionDetailResponse {
Some((version, assets, download)) => Some(DocumentVersionDetailResponse {
version,
assets,
download_path: build_download_path(self.state, &doc, user_id)?,
download,
}),
None => None,
};
@@ -1339,7 +1411,7 @@ impl<'a> DocumentsService<'a> {
filename: doc.filename,
title: doc.title,
original_name: doc.original_name,
content_type: doc.content_type,
mime_type: doc.mime_type,
folder_id: doc.folder_id,
created_at: to_iso(doc.created_at),
updated_at: to_iso(doc.updated_at),
@@ -1384,13 +1456,26 @@ impl<'a> DocumentsService<'a> {
checksum = %checksum_hex,
"upload rejected because document already exists",
);
return Err(
AppError::conflict("a document with the same contents already exists")
.with_code("duplicate_document")
.with_details(json!({
"conflict_document_id": document.id,
})),
);
let mut message = String::from("a document with the same contents already exists");
let mut details = json!({
"conflict_document_id": document.id,
});
if let Some(deleted_at) = document.deleted_at {
message.push_str(". Note: the existing document is currently in the trash.");
if let Some(obj) = details.as_object_mut() {
obj.insert("conflict_document_in_trash".to_string(), Value::Bool(true));
obj.insert(
"conflict_document_deleted_at".to_string(),
Value::String(to_iso(deleted_at)),
);
}
}
return Err(AppError::conflict(message)
.with_code("duplicate_document")
.with_details(details));
}
if let Some(issued_at) = issued_at_override {
@@ -1444,7 +1529,8 @@ impl<'a> DocumentsService<'a> {
.cloned()
.unwrap_or_else(|| (Vec::new(), Vec::new()));
let assets = load_asset_responses_with_conn(conn, tenant_id, version.id)?;
let assets = self.load_asset_responses(conn, tenant_id, version.id, user_id)?;
let download = build_download_link(self.state, &document, version.id, user_id)?;
let version_response = to_version_response(version.clone());
info!(
@@ -1459,18 +1545,74 @@ impl<'a> DocumentsService<'a> {
document,
tags,
correspondents_list,
Some((version_response, assets)),
Some((version_response, assets, download)),
)?,
};
Ok(Some(detail))
}
}
fn presign_disposition_for_asset(
asset: &DocumentAsset,
object: &DocumentAssetObject,
) -> Option<String> {
let filename = format!("{}-{}", asset.asset_type, object.ordinal);
inline_content_disposition(&filename)
fn asset_download_link(
&self,
asset_id: Uuid,
tenant_id: Uuid,
user_id: Uuid,
) -> AppResult<DownloadLink> {
let token = self
.state
.jwt
.generate_asset_download_token(asset_id, user_id, tenant_id)
.map_err(|err| {
error!(error = ?err, "failed to issue asset download token");
AppError::internal("failed to issue asset download token")
})?;
let expires_at = Utc::now()
.checked_add_signed(ChronoDuration::minutes(
self.state.config.download_token_expiry_minutes,
))
.ok_or_else(|| AppError::internal("failed to compute download expiry"))?
.timestamp_millis();
Ok(DownloadLink {
url: format!("/api/download/{token}"),
expires_at,
})
}
fn asset_response(
&self,
asset: DocumentAsset,
tenant_id: Uuid,
user_id: Uuid,
) -> AppResult<DocumentAssetResponse> {
let download = self.asset_download_link(asset.id, tenant_id, user_id)?;
Ok(DocumentAssetResponse {
id: asset.id,
asset_type: asset.asset_type,
mime_type: asset.mime_type,
metadata: asset.metadata,
download: Some(download),
})
}
fn load_asset_responses(
&self,
conn: &mut PgPooledConnection,
tenant_id: Uuid,
version_id: Uuid,
user_id: Uuid,
) -> AppResult<Vec<DocumentAssetResponse>> {
let assets: Vec<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq(version_id))
.filter(document_assets::tenant_id.eq(tenant_id))
.order(document_assets::created_at.asc())
.load(conn)?;
assets
.into_iter()
.map(|asset| self.asset_response(asset, tenant_id, user_id))
.collect()
}
}
+2 -1
View File
@@ -555,10 +555,11 @@ impl<'a> FolderService<'a> {
pub fn hydrate_documents(
&self,
conn: &mut PgPooledConnection,
tenant_id: Uuid,
user_id: Uuid,
docs: Vec<Document>,
) -> AppResult<Vec<DocumentResponse>> {
DocumentsService::new(self.state).hydrate_documents(conn, user_id, docs)
DocumentsService::new(self.state).hydrate_documents(conn, tenant_id, user_id, docs)
}
}
+1
View File
@@ -6,3 +6,4 @@ pub mod folders;
pub mod helpers;
pub mod profile;
pub mod tags;
pub mod tenants;
+260
View File
@@ -0,0 +1,260 @@
use chrono::Utc;
use diesel::prelude::*;
use serde::{Deserialize, Serialize};
use utoipa::ToSchema;
use uuid::Uuid;
use crate::auth::AuthenticatedUser;
use crate::error::{AppError, AppResult};
use crate::http::responders::{ok_json, JsonResponse};
use crate::models::ApiCapability;
use crate::schema::{
capability_sets::dsl as cs_dsl, user_memberships::dsl as memberships_dsl,
user_sessions::dsl as session_dsl, users::dsl as users_dsl,
};
use crate::state::{AppState, PgPooledConnection};
#[derive(Deserialize, ToSchema)]
pub struct UpdateTenantRequest {
#[schema(example = "Acme Inc.")]
pub name: String,
}
#[derive(Deserialize, ToSchema)]
pub struct UpdateTenantUserRequest {
#[schema(example = "a2f1bc73-4c90-4bb9-9da9-1c5d04be12ac")]
pub capability_set_id: Uuid,
}
#[derive(Serialize, ToSchema)]
#[schema(example = json!({
"user_id": "11111111-2222-3333-4444-555555555555",
"username": "cfo",
"capability_set_id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
"capability_set_slug": "owner"
}))]
pub struct TenantUserSummary {
pub user_id: Uuid,
pub username: String,
pub capability_set_id: Option<Uuid>,
pub capability_set_slug: Option<String>,
}
#[derive(Serialize, ToSchema)]
#[schema(example = json!({
"users": [
{
"user_id": "11111111-2222-3333-4444-555555555555",
"username": "alice",
"capability_set_id": "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee",
"capability_set_slug": "owner"
},
{
"user_id": "66666666-7777-8888-9999-000000000000",
"username": "bob",
"capability_set_id": "bbbbbbbb-cccc-dddd-eeee-ffffffffffff",
"capability_set_slug": "user"
}
]
}))]
pub struct TenantUserListResponse {
pub users: Vec<TenantUserSummary>,
}
pub struct TenantApiService<'a> {
state: &'a AppState,
}
impl<'a> TenantApiService<'a> {
pub fn new(state: &'a AppState) -> Self {
Self { state }
}
pub fn update_name(
&self,
user: AuthenticatedUser,
tenant_id: Uuid,
payload: UpdateTenantRequest,
) -> AppResult<JsonResponse<crate::services::auth::TenantSnippet>> {
self.ensure_can_manage(&user, tenant_id)?;
let tenant = self.state.tenants.update_name(tenant_id, &payload.name)?;
ok_json(crate::services::auth::TenantSnippet {
id: tenant.id,
name: tenant.name,
})
}
pub fn list_users(
&self,
user: &AuthenticatedUser,
tenant_id: Uuid,
) -> AppResult<JsonResponse<TenantUserListResponse>> {
self.ensure_can_manage(user, tenant_id)?;
let mut conn = self.state.db_for_tenant(tenant_id)?;
let rows: Vec<(Uuid, String, Option<Uuid>, Option<String>)> =
memberships_dsl::user_memberships
.inner_join(users_dsl::users.on(users_dsl::id.eq(memberships_dsl::user_id)))
.left_join(
cs_dsl::capability_sets
.on(cs_dsl::id.nullable().eq(memberships_dsl::capability_set_id)),
)
.select((
users_dsl::id,
users_dsl::username,
memberships_dsl::capability_set_id,
cs_dsl::slug.nullable(),
))
.order(users_dsl::username.asc())
.load(&mut conn)?;
let users = rows
.into_iter()
.map(
|(user_id, username, capability_set_id, capability_set_slug)| TenantUserSummary {
user_id,
username,
capability_set_id,
capability_set_slug,
},
)
.collect();
ok_json(TenantUserListResponse { users })
}
pub fn get_user(
&self,
user: &AuthenticatedUser,
tenant_id: Uuid,
target_user_id: Uuid,
) -> AppResult<JsonResponse<TenantUserSummary>> {
self.ensure_can_manage(user, tenant_id)?;
let mut conn = self.state.db_for_tenant(tenant_id)?;
let summary = self.load_membership_summary(&mut conn, tenant_id, target_user_id)?;
ok_json(summary)
}
pub fn update_user(
&self,
user: &AuthenticatedUser,
tenant_id: Uuid,
target_user_id: Uuid,
payload: UpdateTenantUserRequest,
) -> AppResult<JsonResponse<TenantUserSummary>> {
self.ensure_can_manage(user, tenant_id)?;
let mut conn = self.state.db_for_tenant(tenant_id)?;
let capability_set_id = self.resolve_capability_set_id(&mut conn, tenant_id, &payload)?;
let updated = diesel::update(
memberships_dsl::user_memberships
.filter(memberships_dsl::tenant_id.eq(tenant_id))
.filter(memberships_dsl::user_id.eq(target_user_id)),
)
.set((
memberships_dsl::capability_set_id.eq(Some(capability_set_id)),
memberships_dsl::updated_at.eq(Utc::now().naive_utc()),
))
.execute(&mut conn)?;
if updated == 0 {
return Err(AppError::not_found());
}
let summary = self.load_membership_summary(&mut conn, tenant_id, target_user_id)?;
ok_json(summary)
}
pub fn remove_user(
&self,
user: &AuthenticatedUser,
tenant_id: Uuid,
target_user_id: Uuid,
) -> AppResult<()> {
self.ensure_can_manage(user, tenant_id)?;
let mut conn = self.state.db_for_tenant(tenant_id)?;
let removed = diesel::delete(
memberships_dsl::user_memberships
.filter(memberships_dsl::tenant_id.eq(tenant_id))
.filter(memberships_dsl::user_id.eq(target_user_id)),
)
.execute(&mut conn)?;
if removed == 0 {
return Err(AppError::not_found());
}
diesel::delete(
session_dsl::user_sessions
.filter(session_dsl::tenant_id.eq(tenant_id))
.filter(session_dsl::user_id.eq(target_user_id)),
)
.execute(&mut conn)?;
Ok(())
}
fn ensure_can_manage(&self, user: &AuthenticatedUser, tenant_id: Uuid) -> AppResult<()> {
if user.tenant_id != tenant_id {
return Err(AppError::forbidden("cannot manage another tenant"));
}
if !user.capabilities.contains(&ApiCapability::TenantsWrite) {
return Err(AppError::forbidden("missing tenants:write capability"));
}
Ok(())
}
fn resolve_capability_set_id(
&self,
conn: &mut PgPooledConnection,
tenant_id: Uuid,
payload: &UpdateTenantUserRequest,
) -> AppResult<Uuid> {
let exists = cs_dsl::capability_sets
.filter(cs_dsl::tenant_id.eq(tenant_id))
.filter(cs_dsl::id.eq(payload.capability_set_id))
.select(cs_dsl::id)
.first::<Uuid>(conn)
.optional()?;
exists.ok_or_else(AppError::not_found)
}
fn load_membership_summary(
&self,
conn: &mut PgPooledConnection,
tenant_id: Uuid,
user_id: Uuid,
) -> AppResult<TenantUserSummary> {
let row = memberships_dsl::user_memberships
.filter(memberships_dsl::tenant_id.eq(tenant_id))
.filter(memberships_dsl::user_id.eq(user_id))
.inner_join(users_dsl::users.on(users_dsl::id.eq(memberships_dsl::user_id)))
.left_join(
cs_dsl::capability_sets
.on(cs_dsl::id.nullable().eq(memberships_dsl::capability_set_id)),
)
.select((
users_dsl::id,
users_dsl::username,
memberships_dsl::capability_set_id,
cs_dsl::slug.nullable(),
))
.first::<(Uuid, String, Option<Uuid>, Option<String>)>(conn)
.optional()?;
match row {
Some((user_id, username, capability_set_id, capability_set_slug)) => {
Ok(TenantUserSummary {
user_id,
username,
capability_set_id,
capability_set_slug,
})
}
None => Err(AppError::not_found()),
}
}
}
+10 -5
View File
@@ -11,6 +11,7 @@ use crate::{
config::AppConfig,
db::PgPool,
error::{AppError, AppResult},
issued_at::IssuedAtSettings,
storage::{ObjectStorage, TenantStorage},
tenants::{apply_tenant_guc, clear_tenant_context, clear_user_guc, TenantService},
};
@@ -25,6 +26,7 @@ pub struct AppState {
pub jwt: JwtService,
pub tenants: TenantService,
pub passkeys: Option<PasskeyService>,
issued_at: Arc<IssuedAtSettings>,
}
impl AppState {
@@ -34,11 +36,8 @@ impl AppState {
) -> anyhow::Result<Self> {
let pool_size = pool_size_override.unwrap_or(config.database_max_pool_size);
let pool = crate::db::init_pool_with_size(&config.database_url, pool_size)?;
let s3_client = crate::s3::build_client(&config).await?;
let storage = Arc::new(crate::storage::S3Storage::new(
s3_client,
config.s3_bucket.clone(),
));
let bucket = crate::s3::build_bucket(&config)?;
let storage = Arc::new(crate::storage::S3Storage::new(bucket));
let jwt = crate::auth::jwt::JwtService::from_config(&config)?;
Ok(Self::new(pool, config, storage, jwt))
@@ -50,6 +49,7 @@ impl AppState {
storage: Arc<dyn ObjectStorage>,
jwt: JwtService,
) -> Self {
let issued_at = Arc::new(IssuedAtSettings::from_config(&config));
let config = Arc::new(config);
let tenants = TenantService::new(pool.clone());
@@ -68,6 +68,7 @@ impl AppState {
jwt,
tenants,
passkeys,
issued_at,
}
}
@@ -99,4 +100,8 @@ impl AppState {
AppError::internal("tenant storage error")
})
}
pub fn issued_at_settings(&self) -> Arc<IssuedAtSettings> {
self.issued_at.clone()
}
}
+59 -56
View File
@@ -1,12 +1,10 @@
use std::time::Duration;
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use anyhow::{anyhow, Context, Result};
use async_trait::async_trait;
use aws_sdk_s3::presigning::PresigningConfig;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::Client as S3Client;
use s3::bucket::Bucket;
use crate::models::Tenant;
@@ -29,20 +27,22 @@ pub trait ObjectStorage: Send + Sync + 'static {
async fn get_object(&self, key: &str) -> Result<Vec<u8>>;
async fn get_object_range(&self, key: &str, start: u64, end: Option<u64>) -> Result<Vec<u8>>;
async fn delete_object(&self, key: &str) -> Result<()>;
}
pub struct S3Storage {
client: S3Client,
bucket: String,
bucket: Bucket,
}
impl S3Storage {
pub fn new(client: S3Client, bucket: impl Into<String>) -> Self {
Self {
client,
bucket: bucket.into(),
}
pub fn new(bucket: Bucket) -> Self {
Self { bucket }
}
fn default_content_type(content_type: Option<String>) -> String {
content_type.unwrap_or_else(|| "application/octet-stream".to_string())
}
}
@@ -55,23 +55,19 @@ impl ObjectStorage for S3Storage {
content_type: Option<String>,
content_disposition: Option<String>,
) -> Result<()> {
let mut request = self
.client
.put_object()
.bucket(&self.bucket)
.key(key)
.body(ByteStream::from(bytes));
let mut builder = self
.bucket
.put_object_builder(key, &bytes)
.with_content_type(Self::default_content_type(content_type));
if let Some(content_type) = content_type {
request = request.content_type(content_type);
if let Some(disposition) = content_disposition {
builder = builder
.with_content_disposition(disposition)
.context("invalid content disposition header")?;
}
if let Some(content_disposition) = content_disposition {
request = request.content_disposition(content_disposition);
}
request
.send()
builder
.execute()
.await
.context("failed to upload object to S3")?;
@@ -84,51 +80,44 @@ impl ObjectStorage for S3Storage {
expires_in: Duration,
response_content_disposition: Option<&str>,
) -> Result<String> {
let presign_config = PresigningConfig::builder()
.expires_in(expires_in)
.build()
.context("failed to build S3 presigning config")?;
let expiry_secs =
u32::try_from(expires_in.as_secs()).context("presign expiry exceeds u32 range")?;
let mut request = self.client.get_object().bucket(&self.bucket).key(key);
let mut queries = HashMap::new();
if let Some(value) = response_content_disposition {
request = request.response_content_disposition(value);
queries.insert(
"response-content-disposition".to_string(),
value.to_string(),
);
}
let presigned = request
.presigned(presign_config)
self.bucket
.presign_get(key, expiry_secs, (!queries.is_empty()).then_some(queries))
.await
.context("failed to generate presigned download URL")?;
Ok(presigned.uri().to_string())
.context("failed to generate presigned download URL")
}
async fn get_object(&self, key: &str) -> Result<Vec<u8>> {
let response = self
.client
.get_object()
.bucket(&self.bucket)
.key(key)
.send()
let data = self
.bucket
.get_object(key)
.await
.context("failed to download object from S3")?;
Ok(data.into_bytes().to_vec())
}
let bytes = response
.body
.collect()
async fn get_object_range(&self, key: &str, start: u64, end: Option<u64>) -> Result<Vec<u8>> {
let data = self
.bucket
.get_object_range(key, start, end)
.await
.context("failed to read object stream")?
.into_bytes()
.to_vec();
Ok(bytes)
.context("failed to download ranged object from S3")?;
Ok(data.into_bytes().to_vec())
}
async fn delete_object(&self, key: &str) -> Result<()> {
self.client
.delete_object()
.bucket(&self.bucket)
.key(key)
.send()
self.bucket
.delete_object(key)
.await
.context("failed to delete object from S3")?;
Ok(())
@@ -155,6 +144,10 @@ impl TenantStorage {
format!("{}{}", self.root, key)
}
pub fn root_prefix(&self) -> &str {
&self.root
}
pub async fn put_object(
&self,
key: &str,
@@ -185,6 +178,16 @@ impl TenantStorage {
self.inner.get_object(&qualified).await
}
pub async fn get_object_range(
&self,
key: &str,
start: u64,
end: Option<u64>,
) -> Result<Vec<u8>> {
let qualified = self.qualify(key);
self.inner.get_object_range(&qualified, start, end).await
}
pub async fn delete_object(&self, key: &str) -> Result<()> {
let qualified = self.qualify(key);
self.inner.delete_object(&qualified).await
+17
View File
@@ -24,6 +24,13 @@ impl TenantRepository {
.first(conn)
.map_err(Into::into)
}
pub fn update_name(conn: &mut PgConnection, tenant_id: Uuid, name: &str) -> AppResult<Tenant> {
diesel::update(dsl::tenants.find(tenant_id))
.set(dsl::name.eq(name))
.execute(conn)?;
Self::get_by_id(conn, tenant_id)
}
}
#[derive(Clone)]
@@ -128,6 +135,16 @@ impl TenantService {
TenantRepository::get_by_id(conn, id)
}
pub fn update_name(&self, tenant_id: Uuid, name: &str) -> AppResult<Tenant> {
let mut conn = self.pool.get().map_err(|err| {
tracing::error!(error = ?err, "database pool error");
AppError::internal("database pool error")
})?;
let normalized = normalize_tenant_name(name)?;
TenantRepository::update_name(&mut conn, tenant_id, &normalized)
}
}
pub fn apply_tenant_guc(conn: &mut PgConnection, tenant_id: Uuid) -> AppResult<()> {
+935
View File
@@ -0,0 +1,935 @@
use std::collections::HashMap;
use std::env;
use std::sync::Arc;
use std::time::Duration;
use crate::auth::capability_sets::{
ensure_capability_set, owner_capabilities, readonly_capabilities, user_capabilities,
webdav_capabilities,
};
use crate::auth::jwt::{AccessTokenContext, JwtService, PrincipalKind};
use crate::config::AppConfig;
use crate::db::{self, PgPool};
use crate::migrations::MIGRATIONS;
use crate::models::{
Job, NewUser, NewUserMembership, NewUserPasskey, NewUserSession, Tenant, TenantStatus, User,
UserMembership,
};
use crate::routes;
use crate::schema::user_sessions::dsl as session_dsl;
use crate::state::AppState;
use crate::storage::ObjectStorage;
use anyhow::{anyhow, ensure, Context, Result};
use async_trait::async_trait;
use axum::body::Body;
use axum::http::{header, Method, Request};
use axum::Router;
use chrono::{Duration as ChronoDuration, Utc};
use diesel::connection::SimpleConnection;
use diesel::prelude::*;
use diesel::OptionalExtension;
use diesel::PgConnection;
use diesel_migrations::MigrationHarness;
use http_body_util::BodyExt;
use once_cell::sync::Lazy;
use rand::{rngs::OsRng, TryRngCore};
use serde::Serialize;
use serde_json::{self, json};
use sha2::{Digest, Sha256};
use tokio::sync::Mutex;
use tower::util::ServiceExt;
use uuid::Uuid;
const RESET_DATABASE_SQL: &str = "DROP SCHEMA IF EXISTS tenant CASCADE;\n\
DROP SCHEMA IF EXISTS shared CASCADE;\n\
DROP SCHEMA IF EXISTS public CASCADE;\n\
CREATE SCHEMA public;\n\
GRANT ALL ON SCHEMA public TO public;";
static DB_LOCK: Lazy<Mutex<()>> = Lazy::new(|| Mutex::new(()));
const TEST_TENANT_NAME: &str = "test_tenant";
#[derive(Clone, Copy, Debug)]
pub enum TestUserRole {
Owner,
Member,
WebDav,
}
#[derive(Clone)]
pub struct StoredObject {
pub key: String,
pub bytes: Vec<u8>,
pub content_type: Option<String>,
pub content_disposition: Option<String>,
}
#[derive(Default)]
pub struct FakeStorage {
objects: Mutex<HashMap<String, StoredObject>>,
}
#[async_trait]
impl ObjectStorage for FakeStorage {
async fn put_object(
&self,
key: &str,
bytes: Vec<u8>,
content_type: Option<String>,
content_disposition: Option<String>,
) -> Result<()> {
let stored = StoredObject {
key: key.to_string(),
bytes,
content_type,
content_disposition,
};
let mut guard = self.objects.lock().await;
guard.insert(stored.key.clone(), stored);
Ok(())
}
async fn presign_get_object(
&self,
key: &str,
expires_in: Duration,
_response_content_disposition: Option<&str>,
) -> Result<String> {
let guard = self.objects.lock().await;
ensure!(guard.contains_key(key), "object {key} missing");
Ok(format!(
"https://fake-storage/{key}?expires_in={}",
expires_in.as_secs()
))
}
async fn get_object(&self, key: &str) -> Result<Vec<u8>> {
let guard = self.objects.lock().await;
guard
.get(key)
.map(|obj| obj.bytes.clone())
.ok_or_else(|| anyhow!("object {key} missing"))
}
async fn get_object_range(
&self,
key: &str,
start: u64,
end: Option<u64>,
) -> Result<Vec<u8>> {
let guard = self.objects.lock().await;
let bytes = guard
.get(key)
.map(|obj| obj.bytes.clone())
.ok_or_else(|| anyhow!("object {key} missing"))?;
let start_idx = start as usize;
let end_idx = end.map(|idx| idx.saturating_add(1) as usize).unwrap_or(bytes.len());
if start_idx >= bytes.len() {
return Ok(Vec::new());
}
Ok(bytes[start_idx..end_idx.min(bytes.len())].to_vec())
}
async fn delete_object(&self, key: &str) -> Result<()> {
let mut guard = self.objects.lock().await;
guard.remove(key);
Ok(())
}
}
impl FakeStorage {
pub async fn get(&self, key: &str) -> Option<StoredObject> {
let guard = self.objects.lock().await;
guard.get(key).cloned()
}
pub async fn object_count(&self) -> usize {
let guard = self.objects.lock().await;
guard.len()
}
pub async fn object_count_with_prefix(&self, prefix: &str) -> usize {
let guard = self.objects.lock().await;
guard.keys().filter(|key| key.starts_with(prefix)).count()
}
pub async fn contains_key(&self, key: &str) -> bool {
let guard = self.objects.lock().await;
guard.contains_key(key)
}
pub async fn keys_with_prefix(&self, prefix: &str) -> Vec<String> {
let guard = self.objects.lock().await;
guard
.keys()
.filter(|key| key.starts_with(prefix))
.cloned()
.collect()
}
}
pub struct TestApp {
pub state: AppState,
router: Router,
storage: Arc<FakeStorage>,
}
impl TestApp {
pub async fn new() -> Result<Self> {
Self::with_config(|_| {}).await
}
pub async fn with_config<F>(configure: F) -> Result<Self>
where
F: FnOnce(&mut AppConfig),
{
let database_url = env::var("TEST_DATABASE_URL")
.context("TEST_DATABASE_URL must be set for integration tests")?;
let mut config = AppConfig {
database_url: database_url.clone(),
migrations_database_url: None,
database_max_pool_size: db::DEFAULT_MAX_POOL_SIZE,
server_host: "127.0.0.1".to_string(),
server_port: 0,
webdav_host: "127.0.0.1".to_string(),
webdav_port: 0,
jwt_secret: "test-secret".to_string(),
jwt_issuer: "test-issuer".to_string(),
jwt_audience: "test-audience".to_string(),
jwt_expiry_minutes: 60,
download_token_audience: "test-download".to_string(),
download_token_expiry_minutes: 60,
refresh_token_expiry_days: 30,
refresh_cookie_secure: false,
refresh_cookie_domain: None,
cors_allowed_origin: None,
proxy_downloads: false,
aws_endpoint_url: None,
aws_access_key_id: None,
aws_secret_access_key: None,
aws_region: "us-east-1".to_string(),
s3_bucket: "test-bucket".to_string(),
quickwit_endpoint: None,
quickwit_index: None,
worker_max_document_bytes: 200 * 1024 * 1024,
upload_body_limit_bytes: 128 * 1024 * 1024,
service_timezone: "UTC".to_string(),
issued_at_date_order: "DMY".to_string(),
issued_at_filename_date_order: None,
issued_at_date_parser_locales: Vec::new(),
issued_at_ignore_dates: Vec::new(),
webauthn_rp_id: Some("localhost".to_string()),
webauthn_origin: Some("http://localhost".to_string()),
webauthn_rp_name: "Papercrate".to_string(),
};
configure(&mut config);
let pool = db::init_pool_with_size(&config.database_url, config.database_max_pool_size)?;
prepare_database(&pool).await?;
let storage = Arc::new(FakeStorage::default());
let storage_for_state: Arc<dyn ObjectStorage> = storage.clone();
let jwt = JwtService::from_config(&config)?;
let state = AppState::new(pool.clone(), config, storage_for_state, jwt);
let router = routes::create_router(state.clone());
let app = Self {
state,
router,
storage,
};
app.ensure_default_tenant().await?;
Ok(app)
}
pub async fn cleanup(&self) -> Result<()> {
let pool = self.state.pool.clone();
let _ = tokio::task::spawn_blocking(move || -> Result<()> {
let mut conn = pool
.get()
.map_err(|err| anyhow!("failed to get cleanup connection: {err}"))?;
truncate_all(&mut conn)?;
Ok(())
})
.await
.context("cleanup task panicked")?;
self.ensure_default_tenant().await?;
Ok(())
}
pub async fn tenant_id(&self) -> Result<Uuid> {
self.ensure_default_tenant().await
}
pub fn storage(&self) -> Arc<FakeStorage> {
self.storage.clone()
}
pub async fn storage_key_for(&self, key: &str) -> Result<String> {
self.ensure_default_tenant().await?;
let tenant = self
.state
.tenants
.get_by_name(TEST_TENANT_NAME)
.map_err(|err| anyhow!("default tenant not found: {:?}", err))?;
let root = tenant
.storage_root
.clone()
.ok_or_else(|| anyhow!("default tenant missing storage root"))?;
Ok(format!("{}{}", root, key))
}
pub async fn insert_user(&self, username: &str, role: TestUserRole) -> Result<Uuid> {
let username = username.to_string();
let tenant_id = self.ensure_default_tenant().await?;
let user_id = self
.with_conn(move |conn| {
let user = NewUser {
id: Uuid::new_v4(),
username,
};
diesel::insert_into(crate::schema::users::table)
.values(&user)
.execute(conn)
.context("failed to insert user")?;
let capabilities = match role {
TestUserRole::Owner => owner_capabilities(),
TestUserRole::Member => user_capabilities(),
TestUserRole::WebDav => webdav_capabilities(),
};
let capability_set = ensure_capability_set(conn, tenant_id, capabilities)
.map_err(|err| anyhow!("failed to ensure capability set: {:?}", err))?;
let membership = NewUserMembership {
id: Uuid::new_v4(),
user_id: user.id,
tenant_id,
capability_set_id: Some(capability_set.id),
};
diesel::insert_into(crate::schema::user_memberships::table)
.values(&membership)
.execute(conn)
.context("failed to insert user membership")?;
Ok(user.id)
})
.await?;
Ok(user_id)
}
pub async fn insert_passkey(&self, user_id: Uuid, nickname: Option<&str>) -> Result<Uuid> {
let passkey_id = Uuid::new_v4();
let nickname = nickname.map(|value| value.to_string());
self.with_conn(move |conn| {
let credential_id = passkey_id.as_bytes().to_vec();
let public_key = passkey_id.as_bytes().iter().copied().collect::<Vec<u8>>();
let passkey = NewUserPasskey {
id: passkey_id,
user_id,
credential_id,
public_key,
credential: json!({ "dummy": passkey_id.to_string() }),
sign_count: 0,
transports: vec![Some("usb".to_string())],
aaguid: None,
nickname,
};
diesel::insert_into(crate::schema::user_passkeys::table)
.values(&passkey)
.execute(conn)
.context("failed to insert passkey")?;
Ok(passkey_id)
})
.await
}
async fn ensure_default_tenant(&self) -> Result<Uuid> {
let name_value = TEST_TENANT_NAME.to_string();
let quickwit_enabled = self.state.config.quickwit_endpoint.is_some();
let tenant_id = self
.with_conn(move |conn| {
use crate::schema::tenants::dsl as tenants_dsl;
let existing = tenants_dsl::tenants
.filter(tenants_dsl::name.eq(&name_value))
.first::<Tenant>(conn)
.optional()
.context("failed to load default tenant")?;
let tenant_id = if let Some(current) = existing {
let desired_root = current
.storage_root
.clone()
.filter(|root| root.ends_with('/'))
.unwrap_or_else(|| format!("test-tenants/{}/", current.id));
if current.storage_root.as_deref() != Some(desired_root.as_str()) {
diesel::update(tenants_dsl::tenants.filter(tenants_dsl::id.eq(current.id)))
.set(tenants_dsl::storage_root.eq(Some(desired_root)))
.execute(conn)
.context("failed to update default tenant storage root")?;
}
current.id
} else {
let new_id = Uuid::new_v4();
let root = format!("test-tenants/{}/", new_id);
let quickwit_value = if quickwit_enabled {
Some(format!("documents-{}", new_id))
} else {
None
};
diesel::insert_into(tenants_dsl::tenants)
.values((
tenants_dsl::id.eq(new_id),
tenants_dsl::name.eq(&name_value),
tenants_dsl::storage_root.eq(Some(root)),
tenants_dsl::quickwit_index.eq(quickwit_value),
tenants_dsl::status.eq(TenantStatus::Active),
))
.execute(conn)
.context("failed to insert default tenant")?;
new_id
};
Ok(tenant_id)
})
.await?;
let mut conn = self
.state
.db_for_tenant(tenant_id)
.map_err(|err| anyhow!("failed to scope tenant connection: {err:?}"))?;
ensure_capability_set(&mut conn, tenant_id, owner_capabilities())
.map_err(|err| anyhow!("ensure owner capability set: {err:?}"))?;
ensure_capability_set(&mut conn, tenant_id, user_capabilities())
.map_err(|err| anyhow!("ensure user capability set: {err:?}"))?;
ensure_capability_set(&mut conn, tenant_id, readonly_capabilities())
.map_err(|err| anyhow!("ensure readonly capability set: {err:?}"))?;
ensure_capability_set(&mut conn, tenant_id, webdav_capabilities())
.map_err(|err| anyhow!("ensure webdav capability set: {err:?}"))?;
Ok(tenant_id)
}
pub async fn login_token(&self, username: &str, _password: &str) -> Result<String> {
let (access_token, _, _) = self.create_session(username).await?;
Ok(access_token)
}
pub async fn create_session(&self, username: &str) -> Result<(String, String, Uuid)> {
let username = username.to_string();
let state = self.state.clone();
self.with_conn(move |conn| {
use crate::schema::capability_sets::dsl as capability_sets_dsl;
use crate::schema::tenants::dsl as tenants_dsl;
use crate::schema::user_memberships::dsl as memberships_dsl;
use crate::schema::users::dsl as users_dsl;
let user: User = users_dsl::users
.filter(users_dsl::username.eq(&username))
.first(conn)?;
let membership: UserMembership = memberships_dsl::user_memberships
.filter(memberships_dsl::user_id.eq(user.id))
.first(conn)?;
let tenant: Tenant = tenants_dsl::tenants
.find(membership.tenant_id)
.first(conn)?;
let capability_set_id = membership
.capability_set_id
.ok_or_else(|| anyhow!("membership missing capability set"))?;
let cap_version = capability_sets_dsl::capability_sets
.find(capability_set_id)
.select(capability_sets_dsl::cap_version)
.first::<i32>(conn)?;
let now = Utc::now();
let session_id = Uuid::new_v4();
let access_token = state
.jwt
.generate_token(AccessTokenContext {
user_id: user.id,
tenant_id: tenant.id,
username: user.username.clone(),
principal_kind: PrincipalKind::UserSession,
principal_id: session_id,
capability_set_id,
cap_version,
})
.map_err(|err| anyhow!(err))?;
let session_value = generate_session_token();
let session_hash = hash_session_token(&session_value);
let refresh_expires_at =
now + ChronoDuration::days(state.config.refresh_token_expiry_days);
let new_session = NewUserSession {
id: session_id,
user_id: user.id,
token_hash: session_hash,
issued_at: now.naive_utc(),
expires_at: refresh_expires_at.naive_utc(),
tenant_id: tenant.id,
};
diesel::insert_into(session_dsl::user_sessions)
.values(&new_session)
.execute(conn)?;
let cookie = format!("refresh_token={session_value}");
Ok((access_token, cookie, tenant.id))
})
.await
}
pub async fn clear_jobs(&self) -> Result<()> {
self.with_conn(|conn| {
use crate::schema::jobs::dsl::jobs as jobs_table;
diesel::delete(jobs_table)
.execute(conn)
.context("failed to clear jobs")?;
Ok(())
})
.await
}
pub async fn jobs_by_type(&self, ty: &str) -> Result<Vec<Job>> {
let ty = ty.to_string();
self.with_conn(move |conn| {
use crate::schema::jobs::dsl::{job_type as job_type_col, jobs as jobs_table};
let rows = jobs_table
.filter(job_type_col.eq(&ty))
.load::<Job>(conn)
.context("failed to load jobs")?;
Ok(rows)
})
.await
}
pub async fn post_json<T: Serialize + ?Sized>(
&self,
path: &str,
payload: &T,
token: Option<&str>,
) -> Result<hyper::Response<Body>> {
self.post_json_with_cookie(path, payload, token, None).await
}
pub async fn post_json_with_cookie<T: Serialize + ?Sized>(
&self,
path: &str,
payload: &T,
token: Option<&str>,
cookie: Option<&str>,
) -> Result<hyper::Response<Body>> {
let body = serde_json::to_vec(payload)?;
let mut builder = Request::builder()
.method(Method::POST)
.uri(path)
.header("content-type", "application/json");
if let Some(token) = token {
builder = builder.header("authorization", format!("Bearer {token}"));
}
if let Some(cookie) = cookie {
builder = builder.header(header::COOKIE, cookie);
}
let request = builder.body(Body::from(body))?;
Ok(self
.router
.clone()
.oneshot(request)
.await
.expect("infallible response"))
}
pub async fn patch_json<T: Serialize + ?Sized>(
&self,
path: &str,
payload: &T,
token: Option<&str>,
) -> Result<hyper::Response<Body>> {
let body = serde_json::to_vec(payload)?;
let mut builder = Request::builder()
.method(Method::PATCH)
.uri(path)
.header("content-type", "application/json");
if let Some(token) = token {
builder = builder.header("authorization", format!("Bearer {token}"));
}
let request = builder.body(Body::from(body))?;
Ok(self
.router
.clone()
.oneshot(request)
.await
.expect("infallible response"))
}
pub async fn get(&self, path: &str, token: Option<&str>) -> Result<hyper::Response<Body>> {
let mut builder = Request::builder().method(Method::GET).uri(path);
if let Some(token) = token {
builder = builder.header("authorization", format!("Bearer {token}"));
}
let request = builder.body(Body::empty())?;
Ok(self
.router
.clone()
.oneshot(request)
.await
.expect("infallible response"))
}
pub async fn delete(&self, path: &str, token: Option<&str>) -> Result<hyper::Response<Body>> {
let builder = Request::builder().method(Method::DELETE).uri(path);
let builder = if let Some(token) = token {
builder.header("authorization", format!("Bearer {token}"))
} else {
builder
};
let request = builder.body(Body::empty())?;
Ok(self
.router
.clone()
.oneshot(request)
.await
.expect("infallible response"))
}
pub async fn upload_document(
&self,
path: &str,
filename: &str,
content_type: &str,
data: &[u8],
folder_id: Option<Uuid>,
token: &str,
) -> Result<hyper::Response<Body>> {
let extras = UploadExtras::empty();
self.upload_document_with_extras(
path,
filename,
content_type,
data,
folder_id,
extras,
token,
)
.await
}
pub async fn upload_document_with_options(
&self,
path: &str,
filename: &str,
content_type: &str,
data: &[u8],
folder_id: Option<Uuid>,
title: Option<&str>,
metadata_json: Option<&str>,
token: &str,
) -> Result<hyper::Response<Body>> {
let extras = UploadExtras {
title,
metadata_json,
tag_ids_json: None,
correspondents_json: None,
issued_at: None,
skip_existing: None,
};
self.upload_document_with_extras(
path,
filename,
content_type,
data,
folder_id,
extras,
token,
)
.await
}
pub async fn upload_document_with_extras(
&self,
path: &str,
filename: &str,
content_type: &str,
data: &[u8],
folder_id: Option<Uuid>,
extras: UploadExtras<'_>,
token: &str,
) -> Result<hyper::Response<Body>> {
let boundary = format!("boundary-{}", Uuid::new_v4());
let mut body = Vec::new();
body.extend(format!("--{boundary}\r\n").as_bytes());
body.extend(
format!(
"Content-Disposition: form-data; name=\"file\"; filename=\"{}\"\r\n",
filename
)
.as_bytes(),
);
body.extend(format!("Content-Type: {}\r\n\r\n", content_type).as_bytes());
body.extend(data);
body.extend(b"\r\n");
if let Some(folder) = folder_id {
body.extend(format!("--{boundary}\r\n").as_bytes());
body.extend(b"Content-Disposition: form-data; name=\"folder_id\"\r\n\r\n");
body.extend(folder.to_string().as_bytes());
body.extend(b"\r\n");
}
if let Some(title_value) = extras.title {
body.extend(format!("--{boundary}\r\n").as_bytes());
body.extend(b"Content-Disposition: form-data; name=\"title\"\r\n\r\n");
body.extend(title_value.as_bytes());
body.extend(b"\r\n");
}
if let Some(metadata_value) = extras.metadata_json {
body.extend(format!("--{boundary}\r\n").as_bytes());
body.extend(b"Content-Disposition: form-data; name=\"metadata\"\r\n\r\n");
body.extend(metadata_value.as_bytes());
body.extend(b"\r\n");
}
if let Some(tag_ids_value) = extras.tag_ids_json {
body.extend(format!("--{boundary}\r\n").as_bytes());
body.extend(b"Content-Disposition: form-data; name=\"tag_ids\"\r\n\r\n");
body.extend(tag_ids_value.as_bytes());
body.extend(b"\r\n");
}
if let Some(correspondents_value) = extras.correspondents_json {
body.extend(format!("--{boundary}\r\n").as_bytes());
body.extend(b"Content-Disposition: form-data; name=\"correspondents\"\r\n\r\n");
body.extend(correspondents_value.as_bytes());
body.extend(b"\r\n");
}
if let Some(issued_at_value) = extras.issued_at {
body.extend(format!("--{boundary}\r\n").as_bytes());
body.extend(b"Content-Disposition: form-data; name=\"issued_at\"\r\n\r\n");
body.extend(issued_at_value.as_bytes());
body.extend(b"\r\n");
}
if let Some(skip_flag) = extras.skip_existing {
body.extend(format!("--{boundary}\r\n").as_bytes());
body.extend(b"Content-Disposition: form-data; name=\"skip_existing\"\r\n\r\n");
body.extend(if skip_flag {
b"true".as_ref()
} else {
b"false".as_ref()
});
body.extend(b"\r\n");
}
body.extend(format!("--{boundary}--\r\n").as_bytes());
let builder = Request::builder()
.method(Method::POST)
.uri(path)
.header(
"content-type",
format!("multipart/form-data; boundary={boundary}"),
)
.header("authorization", format!("Bearer {token}"));
let request = builder.body(Body::from(body))?;
Ok(self
.router
.clone()
.oneshot(request)
.await
.expect("infallible response"))
}
pub async fn with_conn<F, T>(&self, f: F) -> Result<T>
where
F: FnOnce(&mut PgConnection) -> Result<T> + Send + 'static,
T: Send + 'static,
{
let pool = self.state.pool.clone();
tokio::task::spawn_blocking(move || {
let mut conn = pool
.get()
.map_err(|err| anyhow!("failed to get database connection: {err}"))?;
f(&mut conn)
})
.await
.context("connection task panicked")?
}
}
pub struct UploadExtras<'a> {
pub title: Option<&'a str>,
pub metadata_json: Option<&'a str>,
pub tag_ids_json: Option<&'a str>,
pub correspondents_json: Option<&'a str>,
pub issued_at: Option<&'a str>,
pub skip_existing: Option<bool>,
}
impl<'a> UploadExtras<'a> {
pub fn empty() -> Self {
Self {
title: None,
metadata_json: None,
tag_ids_json: None,
correspondents_json: None,
issued_at: None,
skip_existing: None,
}
}
}
pub async fn acquire_db_lock() -> tokio::sync::MutexGuard<'static, ()> {
DB_LOCK.lock().await
}
pub async fn body_to_vec(body: Body) -> Result<Vec<u8>> {
let collected = body
.collect()
.await
.map_err(|err| anyhow!("failed to read response body: {err}"))?;
Ok(collected.to_bytes().to_vec())
}
#[cfg(test)]
mod helper_tests {
use super::*;
#[tokio::test]
async fn create_session_and_login_token_provide_access() -> Result<()> {
let _lock = acquire_db_lock().await;
let app = TestApp::new().await?;
let username = "helper-login";
let password = "irrelevant";
app.insert_user(username, TestUserRole::Owner).await?;
let (access, refresh, refresh_id) = app.create_session(username).await?;
assert!(!access.is_empty(), "access token should not be empty");
assert!(!refresh.is_empty(), "refresh token should not be empty");
assert_ne!(
refresh_id,
Uuid::nil(),
"refresh token id should be assigned"
);
let bearer = app.login_token(username, password).await?;
assert!(!bearer.is_empty(), "login_token must yield bearer");
app.cleanup().await?;
Ok(())
}
#[tokio::test]
async fn insert_passkey_and_upload_with_options_succeeds() -> Result<()> {
let _lock = acquire_db_lock().await;
let app = TestApp::new().await?;
let username = "helper-passkey";
let password = "unused";
let user_id = app.insert_user(username, TestUserRole::Owner).await?;
let passkey_id = app.insert_passkey(user_id, Some("Laptop")).await?;
assert_ne!(passkey_id, Uuid::nil());
let bearer = app.login_token(username, password).await?;
let response = app
.upload_document_with_options(
"/api/documents",
"helper.txt",
"text/plain",
b"helper-content",
None,
Some("Helper Note"),
Some("{\"category\":\"note\"}"),
&bearer,
)
.await?;
assert!(response.status().is_success());
app.cleanup().await?;
Ok(())
}
}
async fn prepare_database(pool: &PgPool) -> Result<()> {
let pool = pool.clone();
tokio::task::spawn_blocking(move || -> Result<()> {
let mut conn = pool
.get()
.map_err(|err| anyhow!("failed to acquire connection: {err}"))?;
conn.batch_execute(RESET_DATABASE_SQL)
.map_err(|err| anyhow!("failed to reset schema: {err}"))?;
conn.batch_execute("DROP TABLE IF EXISTS __diesel_schema_migrations;")
.map_err(|err| anyhow!("failed to drop diesel schema table: {err}"))?;
conn.run_pending_migrations(MIGRATIONS)
.map_err(|err| anyhow!("failed to run migrations: {err}"))?;
truncate_all(&mut conn)?;
Ok(())
})
.await
.context("migration task panicked")?
}
fn truncate_all(conn: &mut PgConnection) -> Result<()> {
conn.batch_execute(
"TRUNCATE TABLE \
tenant.document_assets, \
tenant.document_correspondents, \
tenant.correspondents, \
tenant.document_tags, \
tenant.document_versions, \
tenant.documents, \
tenant.folders, \
shared.jobs, \
tenant.user_sessions, \
tenant.tags, \
tenant.api_tokens, \
shared.webauthn_challenges, \
shared.user_passkeys, \
tenant.user_memberships, \
shared.users, \
shared.magic_tokens, \
shared.tenants \
RESTART IDENTITY CASCADE;",
)
.context("failed to truncate tables")?;
Ok(())
}
fn generate_session_token() -> String {
let mut bytes = [0u8; 32];
OsRng
.try_fill_bytes(&mut bytes)
.expect("failed to read random bytes");
hex::encode(bytes)
}
fn hash_session_token(value: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(value.as_bytes());
hex::encode(hasher.finalize())
}
+32 -1
View File
@@ -10,13 +10,44 @@ pub fn inline_content_disposition(filename: &str) -> Option<String> {
.chars()
.map(|ch| match ch {
'"' | '\\' => '_',
c if !c.is_ascii() => '_',
_ => ch,
})
.collect();
let encoded = utf8_percent_encode(&sanitized, NON_ALPHANUMERIC);
let encoded = utf8_percent_encode(filename, NON_ALPHANUMERIC);
Some(format!(
"inline; filename=\"{}\"; filename*=UTF-8''{}",
sanitized, encoded
))
}
#[cfg(test)]
mod tests {
use super::*;
use reqwest::header::HeaderValue;
#[test]
fn test_inline_content_disposition_header_validity() {
// Test with a filename containing non-ASCII characters
let filename = "Täst.pdf";
let disposition = inline_content_disposition(filename).unwrap();
println!("Disposition: {}", disposition);
// This should fail if the sanitized part contains non-ASCII characters
// and we try to create a HeaderValue from it.
let result = HeaderValue::from_str(&disposition);
if let Ok(val) = result {
// Check if to_str succeeds (it should now!)
let to_str_res = val.to_str();
assert!(to_str_res.is_ok(), "HeaderValue::to_str should succeed for sanitized filename");
let disposition_str = to_str_res.unwrap();
assert!(disposition_str.contains("filename=\"T_st.pdf\""), "Filename should be sanitized");
assert!(disposition_str.contains("filename*=UTF-8''T%C3%A4st%2Epdf"), "UTF-8 filename should be preserved");
} else {
panic!("HeaderValue rejected the string: {:?}", result.err());
}
}
}
+4 -24
View File
@@ -48,8 +48,8 @@ fn document_asset_type_prefix(document_id: Uuid, version_number: i32, asset_type
)
}
/// Returns the storage prefix under which the asset objects for a type/id pair live.
pub fn document_asset_object_prefix(
/// Returns the storage key for an asset (single object).
pub fn document_asset_key(
document_id: Uuid,
version_number: i32,
asset_type: &str,
@@ -62,21 +62,6 @@ pub fn document_asset_object_prefix(
)
}
/// Returns the full storage key for a specific asset object (ordinal).
pub fn document_asset_object_key(
document_id: Uuid,
version_number: i32,
asset_type: &str,
asset_id: Uuid,
ordinal: i32,
) -> String {
format!(
"{}/{}",
document_asset_object_prefix(document_id, version_number, asset_type, asset_id),
ordinal
)
}
#[cfg(test)]
mod tests {
use super::*;
@@ -103,13 +88,8 @@ mod tests {
);
assert_eq!(
document_asset_object_prefix(document_id, 3, "preview", asset_id),
format!("documents/{document_id}/v3/assets/preview/{asset_id}")
);
assert_eq!(
document_asset_object_key(document_id, 3, "preview", asset_id, 2),
format!("documents/{document_id}/v3/assets/preview/{asset_id}/2")
document_asset_key(document_id, 3, "thumbnail", asset_id),
format!("documents/{document_id}/v3/assets/thumbnail/{asset_id}")
);
}
}
+205 -103
View File
@@ -2,22 +2,28 @@ use std::{collections::HashSet, sync::Arc, time::Duration};
use async_trait::async_trait;
use diesel::prelude::*;
use infer;
use serde::Deserialize;
use serde_json::json;
use tokio::task;
use tracing::{error, warn};
use uuid::Uuid;
use super::ocr::{document_is_pdf, OCR_TEXT_ASSET_TYPE};
use crate::{
jobs::{enqueue_job, JOB_ANALYZE_DOCUMENT, JOB_GENERATE_OCR_TEXT, JOB_GENERATE_THUMBNAILS},
models::{Document, DocumentAsset, DocumentVersion},
schema::{document_assets, document_versions, documents},
state::AppState,
auth::ensure_active_tenant, jobs::JOB_ANALYZE_DOCUMENT, models::Document, state::AppState,
storage::TenantStorage,
};
use super::{JobExecution, JobHandler};
use super::{
index::IndexDocumentTask,
issued_at::DetermineIssuedAtTask,
job_execution_from_task_error,
ocr::{GenerateOcrTask, TEXT_CONTENT_ASSET_TYPE},
taskflow::{
document::DocumentVersionTaskContext, BoxedTask, Task, TaskError, TaskExecutor,
TaskPlanner, TaskResult,
},
thumbnails::GenerateThumbnailsTask,
JobExecution, JobHandler,
};
#[derive(Debug, Deserialize)]
struct AnalyzePayload {
@@ -45,8 +51,23 @@ impl JobHandler for AnalyzeDocumentJob {
&self,
state: Arc<AppState>,
job: crate::models::Job,
_storage: TenantStorage,
storage: TenantStorage,
) -> JobExecution {
let tenant_id = match job.tenant_id {
Some(id) => id,
None => {
return JobExecution::Failed {
error: "job is no longer associated with a tenant".to_string(),
}
}
};
if let Err(err) = ensure_active_tenant(&state, tenant_id) {
return JobExecution::Failed {
error: err.to_string(),
};
}
let payload: AnalyzePayload = match serde_json::from_value(job.payload.clone()) {
Ok(payload) => payload,
Err(err) => {
@@ -56,114 +77,155 @@ impl JobHandler for AnalyzeDocumentJob {
}
};
let state_clone = state.clone();
let tenant_id = job.tenant_id;
match task::spawn_blocking(move || analyze_document(state_clone, tenant_id, payload)).await
{
Ok(Ok(execution)) => execution,
Ok(Err(err)) => {
warn!(job_id = %job.id, error = %err, "analyze job will retry");
JobExecution::Retry {
delay: Duration::from_secs(30),
error: err,
}
}
Err(join_err) => {
error!(job_id = %job.id, error = %join_err, "analyze task panicked");
JobExecution::Retry {
delay: Duration::from_secs(60),
error: format!("worker panicked: {join_err}"),
}
}
let mut context = DocumentVersionTaskContext::new(
job.id,
JOB_ANALYZE_DOCUMENT,
tenant_id,
payload.document_id,
payload.document_version_id,
payload.force,
state.config.worker_max_document_bytes,
state.clone(),
storage,
);
let planner = AnalyzePlanner::new(payload.force, state.clone());
match TaskExecutor::run(&planner, &mut context).await {
Ok(()) => JobExecution::Success,
Err(err) => job_execution_from_task_error(err),
}
}
}
fn analyze_document(
struct AnalyzePlanner {
force: bool,
state: Arc<AppState>,
tenant_id: Uuid,
payload: AnalyzePayload,
) -> Result<JobExecution, String> {
let mut conn = state
.db_for_tenant(tenant_id)
.map_err(|err| format!("{err:?}"))?;
}
let version: DocumentVersion = document_versions::table
.find(payload.document_version_id)
.first(&mut conn)
.map_err(|err| {
format!(
"failed to load document_version {} for tenant {}: {err:?}",
payload.document_version_id, tenant_id
)
})?;
const MIME_SNIFF_BYTES: usize = 8192;
if version.document_id != payload.document_id {
return Err("document/version mismatch".into());
impl AnalyzePlanner {
fn new(force: bool, state: Arc<AppState>) -> Self {
Self { force, state }
}
}
let document: Document = documents::table
.find(payload.document_id)
.first(&mut conn)
.map_err(|err| {
format!(
"failed to load document {} for tenant {}: {err:?}",
payload.document_id, tenant_id
)
})?;
#[async_trait]
impl TaskPlanner<DocumentVersionTaskContext> for AnalyzePlanner {
async fn plan(
&self,
ctx: &mut DocumentVersionTaskContext,
) -> TaskResult<Vec<BoxedTask<DocumentVersionTaskContext>>> {
let document = ctx.document().await?.clone();
let mut tasks: Vec<BoxedTask<DocumentVersionTaskContext>> = Vec::new();
let tenant_id = document.tenant_id;
tasks.push(Box::new(EnsureMimeTask));
let (supported, _reason) = determine_thumbnail_support(&document);
let ocr_supported = document_is_pdf(&document);
let existing_ocr: Option<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq(payload.document_version_id))
.filter(document_assets::asset_type.eq(OCR_TEXT_ASSET_TYPE))
.filter(document_assets::tenant_id.eq(tenant_id))
.first(&mut conn)
.optional()
.map_err(|err| format!("{err:?}"))?;
let skip_ocr = existing_ocr.is_some() && !payload.force;
if supported {
let enqueue_result = enqueue_job(
&mut conn,
tenant_id,
JOB_GENERATE_THUMBNAILS,
json!({
"document_id": payload.document_id,
"document_version_id": payload.document_version_id,
"force": payload.force,
}),
None,
);
if let Err(err) = enqueue_result {
return Err(err.to_string());
let (thumbnail_supported, _) = determine_thumbnail_support(&document);
if thumbnail_supported {
tasks.push(Box::new(GenerateThumbnailsTask::new(self.force)));
}
}
if ocr_supported && !skip_ocr {
let enqueue_result = enqueue_job(
&mut conn,
tenant_id,
JOB_GENERATE_OCR_TEXT,
json!({
"document_id": payload.document_id,
"document_version_id": payload.document_version_id,
"force": payload.force,
}),
None,
);
let existing_ocr = ctx.asset(TEXT_CONTENT_ASSET_TYPE).await?.is_some();
let mut should_index = existing_ocr;
if let Err(err) = enqueue_result {
return Err(err.to_string());
if document_supports_ocr(&document) {
if self.force || !existing_ocr {
tasks.push(Box::new(GenerateOcrTask::new(
self.force,
self.state.clone(),
)));
should_index = true;
}
}
tasks.push(Box::new(DetermineIssuedAtTask::new()));
if should_index {
tasks.push(Box::new(IndexDocumentTask::new()));
}
Ok(tasks)
}
}
struct EnsureMimeTask;
#[async_trait]
impl Task<DocumentVersionTaskContext> for EnsureMimeTask {
fn name(&self) -> &'static str {
"ensure-mime-type"
}
Ok(JobExecution::Success)
async fn execute(&self, ctx: &mut DocumentVersionTaskContext) -> TaskResult<()> {
let document = ctx.document().await?.clone();
let current = document.mime_type.clone();
let guessed = guess_mime_type(ctx, &document).await?;
let desired = match guessed {
Some(mime) if current.as_deref() != Some(mime.as_str()) => Some(mime),
_ => None,
};
if let Some(new_mime) = desired {
update_document_mime(ctx, document.id, new_mime).await?;
}
Ok(())
}
}
async fn guess_mime_type(
ctx: &mut DocumentVersionTaskContext,
document: &Document,
) -> TaskResult<Option<String>> {
let bytes = ctx.object_head(MIME_SNIFF_BYTES).await?;
Ok(sniff_mime(&bytes, &document.original_name))
}
fn sniff_mime(bytes: &[u8], original_name: &str) -> Option<String> {
if let Some(kind) = infer::get(bytes) {
return Some(kind.mime_type().to_string());
}
mime_guess::from_path(original_name)
.first_raw()
.map(|value| value.to_string())
}
async fn update_document_mime(
ctx: &mut DocumentVersionTaskContext,
document_id: Uuid,
mime_type: String,
) -> TaskResult<()> {
let tenant_id = ctx.tenant_id();
let state = ctx.state().clone();
let mime_type_clone = mime_type.clone();
task::spawn_blocking(move || -> Result<(), String> {
let mut conn = state
.db_for_tenant(tenant_id)
.map_err(|err| format!("{err:?}"))?;
diesel::update(
crate::schema::documents::table.filter(crate::schema::documents::id.eq(document_id)),
)
.set(crate::schema::documents::mime_type.eq(Some(mime_type_clone)))
.execute(&mut conn)
.map_err(|err| format!("{err:?}"))
.map(|_| ())
})
.await
.map_err(|err| {
TaskError::retry(
Duration::from_secs(60),
format!("mime update task panicked: {err}"),
)
})?
.map_err(|err| TaskError::retry(Duration::from_secs(30), err))?;
ctx.set_document_mime(Some(mime_type));
Ok(())
}
pub(crate) fn determine_thumbnail_support(document: &Document) -> (bool, Option<String>) {
@@ -175,12 +237,18 @@ pub(crate) fn determine_thumbnail_support(document: &Document) -> (bool, Option<
"image/bmp",
"image/webp",
"application/pdf",
"video/mp4",
"video/quicktime",
"video/webm",
"video/x-msvideo",
"video/x-ms-wmv",
"video/x-matroska",
]
.into_iter()
.collect();
if let Some(ref content_type) = document.content_type {
if supported_mimes.contains(content_type.as_str()) {
if let Some(ref mime_type) = document.mime_type {
if supported_mimes.contains(mime_type.as_str()) {
return (true, None);
}
}
@@ -192,7 +260,8 @@ pub(crate) fn determine_thumbnail_support(document: &Document) -> (bool, Option<
.map(|ext| ext.to_ascii_lowercase())
{
let supported_exts = [
"jpg", "jpeg", "png", "gif", "tif", "tiff", "bmp", "webp", "pdf",
"jpg", "jpeg", "png", "gif", "tif", "tiff", "bmp", "webp", "pdf", "mp4", "m4v", "mov",
"webm", "mkv", "avi", "wmv",
];
if supported_exts.contains(&ext.as_str()) {
return (true, None);
@@ -204,3 +273,36 @@ pub(crate) fn determine_thumbnail_support(document: &Document) -> (bool, Option<
Some("content type not supported for thumbnails".into()),
)
}
fn document_supports_ocr(document: &Document) -> bool {
document
.mime_type
.as_deref()
.map(|mime| mime.eq_ignore_ascii_case("application/pdf"))
.unwrap_or_else(|| {
document
.original_name
.rsplit('.')
.next()
.map(|ext| ext.eq_ignore_ascii_case("pdf"))
.unwrap_or(false)
})
}
#[cfg(test)]
mod tests {
use super::sniff_mime;
#[test]
fn sniff_mime_prefers_magic_bytes() {
const PNG_HEADER: [u8; 8] = [0x89, b'P', b'N', b'G', 0x0D, 0x0A, 0x1A, 0x0A];
let mime = sniff_mime(&PNG_HEADER, "file.txt");
assert_eq!(mime.as_deref(), Some("image/png"));
}
#[test]
fn sniff_mime_falls_back_to_extension() {
let mime = sniff_mime(b"not enough to detect", "video.mp4");
assert_eq!(mime.as_deref(), Some("video/mp4"));
}
}
+7 -30
View File
@@ -3,14 +3,13 @@ use std::collections::HashMap;
use diesel::{prelude::*, PgConnection};
use uuid::Uuid;
use crate::models::{Document, DocumentAsset, DocumentAssetObject, DocumentVersion};
use crate::schema::{document_asset_objects, document_assets, document_versions, documents};
use crate::models::{Document, DocumentAsset, DocumentVersion};
use crate::schema::{document_assets, document_versions, documents};
use crate::state::AppState;
pub(crate) struct DocumentVersionContext {
pub(crate) struct LoadedDocumentVersion {
pub document: Document,
pub version: DocumentVersion,
pub tenant_id: Uuid,
}
pub(crate) fn load_document_version(
@@ -18,7 +17,7 @@ pub(crate) fn load_document_version(
tenant_id: Uuid,
document_id: Uuid,
version_id: Uuid,
) -> Result<DocumentVersionContext, String> {
) -> Result<LoadedDocumentVersion, String> {
let mut conn = state
.db_for_tenant(tenant_id)
.map_err(|err| format!("{err:?}"))?;
@@ -37,16 +36,11 @@ pub(crate) fn load_document_version(
.first(&mut conn)
.map_err(|err| format!("{err:?}"))?;
Ok(DocumentVersionContext {
document,
version,
tenant_id,
})
Ok(LoadedDocumentVersion { document, version })
}
pub(crate) struct LoadedAsset {
pub struct LoadedAsset {
pub asset: DocumentAsset,
pub objects: Vec<DocumentAssetObject>,
}
pub(crate) fn load_version_assets(
@@ -70,26 +64,9 @@ pub(crate) fn load_version_assets(
.load(conn)
.map_err(|err| format!("{err:?}"))?;
let asset_ids: Vec<Uuid> = assets.iter().map(|asset| asset.id).collect();
let mut object_map: HashMap<Uuid, Vec<DocumentAssetObject>> = HashMap::new();
if !asset_ids.is_empty() {
let objects: Vec<DocumentAssetObject> = document_asset_objects::table
.filter(document_asset_objects::asset_id.eq_any(&asset_ids))
.filter(document_asset_objects::tenant_id.eq(tenant_id))
.order(document_asset_objects::ordinal.asc())
.load(conn)
.map_err(|err| format!("{err:?}"))?;
for object in objects {
object_map.entry(object.asset_id).or_default().push(object);
}
}
let mut result = HashMap::with_capacity(assets.len());
for asset in assets {
let objects = object_map.remove(&asset.id).unwrap_or_default();
result.insert(asset.asset_type.clone(), LoadedAsset { asset, objects });
result.insert(asset.asset_type.clone(), LoadedAsset { asset });
}
Ok(result)
+49 -183
View File
@@ -1,213 +1,79 @@
use std::sync::Arc;
use std::time::Duration;
use async_trait::async_trait;
use diesel::prelude::*;
use reqwest::Client;
use serde::Deserialize;
use tokio::task;
use tracing::{error, warn};
use uuid::Uuid;
use crate::{
documents::search::{build_quickwit_ingest_record, quickwit_ingest},
jobs::JOB_INDEX_DOCUMENT_TEXT,
models::{Document, DocumentVersion},
schema::{document_asset_objects, document_assets, document_versions, documents},
state::AppState,
storage::TenantStorage,
};
use crate::documents::search::{build_quickwit_ingest_record, quickwit_ingest};
use super::{
fetch_version_object, handle_fetch_error, ocr::OCR_TEXT_ASSET_TYPE, JobExecution, JobHandler,
ocr::TEXT_CONTENT_ASSET_TYPE,
taskflow::{document::DocumentVersionTaskContext, Task, TaskError, TaskResult},
};
#[derive(Debug, Deserialize)]
struct IndexPayload {
document_id: Uuid,
document_version_id: Uuid,
}
pub struct IndexDocumentTask;
pub struct IndexDocumentTextJob;
impl IndexDocumentTextJob {
impl IndexDocumentTask {
pub fn new() -> Self {
Self
}
}
#[async_trait]
impl JobHandler for IndexDocumentTextJob {
fn job_type(&self) -> &'static str {
JOB_INDEX_DOCUMENT_TEXT
impl Task<DocumentVersionTaskContext> for IndexDocumentTask {
fn name(&self) -> &'static str {
"index-document-text"
}
async fn handle(
&self,
state: Arc<AppState>,
job: crate::models::Job,
storage: TenantStorage,
) -> JobExecution {
let payload: IndexPayload = match serde_json::from_value(job.payload.clone()) {
Ok(payload) => payload,
Err(err) => {
return JobExecution::Failed {
error: format!("invalid index payload: {err}"),
}
}
};
async fn execute(&self, ctx: &mut DocumentVersionTaskContext) -> TaskResult<()> {
let state = ctx.state().clone();
let quickwit_endpoint = state
.config
.quickwit_endpoint
.clone()
.ok_or_else(|| TaskError::fail("quickwit endpoint missing"))?;
let quickwit_endpoint = match &state.config.quickwit_endpoint {
Some(endpoint) => endpoint.clone(),
None => {
return JobExecution::Failed {
error: "quickwit endpoint missing".into(),
};
}
};
let tenant = state.tenants.get_by_id(ctx.tenant_id()).map_err(|err| {
TaskError::retry(
Duration::from_secs(30),
format!("failed to load tenant: {err:?}"),
)
})?;
let tenant = match state.tenants.get_by_id(job.tenant_id) {
Ok(tenant) => tenant,
Err(err) => {
warn!(job_id = %job.id, error = ?err, "failed to load tenant for indexing");
return JobExecution::Retry {
delay: Duration::from_secs(30),
error: format!("failed to load tenant: {err:?}"),
};
}
};
let quickwit_index = tenant
.quickwit_index
.clone()
.ok_or_else(|| TaskError::fail("tenant quickwit index not configured"))?;
let quickwit_index = match tenant.quickwit_index.clone() {
Some(index) => index,
None => {
return JobExecution::Failed {
error: "tenant quickwit index not configured".into(),
};
}
};
let asset = ctx
.asset(TEXT_CONTENT_ASSET_TYPE)
.await?
.ok_or_else(|| TaskError::fail("missing OCR text asset"))?;
let client = Client::new();
let s3_key = asset.asset.s3_key.clone();
let bytes = ctx.storage().get_object(&s3_key).await.map_err(|err| {
TaskError::retry(
Duration::from_secs(30),
format!("failed to download ocr text: {err}"),
)
})?;
let state_clone = state.clone();
let tenant_id = job.tenant_id;
let context =
match task::spawn_blocking(move || load_context(state_clone, tenant_id, payload)).await
{
Ok(Ok(ctx)) => ctx,
Ok(Err(err)) => {
warn!(job_id = %job.id, error = %err, "index job will retry");
return JobExecution::Retry {
delay: Duration::from_secs(30),
error: err,
};
}
Err(join_err) => {
error!(job_id = %job.id, error = %join_err, "index task panicked");
return JobExecution::Retry {
delay: Duration::from_secs(60),
error: format!("worker panicked: {join_err}"),
};
}
};
if context.text_s3_key.is_none() {
warn!(job_id = %job.id, "missing OCR text asset; failing indexing job");
return JobExecution::Failed {
error: "missing OCR text asset".into(),
};
}
let s3_key = context.text_s3_key.unwrap();
let bytes = match fetch_version_object(
&context.version,
&storage,
&s3_key,
state.config.worker_max_document_bytes,
)
.await
{
Ok(bytes) => bytes,
Err(err) => return handle_fetch_error(&job, err, "failed to download ocr text"),
};
let text = match String::from_utf8(bytes) {
Ok(text) => text,
Err(err) => {
warn!(job_id = %job.id, error = %err, "ocr text not valid UTF-8");
return JobExecution::Failed {
error: "ocr text not valid UTF-8".into(),
};
}
};
let text = String::from_utf8(bytes)
.map_err(|err| TaskError::fail(format!("ocr text not valid UTF-8: {err}")))?;
if text.trim().is_empty() {
warn!(job_id = %job.id, "ocr text empty; skipping");
return JobExecution::Failed {
error: "ocr text empty".into(),
};
return Err(TaskError::fail("ocr text empty"));
}
let record =
build_quickwit_ingest_record(&context.document, &context.version, job.tenant_id, &text);
let document = ctx.document().await?.clone();
let version = ctx.version().await?.clone();
match quickwit_ingest(&client, &quickwit_endpoint, &quickwit_index, &[record]).await {
Ok(()) => JobExecution::Success,
Err(err) => {
warn!(job_id = %job.id, error = %err, "quickwit ingest failed");
JobExecution::Retry {
delay: Duration::from_secs(30),
error: err.to_string(),
}
}
}
let record = build_quickwit_ingest_record(&document, &version, ctx.tenant_id(), &text);
let client = Client::new();
quickwit_ingest(&client, &quickwit_endpoint, &quickwit_index, &[record])
.await
.map_err(|err| TaskError::retry(Duration::from_secs(30), err.to_string()))?;
Ok(())
}
}
struct IndexContext {
document: Document,
version: DocumentVersion,
text_s3_key: Option<String>,
}
fn load_context(
state: Arc<AppState>,
tenant_id: Uuid,
payload: IndexPayload,
) -> Result<IndexContext, String> {
let mut conn = state
.db_for_tenant(tenant_id)
.map_err(|err| format!("{err:?}"))?;
let version: DocumentVersion = document_versions::table
.find(payload.document_version_id)
.first(&mut conn)
.map_err(|err| format!("{err:?}"))?;
if version.document_id != payload.document_id {
return Err("document/version mismatch".into());
}
let document: Document = documents::table
.find(payload.document_id)
.first(&mut conn)
.map_err(|err| format!("{err:?}"))?;
let text_s3_key: Option<String> = document_asset_objects::table
.inner_join(
document_assets::table.on(document_asset_objects::asset_id.eq(document_assets::id)),
)
.filter(document_assets::document_version_id.eq(payload.document_version_id))
.filter(document_assets::asset_type.eq(OCR_TEXT_ASSET_TYPE))
.filter(document_assets::tenant_id.eq(tenant_id))
.filter(document_asset_objects::ordinal.eq(1))
.filter(document_asset_objects::tenant_id.eq(tenant_id))
.select(document_asset_objects::s3_key)
.first(&mut conn)
.optional()
.map_err(|err| format!("{err:?}"))?;
Ok(IndexContext {
document,
version,
text_s3_key,
})
}
+695
View File
@@ -0,0 +1,695 @@
use std::time::Duration;
use async_trait::async_trait;
use chrono::{DateTime, Datelike, NaiveDate, NaiveDateTime, TimeZone, Utc};
use diesel::prelude::*;
use once_cell::sync::Lazy;
use regex::Match;
use regex::Regex;
use serde_json::Value;
use tracing::{info, warn};
use uuid::Uuid;
use crate::issued_at::{DateOrder, IssuedAtSettings};
use crate::schema::documents::dsl as documents_dsl;
use crate::workers::ocr::TEXT_CONTENT_ASSET_TYPE;
use crate::workers::taskflow::document::DocumentVersionTaskContext;
use crate::workers::taskflow::{Task, TaskContext, TaskError, TaskResult};
#[path = "issued_at_months.rs"]
mod issued_at_months;
const MAX_FILENAME_CHARS: usize = 256;
const MAX_TEXT_CHARS: usize = 50_000;
const DATE_SEP_PATTERN: &str = r"[\s._/\-]+";
static YMD_RE: Lazy<Regex> =
Lazy::new(|| Regex::new(r"(?u)\b(\d{4})[./-](\d{1,2})[./-](\d{1,2})\b").expect("ymd regex"));
static NUMERIC_RE: Lazy<Regex> = Lazy::new(|| {
Regex::new(r"(?u)\b(\d{1,2})[./-](\d{1,2})[./-](\d{2,4})\b").expect("numeric regex")
});
static DAY_MONTH_RE: Lazy<Regex> = Lazy::new(|| {
Regex::new(&format!(
r"(?u)\b(\d{{1,2}})(?:st|nd|rd|th)?{SEP}({MONTH_PATTERN}){SEP}(\d{{2,4}})\b",
SEP = DATE_SEP_PATTERN,
MONTH_PATTERN = month_pattern()
))
.expect("day month regex")
});
static MONTH_DAY_RE: Lazy<Regex> = Lazy::new(|| {
Regex::new(&format!(
r"(?u)\b({MONTH_PATTERN}){SEP}(\d{{1,2}})(?:st|nd|rd|th)?(?:,)?{SEP}(\d{{2,4}})\b",
SEP = DATE_SEP_PATTERN,
MONTH_PATTERN = month_pattern()
))
.expect("month day regex")
});
static MONTH_YEAR_RE: Lazy<Regex> = Lazy::new(|| {
Regex::new(&format!(
r"(?u)\b({MONTH_PATTERN})[\s._-]*(\d{{4}})\b",
MONTH_PATTERN = month_pattern()
))
.expect("month year regex")
});
fn month_pattern() -> &'static str {
issued_at_months::pattern()
}
pub struct DetermineIssuedAtTask;
impl DetermineIssuedAtTask {
pub fn new() -> Self {
Self
}
}
#[async_trait]
impl Task<DocumentVersionTaskContext> for DetermineIssuedAtTask {
fn name(&self) -> &'static str {
"determine-issued-at"
}
async fn execute(&self, ctx: &mut DocumentVersionTaskContext) -> TaskResult<()> {
let document = ctx.document().await?.clone();
if document.issued_at.is_some() {
return Ok(());
}
let version = ctx.version().await?.clone();
let settings = ctx.state().issued_at_settings();
let now_utc = Utc::now();
let parser_hint = parser_supplied_date(&document.metadata, &version.metadata, &settings)
.and_then(|dt| settings.normalize_datetime(dt, now_utc));
let filename_candidate = settings.filename_date_order().and_then(|order| {
let normalized = normalize_content(&document.original_name, MAX_FILENAME_CHARS);
find_date_in_text(&normalized, order, &settings, now_utc)
});
let text_candidate = load_document_text(ctx).await?.and_then(|text| {
let normalized = normalize_content(&text, MAX_TEXT_CHARS);
find_date_in_text(&normalized, settings.date_order(), &settings, now_utc)
});
if let Some((final_date, source)) = parser_hint
.map(|dt| (dt, IssuedAtSource::Parser))
.or_else(|| filename_candidate.map(|dt| (dt, IssuedAtSource::Filename)))
.or_else(|| text_candidate.map(|dt| (dt, IssuedAtSource::Text)))
{
persist_issued_at(ctx, document.id, final_date.naive_utc()).await?;
info!(
job_id = %ctx.job_id(),
document_id = %document.id,
issued_at = %final_date,
source = source.as_ref(),
"issued_at determined"
);
} else {
info!(
job_id = %ctx.job_id(),
document_id = %document.id,
"no issued_at signals discovered; leaving unset"
);
}
Ok(())
}
}
async fn persist_issued_at(
ctx: &DocumentVersionTaskContext,
document_id: Uuid,
issued_at: NaiveDateTime,
) -> TaskResult<()> {
let tenant_id = ctx.tenant_id();
let state = ctx.state().clone();
let result = tokio::task::spawn_blocking(move || -> Result<(), String> {
let mut conn = state
.db_for_tenant(tenant_id)
.map_err(|err| format!("failed to scope connection: {err:?}"))?;
diesel::update(
documents_dsl::documents
.filter(documents_dsl::tenant_id.eq(tenant_id))
.filter(documents_dsl::id.eq(document_id)),
)
.set(documents_dsl::issued_at.eq(issued_at))
.execute(&mut conn)
.map_err(|err| format!("failed to update issued_at: {err}"))?;
Ok(())
})
.await
.map_err(|err| {
TaskError::retry(
Duration::from_secs(30),
format!("issued_at task panicked: {err}"),
)
})?;
result.map_err(|err| TaskError::retry(Duration::from_secs(30), err))?;
Ok(())
}
fn parser_supplied_date(
document_meta: &Value,
version_meta: &Value,
settings: &IssuedAtSettings,
) -> Option<DateTime<Utc>> {
metadata_datetime(document_meta)
.or_else(|| metadata_datetime(version_meta))
.and_then(|raw| parse_hint_datetime(raw, settings))
}
fn metadata_datetime(value: &Value) -> Option<&str> {
match value {
Value::String(s) => Some(s.as_str()),
Value::Object(map) => {
for key in [
"issued_at_override",
"issued_at",
"source_date",
"created_at",
] {
if let Some(Value::String(s)) = map.get(key) {
return Some(s.as_str());
}
}
if let Some(Value::Object(parser)) = map.get("parser") {
if let Some(Value::String(s)) = parser.get("issued_at") {
return Some(s.as_str());
}
}
None
}
_ => None,
}
}
fn parse_hint_datetime(raw: &str, settings: &IssuedAtSettings) -> Option<DateTime<Utc>> {
if let Ok(dt) = DateTime::parse_from_rfc3339(raw) {
return Some(dt.with_timezone(&Utc));
}
if let Ok(date) = NaiveDate::parse_from_str(raw, "%Y-%m-%d") {
return settings
.timezone()
.with_ymd_and_hms(date.year(), date.month(), date.day(), 0, 0, 0)
.single()
.map(|dt| dt.with_timezone(&Utc));
}
None
}
fn normalize_content(input: &str, limit: usize) -> String {
let truncated: String = input.chars().take(limit).collect();
let mut normalized = String::with_capacity(truncated.len());
for ch in truncated.chars() {
if ch.is_control() {
normalized.push(' ');
} else {
normalized.extend(ch.to_lowercase());
}
}
normalized
}
fn find_date_in_text(
text: &str,
order: DateOrder,
settings: &IssuedAtSettings,
now_utc: DateTime<Utc>,
) -> Option<DateTime<Utc>> {
collect_matches_with_spans(text, order, settings, now_utc)
.into_iter()
.map(|(_, dt)| dt)
.next()
}
fn collect_matches_with_spans(
text: &str,
order: DateOrder,
settings: &IssuedAtSettings,
now_utc: DateTime<Utc>,
) -> Vec<(usize, DateTime<Utc>)> {
let mut matches: Vec<(usize, usize, DateTime<Utc>)> = Vec::new();
let mut push_date = |span: Option<Match>, date: NaiveDate| {
if let Some(dt) = settings.normalize_naive(date, now_utc) {
if let Some(span) = span {
let start = span.start();
let end = span.end();
if let Some(existing) =
matches
.iter_mut()
.find(|(existing_start, existing_end, _)| {
*existing_start != usize::MAX
&& start < *existing_end
&& *existing_start < end
})
{
let existing_len = existing.1.saturating_sub(existing.0);
let new_len = end.saturating_sub(start);
if new_len > existing_len {
*existing = (start, end, dt);
}
return;
}
matches.push((start, end, dt));
} else {
matches.push((usize::MAX, usize::MAX, dt));
}
}
};
for caps in YMD_RE.captures_iter(text) {
let (Some(year_match), Some(month_match), Some(day_match)) =
(caps.get(1), caps.get(2), caps.get(3))
else {
continue;
};
let year = match year_match.as_str().parse::<i32>() {
Ok(value) => value,
Err(_) => continue,
};
let month = match month_match.as_str().parse::<u32>() {
Ok(value) => value,
Err(_) => continue,
};
let day = match day_match.as_str().parse::<u32>() {
Ok(value) => value,
Err(_) => continue,
};
if let Some(date) = NaiveDate::from_ymd_opt(year, month, day) {
push_date(caps.get(0), date);
}
}
for caps in NUMERIC_RE.captures_iter(text) {
let (Some(first_match), Some(second_match), Some(year_match)) =
(caps.get(1), caps.get(2), caps.get(3))
else {
continue;
};
let first = match first_match.as_str().parse::<u32>() {
Ok(value) => value,
Err(_) => continue,
};
let second = match second_match.as_str().parse::<u32>() {
Ok(value) => value,
Err(_) => continue,
};
let year_raw = year_match.as_str();
let mut year = match year_raw.parse::<i32>() {
Ok(value) => value,
Err(_) => continue,
};
if year_raw.len() == 2 {
year += if year >= 70 { 1900 } else { 2000 };
}
// NUMERIC_RE always captures a day-first form (dd[sep]mm[sep]yy(yy));
// YMD layouts are handled earlier by YMD_RE, so YMD here is treated the
// same as DMY to avoid mis-parsing strings like 01-07-2024.
let (day, month) = match order {
DateOrder::Dmy | DateOrder::Ymd => (first, second),
DateOrder::Mdy => (second, first),
};
if let Some(date) = NaiveDate::from_ymd_opt(year, month, day) {
push_date(caps.get(0), date);
}
}
for caps in DAY_MONTH_RE.captures_iter(text) {
let (Some(day_match), Some(month_match), Some(year_match)) =
(caps.get(1), caps.get(2), caps.get(3))
else {
continue;
};
let day_str = day_match
.as_str()
.trim_matches(|c: char| !c.is_ascii_digit());
let day = match day_str.parse::<u32>() {
Ok(value) => value,
Err(_) => continue,
};
let Some(month) = month_name_to_number(month_match.as_str(), settings) else {
continue;
};
if year_match.as_str().len() < 3 {
continue;
}
let Some(year) = normalize_year(year_match.as_str()) else {
continue;
};
if let Some(date) = NaiveDate::from_ymd_opt(year, month, day) {
push_date(caps.get(0), date);
}
}
for caps in MONTH_DAY_RE.captures_iter(text) {
let (Some(month_match), Some(day_match), Some(year_match)) =
(caps.get(1), caps.get(2), caps.get(3))
else {
continue;
};
let Some(month) = month_name_to_number(month_match.as_str(), settings) else {
continue;
};
let day_str = day_match
.as_str()
.trim_matches(|c: char| !c.is_ascii_digit());
let day = match day_str.parse::<u32>() {
Ok(value) => value,
Err(_) => continue,
};
let Some(year) = normalize_year(year_match.as_str()) else {
continue;
};
if let Some(date) = NaiveDate::from_ymd_opt(year, month, day) {
push_date(caps.get(0), date);
}
}
for caps in MONTH_YEAR_RE.captures_iter(text) {
let (Some(month_match), Some(year_match)) = (caps.get(1), caps.get(2)) else {
continue;
};
let Some(month) = month_name_to_number(month_match.as_str(), settings) else {
continue;
};
let year = match year_match.as_str().parse::<i32>() {
Ok(value) => value,
Err(_) => continue,
};
if let Some(date) = NaiveDate::from_ymd_opt(year, month, 1) {
push_date(caps.get(0), date);
}
}
matches.sort_by_key(|(start, _, _)| *start);
matches
.into_iter()
.map(|(start, _, dt)| (start, dt))
.collect()
}
fn normalize_year(raw: &str) -> Option<i32> {
if raw.len() == 2 {
let mut year = raw.parse::<i32>().ok()?;
year += if year >= 70 { 1900 } else { 2000 };
Some(year)
} else {
raw.parse::<i32>().ok()
}
}
fn month_name_to_number(value: &str, settings: &IssuedAtSettings) -> Option<u32> {
let normalized = value.trim();
let variant = issued_at_months::variants()
.iter()
.find(|entry| entry.name == normalized)?;
if settings.locales().is_empty() || variant.locales.is_empty() {
return Some(variant.month);
}
if variant
.locales
.iter()
.any(|locale| settings.locales().contains(*locale))
{
Some(variant.month)
} else {
None
}
}
async fn load_document_text(ctx: &mut DocumentVersionTaskContext) -> TaskResult<Option<String>> {
let object_key = {
let asset = ctx.asset(TEXT_CONTENT_ASSET_TYPE).await?;
asset.map(|a| a.asset.s3_key.clone())
};
let Some(key) = object_key else {
return Ok(None);
};
match ctx.storage().get_object(&key).await {
Ok(bytes) => match String::from_utf8(bytes) {
Ok(mut text) => {
if text.len() > MAX_TEXT_CHARS {
text.truncate(MAX_TEXT_CHARS);
}
Ok(Some(text))
}
Err(err) => {
warn!(error = %err, "ocr text asset not valid utf-8");
Ok(None)
}
},
Err(err) => {
warn!(error = %err, "failed to download ocr text for issued_at extractor");
Ok(None)
}
}
}
#[derive(Copy, Clone)]
enum IssuedAtSource {
Parser,
Filename,
Text,
}
impl IssuedAtSource {
fn as_ref(&self) -> &'static str {
match self {
IssuedAtSource::Parser => "parser",
IssuedAtSource::Filename => "filename",
IssuedAtSource::Text => "text",
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::config::AppConfig;
use serde::Deserialize;
use serde_yaml::Value as YamlValue;
use once_cell::sync::Lazy;
#[derive(Clone, Deserialize)]
struct CaseSuite {
cases: Vec<CaseDefinition>,
}
#[derive(Clone, Deserialize)]
struct CaseDefinition {
name: String,
parser: String,
#[serde(default)]
filename: Option<String>,
#[serde(default)]
content: Option<String>,
#[serde(default)]
settings: CaseSettings,
expected: ExpectedCase,
}
#[derive(Clone, Default, Deserialize)]
struct CaseSettings {
#[serde(rename = "DATE_PARSER_LANGUAGES", default)]
date_parser_languages: Vec<String>,
#[serde(rename = "FILENAME_DATE_ORDER")]
filename_date_order: Option<String>,
#[serde(rename = "DATE_ORDER")]
date_order: Option<String>,
#[serde(rename = "IGNORE_DATES", default)]
ignore_dates: Vec<String>,
}
#[derive(Clone, Deserialize)]
struct ExpectedCase {
mode: ExpectedMode,
#[serde(default)]
value: Option<YamlValue>,
}
#[derive(Clone, Deserialize, PartialEq)]
#[serde(rename_all = "lowercase")]
enum ExpectedMode {
None,
Single,
Multiple,
}
static CASES: Lazy<CaseSuite> = Lazy::new(|| {
let raw = include_str!("../../tests/data/issued_at_cases.yaml");
serde_yaml::from_str(raw).expect("failed to parse issued_at cases")
});
pub(crate) fn run_named_case(name: &str) {
let case = CASES
.cases
.iter()
.find(|case| case.name == name)
.unwrap_or_else(|| panic!("case '{}' not found", name))
.clone();
run_case(case);
}
fn run_case(case: CaseDefinition) {
let mut config = base_config();
if let Some(order) = case.settings.date_order {
config.issued_at_date_order = order;
}
config.issued_at_filename_date_order = case.settings.filename_date_order;
config.issued_at_date_parser_locales = case.settings.date_parser_languages;
config.issued_at_ignore_dates = case.settings.ignore_dates;
let settings = IssuedAtSettings::from_config(&config);
let now_utc = Utc.with_ymd_and_hms(2025, 1, 1, 0, 0, 0).single().unwrap();
let mut matches = Vec::new();
if let Some(filename) = &case.filename {
if let Some(order) = settings.filename_date_order() {
let normalized = normalize_content(filename, MAX_FILENAME_CHARS);
matches.extend(
collect_matches_with_spans(&normalized, order, &settings, now_utc)
.into_iter()
.map(|(_, dt)| dt.date_naive()),
);
}
}
if let Some(content) = &case.content {
let normalized = normalize_content(content, MAX_TEXT_CHARS);
matches.extend(
collect_matches_with_spans(&normalized, settings.date_order(), &settings, now_utc)
.into_iter()
.map(|(_, dt)| dt.date_naive()),
);
}
let actual: Vec<String> = matches
.into_iter()
.map(|date| date.format("%Y-%m-%d").to_string())
.collect();
match case.parser.as_str() {
"parse_date" => match case.expected.mode {
ExpectedMode::None => assert!(
actual.is_empty(),
"case '{}' expected no matches, got {:?}",
case.name,
actual
),
ExpectedMode::Single => {
let expected = case.expected.single_value();
assert!(
expected.is_some(),
"case '{}' is missing expected single value",
case.name
);
assert_eq!(
actual.first(),
expected.as_ref(),
"case '{}' single mismatch",
case.name
);
}
ExpectedMode::Multiple => panic!(
"case '{}' declares parse_date but expects multiple results",
case.name
),
},
"parse_date_generator" => {
let expected = case.expected.multiple_values().unwrap_or_default();
assert_eq!(expected, actual, "case '{}' multiple mismatch", case.name);
}
other => panic!("unsupported parser '{}' in case {}", other, case.name),
}
}
impl ExpectedCase {
fn single_value(&self) -> Option<String> {
match self.value.as_ref()? {
YamlValue::String(value) => Some(value.clone()),
other => Some(other.as_str()?.to_string()),
}
}
fn multiple_values(&self) -> Option<Vec<String>> {
let list = match self.value.as_ref()? {
YamlValue::Sequence(seq) => seq,
_ => return None,
};
Some(
list.iter()
.filter_map(|value| value.as_str().map(|s| s.to_string()))
.collect(),
)
}
}
fn base_config() -> AppConfig {
AppConfig {
database_url: "postgres://test".to_string(),
migrations_database_url: None,
database_max_pool_size: 5,
server_host: "127.0.0.1".to_string(),
server_port: 0,
webdav_host: "127.0.0.1".to_string(),
webdav_port: 0,
jwt_secret: "secret".to_string(),
jwt_issuer: "issuer".to_string(),
jwt_audience: "audience".to_string(),
jwt_expiry_minutes: 60,
download_token_audience: "download".to_string(),
download_token_expiry_minutes: 60,
refresh_token_expiry_days: 30,
refresh_cookie_secure: false,
refresh_cookie_domain: None,
cors_allowed_origin: None,
proxy_downloads: false,
aws_endpoint_url: None,
aws_access_key_id: None,
aws_secret_access_key: None,
aws_region: "us-east-1".to_string(),
s3_bucket: "bucket".to_string(),
quickwit_endpoint: None,
quickwit_index: None,
worker_max_document_bytes: 100 * 1024 * 1024,
upload_body_limit_bytes: 64 * 1024 * 1024,
service_timezone: "UTC".to_string(),
issued_at_date_order: "DMY".to_string(),
issued_at_filename_date_order: None,
issued_at_date_parser_locales: Vec::new(),
issued_at_ignore_dates: Vec::new(),
webauthn_rp_id: Some("localhost".to_string()),
webauthn_origin: Some("http://localhost".to_string()),
webauthn_rp_name: "Papercrate".to_string(),
}
}
include!(concat!(env!("OUT_DIR"), "/issued_at_generated_tests.rs"));
#[test]
fn month_name_lookup_handles_turkish_variants() {
let settings = IssuedAtSettings::from_config(&base_config());
assert_eq!(month_name_to_number("şubat", &settings), Some(2));
assert_eq!(month_name_to_number("subat", &settings), Some(2));
}
#[test]
fn locale_filter_limits_month_names() {
let mut config = base_config();
config.issued_at_date_parser_locales = vec!["tr".into()];
let settings = IssuedAtSettings::from_config(&config);
assert_eq!(month_name_to_number("january", &settings), None);
assert_eq!(month_name_to_number("şubat", &settings), Some(2));
}
}
+15
View File
@@ -0,0 +1,15 @@
pub(super) struct MonthVariant {
pub name: &'static str,
pub month: u32,
pub locales: &'static [&'static str],
}
include!(concat!(env!("OUT_DIR"), "/issued_at_months.rs"));
pub(super) fn pattern() -> &'static str {
MONTH_PATTERN
}
pub(super) fn variants() -> &'static [MonthVariant] {
MONTH_VARIANTS
}
+25 -37
View File
@@ -11,16 +11,19 @@ use crate::{
state::AppState,
storage::TenantStorage,
};
use taskflow::TaskError;
pub mod analyze;
pub mod common;
pub mod index;
pub mod issued_at;
pub mod ocr;
pub mod purge;
pub mod taskflow;
pub mod tenants;
pub mod thumbnails;
use tenants::ProvisionTenantJob;
use tenants::{DeleteTenantJob, ProvisionTenantJob};
#[derive(Debug)]
pub enum JobExecution {
@@ -90,8 +93,20 @@ impl Worker {
drop(conn);
if let Some(job) = job_opt {
let tenant_id = match job.tenant_id {
Some(id) => id,
None => {
warn!(job_id = %job.id, job_type = %job.job_type, "job detached from tenant; marking failed");
if let Ok(mut conn) = self.state.db_unscoped() {
let _ =
mark_job_failed(&mut conn, job.id, "job detached from tenant context");
}
return Ok(true);
}
};
if let Some(handler) = self.handlers.get(job.job_type.as_str()) {
let execution = match self.state.storage_for_tenant(job.tenant_id) {
let execution = match self.state.storage_for_tenant(tenant_id) {
Ok(storage) => {
handler
.handle(self.state.clone(), job.clone(), storage)
@@ -148,14 +163,19 @@ impl Worker {
pub fn default_handlers() -> Vec<Arc<dyn JobHandler>> {
vec![
Arc::new(analyze::AnalyzeDocumentJob::new()),
Arc::new(thumbnails::GenerateThumbnailsJob::new()),
Arc::new(ocr::GenerateOcrTextJob::new()),
Arc::new(purge::PurgeDocumentJob::new()),
Arc::new(index::IndexDocumentTextJob::new()),
Arc::new(ProvisionTenantJob::new()),
Arc::new(DeleteTenantJob::new()),
]
}
pub(crate) fn job_execution_from_task_error(error: TaskError) -> JobExecution {
match error {
TaskError::Fail { error } => JobExecution::Failed { error },
TaskError::Retry { delay, error } => JobExecution::Retry { delay, error },
}
}
pub(crate) fn check_worker_document_limit(
size_bytes: i64,
limit_bytes: u64,
@@ -187,35 +207,3 @@ pub(crate) async fn fetch_version_object(
.await
.map_err(FetchVersionError::Storage)
}
pub(crate) fn handle_fetch_error(
job: &crate::models::Job,
err: FetchVersionError,
message: &str,
) -> JobExecution {
match err {
FetchVersionError::TooLarge { size, limit } => {
warn!(
job_id = %job.id,
size_bytes = size,
limit_bytes = limit,
"document exceeds worker size limit"
);
JobExecution::Failed {
error: format!("document size {size} bytes exceeds worker limit of {limit} bytes"),
}
}
FetchVersionError::Storage(err) => {
warn!(
job_id = %job.id,
error = %err,
context = message,
"failed to fetch object for worker"
);
JobExecution::Retry {
delay: Duration::from_secs(30),
error: err.to_string(),
}
}
}
}
+226 -337
View File
@@ -10,174 +10,84 @@ use async_trait::async_trait;
use chrono::Utc;
use diesel::{pg::upsert::excluded, prelude::*};
use pdfium_render::prelude::*;
use serde::Deserialize;
use serde_json::json;
use tempfile::NamedTempFile;
use tokio::task;
use tracing::{error, info, warn};
use tracing::{info, warn};
use uuid::Uuid;
use crate::{
documents::asset::delete_asset,
error::AppResult,
jobs::{enqueue_job, JOB_GENERATE_OCR_TEXT, JOB_INDEX_DOCUMENT_TEXT},
models::{
Document, DocumentAsset, DocumentAssetObject, DocumentVersion, NewDocumentAsset,
NewDocumentAssetObject,
},
schema::{document_asset_objects, document_assets},
models::{Document, DocumentAsset, DocumentVersion, NewDocumentAsset},
schema::document_assets,
state::AppState,
storage::TenantStorage,
utils::storage_paths::document_asset_object_prefix,
utils::storage_paths::document_asset_key,
};
use super::{
common::{load_document_version, load_version_assets},
fetch_version_object, handle_fetch_error, JobExecution, JobHandler,
use super::taskflow::{
document::DocumentVersionTaskContext, Task, TaskContext, TaskError, TaskResult,
};
pub const OCR_TEXT_ASSET_TYPE: &str = "ocr-text";
pub const TEXT_CONTENT_ASSET_TYPE: &str = "text-content";
const MIN_TEXT_LENGTH: usize = 50;
#[derive(Clone, Debug, Deserialize)]
struct OcrPayload {
document_id: Uuid,
document_version_id: Uuid,
#[serde(default)]
pub struct GenerateOcrTask {
force: bool,
state: Arc<AppState>,
}
pub struct GenerateOcrTextJob;
impl GenerateOcrTextJob {
pub fn new() -> Self {
Self
impl GenerateOcrTask {
pub fn new(force: bool, state: Arc<AppState>) -> Self {
Self { force, state }
}
}
#[async_trait]
impl JobHandler for GenerateOcrTextJob {
fn job_type(&self) -> &'static str {
JOB_GENERATE_OCR_TEXT
impl Task<DocumentVersionTaskContext> for GenerateOcrTask {
fn name(&self) -> &'static str {
"generate-ocr-text"
}
async fn handle(
&self,
state: Arc<AppState>,
job: crate::models::Job,
storage: TenantStorage,
) -> JobExecution {
let payload: OcrPayload = match serde_json::from_value(job.payload.clone()) {
Ok(payload) => payload,
Err(err) => {
return JobExecution::Failed {
error: format!("invalid OCR payload: {err}"),
}
}
};
let state_clone = state.clone();
let payload_clone = payload.clone();
let tenant_id = job.tenant_id;
let context = match task::spawn_blocking(move || {
load_ocr_context(state_clone, tenant_id, payload_clone)
})
.await
{
Ok(Ok(ctx)) => ctx,
Ok(Err(err)) => {
warn!(job_id = %job.id, error = %err, "ocr job will retry");
return JobExecution::Retry {
delay: Duration::from_secs(30),
error: err,
};
}
Err(join_err) => {
error!(job_id = %job.id, error = %join_err, "ocr task panicked");
return JobExecution::Retry {
delay: Duration::from_secs(60),
error: format!("worker panicked: {join_err}"),
};
}
};
async fn execute(&self, ctx: &mut DocumentVersionTaskContext) -> TaskResult<()> {
let context = build_ocr_context(ctx, self.force).await?;
if context.skip {
info!(job_id = %job.id, "ocr already present; skipping");
return JobExecution::Success;
info!(job_id = %ctx.job_id(), "ocr already present; skipping");
return Ok(());
}
let bytes = match fetch_version_object(
&context.version,
&storage,
&context.version.s3_key,
state.config.worker_max_document_bytes,
)
.await
{
Ok(bytes) => bytes,
Err(err) => return handle_fetch_error(&job, err, "failed to fetch document for ocr"),
};
let doc_meta = PdfDocumentMeta {
content_type: context.document.content_type.clone(),
let bytes = ctx.buffered_object().await?.to_vec();
let meta = PdfDocumentMeta {
mime_type: context.document.mime_type.clone(),
original_name: context.document.original_name.clone(),
};
let generation =
match task::spawn_blocking(move || generate_ocr_text(&doc_meta, &bytes)).await {
Ok(result) => result,
Err(join_err) => {
error!(job_id = %job.id, error = %join_err, "ocr text task panicked");
return JobExecution::Retry {
delay: Duration::from_secs(60),
error: format!("worker panicked: {join_err}"),
};
}
};
let generation = task::spawn_blocking(move || generate_ocr_text(&meta, &bytes))
.await
.map_err(|err| {
TaskError::retry(
Duration::from_secs(60),
format!("ocr text task panicked: {err}"),
)
})?;
let Some(generation) = generation else {
warn!(job_id = %job.id, "no text extracted from document; failing job");
return JobExecution::Failed {
error: "no text extracted and OCR unavailable".into(),
};
warn!(job_id = %ctx.job_id(), "no text extracted from document; failing job");
return Err(TaskError::fail("no text extracted and OCR unavailable"));
};
if let Some(existing_asset) = &context.existing_asset {
for object in &context.existing_objects {
if let Err(err) = storage.delete_object(&object.s3_key).await {
warn!(job_id = %job.id, error = %err, s3_key = %object.s3_key, "failed to delete existing ocr asset object");
}
}
let tenant_id = context.document.tenant_id;
let asset_id = existing_asset.id;
let state_clone = state.clone();
match task::spawn_blocking(move || -> AppResult<()> {
let mut conn = state_clone.db_for_tenant(tenant_id)?;
delete_asset(&mut conn, tenant_id, asset_id)
})
.await
{
Ok(Ok(())) => {}
Ok(Err(err)) => {
warn!(job_id = %job.id, error = ?err, asset_id = %asset_id, "failed to remove ocr asset metadata after deletion");
}
Err(join_err) => {
warn!(job_id = %job.id, error = %join_err, asset_id = %asset_id, "failed to remove ocr asset metadata: task panicked");
}
}
}
remove_existing_ocr_asset(ctx, &context).await;
let asset_id = Uuid::new_v4();
let s3_key = document_asset_object_prefix(
let s3_key = document_asset_key(
context.document.id,
context.version.version_number,
OCR_TEXT_ASSET_TYPE,
TEXT_CONTENT_ASSET_TYPE,
asset_id,
);
if let Err(err) = storage
ctx.storage()
.put_object(
&s3_key,
generation.text.into_bytes(),
@@ -185,112 +95,163 @@ impl JobHandler for GenerateOcrTextJob {
None,
)
.await
{
warn!(job_id = %job.id, error = %err, "failed to upload ocr text");
return JobExecution::Retry {
delay: Duration::from_secs(30),
error: err.to_string(),
};
}
.map_err(|err| TaskError::retry(Duration::from_secs(30), err.to_string()))?;
let state_clone = state.clone();
match task::spawn_blocking(move || {
persist_ocr_metadata(state_clone, &context, asset_id, &s3_key, generation.source)
let state = self.state.clone();
task::spawn_blocking(move || {
persist_ocr_metadata(state, &context, asset_id, &s3_key, generation.source)
})
.await
{
Ok(Ok(())) => {
if let Err(err) = enqueue_index_job(&state, job.tenant_id, &payload) {
warn!(job_id = %job.id, error = %err, "failed to enqueue index job");
}
JobExecution::Success
}
Ok(Err(err)) => {
warn!(job_id = %job.id, error = %err, "failed to persist ocr metadata");
JobExecution::Retry {
delay: Duration::from_secs(30),
error: err,
}
}
Err(join_err) => {
error!(job_id = %job.id, error = %join_err, "ocr metadata task panicked");
JobExecution::Retry {
delay: Duration::from_secs(30),
error: format!("metadata update panic: {join_err}"),
}
}
}
}
}
.map_err(|err| {
TaskError::retry(
Duration::from_secs(60),
format!("ocr metadata task panicked: {err}"),
)
})?
.map_err(|err| TaskError::retry(Duration::from_secs(30), err))?;
struct PdfDocumentMeta {
content_type: Option<String>,
original_name: String,
ctx.invalidate_asset_cache();
Ok(())
}
}
struct OcrContext {
document: Document,
version: DocumentVersion,
existing_asset: Option<DocumentAsset>,
existing_objects: Vec<DocumentAssetObject>,
skip: bool,
}
struct OcrGeneration {
text: String,
source: &'static str,
}
async fn build_ocr_context(
ctx: &mut DocumentVersionTaskContext,
force: bool,
) -> TaskResult<OcrContext> {
let document = ctx.document().await?.clone();
let version = ctx.version().await?.clone();
let asset = ctx.asset(TEXT_CONTENT_ASSET_TYPE).await?;
fn load_ocr_context(
state: Arc<AppState>,
tenant_id: Uuid,
payload: OcrPayload,
) -> Result<OcrContext, String> {
let base = load_document_version(
state.as_ref(),
tenant_id,
payload.document_id,
payload.document_version_id,
)?;
let existing_asset = asset.map(|asset| asset.asset.clone());
let mut conn = state
.db_for_tenant(base.tenant_id)
.map_err(|err| format!("{err:?}"))?;
let mut assets = load_version_assets(
&mut conn,
base.tenant_id,
base.version.id,
&[OCR_TEXT_ASSET_TYPE],
)?;
let (existing_asset, existing_objects) = assets
.remove(OCR_TEXT_ASSET_TYPE)
.map(|entry| (Some(entry.asset), entry.objects))
.unwrap_or((None, Vec::new()));
let is_pdf = document_is_pdf(&base.document);
if !is_pdf {
if !document_is_pdf(&document) {
return Ok(OcrContext {
document: base.document,
version: base.version,
document,
version,
existing_asset,
existing_objects,
skip: true,
});
}
let skip = existing_asset.is_some() && !payload.force;
let skip = existing_asset.is_some() && !force;
Ok(OcrContext {
document: base.document,
version: base.version,
document,
version,
existing_asset,
existing_objects,
skip,
})
}
async fn remove_existing_ocr_asset(ctx: &DocumentVersionTaskContext, context: &OcrContext) {
if let Some(existing_asset) = &context.existing_asset {
if let Err(err) = ctx.storage().delete_object(&existing_asset.s3_key).await {
warn!(
job_id = %ctx.job_id(),
error = %err,
s3_key = %existing_asset.s3_key,
"failed to delete existing ocr asset object"
);
}
let tenant_id = context.document.tenant_id;
let asset_id = existing_asset.id;
let state = ctx.state().clone();
match task::spawn_blocking(move || -> AppResult<()> {
let mut conn = state.db_for_tenant(tenant_id)?;
delete_asset(&mut conn, tenant_id, asset_id)
})
.await
{
Ok(Ok(())) => {}
Ok(Err(err)) => {
warn!(
job_id = %ctx.job_id(),
error = ?err,
asset_id = %asset_id,
"failed to remove ocr asset metadata after deletion"
);
}
Err(join_err) => {
warn!(
job_id = %ctx.job_id(),
error = %join_err,
asset_id = %asset_id,
"failed to remove ocr asset metadata: task panicked"
);
}
}
}
}
fn persist_ocr_metadata(
state: Arc<AppState>,
context: &OcrContext,
asset_id: Uuid,
s3_key: &str,
source: OcrSource,
) -> Result<(), String> {
let tenant_id = context.document.tenant_id;
let mut conn = state
.db_for_tenant(tenant_id)
.map_err(|err| format!("{err:?}"))?;
let document_version_id = context.version.id;
let existing_asset = context.existing_asset.as_ref().map(|asset| asset.id);
if let Some(existing_asset) = existing_asset {
diesel::delete(
document_assets::table
.filter(document_assets::id.eq(existing_asset))
.filter(document_assets::tenant_id.eq(tenant_id)),
)
.execute(&mut conn)
.map_err(|err| format!("{err:?}"))?;
}
let metadata = json!({
"source": source.to_string(),
"generated_at": Utc::now().to_rfc3339(),
});
let new_asset = NewDocumentAsset {
id: asset_id,
document_version_id,
asset_type: TEXT_CONTENT_ASSET_TYPE.to_string(),
mime_type: "text/plain".to_string(),
metadata,
s3_key: s3_key.to_string(),
tenant_id,
};
diesel::insert_into(document_assets::table)
.values(&new_asset)
.on_conflict((
document_assets::document_version_id,
document_assets::asset_type,
))
.do_update()
.set((
document_assets::mime_type.eq(excluded(document_assets::mime_type)),
document_assets::metadata.eq(excluded(document_assets::metadata)),
document_assets::s3_key.eq(excluded(document_assets::s3_key)),
document_assets::id.eq(excluded(document_assets::id)),
))
.execute(&mut conn)
.map_err(|err| format!("{err:?}"))?;
Ok(())
}
fn generate_ocr_text(meta: &PdfDocumentMeta, bytes: &[u8]) -> Option<OcrGeneration> {
if !document_meta_is_pdf(meta) {
return None;
@@ -300,7 +261,7 @@ fn generate_ocr_text(meta: &PdfDocumentMeta, bytes: &[u8]) -> Option<OcrGenerati
if text.trim().chars().count() >= MIN_TEXT_LENGTH {
return Some(OcrGeneration {
text,
source: "pdf-text",
source: OcrSource::PdfText,
});
}
}
@@ -308,20 +269,60 @@ fn generate_ocr_text(meta: &PdfDocumentMeta, bytes: &[u8]) -> Option<OcrGenerati
match run_ocr(bytes) {
Ok(Some(text)) => Some(OcrGeneration {
text,
source: "ocr",
source: OcrSource::Ocr,
}),
Ok(None) => None,
Err(OcrError::BinaryMissing) => {
warn!("ocrmypdf not installed; cannot perform OCR");
warn!("ocrmypdf binary not found; OCR unavailable");
None
}
Err(err) => {
warn!(error = ?err, "ocr command failed");
warn!(error = %err, "ocr command failed");
None
}
}
}
struct PdfDocumentMeta {
mime_type: Option<String>,
original_name: String,
}
struct OcrGeneration {
text: String,
source: OcrSource,
}
#[derive(Clone, Copy)]
enum OcrSource {
PdfText,
Ocr,
}
impl fmt::Display for OcrSource {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
OcrSource::PdfText => write!(f, "pdf-text"),
OcrSource::Ocr => write!(f, "ocr"),
}
}
}
#[derive(Debug)]
enum OcrError {
BinaryMissing,
Failed(String),
}
impl fmt::Display for OcrError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
OcrError::BinaryMissing => write!(f, "ocrmypdf binary not found"),
OcrError::Failed(msg) => write!(f, "ocr failed: {msg}"),
}
}
}
fn extract_pdf_text(bytes: &[u8]) -> Result<String, String> {
let pdfium = Pdfium::default();
let document = pdfium
@@ -345,21 +346,6 @@ fn extract_pdf_text(bytes: &[u8]) -> Result<String, String> {
Ok(combined)
}
#[derive(Debug)]
enum OcrError {
BinaryMissing,
Failed(String),
}
impl fmt::Display for OcrError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
OcrError::BinaryMissing => write!(f, "ocrmypdf binary not found"),
OcrError::Failed(msg) => write!(f, "ocr failed: {msg}"),
}
}
}
fn run_ocr(bytes: &[u8]) -> Result<Option<String>, OcrError> {
let mut input = NamedTempFile::new().map_err(|err| OcrError::Failed(err.to_string()))?;
input
@@ -408,121 +394,9 @@ fn run_ocr(bytes: &[u8]) -> Result<Option<String>, OcrError> {
}
}
fn persist_ocr_metadata(
state: Arc<AppState>,
context: &OcrContext,
asset_id: Uuid,
s3_key: &str,
source: &'static str,
) -> Result<(), String> {
let tenant_id = context.document.tenant_id;
let mut conn = state
.db_for_tenant(tenant_id)
.map_err(|err| format!("{err:?}"))?;
if let Some(existing_asset) = &context.existing_asset {
diesel::delete(document_assets::table.filter(document_assets::id.eq(existing_asset.id)))
.execute(&mut conn)
.map_err(|err| format!("{err:?}"))?;
}
let new_asset = NewDocumentAsset {
id: asset_id,
document_version_id: context.version.id,
asset_type: OCR_TEXT_ASSET_TYPE.to_string(),
mime_type: "text/plain".to_string(),
metadata: json!({
"generated_at": Utc::now().to_rfc3339(),
"source": source,
}),
cardinality: Some(1),
tenant_id,
};
diesel::insert_into(document_assets::table)
.values(&new_asset)
.on_conflict((
document_assets::document_version_id,
document_assets::asset_type,
))
.do_update()
.set((
document_assets::mime_type.eq(excluded(document_assets::mime_type)),
document_assets::metadata.eq(excluded(document_assets::metadata)),
document_assets::cardinality.eq(excluded(document_assets::cardinality)),
))
.execute(&mut conn)
.map_err(|err| format!("{err:?}"))?;
let existing_object_id: Option<Uuid> = document_asset_objects::table
.filter(document_asset_objects::asset_id.eq(asset_id))
.filter(document_asset_objects::ordinal.eq(1))
.filter(document_asset_objects::tenant_id.eq(tenant_id))
.select(document_asset_objects::id)
.first(&mut conn)
.optional()
.map_err(|err| format!("{err:?}"))?;
let object_id = existing_object_id.unwrap_or_else(Uuid::new_v4);
let new_object = NewDocumentAssetObject {
id: object_id,
asset_id,
ordinal: 1,
s3_key: s3_key.to_string(),
metadata: json!({}),
tenant_id,
};
diesel::insert_into(document_asset_objects::table)
.values(&new_object)
.on_conflict((
document_asset_objects::asset_id,
document_asset_objects::ordinal,
))
.do_update()
.set((
document_asset_objects::s3_key.eq(excluded(document_asset_objects::s3_key)),
document_asset_objects::metadata.eq(excluded(document_asset_objects::metadata)),
))
.execute(&mut conn)
.map_err(|err| format!("{err:?}"))?;
Ok(())
}
fn enqueue_index_job(
state: &AppState,
tenant_id: Uuid,
payload: &OcrPayload,
) -> Result<(), String> {
let mut conn = state
.db_for_tenant(tenant_id)
.map_err(|err| format!("{err:?}"))?;
enqueue_job(
&mut conn,
tenant_id,
JOB_INDEX_DOCUMENT_TEXT,
json!({
"document_id": payload.document_id,
"document_version_id": payload.document_version_id,
}),
None,
)
.map(|_| ())
.map_err(|err| err.to_string())
}
pub fn document_is_pdf(document: &Document) -> bool {
document_meta_is_pdf(&PdfDocumentMeta {
content_type: document.content_type.clone(),
original_name: document.original_name.clone(),
})
}
fn document_meta_is_pdf(meta: &PdfDocumentMeta) -> bool {
if let Some(content_type) = &meta.content_type {
if content_type.eq_ignore_ascii_case("application/pdf") {
if let Some(mime_type) = &meta.mime_type {
if mime_type.eq_ignore_ascii_case("application/pdf") {
return true;
}
}
@@ -533,3 +407,18 @@ fn document_meta_is_pdf(meta: &PdfDocumentMeta) -> bool {
.map(|ext| ext.eq_ignore_ascii_case("pdf"))
.unwrap_or(false)
}
fn document_is_pdf(document: &Document) -> bool {
document
.mime_type
.as_deref()
.map(|mime| mime.eq_ignore_ascii_case("application/pdf"))
.unwrap_or_else(|| {
document
.original_name
.rsplit('.')
.next()
.map(|ext| ext.eq_ignore_ascii_case("pdf"))
.unwrap_or(false)
})
}
+131 -72
View File
@@ -6,16 +6,20 @@ use async_trait::async_trait;
use diesel::prelude::*;
use diesel::result::Error as DieselError;
use serde::Deserialize;
use tracing::{error, warn};
use uuid::Uuid;
use crate::auth::ensure_active_tenant;
use crate::jobs::JOB_PURGE_DOCUMENT;
use crate::models::{Document, DocumentVersion};
use crate::schema::{document_asset_objects, document_assets, document_versions};
use crate::schema::{document_assets, document_versions};
use crate::state::AppState;
use crate::storage::TenantStorage;
use super::{JobExecution, JobHandler};
use super::{
job_execution_from_task_error,
taskflow::{BoxedTask, Task, TaskContext, TaskError, TaskExecutor, TaskPlanner, TaskResult},
JobExecution, JobHandler,
};
#[derive(Debug, Deserialize)]
struct PurgeDocumentPayload {
@@ -49,6 +53,21 @@ impl JobHandler for PurgeDocumentJob {
job: crate::models::Job,
storage: TenantStorage,
) -> JobExecution {
let tenant_id = match job.tenant_id {
Some(id) => id,
None => {
return JobExecution::Failed {
error: "job is no longer associated with a tenant".to_string(),
}
}
};
if let Err(err) = ensure_active_tenant(&state, tenant_id) {
return JobExecution::Failed {
error: err.to_string(),
};
}
let payload: PurgeDocumentPayload = match serde_json::from_value(job.payload.clone()) {
Ok(payload) => payload,
Err(err) => {
@@ -58,70 +77,120 @@ impl JobHandler for PurgeDocumentJob {
}
};
let tenant_id = job.tenant_id;
let document_id = payload.document_id;
let state_for_prepare = state.clone();
let mut context = PurgeTaskContext::new(
job.id,
JOB_PURGE_DOCUMENT,
tenant_id,
payload.document_id,
state.clone(),
storage,
);
let planner = PurgePlanner;
match TaskExecutor::run(&planner, &mut context).await {
Ok(()) => JobExecution::Success,
Err(err) => job_execution_from_task_error(err),
}
}
}
struct PurgeTaskContext {
job_id: Uuid,
job_type: &'static str,
tenant_id: Uuid,
document_id: Uuid,
state: Arc<AppState>,
storage: TenantStorage,
}
impl PurgeTaskContext {
fn new(
job_id: Uuid,
job_type: &'static str,
tenant_id: Uuid,
document_id: Uuid,
state: Arc<AppState>,
storage: TenantStorage,
) -> Self {
Self {
job_id,
job_type,
tenant_id,
document_id,
state,
storage,
}
}
}
impl TaskContext for PurgeTaskContext {
fn job_id(&self) -> Uuid {
self.job_id
}
fn job_type(&self) -> &'static str {
self.job_type
}
}
struct PurgePlanner;
#[async_trait]
impl TaskPlanner<PurgeTaskContext> for PurgePlanner {
async fn plan(
&self,
_ctx: &mut PurgeTaskContext,
) -> TaskResult<Vec<BoxedTask<PurgeTaskContext>>> {
Ok(vec![Box::new(PurgeTask)])
}
}
struct PurgeTask;
#[async_trait]
impl Task<PurgeTaskContext> for PurgeTask {
fn name(&self) -> &'static str {
"purge-document"
}
async fn execute(&self, ctx: &mut PurgeTaskContext) -> TaskResult<()> {
let tenant_id = ctx.tenant_id;
let document_id = ctx.document_id;
let state = ctx.state.clone();
let preparation = tokio::task::spawn_blocking(move || {
prepare_purge_context(state_for_prepare, tenant_id, document_id)
prepare_purge_context(state, tenant_id, document_id)
})
.await;
.await
.map_err(|err| {
TaskError::retry(
Duration::from_secs(60),
format!("purge preparation panicked: {err}"),
)
})?;
let context = match preparation {
Ok(Ok(Some(ctx))) => ctx,
Ok(Ok(None)) => {
// Document already gone or restored; nothing to do.
return JobExecution::Success;
}
Ok(Err(err)) => {
warn!(job_id = %job.id, error = %err, "purge preparation failed");
return JobExecution::Retry {
delay: Duration::from_secs(30),
error: err,
};
}
Err(join_err) => {
error!(job_id = %job.id, error = %join_err, "purge preparation task panicked");
return JobExecution::Retry {
delay: Duration::from_secs(60),
error: format!("purge preparation panicked: {join_err}"),
};
}
let Some(context) =
preparation.map_err(|err| TaskError::retry(Duration::from_secs(30), err))?
else {
return Ok(());
};
if let Err(err) = delete_storage_objects(&storage, &context).await {
warn!(job_id = %job.id, error = %err, "failed to delete storage objects for purge");
return JobExecution::Retry {
delay: Duration::from_secs(30),
error: err,
};
}
delete_storage_objects(&ctx.storage, &context)
.await
.map_err(|err| TaskError::retry(Duration::from_secs(30), err))?;
let PurgeContext { document_id, .. } = context;
let state_for_finalize = state.clone();
let state = ctx.state.clone();
tokio::task::spawn_blocking(move || finalize_purge(state, tenant_id, context.document_id))
.await
.map_err(|err| {
TaskError::retry(
Duration::from_secs(60),
format!("purge finalize panicked: {err}"),
)
})?
.map_err(|err| TaskError::retry(Duration::from_secs(30), err))?;
let finalize = tokio::task::spawn_blocking(move || {
finalize_purge(state_for_finalize, tenant_id, document_id)
})
.await;
match finalize {
Ok(Ok(())) => JobExecution::Success,
Ok(Err(err)) => {
warn!(job_id = %job.id, error = %err, "failed to finalize purge");
JobExecution::Retry {
delay: Duration::from_secs(30),
error: err,
}
}
Err(join_err) => {
error!(job_id = %job.id, error = %join_err, "purge finalize task panicked");
JobExecution::Retry {
delay: Duration::from_secs(60),
error: format!("purge finalize panicked: {join_err}"),
}
}
}
Ok(())
}
}
@@ -166,21 +235,11 @@ fn prepare_purge_context(
let asset_keys = if version_ids.is_empty() {
Vec::new()
} else {
let asset_ids: Vec<Uuid> = document_assets::table
document_assets::table
.filter(document_assets::document_version_id.eq_any(&version_ids))
.filter(document_assets::tenant_id.eq(tenant_id))
.select(document_assets::id)
.load(conn)?;
if asset_ids.is_empty() {
Vec::new()
} else {
document_asset_objects::table
.filter(document_asset_objects::asset_id.eq_any(&asset_ids))
.filter(document_asset_objects::tenant_id.eq(tenant_id))
.select(document_asset_objects::s3_key)
.load(conn)?
}
.select(document_assets::s3_key)
.load(conn)?
};
Ok(Some(PurgeContext {
+225
View File
@@ -0,0 +1,225 @@
use std::collections::HashMap;
use std::sync::Arc;
use std::time::Duration;
use tokio::task;
use uuid::Uuid;
use crate::models::{Document, DocumentVersion};
use crate::state::AppState;
use crate::storage::TenantStorage;
use crate::workers::common::{load_document_version, load_version_assets, LoadedAsset};
use crate::workers::{check_worker_document_limit, fetch_version_object, FetchVersionError};
use super::{TaskContext, TaskError, TaskResult};
const BLOCKING_RETRY_DELAY: Duration = Duration::from_secs(60);
const DEFAULT_RETRY_DELAY: Duration = Duration::from_secs(30);
pub struct DocumentVersionTaskContext {
job_id: Uuid,
job_type: &'static str,
tenant_id: Uuid,
document_id: Uuid,
document_version_id: Uuid,
force: bool,
max_document_bytes: u64,
state: Arc<AppState>,
storage: TenantStorage,
document: Option<Document>,
version: Option<DocumentVersion>,
assets: Option<HashMap<String, LoadedAsset>>, // keyed by asset_type
object_bytes: Option<Vec<u8>>,
}
impl DocumentVersionTaskContext {
#[allow(clippy::too_many_arguments)]
pub fn new(
job_id: Uuid,
job_type: &'static str,
tenant_id: Uuid,
document_id: Uuid,
document_version_id: Uuid,
force: bool,
max_document_bytes: u64,
state: Arc<AppState>,
storage: TenantStorage,
) -> Self {
Self {
job_id,
job_type,
tenant_id,
document_id,
document_version_id,
force,
max_document_bytes,
state,
storage,
document: None,
version: None,
assets: None,
object_bytes: None,
}
}
pub fn tenant_id(&self) -> Uuid {
self.tenant_id
}
pub fn document_id(&self) -> Uuid {
self.document_id
}
pub fn version_id(&self) -> Uuid {
self.document_version_id
}
pub fn force(&self) -> bool {
self.force
}
pub fn storage(&self) -> &TenantStorage {
&self.storage
}
pub fn invalidate_asset_cache(&mut self) {
self.assets = None;
}
pub fn set_document_mime(&mut self, mime: Option<String>) {
if let Some(document) = self.document.as_mut() {
document.mime_type = mime;
}
}
pub fn state(&self) -> &Arc<AppState> {
&self.state
}
pub fn max_document_bytes(&self) -> u64 {
self.max_document_bytes
}
pub async fn document(&mut self) -> TaskResult<&Document> {
self.ensure_document_loaded().await?;
Ok(self.document.as_ref().expect("document hydrated"))
}
pub async fn version(&mut self) -> TaskResult<&DocumentVersion> {
self.ensure_document_loaded().await?;
Ok(self.version.as_ref().expect("version hydrated"))
}
pub async fn assets(&mut self) -> TaskResult<&HashMap<String, LoadedAsset>> {
if self.assets.is_none() {
let tenant_id = self.tenant_id;
let version_id = self.document_version_id;
let state = self.state.clone();
let result = task::spawn_blocking(move || {
let mut conn = state
.db_for_tenant(tenant_id)
.map_err(|err| format!("failed to scope tenant connection: {err:?}"))?;
load_version_assets(&mut conn, tenant_id, version_id, &[])
})
.await
.map_err(|err| {
TaskError::retry(
BLOCKING_RETRY_DELAY,
format!("asset load task panicked: {err}"),
)
})?
.map_err(|err| TaskError::retry(DEFAULT_RETRY_DELAY, err))?;
self.assets = Some(result);
}
Ok(self.assets.as_ref().expect("asset map hydrated"))
}
pub async fn asset(&mut self, asset_type: &str) -> TaskResult<Option<&LoadedAsset>> {
let assets = self.assets().await?;
Ok(assets.get(asset_type))
}
pub async fn buffered_object(&mut self) -> TaskResult<&[u8]> {
if self.object_bytes.is_some() {
return Ok(self.object_bytes.as_deref().expect("bytes present"));
}
let version = self.version().await?.clone();
let bytes = fetch_version_object(
&version,
&self.storage,
&version.s3_key,
self.max_document_bytes,
)
.await
.map_err(|err| match err {
FetchVersionError::TooLarge { size, limit } => TaskError::fail(format!(
"document size {size} bytes exceeds worker limit of {limit} bytes"
)),
FetchVersionError::Storage(err) => TaskError::retry(
DEFAULT_RETRY_DELAY,
format!("failed to fetch object: {err}"),
),
})?;
self.object_bytes = Some(bytes);
Ok(self.object_bytes.as_deref().expect("bytes hydrated"))
}
pub async fn object_head(&mut self, max_bytes: usize) -> TaskResult<Vec<u8>> {
let version = self.version().await?.clone();
check_worker_document_limit(version.size_bytes, self.max_document_bytes).map_err(
|(size, limit)| {
TaskError::fail(format!(
"document size {size} bytes exceeds worker limit of {limit} bytes"
))
},
)?;
let end = max_bytes.saturating_sub(1) as u64;
self.storage
.get_object_range(&version.s3_key, 0, Some(end))
.await
.map_err(|err| {
TaskError::retry(
DEFAULT_RETRY_DELAY,
format!("failed to fetch ranged object: {err}"),
)
})
}
async fn ensure_document_loaded(&mut self) -> TaskResult<()> {
if self.document.is_some() && self.version.is_some() {
return Ok(());
}
let tenant_id = self.tenant_id;
let document_id = self.document_id;
let version_id = self.document_version_id;
let state = self.state.clone();
let loaded = task::spawn_blocking(move || {
load_document_version(state.as_ref(), tenant_id, document_id, version_id)
})
.await
.map_err(|err| {
TaskError::retry(
BLOCKING_RETRY_DELAY,
format!("document load task panicked: {err}"),
)
})?
.map_err(|err| TaskError::fail(format!("failed to load document context: {err}")))?;
self.document = Some(loaded.document);
self.version = Some(loaded.version);
Ok(())
}
}
impl TaskContext for DocumentVersionTaskContext {
fn job_id(&self) -> Uuid {
self.job_id
}
fn job_type(&self) -> &'static str {
self.job_type
}
}
+85
View File
@@ -0,0 +1,85 @@
use std::time::Duration;
use async_trait::async_trait;
use thiserror::Error;
use tracing::info;
use uuid::Uuid;
pub mod document;
pub type TaskResult<T> = Result<T, TaskError>;
#[derive(Debug, Error)]
pub enum TaskError {
#[error("{error}")]
Fail { error: String },
#[error("{error}")]
Retry { delay: Duration, error: String },
}
impl TaskError {
pub fn fail(error: impl Into<String>) -> Self {
Self::Fail {
error: error.into(),
}
}
pub fn retry(delay: Duration, error: impl Into<String>) -> Self {
Self::Retry {
delay,
error: error.into(),
}
}
}
pub trait TaskContext: Send + Sync {
fn job_id(&self) -> Uuid;
fn job_type(&self) -> &'static str;
}
#[async_trait]
pub trait Task<Ctx>: Send + Sync
where
Ctx: TaskContext,
{
fn name(&self) -> &'static str;
async fn execute(&self, ctx: &mut Ctx) -> TaskResult<()>;
}
pub type BoxedTask<Ctx> = Box<dyn Task<Ctx> + Send + Sync>;
#[async_trait]
pub trait TaskPlanner<Ctx>: Send + Sync
where
Ctx: TaskContext,
{
async fn plan(&self, ctx: &mut Ctx) -> TaskResult<Vec<BoxedTask<Ctx>>>;
}
pub struct TaskExecutor;
impl TaskExecutor {
pub async fn run<P, C>(planner: &P, ctx: &mut C) -> TaskResult<()>
where
P: TaskPlanner<C>,
C: TaskContext,
{
let tasks = planner.plan(ctx).await?;
for task in tasks {
info!(
job_id = %ctx.job_id(),
job_type = ctx.job_type(),
task = task.name(),
"starting job task"
);
task.execute(ctx).await?;
info!(
job_id = %ctx.job_id(),
job_type = ctx.job_type(),
task = task.name(),
"finished job task"
);
}
Ok(())
}
}
+550 -158
View File
@@ -1,24 +1,41 @@
use std::sync::Arc;
use std::time::Duration;
use async_trait::async_trait;
use chrono::Utc;
use chrono::{DateTime, Utc};
use diesel::prelude::*;
use diesel::sql_types::Jsonb;
use hmac::{Hmac, Mac};
use reqwest::Client;
use serde::Deserialize;
use serde_json::json;
use tracing::warn;
use uuid::Uuid;
use sha2::Sha256;
use crate::auth::capability_sets::{
ensure_capability_set, owner_capabilities, readonly_capabilities, user_capabilities,
webdav_capabilities,
};
use crate::documents::search::ensure_quickwit_index;
use crate::jobs::JOB_PROVISION_TENANT;
use crate::models::{NewUserMembership, TenantStatus};
use crate::schema::{tenants, user_memberships};
use crate::documents::search::{delete_quickwit_index, ensure_quickwit_index};
use crate::jobs::{JOB_DELETE_TENANT, JOB_PROVISION_TENANT};
use crate::models::{NewUserMembership, Tenant, TenantStatus};
use crate::schema::{
api_tokens, correspondents, document_assets, document_correspondents, document_tags,
document_versions, documents, folders, tags, tenants, user_memberships, user_sessions,
};
use crate::state::AppState;
use crate::tenants::TenantRepository;
use crate::workers::{JobExecution, JobHandler};
use crate::workers::{
job_execution_from_task_error,
taskflow::{BoxedTask, Task, TaskContext, TaskError, TaskExecutor, TaskPlanner, TaskResult},
JobExecution, JobHandler,
};
type HmacSha256 = Hmac<Sha256>;
const DELETE_PROOF_TTL_SECONDS: i64 = 300;
const DELETE_PROOF_VERSION: &str = "v1";
pub struct ProvisionTenantJob;
@@ -40,77 +57,129 @@ impl JobHandler for ProvisionTenantJob {
job: crate::models::Job,
_storage: crate::storage::TenantStorage,
) -> JobExecution {
let mut conn = match state.db_unscoped() {
Ok(conn) => conn,
Err(err) => {
warn!(job_id = %job.id, error = ?err, "failed to get connection for tenant provisioning");
return JobExecution::Retry {
delay: std::time::Duration::from_secs(30),
error: "database connection unavailable".into(),
};
}
};
let tenant = match TenantRepository::get_by_id(&mut conn, job.tenant_id) {
Ok(tenant) => tenant,
Err(err) => {
warn!(job_id = %job.id, error = ?err, "tenant not found for provisioning");
let tenant_id = match job.tenant_id {
Some(id) => id,
None => {
return JobExecution::Failed {
error: "tenant not found".into(),
};
error: "provision job is missing tenant context".to_string(),
}
}
};
let members = ProvisionPayload::from_job(&job).unwrap_or_default();
let mut context = ProvisionContext::new(
job.id,
JOB_PROVISION_TENANT,
tenant_id,
state.clone(),
members,
);
let planner = ProvisionPlanner;
match TaskExecutor::run(&planner, &mut context).await {
Ok(()) => JobExecution::Success,
Err(err) => job_execution_from_task_error(err),
}
}
}
struct ProvisionContext {
job_id: Uuid,
job_type: &'static str,
tenant_id: Uuid,
state: Arc<AppState>,
members: Vec<Uuid>,
}
impl ProvisionContext {
fn new(
job_id: Uuid,
job_type: &'static str,
tenant_id: Uuid,
state: Arc<AppState>,
members: Vec<Uuid>,
) -> Self {
Self {
job_id,
job_type,
tenant_id,
state,
members,
}
}
}
impl TaskContext for ProvisionContext {
fn job_id(&self) -> Uuid {
self.job_id
}
fn job_type(&self) -> &'static str {
self.job_type
}
}
struct ProvisionPlanner;
#[async_trait]
impl TaskPlanner<ProvisionContext> for ProvisionPlanner {
async fn plan(
&self,
_ctx: &mut ProvisionContext,
) -> TaskResult<Vec<BoxedTask<ProvisionContext>>> {
Ok(vec![Box::new(ProvisionTask)])
}
}
struct ProvisionTask;
#[async_trait]
impl Task<ProvisionContext> for ProvisionTask {
fn name(&self) -> &'static str {
"provision-tenant"
}
async fn execute(&self, ctx: &mut ProvisionContext) -> TaskResult<()> {
let mut conn = ctx
.state
.db_unscoped()
.map_err(|err| TaskError::retry(Duration::from_secs(30), format!("{err:?}")))?;
let tenant = TenantRepository::get_by_id(&mut conn, ctx.tenant_id).map_err(|err| {
TaskError::fail(format!("tenant not found for provisioning: {err:?}"))
})?;
drop(conn);
let mut conn = match state.db_for_tenant(tenant.id) {
Ok(conn) => conn,
Err(err) => {
warn!(job_id = %job.id, error = ?err, "failed to scope connection for tenant provisioning");
return JobExecution::Retry {
delay: std::time::Duration::from_secs(30),
error: "tenant connection unavailable".into(),
};
}
};
let mut conn = ctx
.state
.db_for_tenant(tenant.id)
.map_err(|err| TaskError::retry(Duration::from_secs(30), format!("{err:?}")))?;
if tenant.status == TenantStatus::Active {
warn!(
job_id = %job.id,
job_id = %ctx.job_id(),
tenant_id = %tenant.id,
"tenant already active; skipping provisioning"
);
return JobExecution::Success;
return Ok(());
}
if tenant.status != TenantStatus::Creating {
warn!(
job_id = %job.id,
tenant_id = %tenant.id,
status = %tenant.status.as_str(),
"tenant not in creating state; skipping provisioning"
);
return JobExecution::Failed {
error: format!(
"tenant status '{}' not eligible for provisioning",
tenant.status.as_str()
),
};
return Err(TaskError::fail(format!(
"tenant status '{}' not eligible for provisioning",
tenant.status.as_str()
)));
}
let endpoint = match &state.config.quickwit_endpoint {
Some(endpoint) => endpoint.trim_end_matches('/').to_owned(),
None => {
warn!(
job_id = %job.id,
tenant_id = %tenant.id,
"quickwit endpoint not configured; retrying"
);
return JobExecution::Retry {
delay: std::time::Duration::from_secs(30),
error: "quickwit endpoint not configured".into(),
};
}
};
let endpoint = ctx
.state
.config
.quickwit_endpoint
.as_ref()
.map(|value| value.trim_end_matches('/').to_owned())
.ok_or_else(|| {
TaskError::retry(Duration::from_secs(30), "quickwit endpoint not configured")
})?;
let index_id = tenant
.quickwit_index
@@ -119,120 +188,306 @@ impl JobHandler for ProvisionTenantJob {
.unwrap_or_else(|| format!("documents-{}", tenant.id));
let client = Client::new();
if let Err(err) = ensure_quickwit_index(&client, &endpoint, &index_id).await {
warn!(
job_id = %job.id,
tenant_id = %tenant.id,
error = %err,
"failed to ensure quickwit index"
);
return JobExecution::Retry {
delay: std::time::Duration::from_secs(30),
error: err.to_string(),
};
}
ensure_quickwit_index(&client, &endpoint, &index_id)
.await
.map_err(|err| TaskError::retry(Duration::from_secs(30), err.to_string()))?;
let owner_capability_set_id =
match ensure_capability_set(&mut conn, tenant.id, owner_capabilities()) {
Ok(set) => set.id,
Err(err) => {
warn!(
job_id = %job.id,
tenant_id = %tenant.id,
error = ?err,
"failed to ensure owner capability set during provisioning"
);
return JobExecution::Retry {
delay: std::time::Duration::from_secs(30),
error: "owner capability set unavailable".into(),
};
}
ensure_capability_set(&mut conn, tenant.id, owner_capabilities())
.map_err(|_| {
TaskError::retry(Duration::from_secs(30), "owner capability set unavailable")
})?
.id;
ensure_capability_set(&mut conn, tenant.id, user_capabilities()).map_err(|_| {
TaskError::retry(Duration::from_secs(30), "user capability set unavailable")
})?;
ensure_capability_set(&mut conn, tenant.id, readonly_capabilities()).map_err(|_| {
TaskError::retry(
Duration::from_secs(30),
"readonly capability set unavailable",
)
})?;
ensure_capability_set(&mut conn, tenant.id, webdav_capabilities()).map_err(|_| {
TaskError::retry(Duration::from_secs(30), "webdav capability set unavailable")
})?;
for member in &ctx.members {
let new_membership = NewUserMembership {
id: Uuid::new_v4(),
user_id: *member,
tenant_id: tenant.id,
capability_set_id: Some(owner_capability_set_id),
};
if let Err(err) = ensure_capability_set(&mut conn, tenant.id, user_capabilities()) {
warn!(
job_id = %job.id,
tenant_id = %tenant.id,
error = ?err,
"failed to ensure user capability set during provisioning"
);
return JobExecution::Retry {
delay: std::time::Duration::from_secs(30),
error: "user capability set unavailable".into(),
};
}
if let Err(err) = ensure_capability_set(&mut conn, tenant.id, readonly_capabilities()) {
warn!(
job_id = %job.id,
tenant_id = %tenant.id,
error = ?err,
"failed to ensure readonly capability set during provisioning"
);
return JobExecution::Retry {
delay: std::time::Duration::from_secs(30),
error: "readonly capability set unavailable".into(),
};
}
if let Err(err) = ensure_capability_set(&mut conn, tenant.id, webdav_capabilities()) {
warn!(
job_id = %job.id,
tenant_id = %tenant.id,
error = ?err,
"failed to ensure webdav capability set during provisioning"
);
return JobExecution::Retry {
delay: std::time::Duration::from_secs(30),
error: "webdav capability set unavailable".into(),
};
}
if let Some(members) = ProvisionPayload::from_job(&job) {
for member in members {
let new_membership = NewUserMembership {
id: Uuid::new_v4(),
user_id: member,
tenant_id: tenant.id,
capability_set_id: Some(owner_capability_set_id),
};
if let Err(err) = diesel::insert_into(user_memberships::table)
.values(&new_membership)
.on_conflict((user_memberships::user_id, user_memberships::tenant_id))
.do_nothing()
.execute(&mut conn)
{
warn!(
job_id = %job.id,
tenant_id = %tenant.id,
user_id = %member,
error = %err,
"failed to assign initial membership"
);
}
if let Err(err) = diesel::insert_into(user_memberships::table)
.values(&new_membership)
.on_conflict((user_memberships::user_id, user_memberships::tenant_id))
.do_nothing()
.execute(&mut conn)
{
warn!(
job_id = %ctx.job_id(),
tenant_id = %tenant.id,
user_id = %member,
error = %err,
"failed to assign initial membership"
);
}
}
if let Err(err) = diesel::update(tenants::table.find(tenant.id))
diesel::update(tenants::table.find(tenant.id))
.set((
tenants::status.eq(TenantStatus::Active),
tenants::quickwit_index.eq(Some(index_id)),
tenants::updated_at.eq(Utc::now().naive_utc()),
))
.execute(&mut conn)
{
warn!(job_id = %job.id, error = %err, "failed to activate tenant");
return JobExecution::Retry {
delay: std::time::Duration::from_secs(30),
error: format!("failed to update tenant status: {err}"),
};
}
.map_err(|err| {
TaskError::retry(
Duration::from_secs(30),
format!("failed to update tenant status: {err}"),
)
})?;
JobExecution::Success
Ok(())
}
}
async fn delete_tenant(
state: &Arc<AppState>,
storage: crate::storage::TenantStorage,
tenant_id: Uuid,
current_job_id: Uuid,
payload: &DeleteTenantPayload,
) -> Result<(), String> {
let remove_tenant = payload.remove_tenant;
let tenant = {
let mut conn = state
.db_unscoped()
.map_err(|err| format!("failed to get db connection: {err:?}"))?;
TenantRepository::get_by_id(&mut conn, tenant_id)
.map_err(|err| format!("tenant lookup failed: {err:?}"))?
};
if tenant.name != payload.tenant_name {
return Err(format!(
"tenant name mismatch: expected '{}', got '{}'",
payload.tenant_name, tenant.name
));
}
if tenant.status != TenantStatus::Deleting {
return Err(format!(
"tenant status '{}' not eligible for deletion",
tenant.status.as_str()
));
}
match (payload_action_applicable(remove_tenant), payload.action) {
(DeleteAction::Delete, DeleteAction::Delete)
| (DeleteAction::Reset, DeleteAction::Reset) => {}
_ => {
return Err("delete payload action mismatch".into());
}
}
let issued_at = DateTime::parse_from_rfc3339(&payload.issued_at)
.map_err(|_| "invalid issued_at timestamp".to_string())?
.with_timezone(&Utc);
if (Utc::now() - issued_at).num_seconds().abs() > DELETE_PROOF_TTL_SECONDS {
return Err("delete confirmation expired".into());
}
let resolved_final_status = if remove_tenant {
None
} else {
Some(payload.final_status.unwrap_or(FinalTenantStatus::Suspended))
};
let final_status_str = resolved_final_status.map(|s| s.as_str());
let message = build_delete_proof_message(
tenant_id,
&tenant.name,
payload.action,
&payload.nonce,
&payload.issued_at,
final_status_str,
);
verify_delete_proof(&state.config.jwt_secret, &message, &payload.signature)?;
let object_keys = {
let mut conn = state
.db_for_tenant(tenant_id)
.map_err(|err| format!("failed to scope tenant connection: {err:?}"))?;
collect_object_keys(&mut conn)
.map_err(|err| format!("failed to collect storage keys: {err}"))?
};
delete_storage_objects(&storage, &object_keys)
.await
.map_err(|err| format!("failed to delete storage objects: {err}"))?;
reset_quickwit_index(state, &tenant, remove_tenant)
.await
.map_err(|err| format!("quickwit cleanup failed: {err}"))?;
{
let mut conn = state
.db_for_tenant(tenant_id)
.map_err(|err| format!("failed to scope tenant connection: {err:?}"))?;
delete_tenant_rows(&mut conn, tenant_id, remove_tenant)
.map_err(|err| format!("tenant data cleanup failed: {err}"))?;
}
{
let mut conn = state
.db_unscoped()
.map_err(|err| format!("failed to get db connection: {err:?}"))?;
let detach_result = json!({
"tenant": {
"id": tenant.id,
"name": tenant.name,
},
"action": payload.action.as_str(),
"remove_tenant": remove_tenant,
"final_status": final_status_str,
"timestamp": Utc::now().to_rfc3339(),
});
diesel::sql_query(
"UPDATE jobs \
SET tenant_id = NULL, \
result = jsonb_set(COALESCE(result, '{}'::jsonb), '{detached_tenant}', $3::jsonb, true) \
WHERE tenant_id = $1 AND id <> $2",
)
.bind::<diesel::sql_types::Uuid, _>(tenant_id)
.bind::<diesel::sql_types::Uuid, _>(current_job_id)
.bind::<Jsonb, _>(detach_result)
.execute(&mut conn)
.map_err(|err| format!("failed to detach tenant jobs: {err}"))?;
if remove_tenant {
diesel::delete(tenants::table.find(tenant_id))
.execute(&mut conn)
.map_err(|err| format!("failed to delete tenant row: {err}"))?;
} else {
let new_status = match resolved_final_status.unwrap_or(FinalTenantStatus::Suspended) {
FinalTenantStatus::Active => TenantStatus::Active,
FinalTenantStatus::Suspended => TenantStatus::Suspended,
};
diesel::update(tenants::table.find(tenant_id))
.set((
tenants::status.eq(new_status),
tenants::updated_at.eq(Utc::now().naive_utc()),
))
.execute(&mut conn)
.map_err(|err| format!("failed to update tenant status: {err}"))?;
}
}
Ok(())
}
struct TenantObjectKeys {
version_keys: Vec<String>,
asset_keys: Vec<String>,
}
fn collect_object_keys(conn: &mut PgConnection) -> Result<TenantObjectKeys, diesel::result::Error> {
let version_keys = document_versions::table
.select(document_versions::s3_key)
.load::<String>(conn)?;
let asset_keys = document_assets::table
.select(document_assets::s3_key)
.load::<String>(conn)?;
Ok(TenantObjectKeys {
version_keys,
asset_keys,
})
}
async fn delete_storage_objects(
storage: &crate::storage::TenantStorage,
keys: &TenantObjectKeys,
) -> Result<(), String> {
for key in keys.version_keys.iter().chain(keys.asset_keys.iter()) {
storage
.delete_object(key)
.await
.map_err(|err| format!("failed to delete object '{key}': {err}"))?;
}
Ok(())
}
async fn reset_quickwit_index(
state: &Arc<AppState>,
tenant: &Tenant,
remove_index: bool,
) -> Result<(), String> {
let endpoint = match state.config.quickwit_endpoint.as_ref() {
Some(endpoint) => endpoint,
None => return Ok(()),
};
let index_id = match tenant.quickwit_index.as_deref() {
Some(index) => index,
None => return Ok(()),
};
let client = Client::new();
delete_quickwit_index(&client, endpoint, index_id)
.await
.map_err(|err| format!("quickwit delete failed: {err}"))?;
if !remove_index {
ensure_quickwit_index(&client, endpoint, index_id)
.await
.map_err(|err| format!("quickwit ensure failed: {err}"))?;
}
Ok(())
}
fn delete_tenant_rows(
conn: &mut PgConnection,
tenant_id: Uuid,
remove_memberships: bool,
) -> Result<(), diesel::result::Error> {
conn.transaction(|conn| {
diesel::delete(document_assets::table.filter(document_assets::tenant_id.eq(tenant_id)))
.execute(conn)?;
diesel::delete(
document_correspondents::table.filter(document_correspondents::tenant_id.eq(tenant_id)),
)
.execute(conn)?;
diesel::delete(document_tags::table.filter(document_tags::tenant_id.eq(tenant_id)))
.execute(conn)?;
diesel::delete(document_versions::table.filter(document_versions::tenant_id.eq(tenant_id)))
.execute(conn)?;
diesel::delete(documents::table.filter(documents::tenant_id.eq(tenant_id)))
.execute(conn)?;
diesel::delete(folders::table.filter(folders::tenant_id.eq(tenant_id))).execute(conn)?;
diesel::delete(correspondents::table.filter(correspondents::tenant_id.eq(tenant_id)))
.execute(conn)?;
diesel::delete(tags::table.filter(tags::tenant_id.eq(tenant_id))).execute(conn)?;
diesel::delete(user_sessions::table.filter(user_sessions::tenant_id.eq(tenant_id)))
.execute(conn)?;
diesel::delete(api_tokens::table.filter(api_tokens::tenant_id.eq(tenant_id)))
.execute(conn)?;
if remove_memberships {
diesel::delete(
user_memberships::table.filter(user_memberships::tenant_id.eq(tenant_id)),
)
.execute(conn)?;
}
Ok(())
})
}
#[derive(Deserialize, Default)]
struct ProvisionPayload {
#[serde(default)]
@@ -246,3 +501,140 @@ impl ProvisionPayload {
.ok()
}
}
pub struct DeleteTenantJob;
impl DeleteTenantJob {
pub fn new() -> Self {
Self
}
}
pub fn build_delete_proof_message(
tenant_id: Uuid,
tenant_name: &str,
action: DeleteAction,
nonce: &str,
issued_at: &str,
final_status: Option<&str>,
) -> String {
let status = final_status.unwrap_or("none");
format!(
"{}|{}|{}|{}|{}|{}|{}",
DELETE_PROOF_VERSION,
tenant_id,
tenant_name,
action.as_str(),
nonce,
issued_at,
status
)
}
pub fn sign_delete_proof(secret: &str, message: &str) -> Result<String, String> {
let mut mac = HmacSha256::new_from_slice(secret.as_bytes())
.map_err(|err| format!("failed to init hmac: {err}"))?;
mac.update(message.as_bytes());
let bytes = mac.finalize().into_bytes();
Ok(hex::encode(bytes))
}
fn verify_delete_proof(secret: &str, message: &str, signature: &str) -> Result<(), String> {
let signature_bytes = hex::decode(signature)
.map_err(|_| "invalid delete proof signature encoding".to_string())?;
let mut mac = HmacSha256::new_from_slice(secret.as_bytes())
.map_err(|err| format!("failed to init hmac: {err}"))?;
mac.update(message.as_bytes());
mac.verify_slice(&signature_bytes)
.map_err(|_| "delete proof signature mismatch".to_string())
}
#[derive(Debug, Deserialize)]
struct DeleteTenantPayload {
#[serde(default)]
remove_tenant: bool,
#[serde(default)]
final_status: Option<FinalTenantStatus>,
tenant_name: String,
action: DeleteAction,
nonce: String,
issued_at: String,
signature: String,
}
#[derive(Debug, Deserialize, Clone, Copy)]
#[serde(rename_all = "lowercase")]
enum FinalTenantStatus {
Active,
Suspended,
}
impl FinalTenantStatus {
fn as_str(&self) -> &'static str {
match self {
FinalTenantStatus::Active => "active",
FinalTenantStatus::Suspended => "suspended",
}
}
}
#[derive(Debug, Deserialize, Clone, Copy, PartialEq, Eq)]
#[serde(rename_all = "lowercase")]
pub enum DeleteAction {
Delete,
Reset,
}
impl DeleteAction {
pub fn as_str(&self) -> &'static str {
match self {
DeleteAction::Delete => "delete",
DeleteAction::Reset => "reset",
}
}
}
fn payload_action_applicable(remove_tenant: bool) -> DeleteAction {
if remove_tenant {
DeleteAction::Delete
} else {
DeleteAction::Reset
}
}
#[async_trait]
impl JobHandler for DeleteTenantJob {
fn job_type(&self) -> &'static str {
JOB_DELETE_TENANT
}
async fn handle(
&self,
state: Arc<AppState>,
job: crate::models::Job,
storage: crate::storage::TenantStorage,
) -> JobExecution {
let payload: DeleteTenantPayload = match serde_json::from_value(job.payload.clone()) {
Ok(payload) => payload,
Err(err) => {
return JobExecution::Failed {
error: format!("invalid delete tenant payload: {err}"),
}
}
};
let tenant_id = match job.tenant_id {
Some(id) => id,
None => {
return JobExecution::Failed {
error: "delete job is missing tenant context".to_string(),
}
}
};
match delete_tenant(&state, storage, tenant_id, job.id, &payload).await {
Ok(()) => JobExecution::Success,
Err(err) => JobExecution::Failed { error: err },
}
}
}
File diff suppressed because it is too large Load Diff
+42 -51
View File
@@ -1,15 +1,16 @@
mod common;
use anyhow::Result;
use anyhow::{Context, Result};
use axum::body::Body;
use axum::http::{header, Method, Request, StatusCode};
use base64::engine::general_purpose::STANDARD as BASE64;
use base64::Engine;
use common::{acquire_db_lock, body_to_vec, TestApp};
use diesel::prelude::*;
use papercrate::models::ApiToken;
use diesel::OptionalExtension;
use papercrate::auth::capability_sets;
use papercrate::models::{ApiCapability, ApiToken};
use papercrate::routes::webdav;
use papercrate::schema::api_tokens;
use papercrate::schema::capability_sets::dsl as capability_sets_dsl;
use papercrate::test_support::{acquire_db_lock, body_to_vec, TestApp, TestUserRole};
use serde::Deserialize;
use serde_json::json;
use tower::ServiceExt;
@@ -59,12 +60,6 @@ struct TenantView {
name: String,
}
#[derive(Debug, Deserialize)]
struct CapabilitySetSummary {
id: Uuid,
slug: String,
}
#[tokio::test]
async fn api_token_crud_flow() -> Result<()> {
let _guard = acquire_db_lock().await;
@@ -72,7 +67,7 @@ async fn api_token_crud_flow() -> Result<()> {
let username = "alice";
let password = "correct horse battery";
app.insert_user(username, password, "admin").await?;
app.insert_user(username, TestUserRole::Owner).await?;
let access_token = app.login_token(username, password).await?;
let legacy_set_id =
@@ -141,7 +136,7 @@ async fn webdav_basic_auth_uses_api_tokens() -> Result<()> {
let username = "bruce";
let password = "wayne";
app.insert_user(username, password, "admin").await?;
app.insert_user(username, TestUserRole::Owner).await?;
let access_token = app.login_token(username, password).await?;
let legacy_set_id =
@@ -356,46 +351,42 @@ async fn ensure_capability_set_slug(
slug: &str,
capabilities: &[&str],
) -> Result<Uuid> {
if let Some(existing) = find_capability_set_slug(app, access_token, slug).await? {
return Ok(existing);
}
let claims = app
.state
.jwt
.verify_token(access_token)
.context("failed to decode access token claims")?;
let tenant_id = claims.tenant_id;
let slug = slug.to_string();
let desired_capabilities = capabilities
.iter()
.map(|value| {
value
.parse::<ApiCapability>()
.map_err(|err| anyhow::anyhow!("invalid capability '{value}': {err}"))
})
.collect::<Result<Vec<_>>>()?;
let response = app
.post_json(
"/api/capability-sets",
&json!({
"slug": slug,
"capabilities": capabilities,
}),
Some(access_token),
)
.await?;
assert_eq!(response.status(), StatusCode::CREATED);
let body = body_to_vec(response.into_body()).await?;
let summary: CapabilitySetSummary = serde_json::from_slice(&body)?;
Ok(summary.id)
}
let caps_for_insert = desired_capabilities.clone();
app.with_conn(move |conn| {
if let Some(existing) = capability_sets_dsl::capability_sets
.filter(capability_sets_dsl::tenant_id.eq(tenant_id))
.filter(capability_sets_dsl::slug.eq(&slug))
.select(capability_sets_dsl::id)
.first::<Uuid>(conn)
.optional()?
{
return Ok(existing);
}
async fn find_capability_set_slug(
app: &TestApp,
access_token: &str,
slug: &str,
) -> Result<Option<Uuid>> {
let sets = list_capability_sets(app, access_token).await?;
Ok(sets
.into_iter()
.find(|set| set.slug == slug)
.map(|set| set.id))
}
async fn list_capability_sets(
app: &TestApp,
access_token: &str,
) -> Result<Vec<CapabilitySetSummary>> {
let response = app.get("/api/capability-sets", Some(access_token)).await?;
assert_eq!(response.status(), StatusCode::OK);
let body = body_to_vec(response.into_body()).await?;
Ok(serde_json::from_slice(&body)?)
let created =
capability_sets::create_capability_set(conn, tenant_id, &slug, caps_for_insert)
.map_err(|err| {
anyhow::anyhow!("failed to create capability set '{slug}': {err:?}")
})?;
Ok(created.id)
})
.await
}
async fn exchange_token(app: &TestApp, api_token: &str) -> Result<LoginResponseView> {
let response = app
+103 -40
View File
@@ -1,9 +1,6 @@
mod common;
use anyhow::{anyhow, Context, Result};
use axum::http::{header::SET_COOKIE, StatusCode};
use chrono::{Duration as ChronoDuration, Utc};
use common::{acquire_db_lock, body_to_vec, TestApp};
use diesel::prelude::*;
use papercrate::auth::capability_sets::{ensure_capability_set, owner_capabilities};
use papercrate::auth::jwt::{AccessTokenContext, PrincipalKind};
@@ -11,11 +8,16 @@ use papercrate::auth::passkeys::{
PasskeyLoginFinishPayload, PasskeyLoginStartPayload, PasskeyRegistrationFinishPayload,
RegistrationChallengeResponse,
};
use papercrate::models::{NewUserMembership, NewUserSession, TenantStatus, UserPasskey};
use papercrate::models::{
MagicToken, MagicTokenKind, NewUserMembership, NewUserSession, TenantStatus, UserPasskey,
};
use papercrate::openapi::schemas::PasskeySummary;
use papercrate::schema::{capability_sets, tenants, user_memberships, user_sessions, users};
use rand::rngs::OsRng;
use rand::RngCore;
use papercrate::schema::{
capability_sets, magic_tokens::dsl as magic_dsl, tenants, tenants::dsl as tenant_dsl,
user_memberships, user_sessions, users,
};
use papercrate::test_support::{acquire_db_lock, body_to_vec, TestApp, TestUserRole};
use rand::{rngs::OsRng, TryRngCore};
use serde::Deserialize;
use serde_json::json;
use sha2::{Digest, Sha256};
@@ -55,19 +57,6 @@ struct SignupStartResponse {
challenge: RegistrationChallengeResponse,
}
#[derive(Deserialize)]
struct TenantSelectionResponse {
#[serde(rename = "access_token")]
_access_token: String,
#[serde(rename = "tenants")]
_tenants: Vec<TenantSummary>,
}
#[derive(Deserialize)]
struct TenantListResponse {
tenants: Vec<TenantSummary>,
}
#[derive(Deserialize)]
struct TenantSummary {
id: Uuid,
@@ -80,7 +69,7 @@ async fn login_and_me_roundtrip() -> Result<()> {
let app = TestApp::new().await?;
let password = "s3cret";
app.insert_user("alice", password, "admin").await?;
app.insert_user("alice", TestUserRole::Owner).await?;
let (login, _) = login_with_session(&app, "alice", password).await?;
@@ -156,7 +145,7 @@ async fn passkey_register_start_creates_challenge() -> Result<()> {
let app = TestApp::new().await?;
let password = "secret";
app.insert_user("passkey-user", password, "admin").await?;
app.insert_user("passkey-user", TestUserRole::Owner).await?;
let (login, _) = login_with_session(&app, "passkey-user", password).await?;
@@ -197,7 +186,7 @@ async fn passkey_register_finish_rejects_unknown_challenge() -> Result<()> {
let app = TestApp::new().await?;
let password = "secret";
app.insert_user("passkey-register", password, "admin")
app.insert_user("passkey-register", TestUserRole::Owner)
.await?;
let (login, _) = login_with_session(&app, "passkey-register", password).await?;
@@ -226,8 +215,9 @@ async fn passkey_login_start_requires_passkey() -> Result<()> {
let _lock = acquire_db_lock().await;
let app = TestApp::new().await?;
let password = "secret";
app.insert_user("passkey-login", password, "admin").await?;
let _password = "secret";
app.insert_user("passkey-login", TestUserRole::Owner)
.await?;
let payload = PasskeyLoginStartPayload {
username: "passkey-login".to_string(),
@@ -288,7 +278,9 @@ async fn list_passkeys_returns_entries() -> Result<()> {
let app = TestApp::new().await?;
let password = "secret";
let user_id = app.insert_user("passkey-owner", password, "admin").await?;
let user_id = app
.insert_user("passkey-owner", TestUserRole::Owner)
.await?;
app.insert_passkey(user_id, Some("Laptop")).await?;
let (session, _) = login_with_session(&app, "passkey-owner", password).await?;
@@ -314,7 +306,9 @@ async fn delete_passkey_soft_revokes() -> Result<()> {
let app = TestApp::new().await?;
let password = "secret";
let user_id = app.insert_user("passkey-delete", password, "admin").await?;
let user_id = app
.insert_user("passkey-delete", TestUserRole::Owner)
.await?;
let passkey_id = app.insert_passkey(user_id, Some("Phone")).await?;
app.insert_passkey(user_id, Some("Backup")).await?;
let (session, _) = login_with_session(&app, "passkey-delete", password).await?;
@@ -349,7 +343,9 @@ async fn delete_passkey_prevents_last() -> Result<()> {
let app = TestApp::new().await?;
let password = "secret";
let user_id = app.insert_user("passkey-guard", password, "admin").await?;
let user_id = app
.insert_user("passkey-guard", TestUserRole::Owner)
.await?;
let first_id = app.insert_passkey(user_id, Some("Key A")).await?;
let last_id = app.insert_passkey(user_id, Some("Key B")).await?;
let (session, _) = login_with_session(&app, "passkey-guard", password).await?;
@@ -408,8 +404,8 @@ async fn login_rejects_invalid_password() -> Result<()> {
let _lock = acquire_db_lock().await;
let app = TestApp::new().await?;
let password = "valid";
app.insert_user("robin", password, "admin").await?;
let _password = "valid";
app.insert_user("robin", TestUserRole::Owner).await?;
let payload = json!({ "username": "robin", "password": "wrong" });
let response = app.post_json("/api/auth/login", &payload, None).await?;
@@ -428,7 +424,7 @@ async fn refresh_rotates_refresh_token() -> Result<()> {
let app = TestApp::new().await?;
let password = "rotate";
app.insert_user("rita", password, "admin").await?;
app.insert_user("rita", TestUserRole::Owner).await?;
let (login, refresh_cookie) = login_with_session(&app, "rita", password).await?;
@@ -464,7 +460,7 @@ async fn logout_revokes_refresh_token() -> Result<()> {
let app = TestApp::new().await?;
let password = "logout";
app.insert_user("logan", password, "admin").await?;
app.insert_user("logan", TestUserRole::Owner).await?;
let (login, refresh_cookie) = login_with_session(&app, "logan", password).await?;
@@ -510,7 +506,7 @@ async fn login_returns_tenant_selection_when_multiple_memberships() -> Result<()
let app = TestApp::new().await?;
let password = "multipass";
let user_id = app.insert_user("multipass", password, "admin").await?;
let user_id = app.insert_user("multipass", TestUserRole::Owner).await?;
let secondary_name = "secondary".to_string();
let name_for_insert = secondary_name.clone();
@@ -545,16 +541,13 @@ async fn login_returns_tenant_selection_when_multiple_memberships() -> Result<()
let (login, refresh_cookie) = login_with_session(&app, "multipass", password).await?;
let tenants_response = app
.get("/api/auth/tenants", Some(&login.access_token))
.await?;
let tenants_response = app.get("/api/tenants", Some(&login.access_token)).await?;
assert_eq!(tenants_response.status(), StatusCode::OK);
let tenants_body = body_to_vec(tenants_response.into_body()).await?;
let tenant_list: TenantListResponse = serde_json::from_slice(&tenants_body)?;
assert!(tenant_list.tenants.len() >= 2);
let tenant_list: Vec<TenantSummary> = serde_json::from_slice(&tenants_body)?;
assert!(tenant_list.len() >= 2);
let secondary = tenant_list
.tenants
.iter()
.find(|tenant| tenant.name == secondary_name)
.map(|t| t.id)
@@ -671,7 +664,9 @@ fn extract_refresh_cookie(headers: &axum::http::HeaderMap) -> Result<String> {
fn generate_session_token() -> String {
let mut bytes = [0u8; 32];
OsRng.fill_bytes(&mut bytes);
OsRng
.try_fill_bytes(&mut bytes)
.expect("failed to read random bytes");
hex::encode(bytes)
}
@@ -680,3 +675,71 @@ fn hash_session_token(value: &str) -> String {
hasher.update(value.as_bytes());
hex::encode(hasher.finalize())
}
#[tokio::test]
async fn tenant_selection_excludes_inactive_tenants() -> Result<()> {
let _lock = acquire_db_lock().await;
let app = TestApp::new().await?;
let user_id = app.insert_user("tenant-user", TestUserRole::Owner).await?;
let tenant_id = app
.with_conn(|conn| {
let tenant: papercrate::models::Tenant = tenant_dsl::tenants
.filter(tenant_dsl::name.eq("test_tenant"))
.first(conn)?;
Ok::<_, anyhow::Error>(tenant.id)
})
.await?;
app.with_conn(move |conn| {
diesel::update(tenant_dsl::tenants.find(tenant_id))
.set(tenant_dsl::status.eq(TenantStatus::Suspended))
.execute(conn)?;
Ok(())
})
.await?;
let magic_value = "tenant-status-token";
let token_hash = {
let mut hasher = Sha256::new();
hasher.update(magic_value.as_bytes());
hex::encode(hasher.finalize())
};
let user_id_for_token = user_id;
app.with_conn(move |conn| {
let token = MagicToken {
id: Uuid::new_v4(),
user_id: user_id_for_token,
kind: MagicTokenKind::EmailLogin,
token_hash,
metadata: json!({}),
expires_at: (Utc::now() + ChronoDuration::hours(1)).naive_utc(),
max_uses: None,
used_count: 0,
created_at: Utc::now().naive_utc(),
created_by: None,
last_used_at: None,
};
diesel::insert_into(magic_dsl::magic_tokens)
.values(&token)
.execute(conn)?;
Ok(())
})
.await?;
let payload = json!({
"username": "tenant-user",
"magic_token": magic_value,
});
let response = app.post_json("/api/auth/login", &payload, None).await?;
assert_eq!(response.status(), StatusCode::FORBIDDEN);
let body = body_to_vec(response.into_body()).await?;
let err: ApiErrorResponse = serde_json::from_slice(&body)?;
assert_eq!(err.error, "no active tenants available");
app.cleanup().await?;
Ok(())
}
+8 -10
View File
@@ -1,10 +1,8 @@
mod common;
use anyhow::{anyhow, Result};
use axum::http::StatusCode;
use common::TestApp;
use diesel::prelude::*;
use papercrate::models::ApiCapability;
use papercrate::test_support::{acquire_db_lock, body_to_vec, TestApp, TestUserRole};
use serde_json::json;
async fn set_user_capabilities(
@@ -38,11 +36,11 @@ async fn set_user_capabilities(
#[tokio::test]
async fn documents_routes_enforce_capabilities() -> Result<()> {
let _lock = common::acquire_db_lock().await;
let _lock = acquire_db_lock().await;
let app = TestApp::new().await?;
let password = "limited-docs";
let user_id = app.insert_user("limited-docs", password, "admin").await?;
let user_id = app.insert_user("limited-docs", TestUserRole::Owner).await?;
set_user_capabilities(&app, user_id, &[ApiCapability::DocumentsRead]).await?;
let token = app.login_token("limited-docs", password).await?;
@@ -61,12 +59,12 @@ async fn documents_routes_enforce_capabilities() -> Result<()> {
)
.await?;
assert_eq!(upload.status(), StatusCode::FORBIDDEN);
let upload_body = common::body_to_vec(upload.into_body()).await?;
let upload_body = body_to_vec(upload.into_body()).await?;
assert!(String::from_utf8_lossy(&upload_body).contains("missing"));
let capability_sets = app.get("/api/capability-sets", Some(&token)).await?;
assert_eq!(capability_sets.status(), StatusCode::FORBIDDEN);
let caps_body = common::body_to_vec(capability_sets.into_body()).await?;
let caps_body = body_to_vec(capability_sets.into_body()).await?;
assert!(String::from_utf8_lossy(&caps_body).contains("missing"));
app.cleanup().await?;
@@ -75,11 +73,11 @@ async fn documents_routes_enforce_capabilities() -> Result<()> {
#[tokio::test]
async fn capability_set_routes_require_write_privilege() -> Result<()> {
let _lock = common::acquire_db_lock().await;
let _lock = acquire_db_lock().await;
let app = TestApp::new().await?;
let password = "caps-reader";
let user_id = app.insert_user("caps-reader", password, "admin").await?;
let user_id = app.insert_user("caps-reader", TestUserRole::Member).await?;
set_user_capabilities(&app, user_id, &[ApiCapability::CapabilitySetsRead]).await?;
let token = app.login_token("caps-reader", password).await?;
@@ -98,7 +96,7 @@ async fn capability_set_routes_require_write_privilege() -> Result<()> {
)
.await?;
assert_eq!(create.status(), StatusCode::FORBIDDEN);
let create_body = common::body_to_vec(create.into_body()).await?;
let create_body = body_to_vec(create.into_body()).await?;
assert!(String::from_utf8_lossy(&create_body).contains("missing"));
app.cleanup().await?;
+2 -4
View File
@@ -1,8 +1,6 @@
mod common;
use anyhow::Result;
use axum::http::StatusCode;
use common::{acquire_db_lock, body_to_vec, TestApp};
use papercrate::test_support::{acquire_db_lock, body_to_vec, TestApp, TestUserRole};
use serde::Deserialize;
use serde_json::json;
use uuid::Uuid;
@@ -23,7 +21,7 @@ async fn capability_set_crud_flow() -> Result<()> {
let app = TestApp::new().await?;
let password = "caps-admin";
app.insert_user("caps", password, "admin").await?;
app.insert_user("caps", TestUserRole::Owner).await?;
let token = app.login_token("caps", password).await?;
// Initial list should contain system sets.
+1 -881
View File
@@ -1,881 +1 @@
use std::collections::HashMap;
use std::env;
use std::sync::Arc;
use std::time::Duration;
use anyhow::{anyhow, ensure, Context, Result};
use async_trait::async_trait;
use axum::body::Body;
use axum::http::{header, Method, Request};
use axum::Router;
use chrono::{Duration as ChronoDuration, Utc};
use diesel::connection::SimpleConnection;
use diesel::prelude::*;
use diesel::OptionalExtension;
use diesel::PgConnection;
use diesel_migrations::{embed_migrations, EmbeddedMigrations, MigrationHarness};
use http_body_util::BodyExt;
use once_cell::sync::Lazy;
use papercrate::auth::capability_sets::{
ensure_capability_set, owner_capabilities, readonly_capabilities, user_capabilities,
webdav_capabilities,
};
use papercrate::auth::jwt::{AccessTokenContext, JwtService, PrincipalKind};
use papercrate::config::AppConfig;
use papercrate::db::{self, PgPool};
use papercrate::models::{
Job, NewUser, NewUserMembership, NewUserPasskey, NewUserSession, Tenant, TenantStatus, User,
UserMembership,
};
use papercrate::routes;
use papercrate::schema::user_sessions::dsl as session_dsl;
use papercrate::state::AppState;
use papercrate::storage::ObjectStorage;
use rand::rngs::OsRng;
use rand::RngCore;
use serde::Serialize;
use serde_json::{self, json};
use sha2::{Digest, Sha256};
use tokio::sync::Mutex;
use tower::util::ServiceExt;
use uuid::Uuid;
const MIGRATIONS: EmbeddedMigrations = embed_migrations!("migrations");
const RESET_DATABASE_SQL: &str = "DROP SCHEMA IF EXISTS tenant CASCADE;\n\
DROP SCHEMA IF EXISTS shared CASCADE;\n\
DROP SCHEMA IF EXISTS public CASCADE;\n\
CREATE SCHEMA public;\n\
GRANT ALL ON SCHEMA public TO public;";
static DB_LOCK: Lazy<Mutex<()>> = Lazy::new(|| Mutex::new(()));
const TEST_TENANT_NAME: &str = "test_tenant";
#[allow(dead_code)]
#[derive(Clone)]
pub struct StoredObject {
pub key: String,
pub bytes: Vec<u8>,
pub content_type: Option<String>,
pub content_disposition: Option<String>,
}
#[derive(Default)]
pub struct FakeStorage {
objects: Mutex<HashMap<String, StoredObject>>,
}
#[async_trait]
impl ObjectStorage for FakeStorage {
async fn put_object(
&self,
key: &str,
bytes: Vec<u8>,
content_type: Option<String>,
content_disposition: Option<String>,
) -> Result<()> {
let stored = StoredObject {
key: key.to_string(),
bytes,
content_type,
content_disposition,
};
let mut guard = self.objects.lock().await;
guard.insert(stored.key.clone(), stored);
Ok(())
}
async fn presign_get_object(
&self,
key: &str,
expires_in: Duration,
_response_content_disposition: Option<&str>,
) -> Result<String> {
let guard = self.objects.lock().await;
ensure!(guard.contains_key(key), "object {key} missing");
Ok(format!(
"https://fake-storage/{key}?expires_in={}",
expires_in.as_secs()
))
}
async fn get_object(&self, key: &str) -> Result<Vec<u8>> {
let guard = self.objects.lock().await;
guard
.get(key)
.map(|obj| obj.bytes.clone())
.ok_or_else(|| anyhow!("object {key} missing"))
}
async fn delete_object(&self, key: &str) -> Result<()> {
let mut guard = self.objects.lock().await;
guard.remove(key);
Ok(())
}
}
impl FakeStorage {
#[allow(dead_code)]
pub async fn get(&self, key: &str) -> Option<StoredObject> {
let guard = self.objects.lock().await;
guard.get(key).cloned()
}
#[allow(dead_code)]
pub async fn object_count(&self) -> usize {
let guard = self.objects.lock().await;
guard.len()
}
}
pub struct TestApp {
pub state: AppState,
router: Router,
storage: Arc<FakeStorage>,
}
impl TestApp {
pub async fn new() -> Result<Self> {
let database_url = env::var("TEST_DATABASE_URL")
.context("TEST_DATABASE_URL must be set for integration tests")?;
let config = AppConfig {
database_url: database_url.clone(),
database_max_pool_size: db::DEFAULT_MAX_POOL_SIZE,
server_host: "127.0.0.1".to_string(),
server_port: 0,
webdav_host: "127.0.0.1".to_string(),
webdav_port: 0,
jwt_secret: "test-secret".to_string(),
jwt_issuer: "test-issuer".to_string(),
jwt_audience: "test-audience".to_string(),
jwt_expiry_minutes: 60,
download_token_audience: "test-download".to_string(),
download_token_expiry_minutes: 60,
refresh_token_expiry_days: 30,
refresh_cookie_secure: false,
refresh_cookie_domain: None,
cors_allowed_origin: None,
aws_endpoint_url: None,
aws_access_key_id: None,
aws_secret_access_key: None,
aws_region: "us-east-1".to_string(),
s3_bucket: "test-bucket".to_string(),
quickwit_endpoint: None,
quickwit_index: None,
worker_max_document_bytes: 200 * 1024 * 1024,
upload_body_limit_bytes: 128 * 1024 * 1024,
webauthn_rp_id: Some("localhost".to_string()),
webauthn_origin: Some("http://localhost".to_string()),
webauthn_rp_name: "Papercrate".to_string(),
};
let pool = db::init_pool_with_size(&config.database_url, config.database_max_pool_size)?;
prepare_database(&pool).await?;
let storage = Arc::new(FakeStorage::default());
let storage_for_state: Arc<dyn ObjectStorage> = storage.clone();
let jwt = JwtService::from_config(&config)?;
let state = AppState::new(pool.clone(), config, storage_for_state, jwt);
let router = routes::create_router(state.clone());
let app = Self {
state,
router,
storage,
};
app.ensure_default_tenant().await?;
Ok(app)
}
pub async fn cleanup(&self) -> Result<()> {
let pool = self.state.pool.clone();
let _ = tokio::task::spawn_blocking(move || -> Result<()> {
let mut conn = pool
.get()
.map_err(|err| anyhow!("failed to get cleanup connection: {err}"))?;
truncate_all(&mut conn)?;
Ok(())
})
.await
.context("cleanup task panicked")?;
self.ensure_default_tenant().await?;
Ok(())
}
#[allow(dead_code)]
pub fn storage(&self) -> Arc<FakeStorage> {
self.storage.clone()
}
#[allow(dead_code)]
pub async fn storage_key_for(&self, key: &str) -> Result<String> {
self.ensure_default_tenant().await?;
let tenant = self
.state
.tenants
.get_by_name(TEST_TENANT_NAME)
.map_err(|err| anyhow!("default tenant not found: {:?}", err))?;
let root = tenant
.storage_root
.clone()
.ok_or_else(|| anyhow!("default tenant missing storage root"))?;
Ok(format!("{}{}", root, key))
}
pub async fn insert_user(&self, username: &str, _password: &str, role: &str) -> Result<Uuid> {
let username = username.to_string();
let role = role.to_string();
let tenant_id = self.ensure_default_tenant().await?;
let user_id = self
.with_conn(move |conn| {
let user = NewUser {
id: Uuid::new_v4(),
username,
};
diesel::insert_into(papercrate::schema::users::table)
.values(&user)
.execute(conn)
.context("failed to insert user")?;
let capabilities = match role.as_str() {
"admin" => owner_capabilities(),
"webdav" => webdav_capabilities(),
_ => user_capabilities(),
};
let capability_set = ensure_capability_set(conn, tenant_id, capabilities)
.map_err(|err| anyhow!("failed to ensure capability set: {:?}", err))?;
let membership = NewUserMembership {
id: Uuid::new_v4(),
user_id: user.id,
tenant_id,
capability_set_id: Some(capability_set.id),
};
diesel::insert_into(papercrate::schema::user_memberships::table)
.values(&membership)
.execute(conn)
.context("failed to insert user membership")?;
Ok(user.id)
})
.await?;
Ok(user_id)
}
#[cfg(test)]
pub async fn insert_passkey(&self, user_id: Uuid, nickname: Option<&str>) -> Result<Uuid> {
let passkey_id = Uuid::new_v4();
let nickname = nickname.map(|value| value.to_string());
self.with_conn(move |conn| {
let credential_id = passkey_id.as_bytes().to_vec();
let public_key = passkey_id.as_bytes().iter().copied().collect::<Vec<u8>>();
let passkey = NewUserPasskey {
id: passkey_id,
user_id,
credential_id,
public_key,
credential: json!({ "dummy": passkey_id.to_string() }),
sign_count: 0,
transports: vec![Some("usb".to_string())],
aaguid: None,
nickname,
};
diesel::insert_into(papercrate::schema::user_passkeys::table)
.values(&passkey)
.execute(conn)
.context("failed to insert passkey")?;
Ok(passkey_id)
})
.await
}
async fn ensure_default_tenant(&self) -> Result<Uuid> {
let name_value = TEST_TENANT_NAME.to_string();
let quickwit_enabled = self.state.config.quickwit_endpoint.is_some();
let tenant_id = self
.with_conn(move |conn| {
use papercrate::schema::tenants::dsl as tenants_dsl;
let existing = tenants_dsl::tenants
.filter(tenants_dsl::name.eq(&name_value))
.first::<Tenant>(conn)
.optional()
.context("failed to load default tenant")?;
let tenant_id = if let Some(current) = existing {
let desired_root = current
.storage_root
.clone()
.filter(|root| root.ends_with('/'))
.unwrap_or_else(|| format!("test-tenants/{}/", current.id));
if current.storage_root.as_deref() != Some(desired_root.as_str()) {
diesel::update(tenants_dsl::tenants.filter(tenants_dsl::id.eq(current.id)))
.set(tenants_dsl::storage_root.eq(Some(desired_root)))
.execute(conn)
.context("failed to update default tenant storage root")?;
}
current.id
} else {
let new_id = Uuid::new_v4();
let root = format!("test-tenants/{}/", new_id);
let quickwit_value = if quickwit_enabled {
Some(format!("documents-{}", new_id))
} else {
None
};
diesel::insert_into(tenants_dsl::tenants)
.values((
tenants_dsl::id.eq(new_id),
tenants_dsl::name.eq(&name_value),
tenants_dsl::storage_root.eq(Some(root)),
tenants_dsl::quickwit_index.eq(quickwit_value),
tenants_dsl::status.eq(TenantStatus::Active),
))
.execute(conn)
.context("failed to insert default tenant")?;
new_id
};
Ok(tenant_id)
})
.await?;
let mut conn = self
.state
.db_for_tenant(tenant_id)
.map_err(|err| anyhow!("failed to scope tenant connection: {err:?}"))?;
ensure_capability_set(&mut conn, tenant_id, owner_capabilities())
.map_err(|err| anyhow!("ensure owner capability set: {err:?}"))?;
ensure_capability_set(&mut conn, tenant_id, user_capabilities())
.map_err(|err| anyhow!("ensure user capability set: {err:?}"))?;
ensure_capability_set(&mut conn, tenant_id, readonly_capabilities())
.map_err(|err| anyhow!("ensure readonly capability set: {err:?}"))?;
ensure_capability_set(&mut conn, tenant_id, webdav_capabilities())
.map_err(|err| anyhow!("ensure webdav capability set: {err:?}"))?;
Ok(tenant_id)
}
pub async fn login_token(&self, username: &str, _password: &str) -> Result<String> {
let (access_token, _, _) = self.create_session(username).await?;
Ok(access_token)
}
pub async fn create_session(&self, username: &str) -> Result<(String, String, Uuid)> {
let username = username.to_string();
let state = self.state.clone();
self.with_conn(move |conn| {
use papercrate::schema::capability_sets::dsl as capability_sets_dsl;
use papercrate::schema::tenants::dsl as tenants_dsl;
use papercrate::schema::user_memberships::dsl as memberships_dsl;
use papercrate::schema::users::dsl as users_dsl;
let user: User = users_dsl::users
.filter(users_dsl::username.eq(&username))
.first(conn)?;
let membership: UserMembership = memberships_dsl::user_memberships
.filter(memberships_dsl::user_id.eq(user.id))
.first(conn)?;
let tenant: Tenant = tenants_dsl::tenants
.find(membership.tenant_id)
.first(conn)?;
let capability_set_id = membership
.capability_set_id
.ok_or_else(|| anyhow!("membership missing capability set"))?;
let cap_version = capability_sets_dsl::capability_sets
.find(capability_set_id)
.select(capability_sets_dsl::cap_version)
.first::<i32>(conn)?;
let now = Utc::now();
let session_id = Uuid::new_v4();
let access_token = state
.jwt
.generate_token(AccessTokenContext {
user_id: user.id,
tenant_id: tenant.id,
username: user.username.clone(),
principal_kind: PrincipalKind::UserSession,
principal_id: session_id,
capability_set_id,
cap_version,
})
.map_err(|err| anyhow!(err))?;
let session_value = generate_session_token();
let session_hash = hash_session_token(&session_value);
let refresh_expires_at =
now + ChronoDuration::days(state.config.refresh_token_expiry_days);
let new_session = NewUserSession {
id: session_id,
user_id: user.id,
token_hash: session_hash,
issued_at: now.naive_utc(),
expires_at: refresh_expires_at.naive_utc(),
tenant_id: tenant.id,
};
diesel::insert_into(session_dsl::user_sessions)
.values(&new_session)
.execute(conn)?;
let cookie = format!("refresh_token={session_value}");
Ok((access_token, cookie, tenant.id))
})
.await
}
#[allow(dead_code)]
pub async fn clear_jobs(&self) -> Result<()> {
self.with_conn(|conn| {
use papercrate::schema::jobs::dsl::jobs as jobs_table;
diesel::delete(jobs_table)
.execute(conn)
.context("failed to clear jobs")?;
Ok(())
})
.await
}
#[allow(dead_code)]
pub async fn jobs_by_type(&self, ty: &str) -> Result<Vec<Job>> {
let ty = ty.to_string();
self.with_conn(move |conn| {
use papercrate::schema::jobs::dsl::{job_type as job_type_col, jobs as jobs_table};
let rows = jobs_table
.filter(job_type_col.eq(&ty))
.load::<Job>(conn)
.context("failed to load jobs")?;
Ok(rows)
})
.await
}
pub async fn post_json<T: Serialize + ?Sized>(
&self,
path: &str,
payload: &T,
token: Option<&str>,
) -> Result<hyper::Response<Body>> {
self.post_json_with_cookie(path, payload, token, None).await
}
pub async fn post_json_with_cookie<T: Serialize + ?Sized>(
&self,
path: &str,
payload: &T,
token: Option<&str>,
cookie: Option<&str>,
) -> Result<hyper::Response<Body>> {
let body = serde_json::to_vec(payload)?;
let mut builder = Request::builder()
.method(Method::POST)
.uri(path)
.header("content-type", "application/json");
if let Some(token) = token {
builder = builder.header("authorization", format!("Bearer {token}"));
}
if let Some(cookie) = cookie {
builder = builder.header(header::COOKIE, cookie);
}
let request = builder.body(Body::from(body))?;
Ok(self
.router
.clone()
.oneshot(request)
.await
.expect("infallible response"))
}
#[allow(dead_code)]
pub async fn patch_json<T: Serialize + ?Sized>(
&self,
path: &str,
payload: &T,
token: Option<&str>,
) -> Result<hyper::Response<Body>> {
let body = serde_json::to_vec(payload)?;
let mut builder = Request::builder()
.method(Method::PATCH)
.uri(path)
.header("content-type", "application/json");
if let Some(token) = token {
builder = builder.header("authorization", format!("Bearer {token}"));
}
let request = builder.body(Body::from(body))?;
Ok(self
.router
.clone()
.oneshot(request)
.await
.expect("infallible response"))
}
pub async fn get(&self, path: &str, token: Option<&str>) -> Result<hyper::Response<Body>> {
let mut builder = Request::builder().method(Method::GET).uri(path);
if let Some(token) = token {
builder = builder.header("authorization", format!("Bearer {token}"));
}
let request = builder.body(Body::empty())?;
Ok(self
.router
.clone()
.oneshot(request)
.await
.expect("infallible response"))
}
#[allow(dead_code)]
pub async fn delete(&self, path: &str, token: Option<&str>) -> Result<hyper::Response<Body>> {
let builder = Request::builder().method(Method::DELETE).uri(path);
let builder = if let Some(token) = token {
builder.header("authorization", format!("Bearer {token}"))
} else {
builder
};
let request = builder.body(Body::empty())?;
Ok(self
.router
.clone()
.oneshot(request)
.await
.expect("infallible response"))
}
#[allow(dead_code)]
pub async fn upload_document(
&self,
path: &str,
filename: &str,
content_type: &str,
data: &[u8],
folder_id: Option<Uuid>,
token: &str,
) -> Result<hyper::Response<Body>> {
let extras = UploadExtras::empty();
self.upload_document_with_extras(
path,
filename,
content_type,
data,
folder_id,
extras,
token,
)
.await
}
pub async fn upload_document_with_options(
&self,
path: &str,
filename: &str,
content_type: &str,
data: &[u8],
folder_id: Option<Uuid>,
title: Option<&str>,
metadata_json: Option<&str>,
token: &str,
) -> Result<hyper::Response<Body>> {
let extras = UploadExtras {
title,
metadata_json,
tag_ids_json: None,
correspondents_json: None,
issued_at: None,
skip_existing: None,
};
self.upload_document_with_extras(
path,
filename,
content_type,
data,
folder_id,
extras,
token,
)
.await
}
pub async fn upload_document_with_extras(
&self,
path: &str,
filename: &str,
content_type: &str,
data: &[u8],
folder_id: Option<Uuid>,
extras: UploadExtras<'_>,
token: &str,
) -> Result<hyper::Response<Body>> {
let boundary = format!("boundary-{}", Uuid::new_v4());
let mut body = Vec::new();
body.extend(format!("--{boundary}\r\n").as_bytes());
body.extend(
format!(
"Content-Disposition: form-data; name=\"file\"; filename=\"{}\"\r\n",
filename
)
.as_bytes(),
);
body.extend(format!("Content-Type: {}\r\n\r\n", content_type).as_bytes());
body.extend(data);
body.extend(b"\r\n");
if let Some(folder) = folder_id {
body.extend(format!("--{boundary}\r\n").as_bytes());
body.extend(b"Content-Disposition: form-data; name=\"folder_id\"\r\n\r\n");
body.extend(folder.to_string().as_bytes());
body.extend(b"\r\n");
}
if let Some(title_value) = extras.title {
body.extend(format!("--{boundary}\r\n").as_bytes());
body.extend(b"Content-Disposition: form-data; name=\"title\"\r\n\r\n");
body.extend(title_value.as_bytes());
body.extend(b"\r\n");
}
if let Some(metadata_value) = extras.metadata_json {
body.extend(format!("--{boundary}\r\n").as_bytes());
body.extend(b"Content-Disposition: form-data; name=\"metadata\"\r\n\r\n");
body.extend(metadata_value.as_bytes());
body.extend(b"\r\n");
}
if let Some(tag_ids_value) = extras.tag_ids_json {
body.extend(format!("--{boundary}\r\n").as_bytes());
body.extend(b"Content-Disposition: form-data; name=\"tag_ids\"\r\n\r\n");
body.extend(tag_ids_value.as_bytes());
body.extend(b"\r\n");
}
if let Some(correspondents_value) = extras.correspondents_json {
body.extend(format!("--{boundary}\r\n").as_bytes());
body.extend(b"Content-Disposition: form-data; name=\"correspondents\"\r\n\r\n");
body.extend(correspondents_value.as_bytes());
body.extend(b"\r\n");
}
if let Some(issued_at_value) = extras.issued_at {
body.extend(format!("--{boundary}\r\n").as_bytes());
body.extend(b"Content-Disposition: form-data; name=\"issued_at\"\r\n\r\n");
body.extend(issued_at_value.as_bytes());
body.extend(b"\r\n");
}
if let Some(skip_flag) = extras.skip_existing {
body.extend(format!("--{boundary}\r\n").as_bytes());
body.extend(b"Content-Disposition: form-data; name=\"skip_existing\"\r\n\r\n");
body.extend(if skip_flag {
b"true".as_ref()
} else {
b"false".as_ref()
});
body.extend(b"\r\n");
}
body.extend(format!("--{boundary}--\r\n").as_bytes());
let builder = Request::builder()
.method(Method::POST)
.uri(path)
.header(
"content-type",
format!("multipart/form-data; boundary={boundary}"),
)
.header("authorization", format!("Bearer {token}"));
let request = builder.body(Body::from(body))?;
Ok(self
.router
.clone()
.oneshot(request)
.await
.expect("infallible response"))
}
pub async fn with_conn<F, T>(&self, f: F) -> Result<T>
where
F: FnOnce(&mut PgConnection) -> Result<T> + Send + 'static,
T: Send + 'static,
{
let pool = self.state.pool.clone();
tokio::task::spawn_blocking(move || {
let mut conn = pool
.get()
.map_err(|err| anyhow!("failed to get database connection: {err}"))?;
f(&mut conn)
})
.await
.context("connection task panicked")?
}
}
pub struct UploadExtras<'a> {
pub title: Option<&'a str>,
pub metadata_json: Option<&'a str>,
pub tag_ids_json: Option<&'a str>,
pub correspondents_json: Option<&'a str>,
pub issued_at: Option<&'a str>,
pub skip_existing: Option<bool>,
}
impl<'a> UploadExtras<'a> {
pub fn empty() -> Self {
Self {
title: None,
metadata_json: None,
tag_ids_json: None,
correspondents_json: None,
issued_at: None,
skip_existing: None,
}
}
}
pub async fn acquire_db_lock() -> tokio::sync::MutexGuard<'static, ()> {
DB_LOCK.lock().await
}
pub async fn body_to_vec(body: Body) -> Result<Vec<u8>> {
let collected = body
.collect()
.await
.map_err(|err| anyhow!("failed to read response body: {err}"))?;
Ok(collected.to_bytes().to_vec())
}
#[cfg(test)]
mod helper_tests {
use super::*;
#[tokio::test]
async fn create_session_and_login_token_provide_access() -> Result<()> {
let _lock = acquire_db_lock().await;
let app = TestApp::new().await?;
let username = "helper-login";
let password = "irrelevant";
app.insert_user(username, password, "admin").await?;
let (access, refresh, refresh_id) = app.create_session(username).await?;
assert!(!access.is_empty(), "access token should not be empty");
assert!(!refresh.is_empty(), "refresh token should not be empty");
assert_ne!(
refresh_id,
Uuid::nil(),
"refresh token id should be assigned"
);
let bearer = app.login_token(username, password).await?;
assert!(!bearer.is_empty(), "login_token must yield bearer");
app.cleanup().await?;
Ok(())
}
#[tokio::test]
async fn insert_passkey_and_upload_with_options_succeeds() -> Result<()> {
let _lock = acquire_db_lock().await;
let app = TestApp::new().await?;
let username = "helper-passkey";
let password = "unused";
let user_id = app.insert_user(username, password, "admin").await?;
let passkey_id = app.insert_passkey(user_id, Some("Laptop")).await?;
assert_ne!(passkey_id, Uuid::nil());
let bearer = app.login_token(username, password).await?;
let response = app
.upload_document_with_options(
"/api/documents",
"helper.txt",
"text/plain",
b"helper-content",
None,
Some("Helper Note"),
Some("{\"category\":\"note\"}"),
&bearer,
)
.await?;
assert!(response.status().is_success());
app.cleanup().await?;
Ok(())
}
}
async fn prepare_database(pool: &PgPool) -> Result<()> {
let pool = pool.clone();
tokio::task::spawn_blocking(move || -> Result<()> {
let mut conn = pool
.get()
.map_err(|err| anyhow!("failed to acquire connection: {err}"))?;
conn.batch_execute(RESET_DATABASE_SQL)
.map_err(|err| anyhow!("failed to reset schema: {err}"))?;
conn.batch_execute("DROP TABLE IF EXISTS __diesel_schema_migrations;")
.map_err(|err| anyhow!("failed to drop diesel schema table: {err}"))?;
conn.run_pending_migrations(MIGRATIONS)
.map_err(|err| anyhow!("failed to run migrations: {err}"))?;
truncate_all(&mut conn)?;
Ok(())
})
.await
.context("migration task panicked")?
}
fn truncate_all(conn: &mut PgConnection) -> Result<()> {
conn.batch_execute(
"TRUNCATE TABLE \
tenant.document_asset_objects, \
tenant.document_assets, \
tenant.document_correspondents, \
tenant.correspondents, \
tenant.document_tags, \
tenant.document_versions, \
tenant.documents, \
tenant.folders, \
shared.jobs, \
tenant.user_sessions, \
tenant.tags, \
tenant.api_tokens, \
shared.webauthn_challenges, \
shared.user_passkeys, \
tenant.user_memberships, \
shared.users, \
shared.magic_tokens, \
shared.tenants \
RESTART IDENTITY CASCADE;",
)
.context("failed to truncate tables")?;
Ok(())
}
fn generate_session_token() -> String {
let mut bytes = [0u8; 32];
OsRng.fill_bytes(&mut bytes);
hex::encode(bytes)
}
fn hash_session_token(value: &str) -> String {
let mut hasher = Sha256::new();
hasher.update(value.as_bytes());
hex::encode(hasher.finalize())
}
pub use papercrate::test_support::*;
+2 -4
View File
@@ -1,7 +1,5 @@
mod common;
use anyhow::Result;
use common::{acquire_db_lock, body_to_vec, TestApp};
use papercrate::test_support::{acquire_db_lock, body_to_vec, TestApp, TestUserRole};
use serde::Deserialize;
use serde_json::json;
use uuid::Uuid;
@@ -53,7 +51,7 @@ impl TestContext {
let app = TestApp::new().await?;
let username = format!("{prefix}_user");
let password = format!("{prefix}_pw");
app.insert_user(&username, &password, "admin").await?;
app.insert_user(&username, TestUserRole::Owner).await?;
let token = app.login_token(&username, &password).await?;
let first_id =
+584
View File
@@ -0,0 +1,584 @@
# Derived test cases from the Paperless-ngx project (https://github.com/paperless-ngx/paperless-ngx).
# Copyright (c) Paperless-ngx contributors, licensed under the GNU GPL-3.0.
cases:
- name: date_format_1
parser: parse_date
filename: null
content: "lorem ipsum 130218 lorem ipsum"
settings: {}
expected:
mode: none
- name: date_format_2
parser: parse_date
filename: null
content: "lorem ipsum 2018 lorem ipsum"
settings: {}
expected:
mode: none
- name: date_format_3
parser: parse_date
filename: null
content: "lorem ipsum 20180213 lorem ipsum"
settings: {}
expected:
mode: none
- name: date_format_4
parser: parse_date
filename: null
content: "lorem ipsum 13.02.2018 lorem ipsum"
settings: {}
expected:
mode: single
value: 2018-02-13
- name: date_format_5
parser: parse_date
filename: null
content: "lorem ipsum 130218, 2018, 20180213 and lorem 13.02.2018 lorem ipsum"
settings: {}
expected:
mode: single
value: 2018-02-13
- name: date_format_6
parser: parse_date
filename: null
content: |
lorem ipsum
Wohnort
3100
IBAN
AT87 4534
1234
1234 5678
BIC
lorem ipsum
settings: {}
expected:
mode: none
- name: date_format_7
parser: parse_date
filename: null
content: |
lorem ipsum
März 2019
lorem ipsum
settings:
DATE_PARSER_LANGUAGES:
- de
expected:
mode: single
value: 2019-03-01
- name: date_format_8
parser: parse_date
filename: null
content: |
lorem ipsum
Wohnort
3100
IBAN
AT87 4534
1234
1234 5678
BIC
lorem ipsum
März 2020
settings:
DATE_PARSER_LANGUAGES:
- de
expected:
mode: single
value: 2020-03-01
- name: date_format_9
parser: parse_date
filename: null
content: |
lorem ipsum
27. Nullmonth 2020
März 2020
lorem ipsum
settings:
DATE_PARSER_LANGUAGES:
- de
expected:
mode: single
value: 2020-03-01
- name: date_format_10
parser: parse_date
filename: null
content: "Customer Number Currency 22-MAR-2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-22
- name: date_format_11
parser: parse_date
filename: null
content: "Customer Number Currency 22 MAR 2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-22
- name: date_format_12
parser: parse_date
filename: null
content: "Customer Number Currency 22/MAR/2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-22
- name: date_format_13
parser: parse_date
filename: null
content: "Customer Number Currency 22.MAR.2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-22
- name: date_format_14
parser: parse_date
filename: null
content: "Customer Number Currency 22.MAR 2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-22
- name: date_format_15
parser: parse_date
filename: null
content: "Customer Number Currency 22.MAR.22 Credit Card 1934829304"
settings: {}
expected:
mode: none
- name: date_format_16
parser: parse_date
filename: null
content: "Customer Number Currency 22.MAR,22 Credit Card 1934829304"
settings: {}
expected:
mode: none
- name: date_format_17
parser: parse_date
filename: null
content: "Customer Number Currency 22,MAR,2022 Credit Card 1934829304"
settings: {}
expected:
mode: none
- name: date_format_18
parser: parse_date
filename: null
content: "Customer Number Currency 22 MAR,2022 Credit Card 1934829304"
settings: {}
expected:
mode: none
- name: date_format_19
parser: parse_date
filename: null
content: "Customer Number Currency 21st MAR 2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-21
- name: date_format_20
parser: parse_date
filename: null
content: "Customer Number Currency 22nd March 2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-22
- name: date_format_21
parser: parse_date
filename: null
content: "Customer Number Currency 2nd MAR 2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-02
- name: date_format_22
parser: parse_date
filename: null
content: "Customer Number Currency 23rd MAR 2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-23
- name: date_format_23
parser: parse_date
filename: null
content: "Customer Number Currency 24th MAR 2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-24
- name: date_format_24
parser: parse_date
filename: null
content: "Customer Number Currency 21-MAR-2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-21
- name: date_format_25
parser: parse_date
filename: null
content: "Customer Number Currency 25TH MAR 2022 Credit Card 1934829304"
settings: {}
expected:
mode: single
value: 2022-03-25
- name: date_format_26
parser: parse_date
filename: null
content: "CHASE 0 September 25, 2019 JPMorgan Chase Bank, NA. P0 Box 182051"
settings: {}
expected:
mode: single
value: 2019-09-25
- name: numeric_mdy_slash
parser: parse_date
filename: null
content: "03/17/2008"
settings:
DATE_ORDER: MDY
expected:
mode: single
value: 2008-03-17
- name: crazy_date_past
parser: parse_date
filename: null
content: "01-07-0590 00:00:00"
settings: {}
expected:
mode: none
- name: crazy_date_future
parser: parse_date
filename: null
content: "01-07-2350 00:00:00"
settings: {}
expected:
mode: none
- name: crazy_date_with_spaces
parser: parse_date
filename: null
content: "20 408000l 2475"
settings: {}
expected:
mode: none
- name: utf_month_names_decembre
parser: parse_date
filename: null
content: "13 décembre 2023"
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 2023-12-13
- name: utf_month_names_aout
parser: parse_date
filename: null
content: "13 août 2022"
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 2022-08-13
- name: utf_month_names_marz
parser: parse_date
filename: null
content: "11 März 2020"
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 2020-03-11
- name: utf_month_names_ozujka
parser: parse_date
filename: null
content: "17. ožujka 2018."
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 2018-03-17
- name: utf_month_names_veljace
parser: parse_date
filename: null
content: "1. veljače 2016."
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 2016-02-01
- name: utf_month_names_unora
parser: parse_date
filename: null
content: "15. února 1985"
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 1985-02-15
- name: utf_month_names_zari
parser: parse_date
filename: null
content: "30. září 2011"
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 2011-09-30
- name: utf_month_names_kvetna
parser: parse_date
filename: null
content: "28. května 1990"
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 1990-05-28
- name: utf_month_names_grudzien
parser: parse_date
filename: null
content: "1. grudzień 1997"
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 1997-12-01
- name: utf_month_names_subat
parser: parse_date
filename: null
content: "17 Şubat 2024"
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 2024-02-17
- name: utf_month_names_agustos
parser: parse_date
filename: null
content: "30 Ağustos 2012"
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 2012-08-30
- name: utf_month_names_eylul
parser: parse_date
filename: null
content: "17 Eylül 2000"
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
expected:
mode: single
value: 2000-09-17
- name: utf_month_names_oktober
parser: parse_date
filename: null
content: "5. október 1992"
settings:
DATE_PARSER_LANGUAGES:
- fr
- de
- hr
- cs
- pl
- tr
- hu
expected:
mode: single
value: 1992-10-05
- name: multiple_dates
parser: parse_date_generator
filename: null
content: |
This text has multiple dates.
For example 02.02.2018, 22 July 2022 and December 2021.
But not 24-12-9999 because it's in the future...
settings: {}
expected:
mode: multiple
value:
- 2018-02-02
- 2022-07-22
- 2021-12-01
- name: filename_date_parse_valid_ymd
parser: parse_date
filename: /tmp/Scan-2022-04-01.pdf
content: "No date in here"
settings:
FILENAME_DATE_ORDER: YMD
expected:
mode: single
value: 2022-04-01
- name: filename_date_parse_valid_dmy
parser: parse_date
filename: /tmp/Scan-10.01.2021.pdf
content: "No date in here"
settings:
FILENAME_DATE_ORDER: DMY
expected:
mode: single
value: 2021-01-10
- name: filename_date_parse_invalid
parser: parse_date
filename: "/tmp/20 408000l 2475 - test.pdf"
content: "No date in here"
settings:
FILENAME_DATE_ORDER: YMD
expected:
mode: none
- name: filename_date_ignored_use_content
parser: parse_date
filename: /tmp/Scan-2022-04-01.pdf
content: "The matching date is 24.03.2022"
settings:
FILENAME_DATE_ORDER: YMD
IGNORE_DATES:
- 2022-04-01
expected:
mode: single
value: 2022-03-24
- name: ignored_dates_default_order
parser: parse_date
filename: null
content: "lorem ipsum 110319, 20200117 and lorem 13.02.2018 lorem ipsum"
settings:
IGNORE_DATES:
- 2019-11-03
- 2020-01-17
expected:
mode: single
value: 2018-02-13
- name: ignored_dates_order_ymd
parser: parse_date
filename: null
content: "lorem ipsum 190311, 20200117 and lorem 13.02.2018 lorem ipsum"
settings:
FILENAME_DATE_ORDER: YMD
IGNORE_DATES:
- 2019-11-03
- 2020-01-17
expected:
mode: single
value: 2018-02-13
+184 -31
View File
@@ -1,14 +1,14 @@
mod common;
use anyhow::{anyhow, Result};
use axum::http::StatusCode;
use common::{acquire_db_lock, body_to_vec, TestApp, UploadExtras};
use diesel::prelude::*;
use papercrate::test_support::{acquire_db_lock, body_to_vec, TestApp, TestUserRole, UploadExtras};
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use uuid::Uuid;
use papercrate::jobs::{mark_job_succeeded, JOB_PURGE_DOCUMENT};
use papercrate::models::Job;
use papercrate::models::{Job, NewDocumentAsset};
use papercrate::schema::document_assets;
use papercrate::workers::{purge::PurgeDocumentJob, JobExecution, JobHandler};
use std::sync::Arc;
#[derive(Deserialize)]
@@ -46,11 +46,17 @@ struct DocumentVersionPayload {
id: Uuid,
version_number: i32,
size_bytes: i64,
download_path: String,
download: DownloadLinkPayload,
#[serde(default)]
assets: Vec<DocumentAssetInfo>,
}
#[derive(Deserialize)]
struct DownloadLinkPayload {
url: String,
expires_at: i64,
}
#[derive(Deserialize)]
struct DocumentVersionListItem {
id: Uuid,
@@ -64,6 +70,12 @@ struct DocumentAssetInfo {
asset_type: String,
}
#[derive(Deserialize)]
struct AssetProxyDetail {
id: Uuid,
download: Option<DownloadLinkPayload>,
}
#[derive(Deserialize)]
struct DocumentListItem {
id: Uuid,
@@ -151,7 +163,7 @@ async fn upload_and_list_document() -> Result<()> {
let app = TestApp::new().await?;
let password = "passw0rd";
app.insert_user("dana", password, "admin").await?;
app.insert_user("dana", TestUserRole::Owner).await?;
let token = app.login_token("dana", password).await?;
let file_bytes = b"example document body".to_vec();
@@ -186,7 +198,8 @@ async fn upload_and_list_document() -> Result<()> {
.current_version
.as_ref()
.expect("current version detail");
assert!(current_version.download_path.starts_with("/download/"));
assert!(current_version.download.url.starts_with("/api/download/"));
assert!(current_version.download.expires_at > 0);
assert_eq!(current_version.version_number, 1);
assert_eq!(current_version.size_bytes, file_bytes.len() as i64);
assert!(current_version.assets.is_empty());
@@ -217,10 +230,11 @@ async fn upload_and_list_document() -> Result<()> {
.current_version
.as_ref()
.expect("list current version")
.download_path
.starts_with("/download/"));
.download
.url
.starts_with("/api/download/"));
let redirect = app.get(&current_version.download_path, None).await?;
let redirect = app.get(&current_version.download.url, None).await?;
assert_eq!(redirect.status(), StatusCode::TEMPORARY_REDIRECT);
let location = redirect
.headers()
@@ -239,7 +253,7 @@ async fn upload_document_with_custom_title_sets_filename() -> Result<()> {
let app = TestApp::new().await?;
let password = "passw0rd";
app.insert_user("nora", password, "admin").await?;
app.insert_user("nora", TestUserRole::Owner).await?;
let token = app.login_token("nora", password).await?;
let file_bytes = b"example contract body".to_vec();
@@ -277,13 +291,83 @@ async fn upload_document_with_custom_title_sets_filename() -> Result<()> {
Ok(())
}
#[tokio::test]
async fn asset_detail_uses_proxy_urls_when_configured() -> Result<()> {
let _lock = acquire_db_lock().await;
let app = TestApp::with_config(|config| config.proxy_downloads = true).await?;
let tenant_id = app.tenant_id().await?;
let username = "proxy-assets";
let password = "secret";
app.insert_user(username, TestUserRole::Owner).await?;
let token = app.login_token(username, password).await?;
let upload = app
.upload_document(
"/api/documents",
"proxy.pdf",
"application/pdf",
b"dummy",
None,
&token,
)
.await?;
assert_eq!(upload.status(), StatusCode::CREATED);
let body = body_to_vec(upload.into_body()).await?;
let detail: DocumentDetail = serde_json::from_slice(&body)?;
let document = detail.document;
let version = document
.current_version
.as_ref()
.ok_or_else(|| anyhow!("current version missing"))?;
let mut conn = app
.state
.db_for_tenant(tenant_id)
.map_err(|err| anyhow!("tenant connection: {err:?}"))?;
let asset_id = Uuid::new_v4();
let s3_key = "objects/preview.png".to_string();
diesel::insert_into(document_assets::table)
.values(&NewDocumentAsset {
id: asset_id,
document_version_id: version.id,
asset_type: "preview".to_string(),
mime_type: "image/png".to_string(),
metadata: json!({}),
s3_key: s3_key.clone(),
tenant_id,
})
.execute(&mut conn)?;
drop(conn);
let response = app
.get(&format!("/api/assets/{asset_id}"), Some(&token))
.await?;
assert_eq!(response.status(), StatusCode::OK);
let body = body_to_vec(response.into_body()).await?;
let asset_detail: AssetProxyDetail = serde_json::from_slice(&body)?;
assert_eq!(asset_detail.id, asset_id);
let download = asset_detail
.download
.as_ref()
.ok_or_else(|| anyhow!("missing download link"))?;
assert!(download.url.starts_with("/api/download/"));
assert!(download.expires_at > 0);
app.cleanup().await?;
Ok(())
}
#[tokio::test]
async fn document_list_sorting_controls() -> Result<()> {
let _lock = acquire_db_lock().await;
let app = TestApp::new().await?;
let password = "passw0rd";
app.insert_user("sorting", password, "admin").await?;
app.insert_user("sorting", TestUserRole::Owner).await?;
let token = app.login_token("sorting", password).await?;
let first = app
@@ -362,7 +446,7 @@ async fn duplicate_and_restore_document() -> Result<()> {
let app = TestApp::new().await?;
let password = "pass1234";
app.insert_user("sam", password, "admin").await?;
app.insert_user("sam", TestUserRole::Owner).await?;
let token = app.login_token("sam", password).await?;
let payload = b"same bytes".to_vec();
@@ -422,6 +506,29 @@ async fn duplicate_and_restore_document() -> Result<()> {
.await?;
assert_eq!(delete.status(), StatusCode::NO_CONTENT);
let trashed_conflict = app
.upload_document(
"/api/documents",
"dup.bin",
"application/octet-stream",
&payload,
None,
&token,
)
.await?;
assert_eq!(trashed_conflict.status(), StatusCode::CONFLICT);
let trashed_body = body_to_vec(trashed_conflict.into_body()).await?;
let trashed_error: ApiErrorResponse = serde_json::from_slice(&trashed_body)?;
assert_eq!(trashed_error.code.as_deref(), Some("duplicate_document"));
assert!(trashed_error.error.contains("trash"));
let trashed_details = trashed_error.details.as_ref().expect("details present");
assert_eq!(
trashed_details
.get("conflict_document_in_trash")
.and_then(|value| value.as_bool()),
Some(true)
);
let third = app
.upload_document_with_extras(
"/api/documents",
@@ -473,7 +580,7 @@ async fn upload_skips_existing_when_requested() -> Result<()> {
let app = TestApp::new().await?;
let password = "skip-doc";
app.insert_user("skip", password, "admin").await?;
app.insert_user("skip", TestUserRole::Owner).await?;
let token = app.login_token("skip", password).await?;
let primary_tag_payload = CreateTagPayload {
@@ -611,7 +718,7 @@ async fn filter_documents_without_tags() -> Result<()> {
let app = TestApp::new().await?;
let password = "tagfilter";
app.insert_user("tagfilter", password, "admin").await?;
app.insert_user("tagfilter", TestUserRole::Owner).await?;
let token = app.login_token("tagfilter", password).await?;
// Create a tag and upload a document that uses it.
@@ -686,7 +793,7 @@ async fn bulk_move_documents_to_folder() -> Result<()> {
let app = TestApp::new().await?;
let password = "bulkmove";
app.insert_user("mover", password, "admin").await?;
app.insert_user("mover", TestUserRole::Owner).await?;
let token = app.login_token("mover", password).await?;
let alpha = app
@@ -799,7 +906,7 @@ async fn bulk_add_tags_for_selection() -> Result<()> {
let app = TestApp::new().await?;
let password = "bulktags";
app.insert_user("tagger", password, "admin").await?;
app.insert_user("tagger", TestUserRole::Owner).await?;
let token = app.login_token("tagger", password).await?;
let first = app
@@ -939,7 +1046,7 @@ async fn bulk_remove_tags_from_selection() -> Result<()> {
let app = TestApp::new().await?;
let password = "bulktagremove";
app.insert_user("tagrem", password, "admin").await?;
app.insert_user("tagrem", TestUserRole::Owner).await?;
let token = app.login_token("tagrem", password).await?;
let first = app
@@ -1067,7 +1174,7 @@ async fn bulk_reanalyze_selected_documents() -> Result<()> {
let app = TestApp::new().await?;
let password = "subsetrean";
app.insert_user("subset", password, "admin").await?;
app.insert_user("subset", TestUserRole::Owner).await?;
let token = app.login_token("subset", password).await?;
app.clear_jobs().await?;
@@ -1178,7 +1285,7 @@ async fn patch_document_updates_title_and_handles_conflict() -> Result<()> {
let app = TestApp::new().await?;
let password = "patch-title";
app.insert_user("editor", password, "admin").await?;
app.insert_user("editor", TestUserRole::Owner).await?;
let token = app.login_token("editor", password).await?;
let first_upload = app
@@ -1246,7 +1353,7 @@ async fn patch_document_updates_and_clears_issued_at() -> Result<()> {
let app = TestApp::new().await?;
let password = "patch-issued";
app.insert_user("scheduler", password, "admin").await?;
app.insert_user("scheduler", TestUserRole::Owner).await?;
let token = app.login_token("scheduler", password).await?;
let upload = app
@@ -1311,7 +1418,7 @@ async fn patch_document_metadata_merge_and_replace() -> Result<()> {
let app = TestApp::new().await?;
let password = "patch-meta";
app.insert_user("curator", password, "admin").await?;
app.insert_user("curator", TestUserRole::Owner).await?;
let token = app.login_token("curator", password).await?;
let initial_metadata = r#"{"existing":{"keep":true},"other":1}"#;
@@ -1390,7 +1497,7 @@ async fn patch_document_validation_errors() -> Result<()> {
let app = TestApp::new().await?;
let password = "patch-errors";
app.insert_user("auditor", password, "admin").await?;
app.insert_user("auditor", TestUserRole::Owner).await?;
let token = app.login_token("auditor", password).await?;
let upload = app
@@ -1513,7 +1620,7 @@ async fn patch_document_updates_multiple_fields() -> Result<()> {
let app = TestApp::new().await?;
let password = "patch-multi";
app.insert_user("planner", password, "admin").await?;
app.insert_user("planner", TestUserRole::Owner).await?;
let token = app.login_token("planner", password).await?;
let upload = app
@@ -1572,7 +1679,7 @@ async fn list_documents_by_status_filter() -> Result<()> {
let app = TestApp::new().await?;
let password = "statusfilter";
app.insert_user("statususer", password, "admin").await?;
app.insert_user("statususer", TestUserRole::Owner).await?;
let token = app.login_token("statususer", password).await?;
let upload = app
@@ -1620,13 +1727,57 @@ async fn list_documents_by_status_filter() -> Result<()> {
app.cleanup().await?;
Ok(())
}
#[tokio::test]
async fn trash_document_requires_active_state() -> Result<()> {
let _lock = acquire_db_lock().await;
let app = TestApp::new().await?;
let password = "trashstate";
app.insert_user("trashstate", TestUserRole::Owner).await?;
let token = app.login_token("trashstate", password).await?;
let upload = app
.upload_document(
"/api/documents",
"trash-once.txt",
"text/plain",
b"trash",
None,
&token,
)
.await?;
let body = body_to_vec(upload.into_body()).await?;
let detail: DocumentDetail = serde_json::from_slice(&body)?;
let first = app
.post_json(
&format!("/api/documents/{}/trash", detail.document.id),
&json!({}),
Some(&token),
)
.await?;
assert_eq!(first.status(), StatusCode::NO_CONTENT);
let second = app
.post_json(
&format!("/api/documents/{}/trash", detail.document.id),
&json!({}),
Some(&token),
)
.await?;
assert_eq!(second.status(), StatusCode::CONFLICT);
app.cleanup().await?;
Ok(())
}
#[tokio::test]
async fn purge_document_removes_data() -> Result<()> {
let _lock = acquire_db_lock().await;
let app = TestApp::new().await?;
let password = "purge";
app.insert_user("purger", password, "admin").await?;
app.insert_user("purger", TestUserRole::Owner).await?;
let token = app.login_token("purger", password).await?;
let upload = app
@@ -1696,7 +1847,7 @@ async fn purge_document_removes_data() -> Result<()> {
let state = Arc::new(app.state.clone());
let storage = app
.state
.storage_for_tenant(job.tenant_id)
.storage_for_tenant(job.tenant_id.expect("job should have tenant"))
.map_err(|err| anyhow!("tenant storage unavailable: {err:?}"))?;
let execution = handler.handle(state, job.clone(), storage).await;
assert!(matches!(execution, JobExecution::Success));
@@ -1724,7 +1875,8 @@ async fn delete_document_requires_trash() -> Result<()> {
let app = TestApp::new().await?;
let password = "conflict";
app.insert_user("conflict-user", password, "admin").await?;
app.insert_user("conflict-user", TestUserRole::Owner)
.await?;
let token = app.login_token("conflict-user", password).await?;
let upload = app
@@ -1758,7 +1910,7 @@ async fn restore_document_to_original_and_custom_folder() -> Result<()> {
let app = TestApp::new().await?;
let password = "restoretest";
app.insert_user("restorer", password, "admin").await?;
app.insert_user("restorer", TestUserRole::Owner).await?;
let token = app.login_token("restorer", password).await?;
let upload = app
@@ -1861,7 +2013,7 @@ async fn list_document_versions_and_fetch_detail() -> Result<()> {
let app = TestApp::new().await?;
let password = "versionlist";
app.insert_user("versions", password, "admin").await?;
app.insert_user("versions", TestUserRole::Owner).await?;
let token = app.login_token("versions", password).await?;
let upload = app
@@ -1910,7 +2062,8 @@ async fn list_document_versions_and_fetch_detail() -> Result<()> {
let detail_body = body_to_vec(detail_resp.into_body()).await?;
let version_detail: DocumentVersionPayload = serde_json::from_slice(&detail_body)?;
assert_eq!(version_detail.id, version_id);
assert!(version_detail.download_path.starts_with("/download/"));
assert!(version_detail.download.url.starts_with("/api/download/"));
assert!(version_detail.download.expires_at > 0);
assert!(version_detail.assets.is_empty());
app.cleanup().await?;
+84
View File
@@ -0,0 +1,84 @@
use anyhow::Result;
use axum::http::StatusCode;
use papercrate::test_support::{acquire_db_lock, body_to_vec, TestApp, TestUserRole};
use serde::Deserialize;
#[derive(Clone, Deserialize)]
struct DocumentDetail {
document: DocumentInfo,
}
#[derive(Clone, Deserialize)]
struct DocumentInfo {
current_version: Option<DocumentVersion>,
}
#[derive(Clone, Deserialize)]
struct DocumentVersion {
download: DownloadLink,
}
#[derive(Clone, Deserialize)]
struct DownloadLink {
url: String,
expires_at: i64,
}
#[tokio::test]
async fn document_download_redirects_when_proxy_disabled() -> Result<()> {
let _lock = acquire_db_lock().await;
let app = TestApp::new().await?;
let username = "download-user";
let password = "secret";
app.insert_user(username, TestUserRole::Owner).await?;
let token = app.login_token(username, password).await?;
let upload = app
.upload_document(
"/api/documents",
"download.pdf",
"application/pdf",
b"dummy",
None,
&token,
)
.await?;
assert_eq!(upload.status(), StatusCode::CREATED);
let body = body_to_vec(upload.into_body()).await?;
let detail: DocumentDetail = serde_json::from_slice(&body)?;
let download_link = detail
.document
.current_version
.as_ref()
.expect("missing version")
.download
.clone();
assert!(download_link.expires_at > 0);
let download_path = download_link.url.clone();
let redirect = app.get(&download_path, None).await?;
assert_eq!(redirect.status(), StatusCode::TEMPORARY_REDIRECT);
let location = redirect
.headers()
.get("location")
.expect("redirect location header")
.to_str()
.expect("location utf8");
assert!(location.starts_with("https://fake-storage/"));
app.cleanup().await?;
Ok(())
}
#[tokio::test]
async fn download_with_invalid_token_is_rejected() -> Result<()> {
let _lock = acquire_db_lock().await;
let app = TestApp::new().await?;
let response = app.get("/api/download/not-a-token", None).await?;
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
app.cleanup().await?;
Ok(())
}
+8 -10
View File
@@ -1,8 +1,6 @@
mod common;
use anyhow::Result;
use axum::http::StatusCode;
use common::{acquire_db_lock, body_to_vec, TestApp};
use papercrate::test_support::{acquire_db_lock, body_to_vec, TestApp, TestUserRole};
use serde::Deserialize;
use serde::Serialize;
use serde_json::json;
@@ -66,7 +64,7 @@ async fn folder_move_and_delete_flow() -> Result<()> {
let app = TestApp::new().await?;
let password = "folderpass";
app.insert_user("folder-admin", password, "admin").await?;
app.insert_user("folder-admin", TestUserRole::Owner).await?;
let token = app.login_token("folder-admin", password).await?;
let folder_resp = app
@@ -156,7 +154,7 @@ async fn folder_tree_lists_hierarchy() -> Result<()> {
let app = TestApp::new().await?;
let password = "folderpass";
app.insert_user("folder-tree", password, "admin").await?;
app.insert_user("folder-tree", TestUserRole::Owner).await?;
let token = app.login_token("folder-tree", password).await?;
let alpha_resp = app
@@ -221,7 +219,7 @@ async fn update_folder_parent_to_root() -> Result<()> {
let app = TestApp::new().await?;
let password = "rootpass";
app.insert_user("root-admin", password, "admin").await?;
app.insert_user("root-admin", TestUserRole::Owner).await?;
let token = app.login_token("root-admin", password).await?;
// Create a parent folder under root
@@ -293,7 +291,7 @@ async fn ensure_path_creates_nested_folders() -> Result<()> {
let app = TestApp::new().await?;
let password = "pathpass";
app.insert_user("path-admin", password, "admin").await?;
app.insert_user("path-admin", TestUserRole::Owner).await?;
let token = app.login_token("path-admin", password).await?;
let base_path = EnsureFolderPath {
@@ -370,7 +368,7 @@ async fn create_folder_is_idempotent() -> Result<()> {
let app = TestApp::new().await?;
let password = "idempotent";
app.insert_user("folders-idem", password, "admin").await?;
app.insert_user("folders-idem", TestUserRole::Owner).await?;
let token = app.login_token("folders-idem", password).await?;
let payload = CreateFolder {
@@ -414,7 +412,7 @@ async fn ensure_folder_path_is_idempotent() -> Result<()> {
let app = TestApp::new().await?;
let password = "pathpass";
app.insert_user("path-admin", password, "admin").await?;
app.insert_user("path-admin", TestUserRole::Owner).await?;
let token = app.login_token("path-admin", password).await?;
let segments = ["500 Immobilien", "501 Kreuzweg 2", "501.01 Rechtliches"];
@@ -482,7 +480,7 @@ async fn folder_rename_updates_name_and_child_paths() -> Result<()> {
let app = TestApp::new().await?;
let password = "renamepass";
app.insert_user("rename-admin", password, "admin").await?;
app.insert_user("rename-admin", TestUserRole::Owner).await?;
let token = app.login_token("rename-admin", password).await?;
let parent_resp = app
+3 -5
View File
@@ -1,8 +1,5 @@
mod common;
use anyhow::{anyhow, Result};
use axum::http::StatusCode;
use common::{acquire_db_lock, body_to_vec, TestApp};
use diesel::prelude::*;
use papercrate::auth::capability_sets::{ensure_capability_set, owner_capabilities};
use papercrate::models::{NewUser, NewUserMembership, Tag, TenantStatus};
@@ -10,6 +7,7 @@ use papercrate::schema::{
tags::dsl as tags_dsl, tenants::dsl as tenants_dsl, user_memberships::dsl as memberships_dsl,
users::dsl as users_dsl,
};
use papercrate::test_support::{acquire_db_lock, body_to_vec, TestApp, TestUserRole};
use serde::Deserialize;
use serde::Serialize;
use uuid::Uuid;
@@ -57,7 +55,7 @@ async fn tag_assignment_flow() -> Result<()> {
let app = TestApp::new().await?;
let password = "tagpass";
app.insert_user("tagger", password, "admin").await?;
app.insert_user("tagger", TestUserRole::Owner).await?;
let token = app.login_token("tagger", password).await?;
let upload = app
@@ -186,7 +184,7 @@ async fn tags_are_isolated_between_tenants() -> Result<()> {
let app = TestApp::new().await?;
let password_a = "tenant-a";
app.insert_user("alice", password_a, "admin").await?;
app.insert_user("alice", TestUserRole::Owner).await?;
let token_a = app.login_token("alice", password_a).await?;
let shared_label = "Shared Label";
+715
View File
@@ -0,0 +1,715 @@
use anyhow::{anyhow, bail, Result};
use axum::http::StatusCode;
use chrono::{Duration as ChronoDuration, Utc};
use diesel::dsl::{count_star, exists, select};
use diesel::prelude::*;
use papercrate::jobs::{
enqueue_job, mark_job_failed, mark_job_succeeded, JOB_DELETE_TENANT, STATUS_FAILED,
STATUS_SUCCEEDED,
};
use papercrate::models::TenantStatus;
use papercrate::schema::{documents, jobs, tenants, user_memberships};
use papercrate::test_support::{acquire_db_lock, body_to_vec, TestApp, TestUserRole};
use papercrate::workers::tenants::{
build_delete_proof_message, sign_delete_proof, DeleteAction, DeleteTenantJob,
};
use papercrate::workers::{JobExecution, JobHandler};
use serde_json::{json, Value};
use std::sync::Arc;
use uuid::Uuid;
#[tokio::test]
async fn delete_tenant_job_keeps_tenant_when_requested() -> Result<()> {
let _lock = acquire_db_lock().await;
let app = TestApp::new().await?;
let password = "delete-keep";
app.insert_user("tenant-keep", TestUserRole::Owner).await?;
let token = app.login_token("tenant-keep", password).await?;
upload_fixture(&app, &token, "keep.pdf", b"keep").await?;
let tenant_id = default_tenant_id(&app)?;
set_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?;
assert_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?;
let storage = tenant_storage(&app, tenant_id)?;
let storage_prefix = storage.root_prefix().to_string();
let before = tenant_snapshot(&app, tenant_id, &storage_prefix).await?;
assert_eq!(before.doc_count, 1);
assert!(before.membership_count > 0);
assert!(!before.storage_keys.is_empty());
assert_storage_keys_present(&app, &before.storage_keys).await?;
let job = enqueue_delete_job(&app, tenant_id, false).await?;
let job_id = job.id;
let handler = DeleteTenantJob::new();
let state = Arc::new(app.state.clone());
let execution = handler.handle(state, job, storage).await;
assert_job_success(&execution);
record_job_outcome(&app, job_id, &execution).await?;
assert_eq!(fetch_job_status(&app, job_id).await?, STATUS_SUCCEEDED);
let after = tenant_snapshot(&app, tenant_id, &storage_prefix).await?;
assert_eq!(after.doc_count, 0);
assert!(after.membership_count > 0);
assert_storage_keys_absent(&app, &before.storage_keys).await?;
assert_eq!(storage_object_count(&app, &storage_prefix).await?, 0);
assert_eq!(
fetch_tenant_status(&app, tenant_id).await?,
TenantStatus::Suspended
);
assert!(tenant_exists(&app, tenant_id).await?);
app.cleanup().await?;
Ok(())
}
#[tokio::test]
async fn delete_tenant_job_can_reset_tenant_to_active() -> Result<()> {
let _lock = acquire_db_lock().await;
let app = TestApp::new().await?;
let password = "delete-reset";
app.insert_user("tenant-reset", TestUserRole::Owner).await?;
let token = app.login_token("tenant-reset", password).await?;
upload_fixture(&app, &token, "reset.pdf", b"reset").await?;
let tenant_id = default_tenant_id(&app)?;
set_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?;
assert_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?;
let storage = tenant_storage(&app, tenant_id)?;
let storage_prefix = storage.root_prefix().to_string();
let before = tenant_snapshot(&app, tenant_id, &storage_prefix).await?;
assert_eq!(before.doc_count, 1);
assert!(before.membership_count > 0);
assert!(!before.storage_keys.is_empty());
assert_storage_keys_present(&app, &before.storage_keys).await?;
let job = enqueue_delete_job_with_status(&app, tenant_id, false, Some("active")).await?;
let job_id = job.id;
let handler = DeleteTenantJob::new();
let state = Arc::new(app.state.clone());
let execution = handler.handle(state, job, storage).await;
assert_job_success(&execution);
record_job_outcome(&app, job_id, &execution).await?;
assert_eq!(fetch_job_status(&app, job_id).await?, STATUS_SUCCEEDED);
let after = tenant_snapshot(&app, tenant_id, &storage_prefix).await?;
assert_eq!(after.doc_count, 0);
assert!(after.membership_count > 0);
assert_storage_keys_absent(&app, &before.storage_keys).await?;
assert_eq!(storage_object_count(&app, &storage_prefix).await?, 0);
assert_eq!(
fetch_tenant_status(&app, tenant_id).await?,
TenantStatus::Active
);
app.cleanup().await?;
Ok(())
}
#[tokio::test]
async fn delete_tenant_job_removes_tenant_entirely() -> Result<()> {
let _lock = acquire_db_lock().await;
let app = TestApp::new().await?;
let password = "delete-remove";
app.insert_user("tenant-remove", TestUserRole::Owner)
.await?;
let token = app.login_token("tenant-remove", password).await?;
upload_fixture(&app, &token, "remove-1.pdf", b"remove-1").await?;
upload_fixture(&app, &token, "remove-2.pdf", b"remove-2").await?;
let tenant_id = default_tenant_id(&app)?;
set_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?;
assert_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?;
let storage = tenant_storage(&app, tenant_id)?;
let storage_prefix = storage.root_prefix().to_string();
let before = tenant_snapshot(&app, tenant_id, &storage_prefix).await?;
assert!(before.doc_count >= 2);
assert!(before.membership_count > 0);
assert!(before.storage_keys.len() >= 2);
assert_storage_keys_present(&app, &before.storage_keys).await?;
let job = enqueue_delete_job(&app, tenant_id, true).await?;
let job_id = job.id;
let handler = DeleteTenantJob::new();
let state = Arc::new(app.state.clone());
let execution = handler.handle(state, job, storage).await;
assert_job_success(&execution);
record_job_outcome(&app, job_id, &execution).await?;
assert_eq!(fetch_job_status(&app, job_id).await?, STATUS_SUCCEEDED);
let after = tenant_snapshot(&app, tenant_id, &storage_prefix).await?;
assert_eq!(after.doc_count, 0);
assert_eq!(after.membership_count, 0);
assert!(after.storage_keys.is_empty());
assert_storage_keys_absent(&app, &before.storage_keys).await?;
assert_eq!(storage_object_count(&app, &storage_prefix).await?, 0);
assert!(!tenant_exists(&app, tenant_id).await?);
app.cleanup().await?;
Ok(())
}
#[tokio::test]
async fn delete_tenant_job_rejects_invalid_signature() -> Result<()> {
let _lock = acquire_db_lock().await;
let app = TestApp::new().await?;
let password = "delete-invalid";
app.insert_user("tenant-invalid", TestUserRole::Owner)
.await?;
let token = app.login_token("tenant-invalid", password).await?;
upload_fixture(&app, &token, "invalid.pdf", b"invalid").await?;
let tenant_id = default_tenant_id(&app)?;
set_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?;
assert_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?;
let storage = tenant_storage(&app, tenant_id)?;
let storage_prefix = storage.root_prefix().to_string();
let before = tenant_snapshot(&app, tenant_id, &storage_prefix).await?;
assert_eq!(before.doc_count, 1);
assert!(before.membership_count > 0);
assert!(!before.storage_keys.is_empty());
assert_storage_keys_present(&app, &before.storage_keys).await?;
let job = enqueue_delete_job_with_invalid_signature(&app, tenant_id, false).await?;
let job_id = job.id;
let handler = DeleteTenantJob::new();
let state = Arc::new(app.state.clone());
let execution = handler.handle(state, job, storage).await;
match execution {
JobExecution::Failed { ref error } => {
assert!(error.contains("signature"), "unexpected error: {error}");
}
_ => bail!("delete job should fail when signature is invalid"),
}
record_job_outcome(&app, job_id, &execution).await?;
assert_eq!(fetch_job_status(&app, job_id).await?, STATUS_FAILED);
let after = tenant_snapshot(&app, tenant_id, &storage_prefix).await?;
assert_eq!(after.doc_count, before.doc_count);
assert_eq!(after.membership_count, before.membership_count);
assert_storage_keys_present(&app, &before.storage_keys).await?;
assert_eq!(
storage_object_count(&app, &storage_prefix).await?,
before.storage_keys.len()
);
assert_eq!(
fetch_tenant_status(&app, tenant_id).await?,
TenantStatus::Deleting
);
app.cleanup().await?;
Ok(())
}
#[tokio::test]
async fn delete_tenant_job_rejects_invalid_final_status() -> Result<()> {
let _lock = acquire_db_lock().await;
let app = TestApp::new().await?;
let password = "delete-invalid-status";
app.insert_user("tenant-invalid-status", TestUserRole::Owner)
.await?;
let token = app.login_token("tenant-invalid-status", password).await?;
upload_fixture(&app, &token, "invalid-status.pdf", b"payload").await?;
let tenant_id = default_tenant_id(&app)?;
set_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?;
assert_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?;
let storage = tenant_storage(&app, tenant_id)?;
let storage_prefix = storage.root_prefix().to_string();
let before = tenant_snapshot(&app, tenant_id, &storage_prefix).await?;
assert_eq!(before.doc_count, 1);
assert!(before.membership_count > 0);
let job = enqueue_delete_job_with_overrides(
&app,
tenant_id,
false,
None,
PayloadOverrides {
final_status: Some("weird"),
..PayloadOverrides::default()
},
)
.await?;
let job_id = job.id;
let handler = DeleteTenantJob::new();
let state = Arc::new(app.state.clone());
let execution = handler.handle(state, job, storage).await;
match execution {
JobExecution::Failed { ref error } => {
assert!(error.contains("payload"), "unexpected error: {error}");
}
_ => bail!("delete job should fail when final_status is invalid"),
}
record_job_outcome(&app, job_id, &execution).await?;
assert_eq!(fetch_job_status(&app, job_id).await?, STATUS_FAILED);
let after = tenant_snapshot(&app, tenant_id, &storage_prefix).await?;
assert_eq!(after.doc_count, before.doc_count);
assert_eq!(after.membership_count, before.membership_count);
assert_storage_keys_present(&app, &before.storage_keys).await?;
assert_eq!(
fetch_tenant_status(&app, tenant_id).await?,
TenantStatus::Deleting
);
app.cleanup().await?;
Ok(())
}
#[tokio::test]
async fn delete_tenant_job_rejects_malformed_issued_at() -> Result<()> {
let _lock = acquire_db_lock().await;
let app = TestApp::new().await?;
let password = "delete-bad-issued-at";
app.insert_user("tenant-issued", TestUserRole::Owner)
.await?;
let token = app.login_token("tenant-issued", password).await?;
upload_fixture(&app, &token, "issued.pdf", b"issued").await?;
let tenant_id = default_tenant_id(&app)?;
set_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?;
assert_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?;
let storage = tenant_storage(&app, tenant_id)?;
let job = enqueue_delete_job_with_overrides(
&app,
tenant_id,
false,
None,
PayloadOverrides {
issued_at: Some("definitely-not-time"),
..PayloadOverrides::default()
},
)
.await?;
let job_id = job.id;
let handler = DeleteTenantJob::new();
let state = Arc::new(app.state.clone());
let execution = handler.handle(state, job, storage).await;
match execution {
JobExecution::Failed { ref error } => {
assert!(error.contains("issued_at"), "unexpected error: {error}");
}
_ => bail!("delete job should fail when issued_at is malformed"),
}
record_job_outcome(&app, job_id, &execution).await?;
assert_eq!(fetch_job_status(&app, job_id).await?, STATUS_FAILED);
app.cleanup().await?;
Ok(())
}
#[tokio::test]
async fn delete_tenant_job_rejects_stale_confirmation() -> Result<()> {
let _lock = acquire_db_lock().await;
let app = TestApp::new().await?;
let password = "delete-stale";
app.insert_user("tenant-stale", TestUserRole::Owner).await?;
let token = app.login_token("tenant-stale", password).await?;
upload_fixture(&app, &token, "stale.pdf", b"stale").await?;
let tenant_id = default_tenant_id(&app)?;
set_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?;
assert_tenant_status(&app, tenant_id, TenantStatus::Deleting).await?;
let storage = tenant_storage(&app, tenant_id)?;
let stale_time = (Utc::now() - ChronoDuration::minutes(10)).to_rfc3339();
let job = enqueue_delete_job_with_overrides(
&app,
tenant_id,
false,
None,
PayloadOverrides {
issued_at: Some(&stale_time),
..PayloadOverrides::default()
},
)
.await?;
let job_id = job.id;
let handler = DeleteTenantJob::new();
let state = Arc::new(app.state.clone());
let execution = handler.handle(state, job, storage).await;
match execution {
JobExecution::Failed { ref error } => {
assert!(error.contains("expired"), "unexpected error: {error}");
}
_ => bail!("delete job should fail when confirmation is stale"),
}
record_job_outcome(&app, job_id, &execution).await?;
assert_eq!(fetch_job_status(&app, job_id).await?, STATUS_FAILED);
app.cleanup().await?;
Ok(())
}
async fn upload_fixture(app: &TestApp, token: &str, filename: &str, contents: &[u8]) -> Result<()> {
let response = app
.upload_document(
"/api/documents",
filename,
"application/pdf",
contents,
None,
token,
)
.await?;
assert_eq!(response.status(), StatusCode::CREATED);
body_to_vec(response.into_body()).await?;
Ok(())
}
fn default_tenant_id(app: &TestApp) -> Result<Uuid> {
Ok(app
.state
.tenants
.get_by_name("test_tenant")
.map_err(|err| anyhow!("tenant lookup failed: {err:?}"))?
.id)
}
async fn set_tenant_status(app: &TestApp, tenant_id: Uuid, status: TenantStatus) -> Result<()> {
app.with_conn(move |conn| {
diesel::update(tenants::table.find(tenant_id))
.set(tenants::status.eq(status))
.execute(conn)?;
Ok(())
})
.await
}
async fn assert_tenant_status(
app: &TestApp,
tenant_id: Uuid,
expected: TenantStatus,
) -> Result<()> {
let status = fetch_tenant_status(app, tenant_id).await?;
assert_eq!(status, expected);
Ok(())
}
async fn tenant_snapshot(
app: &TestApp,
tenant_id: Uuid,
storage_prefix: &str,
) -> Result<TenantSnapshot> {
let (doc_count, membership_count) = app
.with_conn(move |conn| {
let doc_count: i64 = documents::table
.filter(documents::tenant_id.eq(tenant_id))
.select(count_star())
.get_result(conn)?;
let membership_count: i64 = user_memberships::table
.filter(user_memberships::tenant_id.eq(tenant_id))
.select(count_star())
.get_result(conn)?;
Ok::<_, anyhow::Error>((doc_count, membership_count))
})
.await?;
let storage_keys = app.storage().keys_with_prefix(storage_prefix).await;
Ok(TenantSnapshot {
doc_count,
membership_count,
storage_keys,
})
}
async fn tenant_exists(app: &TestApp, tenant_id: Uuid) -> Result<bool> {
app.with_conn(move |conn| {
let exists_value: bool =
select(exists(tenants::table.filter(tenants::id.eq(tenant_id)))).get_result(conn)?;
Ok(exists_value)
})
.await
}
async fn fetch_tenant_status(app: &TestApp, tenant_id: Uuid) -> Result<TenantStatus> {
app.with_conn(move |conn| {
tenants::table
.find(tenant_id)
.select(tenants::status)
.first(conn)
.map_err(Into::into)
})
.await
}
async fn record_job_outcome(app: &TestApp, job_id: Uuid, execution: &JobExecution) -> Result<()> {
match execution {
JobExecution::Success => {
app.with_conn(move |conn| {
mark_job_succeeded(conn, job_id)
.map_err(|err| anyhow!("mark succeeded failed: {err}"))
})
.await?
}
JobExecution::Failed { error } => {
let error = error.clone();
app.with_conn(move |conn| {
mark_job_failed(conn, job_id, &error)
.map_err(|err| anyhow!("mark failed failed: {err}"))
})
.await?
}
JobExecution::Retry { .. } => bail!("retry outcome not expected in tenant deletion tests"),
}
Ok(())
}
async fn fetch_job_status(app: &TestApp, job_id: Uuid) -> Result<String> {
app.with_conn(move |conn| {
jobs::table
.find(job_id)
.select(jobs::status)
.first::<String>(conn)
.map_err(Into::into)
})
.await
}
fn tenant_storage(app: &TestApp, tenant_id: Uuid) -> Result<papercrate::storage::TenantStorage> {
app.state
.storage_for_tenant(tenant_id)
.map_err(|err| anyhow!("storage unavailable: {err:?}"))
}
async fn storage_object_count(app: &TestApp, prefix: &str) -> Result<usize> {
Ok(app.storage().object_count_with_prefix(prefix).await)
}
async fn assert_storage_keys_present(app: &TestApp, keys: &[String]) -> Result<()> {
let storage = app.storage();
for key in keys {
assert!(
storage.contains_key(key).await,
"expected storage object '{}' to exist",
key
);
}
Ok(())
}
async fn assert_storage_keys_absent(app: &TestApp, keys: &[String]) -> Result<()> {
let storage = app.storage();
for key in keys {
assert!(
!storage.contains_key(key).await,
"expected storage object '{}' to be deleted",
key
);
}
Ok(())
}
async fn enqueue_delete_job(
app: &TestApp,
tenant_id: Uuid,
remove_tenant: bool,
) -> Result<papercrate::models::Job> {
enqueue_delete_job_with_status(app, tenant_id, remove_tenant, None).await
}
async fn enqueue_delete_job_with_status(
app: &TestApp,
tenant_id: Uuid,
remove_tenant: bool,
final_status: Option<&'static str>,
) -> Result<papercrate::models::Job> {
enqueue_delete_job_with_overrides(
app,
tenant_id,
remove_tenant,
final_status,
PayloadOverrides::default(),
)
.await
}
async fn enqueue_delete_job_with_invalid_signature(
app: &TestApp,
tenant_id: Uuid,
remove_tenant: bool,
) -> Result<papercrate::models::Job> {
let mut job = enqueue_delete_job_with_overrides(
app,
tenant_id,
remove_tenant,
None,
PayloadOverrides::default(),
)
.await?;
let job_id = job.id;
let mut payload = job.payload.clone();
payload["signature"] = json!("deadbeefdeadbeefdeadbeefdeadbeef");
let payload_for_db = payload.clone();
app.with_conn(move |conn| {
diesel::update(jobs::table.find(job_id))
.set(jobs::payload.eq(payload_for_db))
.execute(conn)?;
Ok(())
})
.await?;
job.payload = payload;
Ok(job)
}
async fn enqueue_delete_job_with_overrides(
app: &TestApp,
tenant_id: Uuid,
remove_tenant: bool,
final_status: Option<&str>,
overrides: PayloadOverrides<'_>,
) -> Result<papercrate::models::Job> {
let secret = app.state.config.jwt_secret.clone();
let overrides_owned = PayloadOverridesOwned::from(overrides);
let requested_final_status = final_status.map(|value| value.to_string());
app.with_conn(move |conn| {
let tenant_name: String = tenants::table
.find(tenant_id)
.select(tenants::name)
.first(conn)
.map_err(|err| anyhow!("tenant lookup failed: {err}"))?;
let payload = build_signed_delete_payload(
tenant_id,
&tenant_name,
remove_tenant,
requested_final_status.as_deref(),
&secret,
overrides_owned.as_borrowed(),
)?;
enqueue_job(conn, tenant_id, JOB_DELETE_TENANT, payload, None)
.map_err(|err| anyhow!("enqueue failed: {err}"))
})
.await
}
fn build_signed_delete_payload(
tenant_id: Uuid,
tenant_name: &str,
remove_tenant: bool,
requested_final_status: Option<&str>,
secret: &str,
overrides: PayloadOverrides<'_>,
) -> Result<Value> {
let action = if remove_tenant {
DeleteAction::Delete
} else {
DeleteAction::Reset
};
let nonce = overrides
.nonce
.map(|value| value.to_string())
.unwrap_or_else(|| format!("test-delete-nonce-{}", Uuid::new_v4()));
let issued_at = overrides
.issued_at
.map(|value| value.to_string())
.unwrap_or_else(|| Utc::now().to_rfc3339());
let payload_final_status = if remove_tenant {
None
} else if let Some(value) = overrides.final_status {
Some(value.to_string())
} else {
Some(requested_final_status.unwrap_or("suspended").to_string())
};
let message = build_delete_proof_message(
tenant_id,
tenant_name,
action,
&nonce,
&issued_at,
payload_final_status.as_deref(),
);
let signature = sign_delete_proof(secret, &message)
.map_err(|err| anyhow!("failed to sign delete proof: {err}"))?;
let mut payload = json!({
"remove_tenant": remove_tenant,
"tenant_name": tenant_name,
"action": action.as_str(),
"nonce": nonce,
"issued_at": issued_at,
"signature": signature,
});
if let Some(status) = payload_final_status {
payload["final_status"] = json!(status);
}
Ok(payload)
}
struct TenantSnapshot {
doc_count: i64,
membership_count: i64,
storage_keys: Vec<String>,
}
#[derive(Default)]
struct PayloadOverrides<'a> {
final_status: Option<&'a str>,
issued_at: Option<&'a str>,
nonce: Option<&'a str>,
}
fn assert_job_success(execution: &JobExecution) {
match execution {
JobExecution::Success => {}
JobExecution::Failed { error } => panic!("delete job failed unexpectedly: {error}"),
JobExecution::Retry { error, .. } => {
panic!("delete job asked for retry unexpectedly: {error}")
}
}
}
#[derive(Default, Clone)]
struct PayloadOverridesOwned {
final_status: Option<String>,
issued_at: Option<String>,
nonce: Option<String>,
}
impl<'a> From<PayloadOverrides<'a>> for PayloadOverridesOwned {
fn from(value: PayloadOverrides<'a>) -> Self {
Self {
final_status: value.final_status.map(|s| s.to_string()),
issued_at: value.issued_at.map(|s| s.to_string()),
nonce: value.nonce.map(|s| s.to_string()),
}
}
}
impl PayloadOverridesOwned {
fn as_borrowed(&self) -> PayloadOverrides<'_> {
PayloadOverrides {
final_status: self.final_status.as_deref(),
issued_at: self.issued_at.as_deref(),
nonce: self.nonce.as_deref(),
}
}
}
+51
View File
@@ -0,0 +1,51 @@
use anyhow::Result;
use axum::http::StatusCode;
use chrono::Utc;
use diesel::prelude::*;
use papercrate::models::TenantStatus;
use papercrate::schema::tenants::dsl as tenants_dsl;
use papercrate::test_support::{acquire_db_lock, TestApp, TestUserRole};
use serde_json::json;
use uuid::Uuid;
#[tokio::test]
async fn tenant_management_is_scoped_to_memberships() -> Result<()> {
let _lock = acquire_db_lock().await;
let app = TestApp::new().await?;
let username = "tenant-owner";
app.insert_user(username, TestUserRole::Owner).await?;
let token = app.login_token(username, "irrelevant").await?;
let other_tenant_id = app
.with_conn(|conn| {
let other_id = Uuid::new_v4();
let now = Utc::now().naive_utc();
diesel::insert_into(tenants_dsl::tenants)
.values((
tenants_dsl::id.eq(other_id),
tenants_dsl::name.eq(format!("foreign-{other_id}")),
tenants_dsl::storage_root.eq(Some(format!("test-tenants/{other_id}/"))),
tenants_dsl::quickwit_index.eq(None::<String>),
tenants_dsl::config.eq(json!({})),
tenants_dsl::created_at.eq(now),
tenants_dsl::updated_at.eq(now),
tenants_dsl::status.eq(TenantStatus::Active),
tenants_dsl::created_by.eq(None::<Uuid>),
))
.execute(conn)?;
Ok(other_id)
})
.await?;
let response = app
.patch_json(
&format!("/api/tenants/{other_tenant_id}"),
&json!({ "name": "should-not-work" }),
Some(&token),
)
.await?;
assert_eq!(response.status(), StatusCode::FORBIDDEN);
Ok(())
}
+4
View File
@@ -0,0 +1,4 @@
# Branding Assets
- `logo.afphoto`: Affinity Photo source for the Papercrate logo.
- Export updated web assets to `frontend/src/assets/logo.webp` and `frontend/src/assets/logo_small.webp` to keep the UI logos in sync.
Binary file not shown.
+1 -1
View File
@@ -44,7 +44,7 @@ Document Assets
Downloads
---------
- GET /download/:token - Follow a one-time download token; redirects to a pre-signed URL (public token required).
- GET /api/download/:token - Follow a one-time download token; redirects to a pre-signed URL (public token required).
Folders
-------
+15
View File
@@ -0,0 +1,15 @@
# Desktop Workspace Interaction Spec
The desktop workspace should apply the following selection and drag behaviours:
- **Click on a non-selected card**: clear any existing selection, then select the clicked card only.
- **Click on a selected card**: keep the selection and open the detail panel for that card (no selection change).
- **Drag on a non-selected card**: clear the selection, select the dragged card, then drag that single card.
- **Drag on a selected card**: drag the entire current selection without altering which cards are selected.
- **Cmd/Ctrl + click on a non-selected card**: add that card to the existing selection.
- **Cmd/Ctrl + click on a selected card**: expand the selection by adding the stack of cards beneath the clicked card.
- **Cmd/Ctrl + drag on a non-selected card**: replace the current selection with the entire stack beneath the pointer, then drag that stack.
- **Cmd/Ctrl + drag on a selected card**: replace the current selection with the stack beneath the pointer, then drag that stack.
- **Touch long-press**: behaves like a stack-select gesture, expanding the selection to the stack under the pressed card without requiring modifier keys.
These rules ensure the selection model remains predictable while supporting stack-aware gestures unique to the desktop workspace.
+40
View File
@@ -0,0 +1,40 @@
# Job Catalogue
Papercrate stores asynchronous work in the shared `jobs` table. Each job carries a
`tenant_id`, a small JSON payload, and one of the statuses defined in
`backend/src/jobs.rs` (`queued`, `processing`, `succeeded`, `failed`). Workers
continuously reserve jobs by type and execute the appropriate handler. This
document lists every job type that is currently recognized by the backend and
briefly describes what it does.
| Job type | Payload shape | When it is enqueued | Work performed |
| --- | --- | --- | --- |
| `analyze-document` | `{ "document_id": Uuid, "document_version_id": Uuid, "force": bool }` | Uploading a document, calling the re-analyze bulk action, or after a metadata edit (e.g. title change) | Runs the taskflow pipeline (`GenerateThumbnailsTask`, `GenerateOcrTask`, `DetermineIssuedAtTask`, `IndexDocumentTask`) for the specified document version. The handler refuses to run if the tenant is not `Active`. |
| `purge-document` | `{ "document_id": Uuid }` | `DELETE /api/documents/{id}` after the document has been trashed | Removes every version and asset object from tenant storage, deletes database rows (`documents`, `document_versions`, associated assets/tags/correspondents), and leaves the system ready for GC. |
| `provision-tenant` | `{ "members": [Uuid, ...] }` | When a tenant is created with status `creating` | Creates/ensures the tenants Quickwit index, materializes the system capability sets (`owner`, `user`, `readonly`, `webdav`), attaches the initial member list, and flips the tenant status to `active`. |
| `delete-tenant` | `{ "remove_tenant": bool, "tenant_name": string, "action": "delete"\|"reset", "nonce": string, "issued_at": RFC3339 datetime, "signature": hex(HMAC-SHA256), "final_status"?: "active"\|"suspended" }` | Administrative action after a tenant has been marked `deleting` | Deletes all tenant-scoped storage objects, wipes the tenants Quickwit index (and optionally deletes it entirely), truncates the tenant schemas/tables, removes queued jobs for that tenant, and either deletes the tenant row or leaves it in the requested final status (defaults to `suspended`) while recreating an empty Quickwit index. |
## Retired job types
`generate-thumbnails` and `generate-ocr-text` once existed as standalone jobs.
Those behaviors now run as tasks inside `analyze-document`. No worker is
registered for the legacy types; keep them out of new payloads.
### Tenant delete/reset safety checks
The `delete-tenant` job refuses to run without a signed payload. The admin CLI
derives a message of the form `v1|tenant_id|tenant_name|action|nonce|issued_at|final_status`
and signs it with an HMAC-SHA256 key based on the servers JWT secret.
Workers verify the signature, ensure the payload matches the job flags, and
require the `issued_at` timestamp to be no more than five minutes old. This
protects against accidental wipes triggered by stale requests or insufficiently
scoped API calls.
## Operational notes
* Every job handler calls `ensure_active_tenant` (or an equivalent guard) before
touching tenant data. If a tenant is suspended or deleting, the job will fail
immediately.
* Jobs are only enqueued for the tenant they operate on. Consequently, wiping a
tenant with `delete-tenant` also removes any remaining queued jobs for that
tenant so workers do not waste effort on work that can no longer succeed.
-2
View File
@@ -1,2 +0,0 @@
dist
node_modules
-31
View File
@@ -1,31 +0,0 @@
{
"env": {
"browser": true,
"es2021": true
},
"extends": [
"eslint:recommended",
"plugin:react/recommended",
"plugin:react-hooks/recommended"
],
"parserOptions": {
"ecmaFeatures": {
"jsx": true
},
"ecmaVersion": "latest",
"sourceType": "module"
},
"settings": {
"react": {
"version": "detect"
}
},
"rules": {
"no-use-before-define": [
"error",
{ "functions": false, "classes": true, "variables": true }
],
"react/react-in-jsx-scope": "off",
"react/prop-types": "off"
}
}
+6 -3
View File
@@ -1,6 +1,9 @@
# syntax=docker/dockerfile:1
# syntax=docker/dockerfile:1.6
FROM node:20-alpine AS build
ARG NODE_IMAGE=node:20-alpine
ARG NGINX_IMAGE=nginx:alpine
FROM --platform=$BUILDPLATFORM ${NODE_IMAGE} AS build
WORKDIR /app
COPY package.json package-lock.json ./
@@ -9,7 +12,7 @@ RUN npm ci --no-audit --no-fund
COPY . .
RUN npm run build
FROM nginx:alpine
FROM ${NGINX_IMAGE}
WORKDIR /usr/share/nginx/html
COPY --from=build /app/dist ./
+9
View File
@@ -33,6 +33,15 @@ npm run build
- Output written to `dist/`
- Set `API_BASE_URL` in `.env.local` if the API is not served from the same origin.
## Code Quality
```bash
npm run check
```
- Runs TypeScript type checking (`tsc`) and ESLint.
- Use this before committing changes.
## Features
- Finder-style layout: folder tree, document table, and detail pane with metadata & tags
+1
View File
@@ -12,5 +12,6 @@ module.exports = {
runtime: 'automatic',
},
],
'@babel/preset-typescript',
],
};
+3
View File
@@ -1,6 +1,9 @@
#!/bin/sh
set -euo pipefail
# Add mjs to mime.types if not present
sed -i 's|application/javascript|application/javascript mjs|' /etc/nginx/mime.types
API_PROXY_PASS_TRIMMED="${API_PROXY_PASS:-}"
API_PROXY_PASS_TRIMMED="${API_PROXY_PASS_TRIMMED%%/}"
+86
View File
@@ -0,0 +1,86 @@
import js from '@eslint/js';
import pluginReact from 'eslint-plugin-react';
import pluginReactHooks from 'eslint-plugin-react-hooks';
import globals from 'globals';
import tsParser from '@typescript-eslint/parser';
import tsPluginImport from '@typescript-eslint/eslint-plugin';
const tsPlugin = tsPluginImport.default ?? tsPluginImport;
const sharedRules = {
...js.configs.recommended.rules,
...pluginReact.configs.recommended.rules,
...pluginReactHooks.configs.recommended.rules,
'no-use-before-define': [
'error',
{ functions: false, classes: true, variables: true },
],
'react/react-in-jsx-scope': 'off',
'react/prop-types': 'off',
'react-hooks/set-state-in-effect': 'off',
'react-hooks/refs': 'off',
'react-hooks/preserve-manual-memoization': 'off',
};
const sharedLanguageOptions = {
ecmaVersion: 'latest',
sourceType: 'module',
parserOptions: {
ecmaFeatures: {
jsx: true,
},
},
globals: {
...globals.browser,
...globals.node,
},
};
export default [
{
ignores: ['dist', 'node_modules'],
},
{
files: ['src/**/*.{js,jsx}', 'tests/**/*.{js,jsx}'],
languageOptions: sharedLanguageOptions,
plugins: {
react: pluginReact,
'react-hooks': pluginReactHooks,
'@typescript-eslint': tsPlugin,
},
settings: {
react: {
version: 'detect',
},
},
rules: sharedRules,
},
{
files: ['src/**/*.{ts,tsx}'],
languageOptions: {
...sharedLanguageOptions,
parser: tsParser,
},
plugins: {
react: pluginReact,
'react-hooks': pluginReactHooks,
'@typescript-eslint': tsPlugin,
},
settings: {
react: {
version: 'detect',
},
},
rules: {
...sharedRules,
'no-unused-vars': 'off',
'@typescript-eslint/no-unused-vars': [
'warn',
{
argsIgnorePattern: '^_',
varsIgnorePattern: '^_',
},
],
},
},
];
+2043 -986
View File
File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More