quickwit
This commit is contained in:
@@ -2,11 +2,14 @@
|
||||
|
||||
## Backend Integration Tests
|
||||
|
||||
Integration tests require a running Postgres instance. The repository includes a lightweight compose file for local runs:
|
||||
Integration tests require a running Postgres instance (and, optionally, Quickwit for OCR indexing). The repository includes a lightweight compose file for local runs:
|
||||
|
||||
```bash
|
||||
docker compose -f docker-compose.test.yml up -d
|
||||
export TEST_DATABASE_URL=postgres://paperless:paperless_test@localhost:5433/paperless_test
|
||||
# optional, enables Quickwit indexing jobs
|
||||
export QUICKWIT_ENDPOINT=http://localhost:7280
|
||||
export QUICKWIT_INDEX=documents
|
||||
cargo test
|
||||
```
|
||||
|
||||
@@ -21,3 +24,4 @@ The compose service uses tmpfs storage, giving each test run a clean database.
|
||||
## Runtime Dependencies
|
||||
|
||||
- `ocrmypdf` (optional but recommended): Used by the OCR worker to extract text from PDFs when no embedded text layer is available. Ensure it is installed and available on the worker hosts if OCR is desired.
|
||||
- Quickwit (optional): The Quickwit indexer is used to ingest extracted text for search. Set `QUICKWIT_ENDPOINT` and `QUICKWIT_INDEX` in the environment when running workers if you want indexing jobs to run. The local compose file starts a Quickwit instance on `http://localhost:7280` and seeds the `documents` index automatically.
|
||||
|
||||
Generated
+177
-1
@@ -753,6 +753,12 @@ version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9"
|
||||
|
||||
[[package]]
|
||||
name = "cfg_aliases"
|
||||
version = "0.2.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
|
||||
|
||||
[[package]]
|
||||
name = "chrono"
|
||||
version = "0.4.42"
|
||||
@@ -1276,9 +1282,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
|
||||
dependencies = [
|
||||
"cfg-if",
|
||||
"js-sys",
|
||||
"libc",
|
||||
"r-efi",
|
||||
"wasi 0.14.7+wasi-0.2.4",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1550,6 +1558,7 @@ dependencies = [
|
||||
"tokio",
|
||||
"tokio-rustls 0.26.4",
|
||||
"tower-service",
|
||||
"webpki-roots",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1755,6 +1764,16 @@ version = "2.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130"
|
||||
|
||||
[[package]]
|
||||
name = "iri-string"
|
||||
version = "0.7.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "itertools"
|
||||
version = "0.13.0"
|
||||
@@ -1863,6 +1882,12 @@ dependencies = [
|
||||
"hashbrown 0.15.5",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lru-slab"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
|
||||
|
||||
[[package]]
|
||||
name = "matchers"
|
||||
version = "0.2.0"
|
||||
@@ -2108,6 +2133,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"pdfium-render",
|
||||
"rand 0.8.5",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2",
|
||||
@@ -2115,7 +2141,7 @@ dependencies = [
|
||||
"thiserror 1.0.69",
|
||||
"tokio",
|
||||
"tower 0.4.13",
|
||||
"tower-http",
|
||||
"tower-http 0.5.2",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
@@ -2327,6 +2353,61 @@ dependencies = [
|
||||
"num-traits",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quinn"
|
||||
version = "0.11.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"cfg_aliases",
|
||||
"pin-project-lite",
|
||||
"quinn-proto",
|
||||
"quinn-udp",
|
||||
"rustc-hash",
|
||||
"rustls 0.23.32",
|
||||
"socket2 0.6.0",
|
||||
"thiserror 2.0.17",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"web-time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quinn-proto"
|
||||
version = "0.11.13"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"getrandom 0.3.3",
|
||||
"lru-slab",
|
||||
"rand 0.9.2",
|
||||
"ring",
|
||||
"rustc-hash",
|
||||
"rustls 0.23.32",
|
||||
"rustls-pki-types",
|
||||
"slab",
|
||||
"thiserror 2.0.17",
|
||||
"tinyvec",
|
||||
"tracing",
|
||||
"web-time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quinn-udp"
|
||||
version = "0.5.14"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd"
|
||||
dependencies = [
|
||||
"cfg_aliases",
|
||||
"libc",
|
||||
"once_cell",
|
||||
"socket2 0.6.0",
|
||||
"tracing",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quote"
|
||||
version = "1.0.41"
|
||||
@@ -2456,6 +2537,44 @@ version = "0.8.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001"
|
||||
|
||||
[[package]]
|
||||
name = "reqwest"
|
||||
version = "0.12.23"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"futures-core",
|
||||
"http 1.3.1",
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
"hyper 1.7.0",
|
||||
"hyper-rustls 0.27.7",
|
||||
"hyper-util",
|
||||
"js-sys",
|
||||
"log",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"quinn",
|
||||
"rustls 0.23.32",
|
||||
"rustls-pki-types",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_urlencoded",
|
||||
"sync_wrapper",
|
||||
"tokio",
|
||||
"tokio-rustls 0.26.4",
|
||||
"tower 0.5.2",
|
||||
"tower-http 0.6.6",
|
||||
"tower-service",
|
||||
"url",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-sys",
|
||||
"webpki-roots",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rfc6979"
|
||||
version = "0.3.1"
|
||||
@@ -2535,6 +2654,7 @@ checksum = "cd3c25631629d034ce7cd9940adc9d45762d46de2b0f57193c4443b92c6d4d40"
|
||||
dependencies = [
|
||||
"aws-lc-rs",
|
||||
"once_cell",
|
||||
"ring",
|
||||
"rustls-pki-types",
|
||||
"rustls-webpki 0.103.7",
|
||||
"subtle",
|
||||
@@ -2580,6 +2700,7 @@ version = "1.12.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79"
|
||||
dependencies = [
|
||||
"web-time",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
@@ -2938,6 +3059,9 @@ name = "sync_wrapper"
|
||||
version = "1.0.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "synstructure"
|
||||
@@ -3053,6 +3177,21 @@ dependencies = [
|
||||
"zerovec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec"
|
||||
version = "1.10.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa"
|
||||
dependencies = [
|
||||
"tinyvec_macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tinyvec_macros"
|
||||
version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
|
||||
|
||||
[[package]]
|
||||
name = "tokio"
|
||||
version = "1.47.1"
|
||||
@@ -3197,6 +3336,24 @@ dependencies = [
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tower-http"
|
||||
version = "0.6.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2"
|
||||
dependencies = [
|
||||
"bitflags",
|
||||
"bytes",
|
||||
"futures-util",
|
||||
"http 1.3.1",
|
||||
"http-body 1.0.1",
|
||||
"iri-string",
|
||||
"pin-project-lite",
|
||||
"tower 0.5.2",
|
||||
"tower-layer",
|
||||
"tower-service",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tower-layer"
|
||||
version = "0.3.3"
|
||||
@@ -3494,6 +3651,25 @@ dependencies = [
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "web-time"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
|
||||
dependencies = [
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "1.0.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32b130c0d2d49f8b6889abc456e795e82525204f27c42cf767cf0d7734e089b8"
|
||||
dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.62.2"
|
||||
|
||||
@@ -38,6 +38,7 @@ image = { version = "0.25", default-features = false, features = ["png", "jpeg"]
|
||||
pdfium-render = "0.8"
|
||||
mime_guess = "2.0"
|
||||
tempfile = "3.10"
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
|
||||
|
||||
# Error handling
|
||||
thiserror = "1.0"
|
||||
|
||||
@@ -20,6 +20,8 @@ pub struct AppConfig {
|
||||
pub aws_secret_access_key: Option<String>,
|
||||
pub aws_region: String,
|
||||
pub s3_bucket: String,
|
||||
pub quickwit_endpoint: Option<String>,
|
||||
pub quickwit_index: Option<String>,
|
||||
}
|
||||
|
||||
impl AppConfig {
|
||||
@@ -52,6 +54,8 @@ impl AppConfig {
|
||||
let aws_secret_access_key = env::var("AWS_SECRET_ACCESS_KEY").ok();
|
||||
let aws_region = env::var("AWS_REGION").unwrap_or_else(|_| "us-east-1".to_string());
|
||||
let s3_bucket = env::var("S3_BUCKET").context("S3_BUCKET must be set")?;
|
||||
let quickwit_endpoint = env::var("QUICKWIT_ENDPOINT").ok();
|
||||
let quickwit_index = env::var("QUICKWIT_INDEX").ok();
|
||||
|
||||
Ok(Self {
|
||||
database_url,
|
||||
@@ -70,6 +74,8 @@ impl AppConfig {
|
||||
aws_secret_access_key,
|
||||
aws_region,
|
||||
s3_bucket,
|
||||
quickwit_endpoint,
|
||||
quickwit_index,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,6 +18,7 @@ pub const STATUS_FAILED: &str = "failed";
|
||||
pub const JOB_ANALYZE_DOCUMENT: &str = "analyze-document";
|
||||
pub const JOB_GENERATE_THUMBNAILS: &str = "generate-thumbnails";
|
||||
pub const JOB_GENERATE_OCR_TEXT: &str = "generate-ocr-text";
|
||||
pub const JOB_INDEX_DOCUMENT_TEXT: &str = "index-document-text";
|
||||
|
||||
#[derive(Debug, Error)]
|
||||
pub enum JobQueueError {
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use diesel::prelude::*;
|
||||
use reqwest::Client;
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
use tokio::task;
|
||||
use tracing::{error, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{
|
||||
jobs::JOB_INDEX_DOCUMENT_TEXT,
|
||||
models::{Document, DocumentAsset, DocumentVersion},
|
||||
schema::{document_assets, document_versions, documents},
|
||||
state::AppState,
|
||||
};
|
||||
|
||||
use super::{ocr::OCR_TEXT_ASSET_TYPE, JobExecution, JobHandler};
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct IndexPayload {
|
||||
document_id: Uuid,
|
||||
document_version_id: Uuid,
|
||||
}
|
||||
|
||||
pub struct IndexDocumentTextJob;
|
||||
|
||||
impl IndexDocumentTextJob {
|
||||
pub fn new() -> Self {
|
||||
Self
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl JobHandler for IndexDocumentTextJob {
|
||||
fn job_type(&self) -> &'static str {
|
||||
JOB_INDEX_DOCUMENT_TEXT
|
||||
}
|
||||
|
||||
async fn handle(&self, state: Arc<AppState>, job: crate::models::Job) -> JobExecution {
|
||||
let payload: IndexPayload = match serde_json::from_value(job.payload.clone()) {
|
||||
Ok(payload) => payload,
|
||||
Err(err) => {
|
||||
return JobExecution::Failed {
|
||||
error: format!("invalid index payload: {err}"),
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let quickwit_endpoint = match &state.config.quickwit_endpoint {
|
||||
Some(endpoint) => endpoint.clone(),
|
||||
None => {
|
||||
warn!("quickwit endpoint missing; skipping indexing");
|
||||
return JobExecution::Success;
|
||||
}
|
||||
};
|
||||
|
||||
let quickwit_index = match &state.config.quickwit_index {
|
||||
Some(index) => index.clone(),
|
||||
None => {
|
||||
warn!("quickwit index missing; skipping indexing");
|
||||
return JobExecution::Success;
|
||||
}
|
||||
};
|
||||
|
||||
let client = Client::new();
|
||||
|
||||
let state_clone = state.clone();
|
||||
let context = match task::spawn_blocking(move || load_context(state_clone, &payload)).await
|
||||
{
|
||||
Ok(Ok(ctx)) => ctx,
|
||||
Ok(Err(err)) => {
|
||||
warn!(job_id = %job.id, error = %err, "index job will retry");
|
||||
return JobExecution::Retry {
|
||||
delay: Duration::from_secs(30),
|
||||
error: err,
|
||||
};
|
||||
}
|
||||
Err(join_err) => {
|
||||
error!(job_id = %job.id, error = %join_err, "index task panicked");
|
||||
return JobExecution::Retry {
|
||||
delay: Duration::from_secs(60),
|
||||
error: format!("worker panicked: {join_err}"),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
if context.text_asset.is_none() {
|
||||
warn!(job_id = %job.id, "missing OCR text asset; failing indexing job");
|
||||
return JobExecution::Failed {
|
||||
error: "missing OCR text asset".into(),
|
||||
};
|
||||
}
|
||||
|
||||
let asset = context.text_asset.unwrap();
|
||||
let text = match state.storage.get_object(&asset.s3_key).await {
|
||||
Ok(bytes) => match String::from_utf8(bytes) {
|
||||
Ok(text) => text,
|
||||
Err(err) => {
|
||||
warn!(job_id = %job.id, error = %err, "ocr text not valid UTF-8");
|
||||
return JobExecution::Failed {
|
||||
error: "ocr text not valid UTF-8".into(),
|
||||
};
|
||||
}
|
||||
},
|
||||
Err(err) => {
|
||||
warn!(job_id = %job.id, error = %err, "failed to download ocr text");
|
||||
return JobExecution::Retry {
|
||||
delay: Duration::from_secs(30),
|
||||
error: err.to_string(),
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
if text.trim().is_empty() {
|
||||
warn!(job_id = %job.id, "ocr text empty; skipping");
|
||||
return JobExecution::Failed {
|
||||
error: "ocr text empty".into(),
|
||||
};
|
||||
}
|
||||
|
||||
let client = client;
|
||||
let url = format!(
|
||||
"{}/api/v1/{}/ingest?commit=auto",
|
||||
quickwit_endpoint, quickwit_index
|
||||
);
|
||||
let payload = json!({
|
||||
"document_id": context.document.id,
|
||||
"version_id": context.version.id,
|
||||
"title": context.document.title,
|
||||
"text": text,
|
||||
});
|
||||
|
||||
let body = serde_json::to_string(&payload).unwrap();
|
||||
|
||||
match client
|
||||
.post(&url)
|
||||
.header("content-type", "application/x-ndjson")
|
||||
.body(format!("{}\n", body))
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(response) => {
|
||||
if response.status().is_success() {
|
||||
JobExecution::Success
|
||||
} else {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
warn!(job_id = %job.id, %status, %body, "quickwit ingest failed");
|
||||
JobExecution::Retry {
|
||||
delay: Duration::from_secs(30),
|
||||
error: format!("quickwit ingest failed with status {status}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(job_id = %job.id, error = %err, "quickwit request failed");
|
||||
JobExecution::Retry {
|
||||
delay: Duration::from_secs(30),
|
||||
error: err.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct IndexContext {
|
||||
document: Document,
|
||||
version: DocumentVersion,
|
||||
text_asset: Option<DocumentAsset>,
|
||||
}
|
||||
|
||||
fn load_context(state: Arc<AppState>, payload: &IndexPayload) -> Result<IndexContext, String> {
|
||||
let mut conn = state.db().map_err(|err| format!("{err:?}"))?;
|
||||
|
||||
let version: DocumentVersion = document_versions::table
|
||||
.find(payload.document_version_id)
|
||||
.first(&mut conn)
|
||||
.map_err(|err| format!("{err:?}"))?;
|
||||
|
||||
if version.document_id != payload.document_id {
|
||||
return Err("document/version mismatch".into());
|
||||
}
|
||||
|
||||
let document: Document = documents::table
|
||||
.find(payload.document_id)
|
||||
.first(&mut conn)
|
||||
.map_err(|err| format!("{err:?}"))?;
|
||||
|
||||
let text_asset: Option<DocumentAsset> = document_assets::table
|
||||
.filter(document_assets::document_version_id.eq(payload.document_version_id))
|
||||
.filter(document_assets::asset_type.eq(OCR_TEXT_ASSET_TYPE))
|
||||
.first(&mut conn)
|
||||
.optional()
|
||||
.map_err(|err| format!("{err:?}"))?;
|
||||
|
||||
Ok(IndexContext {
|
||||
document,
|
||||
version,
|
||||
text_asset,
|
||||
})
|
||||
}
|
||||
@@ -11,6 +11,7 @@ use crate::{
|
||||
};
|
||||
|
||||
pub mod analyze;
|
||||
pub mod index;
|
||||
pub mod ocr;
|
||||
pub mod thumbnails;
|
||||
|
||||
@@ -130,5 +131,6 @@ pub fn default_handlers() -> Vec<Arc<dyn JobHandler>> {
|
||||
Arc::new(analyze::AnalyzeDocumentJob::new()),
|
||||
Arc::new(thumbnails::GenerateThumbnailsJob::new()),
|
||||
Arc::new(ocr::GenerateOcrTextJob::new()),
|
||||
Arc::new(index::IndexDocumentTextJob::new()),
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
use std::{
|
||||
fmt,
|
||||
fs,
|
||||
fmt, fs,
|
||||
io::{ErrorKind, Write},
|
||||
process::Command,
|
||||
sync::Arc,
|
||||
@@ -19,7 +18,7 @@ use tracing::{error, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{
|
||||
jobs::JOB_GENERATE_OCR_TEXT,
|
||||
jobs::{enqueue_job, JOB_GENERATE_OCR_TEXT, JOB_INDEX_DOCUMENT_TEXT},
|
||||
models::{Document, DocumentAsset, DocumentVersion, NewDocumentAsset},
|
||||
schema::{document_assets, document_versions, documents},
|
||||
state::AppState,
|
||||
@@ -30,7 +29,7 @@ use super::{JobExecution, JobHandler};
|
||||
pub const OCR_TEXT_ASSET_TYPE: &str = "ocr-text";
|
||||
const MIN_TEXT_LENGTH: usize = 50;
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
struct OcrPayload {
|
||||
document_id: Uuid,
|
||||
document_version_id: Uuid,
|
||||
@@ -63,8 +62,10 @@ impl JobHandler for GenerateOcrTextJob {
|
||||
};
|
||||
|
||||
let state_clone = state.clone();
|
||||
let payload_clone = payload.clone();
|
||||
let context =
|
||||
match task::spawn_blocking(move || load_ocr_context(state_clone, &payload)).await {
|
||||
match task::spawn_blocking(move || load_ocr_context(state_clone, &payload_clone)).await
|
||||
{
|
||||
Ok(Ok(ctx)) => ctx,
|
||||
Ok(Err(err)) => {
|
||||
warn!(job_id = %job.id, error = %err, "ocr job will retry");
|
||||
@@ -155,7 +156,15 @@ impl JobHandler for GenerateOcrTextJob {
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(Ok(())) => JobExecution::Success,
|
||||
Ok(Ok(())) => {
|
||||
if state.config.quickwit_endpoint.is_some() && state.config.quickwit_index.is_some()
|
||||
{
|
||||
if let Err(err) = enqueue_index_job(&state, &payload) {
|
||||
warn!(job_id = %job.id, error = %err, "failed to enqueue index job");
|
||||
}
|
||||
}
|
||||
JobExecution::Success
|
||||
}
|
||||
Ok(Err(err)) => {
|
||||
warn!(job_id = %job.id, error = %err, "failed to persist ocr metadata");
|
||||
JobExecution::Retry {
|
||||
@@ -334,7 +343,8 @@ fn run_ocr(bytes: &[u8]) -> Result<Option<String>, OcrError> {
|
||||
)));
|
||||
}
|
||||
|
||||
let text = fs::read_to_string(sidecar.path()).map_err(|err| OcrError::Failed(err.to_string()))?;
|
||||
let text = fs::read_to_string(sidecar.path())
|
||||
.map_err(|err| OcrError::Failed(err.to_string()))?;
|
||||
if text.trim().chars().count() >= MIN_TEXT_LENGTH {
|
||||
Ok(Some(text))
|
||||
} else {
|
||||
@@ -392,6 +402,21 @@ fn persist_ocr_metadata(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn enqueue_index_job(state: &AppState, payload: &OcrPayload) -> Result<(), String> {
|
||||
let mut conn = state.db().map_err(|err| format!("{err:?}"))?;
|
||||
enqueue_job(
|
||||
&mut conn,
|
||||
JOB_INDEX_DOCUMENT_TEXT,
|
||||
json!({
|
||||
"document_id": payload.document_id,
|
||||
"document_version_id": payload.document_version_id,
|
||||
}),
|
||||
None,
|
||||
)
|
||||
.map(|_| ())
|
||||
.map_err(|err| err.to_string())
|
||||
}
|
||||
|
||||
pub fn document_is_pdf(document: &Document) -> bool {
|
||||
document_meta_is_pdf(&PdfDocumentMeta {
|
||||
content_type: document.content_type.clone(),
|
||||
|
||||
@@ -128,6 +128,8 @@ impl TestApp {
|
||||
aws_secret_access_key: None,
|
||||
aws_region: "us-east-1".to_string(),
|
||||
s3_bucket: "test-bucket".to_string(),
|
||||
quickwit_endpoint: None,
|
||||
quickwit_index: None,
|
||||
};
|
||||
|
||||
let pool = db::init_pool(&config.database_url)?;
|
||||
|
||||
@@ -14,3 +14,39 @@ services:
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
quickwit:
|
||||
image: quickwit/quickwit:0.8.2
|
||||
command: ["run"]
|
||||
environment:
|
||||
QW_ENABLE_API_AUTH: "false"
|
||||
QW_DATA_DIR: /quickwit/data
|
||||
ports:
|
||||
- "7280:7280"
|
||||
volumes:
|
||||
- quickwit_test_data:/quickwit/data
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-sf", "http://127.0.0.1:7280/api/v1/version"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
quickwit-create-index:
|
||||
image: quickwit/quickwit:0.8.2
|
||||
depends_on:
|
||||
quickwit:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- quickwit_test_data:/quickwit/data
|
||||
- ./quickwit/documents-index.yaml:/tmp/documents-index.yaml:ro
|
||||
environment:
|
||||
QW_DATA_DIR: /quickwit/data
|
||||
QW_NODE_URI: http://quickwit:7280
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command: >
|
||||
quickwit index list --node-uri http://quickwit:7280 2>/dev/null | grep -q '"documents"' \
|
||||
|| quickwit index create --index-config /tmp/documents-index.yaml --node-uri http://quickwit:7280
|
||||
restart: "no"
|
||||
|
||||
volumes:
|
||||
quickwit_test_data:
|
||||
|
||||
+28
-2
@@ -1,5 +1,3 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16-alpine
|
||||
@@ -48,6 +46,34 @@ services:
|
||||
exit 0;
|
||||
"
|
||||
|
||||
quickwit:
|
||||
image: quickwit/quickwit:0.8.2
|
||||
command: ["run"]
|
||||
environment:
|
||||
QW_ENABLE_API_AUTH: "false"
|
||||
QW_DATA_DIR: /quickwit/data
|
||||
ports:
|
||||
- "7280:7280"
|
||||
volumes:
|
||||
- quickwit_data:/quickwit/data
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-sf", "http://127.0.0.1:7280/api/v1/version"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
|
||||
quickwit-create-index:
|
||||
image: quickwit/quickwit:0.8.2
|
||||
depends_on:
|
||||
quickwit:
|
||||
condition: service_healthy
|
||||
volumes:
|
||||
- quickwit_data:/quickwit/data
|
||||
- ./quickwit/documents-index.yaml:/tmp/documents-index.yaml:ro
|
||||
command: index create --endpoint=http://quickwit:7280 --index-config /tmp/documents-index.yaml
|
||||
restart: "no"
|
||||
|
||||
volumes:
|
||||
postgres_data:
|
||||
minio_data:
|
||||
quickwit_data:
|
||||
|
||||
@@ -2520,7 +2520,7 @@ const handleDownload = useCallback(
|
||||
await api.post(`/documents/${documentId}/assets`, null, {
|
||||
params: { force: true },
|
||||
});
|
||||
setStatusMessage('Thumbnail regeneration requested.', 'info');
|
||||
setStatusMessage('Document re-analysis queued.', 'info');
|
||||
await ensureDocumentDetail(documentId, { force: true });
|
||||
} catch (error) {
|
||||
console.error(error);
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
version: 0.8
|
||||
index_id: documents
|
||||
doc_mapping:
|
||||
tokenizers:
|
||||
- name: substring
|
||||
type: ngram
|
||||
min_gram: 2
|
||||
max_gram: 20
|
||||
prefix_only: false
|
||||
field_mappings:
|
||||
- name: document_id
|
||||
type: text
|
||||
stored: true
|
||||
- name: version_id
|
||||
type: text
|
||||
stored: true
|
||||
- name: title
|
||||
type: text
|
||||
tokenizer: substring
|
||||
stored: true
|
||||
- name: text
|
||||
type: text
|
||||
tokenizer: substring
|
||||
record: position
|
||||
search_settings:
|
||||
default_search_fields: [title, text]
|
||||
Reference in New Issue
Block a user