nils/papercrate
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

quickwit

Browse Source
This commit is contained in:
nils
2025-10-11 16:42:25 +02:00
parent cd27f36e3d
commit 0a669064dd
13 changed files with 521 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/Cargo.lock
Generated
+177 -1
View File
@@ -753,6 +753,12 @@ version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9"
[[package]]
name = "cfg_aliases"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
[[package]]
name = "chrono"
version = "0.4.42"
@@ -1276,9 +1282,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
dependencies = [
"cfg-if",
"js-sys",
"libc",
"r-efi",
"wasi 0.14.7+wasi-0.2.4",
"wasm-bindgen",
]
[[package]]
@@ -1550,6 +1558,7 @@ dependencies = [
"tokio",
"tokio-rustls 0.26.4",
"tower-service",
"webpki-roots",
]
[[package]]
@@ -1755,6 +1764,16 @@ version = "2.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130"
[[package]]
name = "iri-string"
version = "0.7.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dbc5ebe9c3a1a7a5127f920a418f7585e9e758e911d0466ed004f393b0e380b2"
dependencies = [
"memchr",
"serde",
]
[[package]]
name = "itertools"
version = "0.13.0"
@@ -1863,6 +1882,12 @@ dependencies = [
"hashbrown 0.15.5",
]
[[package]]
name = "lru-slab"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
[[package]]
name = "matchers"
version = "0.2.0"
@@ -2108,6 +2133,7 @@ dependencies = [
"once_cell",
"pdfium-render",
"rand 0.8.5",
"reqwest",
"serde",
"serde_json",
"sha2",
@@ -2115,7 +2141,7 @@ dependencies = [
"thiserror 1.0.69",
"tokio",
"tower 0.4.13",
"tower-http",
"tower-http 0.5.2",
"tracing",
"tracing-subscriber",
"uuid",
@@ -2327,6 +2353,61 @@ dependencies = [
"num-traits",
]
[[package]]
name = "quinn"
version = "0.11.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20"
dependencies = [
"bytes",
"cfg_aliases",
"pin-project-lite",
"quinn-proto",
"quinn-udp",
"rustc-hash",
"rustls 0.23.32",
"socket2 0.6.0",
"thiserror 2.0.17",
"tokio",
"tracing",
"web-time",
]
[[package]]
name = "quinn-proto"
version = "0.11.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31"
dependencies = [
"bytes",
"getrandom 0.3.3",
"lru-slab",
"rand 0.9.2",
"ring",
"rustc-hash",
"rustls 0.23.32",
"rustls-pki-types",
"slab",
"thiserror 2.0.17",
"tinyvec",
"tracing",
"web-time",
]
[[package]]
name = "quinn-udp"
version = "0.5.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd"
dependencies = [
"cfg_aliases",
"libc",
"once_cell",
"socket2 0.6.0",
"tracing",
"windows-sys 0.59.0",
]
[[package]]
name = "quote"
version = "1.0.41"
@@ -2456,6 +2537,44 @@ version = "0.8.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001"
[[package]]
name = "reqwest"
version = "0.12.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d429f34c8092b2d42c7c93cec323bb4adeb7c67698f70839adec842ec10c7ceb"
dependencies = [
"base64 0.22.1",
"bytes",
"futures-core",
"http 1.3.1",
"http-body 1.0.1",
"http-body-util",
"hyper 1.7.0",
"hyper-rustls 0.27.7",
"hyper-util",
"js-sys",
"log",
"percent-encoding",
"pin-project-lite",
"quinn",
"rustls 0.23.32",
"rustls-pki-types",
"serde",
"serde_json",
"serde_urlencoded",
"sync_wrapper",
"tokio",
"tokio-rustls 0.26.4",
"tower 0.5.2",
"tower-http 0.6.6",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
"webpki-roots",
]
[[package]]
name = "rfc6979"
version = "0.3.1"
@@ -2535,6 +2654,7 @@ checksum = "cd3c25631629d034ce7cd9940adc9d45762d46de2b0f57193c4443b92c6d4d40"
dependencies = [
"aws-lc-rs",
"once_cell",
"ring",
"rustls-pki-types",
"rustls-webpki 0.103.7",
"subtle",
@@ -2580,6 +2700,7 @@ version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79"
dependencies = [
"web-time",
"zeroize",
]
@@ -2938,6 +3059,9 @@ name = "sync_wrapper"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
dependencies = [
"futures-core",
]
[[package]]
name = "synstructure"
@@ -3053,6 +3177,21 @@ dependencies = [
"zerovec",
]
[[package]]
name = "tinyvec"
version = "1.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa"
dependencies = [
"tinyvec_macros",
]
[[package]]
name = "tinyvec_macros"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
[[package]]
name = "tokio"
version = "1.47.1"
@@ -3197,6 +3336,24 @@ dependencies = [
"tracing",
]
[[package]]
name = "tower-http"
version = "0.6.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2"
dependencies = [
"bitflags",
"bytes",
"futures-util",
"http 1.3.1",
"http-body 1.0.1",
"iri-string",
"pin-project-lite",
"tower 0.5.2",
"tower-layer",
"tower-service",
]
[[package]]
name = "tower-layer"
version = "0.3.3"
@@ -3494,6 +3651,25 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "web-time"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
dependencies = [
"js-sys",
"wasm-bindgen",
]
[[package]]
name = "webpki-roots"
version = "1.0.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32b130c0d2d49f8b6889abc456e795e82525204f27c42cf767cf0d7734e089b8"
dependencies = [
"rustls-pki-types",
]
[[package]]
name = "windows-core"
version = "0.62.2"
backend/Cargo.toml
+1
View File
@@ -38,6 +38,7 @@ image = { version = "0.25", default-features = false, features = ["png", "jpeg"]
pdfium-render = "0.8"
mime_guess = "2.0"
tempfile = "3.10"
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
# Error handling
thiserror = "1.0"
backend/src/config.rs
+6
View File
@@ -20,6 +20,8 @@ pub struct AppConfig {
pub aws_secret_access_key: Option<String>,
pub aws_region: String,
pub s3_bucket: String,
pub quickwit_endpoint: Option<String>,
pub quickwit_index: Option<String>,
}
impl AppConfig {
@@ -52,6 +54,8 @@ impl AppConfig {
let aws_secret_access_key = env::var("AWS_SECRET_ACCESS_KEY").ok();
let aws_region = env::var("AWS_REGION").unwrap_or_else(|_| "us-east-1".to_string());
let s3_bucket = env::var("S3_BUCKET").context("S3_BUCKET must be set")?;
let quickwit_endpoint = env::var("QUICKWIT_ENDPOINT").ok();
let quickwit_index = env::var("QUICKWIT_INDEX").ok();
Ok(Self {
database_url,
@@ -70,6 +74,8 @@ impl AppConfig {
aws_secret_access_key,
aws_region,
s3_bucket,
quickwit_endpoint,
quickwit_index,
})
}
}
backend/src/jobs.rs
+1
View File
@@ -18,6 +18,7 @@ pub const STATUS_FAILED: &str = "failed";
pub const JOB_ANALYZE_DOCUMENT: &str = "analyze-document";
pub const JOB_GENERATE_THUMBNAILS: &str = "generate-thumbnails";
pub const JOB_GENERATE_OCR_TEXT: &str = "generate-ocr-text";
pub const JOB_INDEX_DOCUMENT_TEXT: &str = "index-document-text";
#[derive(Debug, Error)]
pub enum JobQueueError {
backend/src/workers/index.rs
+204
View File
@@ -0,0 +1,204 @@
use std::sync::Arc;
use std::time::Duration;
use async_trait::async_trait;
use diesel::prelude::*;
use reqwest::Client;
use serde::Deserialize;
use serde_json::json;
use tokio::task;
use tracing::{error, warn};
use uuid::Uuid;
use crate::{
jobs::JOB_INDEX_DOCUMENT_TEXT,
models::{Document, DocumentAsset, DocumentVersion},
schema::{document_assets, document_versions, documents},
state::AppState,
};
use super::{ocr::OCR_TEXT_ASSET_TYPE, JobExecution, JobHandler};
#[derive(Debug, Deserialize)]
struct IndexPayload {
document_id: Uuid,
document_version_id: Uuid,
}
pub struct IndexDocumentTextJob;
impl IndexDocumentTextJob {
pub fn new() -> Self {
Self
}
}
#[async_trait]
impl JobHandler for IndexDocumentTextJob {
fn job_type(&self) -> &'static str {
JOB_INDEX_DOCUMENT_TEXT
}
async fn handle(&self, state: Arc<AppState>, job: crate::models::Job) -> JobExecution {
let payload: IndexPayload = match serde_json::from_value(job.payload.clone()) {
Ok(payload) => payload,
Err(err) => {
return JobExecution::Failed {
error: format!("invalid index payload: {err}"),
}
}
};
let quickwit_endpoint = match &state.config.quickwit_endpoint {
Some(endpoint) => endpoint.clone(),
None => {
warn!("quickwit endpoint missing; skipping indexing");
return JobExecution::Success;
}
};
let quickwit_index = match &state.config.quickwit_index {
Some(index) => index.clone(),
None => {
warn!("quickwit index missing; skipping indexing");
return JobExecution::Success;
}
};
let client = Client::new();
let state_clone = state.clone();
let context = match task::spawn_blocking(move || load_context(state_clone, &payload)).await
{
Ok(Ok(ctx)) => ctx,
Ok(Err(err)) => {
warn!(job_id = %job.id, error = %err, "index job will retry");
return JobExecution::Retry {
delay: Duration::from_secs(30),
error: err,
};
}
Err(join_err) => {
error!(job_id = %job.id, error = %join_err, "index task panicked");
return JobExecution::Retry {
delay: Duration::from_secs(60),
error: format!("worker panicked: {join_err}"),
};
}
};
if context.text_asset.is_none() {
warn!(job_id = %job.id, "missing OCR text asset; failing indexing job");
return JobExecution::Failed {
error: "missing OCR text asset".into(),
};
}
let asset = context.text_asset.unwrap();
let text = match state.storage.get_object(&asset.s3_key).await {
Ok(bytes) => match String::from_utf8(bytes) {
Ok(text) => text,
Err(err) => {
warn!(job_id = %job.id, error = %err, "ocr text not valid UTF-8");
return JobExecution::Failed {
error: "ocr text not valid UTF-8".into(),
};
}
},
Err(err) => {
warn!(job_id = %job.id, error = %err, "failed to download ocr text");
return JobExecution::Retry {
delay: Duration::from_secs(30),
error: err.to_string(),
};
}
};
if text.trim().is_empty() {
warn!(job_id = %job.id, "ocr text empty; skipping");
return JobExecution::Failed {
error: "ocr text empty".into(),
};
}
let client = client;
let url = format!(
"{}/api/v1/{}/ingest?commit=auto",
quickwit_endpoint, quickwit_index
);
let payload = json!({
"document_id": context.document.id,
"version_id": context.version.id,
"title": context.document.title,
"text": text,
});
let body = serde_json::to_string(&payload).unwrap();
match client
.post(&url)
.header("content-type", "application/x-ndjson")
.body(format!("{}\n", body))
.send()
.await
{
Ok(response) => {
if response.status().is_success() {
JobExecution::Success
} else {
let status = response.status();
let body = response.text().await.unwrap_or_default();
warn!(job_id = %job.id, %status, %body, "quickwit ingest failed");
JobExecution::Retry {
delay: Duration::from_secs(30),
error: format!("quickwit ingest failed with status {status}"),
}
}
}
Err(err) => {
warn!(job_id = %job.id, error = %err, "quickwit request failed");
JobExecution::Retry {
delay: Duration::from_secs(30),
error: err.to_string(),
}
}
}
}
}
struct IndexContext {
document: Document,
version: DocumentVersion,
text_asset: Option<DocumentAsset>,
}
fn load_context(state: Arc<AppState>, payload: &IndexPayload) -> Result<IndexContext, String> {
let mut conn = state.db().map_err(|err| format!("{err:?}"))?;
let version: DocumentVersion = document_versions::table
.find(payload.document_version_id)
.first(&mut conn)
.map_err(|err| format!("{err:?}"))?;
if version.document_id != payload.document_id {
return Err("document/version mismatch".into());
}
let document: Document = documents::table
.find(payload.document_id)
.first(&mut conn)
.map_err(|err| format!("{err:?}"))?;
let text_asset: Option<DocumentAsset> = document_assets::table
.filter(document_assets::document_version_id.eq(payload.document_version_id))
.filter(document_assets::asset_type.eq(OCR_TEXT_ASSET_TYPE))
.first(&mut conn)
.optional()
.map_err(|err| format!("{err:?}"))?;
Ok(IndexContext {
document,
version,
text_asset,
})
}
backend/src/workers/mod.rs
+2
View File
@@ -11,6 +11,7 @@ use crate::{
};
pub mod analyze;
pub mod index;
pub mod ocr;
pub mod thumbnails;
@@ -130,5 +131,6 @@ pub fn default_handlers() -> Vec<Arc<dyn JobHandler>> {
Arc::new(analyze::AnalyzeDocumentJob::new()),
Arc::new(thumbnails::GenerateThumbnailsJob::new()),
Arc::new(ocr::GenerateOcrTextJob::new()),
Arc::new(index::IndexDocumentTextJob::new()),
]
}
backend/src/workers/ocr.rs
+32 -7
View File
@@ -1,6 +1,5 @@
use std::{
fmt,
fs,
fmt, fs,
io::{ErrorKind, Write},
process::Command,
sync::Arc,
@@ -19,7 +18,7 @@ use tracing::{error, info, warn};
use uuid::Uuid;
use crate::{
jobs::JOB_GENERATE_OCR_TEXT,
jobs::{enqueue_job, JOB_GENERATE_OCR_TEXT, JOB_INDEX_DOCUMENT_TEXT},
models::{Document, DocumentAsset, DocumentVersion, NewDocumentAsset},
schema::{document_assets, document_versions, documents},
state::AppState,
@@ -30,7 +29,7 @@ use super::{JobExecution, JobHandler};
pub const OCR_TEXT_ASSET_TYPE: &str = "ocr-text";
const MIN_TEXT_LENGTH: usize = 50;
#[derive(Debug, Deserialize)]
#[derive(Clone, Debug, Deserialize)]
struct OcrPayload {
document_id: Uuid,
document_version_id: Uuid,
@@ -63,8 +62,10 @@ impl JobHandler for GenerateOcrTextJob {
};
let state_clone = state.clone();
let payload_clone = payload.clone();
let context =
match task::spawn_blocking(move || load_ocr_context(state_clone, &payload)).await {
match task::spawn_blocking(move || load_ocr_context(state_clone, &payload_clone)).await
{
Ok(Ok(ctx)) => ctx,
Ok(Err(err)) => {
warn!(job_id = %job.id, error = %err, "ocr job will retry");
@@ -155,7 +156,15 @@ impl JobHandler for GenerateOcrTextJob {
})
.await
{
Ok(Ok(())) => JobExecution::Success,
Ok(Ok(())) => {
if state.config.quickwit_endpoint.is_some() && state.config.quickwit_index.is_some()
{
if let Err(err) = enqueue_index_job(&state, &payload) {
warn!(job_id = %job.id, error = %err, "failed to enqueue index job");
}
}
JobExecution::Success
}
Ok(Err(err)) => {
warn!(job_id = %job.id, error = %err, "failed to persist ocr metadata");
JobExecution::Retry {
@@ -334,7 +343,8 @@ fn run_ocr(bytes: &[u8]) -> Result<Option<String>, OcrError> {
)));
}
let text = fs::read_to_string(sidecar.path()).map_err(|err| OcrError::Failed(err.to_string()))?;
let text = fs::read_to_string(sidecar.path())
.map_err(|err| OcrError::Failed(err.to_string()))?;
if text.trim().chars().count() >= MIN_TEXT_LENGTH {
Ok(Some(text))
} else {
@@ -392,6 +402,21 @@ fn persist_ocr_metadata(
Ok(())
}
fn enqueue_index_job(state: &AppState, payload: &OcrPayload) -> Result<(), String> {
let mut conn = state.db().map_err(|err| format!("{err:?}"))?;
enqueue_job(
&mut conn,
JOB_INDEX_DOCUMENT_TEXT,
json!({
"document_id": payload.document_id,
"document_version_id": payload.document_version_id,
}),
None,
)
.map(|_| ())
.map_err(|err| err.to_string())
}
pub fn document_is_pdf(document: &Document) -> bool {
document_meta_is_pdf(&PdfDocumentMeta {
content_type: document.content_type.clone(),
backend/tests/common/mod.rs
+2
View File
@@ -128,6 +128,8 @@ impl TestApp {
aws_secret_access_key: None,
aws_region: "us-east-1".to_string(),
s3_bucket: "test-bucket".to_string(),
quickwit_endpoint: None,
quickwit_index: None,
};
let pool = db::init_pool(&config.database_url)?;