quickwit

backend/src/workers/index.rs

@@ -5,12 +5,12 @@ use async_trait::async_trait;
 use diesel::prelude::*;
 use reqwest::Client;
 use serde::Deserialize;
-use serde_json::json;
 use tokio::task;
 use tracing::{error, warn};
 use uuid::Uuid;
 
 use crate::{
+    documents::search::{build_quickwit_ingest_record, quickwit_ingest},
     jobs::JOB_INDEX_DOCUMENT_TEXT,
     models::{Document, DocumentVersion},
     schema::{document_asset_objects, document_assets, document_versions, documents},
@@ -129,7 +129,7 @@ impl JobHandler for IndexDocumentTextJob {
         .await
         {
             Ok(bytes) => bytes,
-            Err(err) => return handle_fetch_error(job, err, "failed to download ocr text"),
+            Err(err) => return handle_fetch_error(&job, err, "failed to download ocr text"),
         };
         let text = match String::from_utf8(bytes) {
             Ok(text) => text,
@@ -148,43 +148,17 @@ impl JobHandler for IndexDocumentTextJob {
             };
         }
 
-        let client = client;
-        let url = format!(
-            "{}/api/v1/{}/ingest?commit=auto",
-            quickwit_endpoint, quickwit_index
+        let record = build_quickwit_ingest_record(
+            &context.document,
+            &context.version,
+            job.tenant_id,
+            &text,
         );
-        let payload = json!({
-            "document_id": context.document.id,
-            "version_id": context.version.id,
-            "tenant_id": job.tenant_id,
-            "title": context.document.title.to_lowercase(),
-            "text": text.to_lowercase()
-        });
 
-        let body = serde_json::to_string(&payload).unwrap();
-
-        match client
-            .post(&url)
-            .header("content-type", "application/x-ndjson")
-            .body(format!("{}\n", body))
-            .send()
-            .await
-        {
-            Ok(response) => {
-                if response.status().is_success() {
-                    JobExecution::Success
-                } else {
-                    let status = response.status();
-                    let body = response.text().await.unwrap_or_default();
-                    warn!(job_id = %job.id, %status, %body, "quickwit ingest failed");
-                    JobExecution::Retry {
-                        delay: Duration::from_secs(30),
-                        error: format!("quickwit ingest failed with status {status}"),
-                    }
-                }
-            }
+        match quickwit_ingest(&client, &quickwit_endpoint, &quickwit_index, &[record]).await {
+            Ok(()) => JobExecution::Success,
             Err(err) => {
-                warn!(job_id = %job.id, error = %err, "quickwit request failed");
+                warn!(job_id = %job.id, error = %err, "quickwit ingest failed");
                 JobExecution::Retry {
                     delay: Duration::from_secs(30),
                     error: err.to_string(),

backend/src/workers/mod.rs

@@ -186,14 +186,14 @@ pub(crate) async fn fetch_version_object(
 }
 
 pub(crate) fn handle_fetch_error(
-    job_id: crate::models::Job,
+    job: &crate::models::Job,
     err: FetchVersionError,
     message: &str,
 ) -> JobExecution {
     match err {
         FetchVersionError::TooLarge { size, limit } => {
             warn!(
-                job_id = %job_id.id,
+                job_id = %job.id,
                 size_bytes = size,
                 limit_bytes = limit,
                 "document exceeds worker size limit"
@@ -205,7 +205,12 @@ pub(crate) fn handle_fetch_error(
             }
         }
         FetchVersionError::Storage(err) => {
-            warn!(job_id = %job_id.id, error = %err, "{message}");
+            warn!(
+                job_id = %job.id,
+                error = %err,
+                context = message,
+                "failed to fetch object for worker"
+            );
             JobExecution::Retry {
                 delay: Duration::from_secs(30),
                 error: err.to_string(),

backend/src/workers/ocr.rs

@@ -108,7 +108,7 @@ impl JobHandler for GenerateOcrTextJob {
         .await
         {
             Ok(bytes) => bytes,
-            Err(err) => return handle_fetch_error(job, err, "failed to fetch document for ocr"),
+            Err(err) => return handle_fetch_error(&job, err, "failed to fetch document for ocr"),
         };
 
         let doc_meta = PdfDocumentMeta {

backend/src/workers/thumbnails.rs

@@ -111,7 +111,7 @@ impl JobHandler for GenerateThumbnailsJob {
         .await
         {
             Ok(bytes) => bytes,
-            Err(err) => return handle_fetch_error(job, err, "thumbnail fetch failed; will retry"),
+            Err(err) => return handle_fetch_error(&job, err, "thumbnail fetch failed; will retry"),
         };
 
         let generation = match generate_preview_and_thumbnail(&initial.document, &bytes) {