search
This commit is contained in:
+350
-13
@@ -10,12 +10,14 @@ use axum::response::IntoResponse;
|
||||
use chrono::{DateTime, NaiveDateTime, Utc};
|
||||
use diesel::dsl::exists;
|
||||
use diesel::{prelude::*, result::DatabaseErrorKind, select, PgConnection};
|
||||
use reqwest::Client;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{json, Value};
|
||||
use sha2::{Digest, Sha256};
|
||||
use tracing::{error, info, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::folders::gather_descendant_folder_ids;
|
||||
use crate::auth::AuthenticatedUser;
|
||||
use crate::error::{AppError, AppResult};
|
||||
use crate::jobs::{enqueue_job, JOB_ANALYZE_DOCUMENT};
|
||||
@@ -30,6 +32,7 @@ use crate::schema::{
|
||||
use crate::state::AppState;
|
||||
|
||||
const PRESIGNED_URL_EXPIRY_SECONDS: u64 = 300;
|
||||
const QUICKWIT_MAX_HITS: usize = 200;
|
||||
pub const CORRESPONDENT_ROLES: &[&str] = &["sender", "receiver", "other"];
|
||||
|
||||
fn normalize_role(value: &str) -> String {
|
||||
@@ -66,6 +69,11 @@ pub struct DocumentListQuery {
|
||||
pub folder_id: Option<Uuid>,
|
||||
#[serde(default)]
|
||||
pub include_deleted: bool,
|
||||
#[serde(default)]
|
||||
pub include_descendants: Option<bool>,
|
||||
pub query: Option<String>,
|
||||
pub tags: Option<String>,
|
||||
pub correspondents: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -328,29 +336,226 @@ pub struct AssignTagsRequest {
|
||||
|
||||
pub async fn list_documents(
|
||||
State(state): State<AppState>,
|
||||
Query(query): Query<DocumentListQuery>,
|
||||
Query(params): Query<DocumentListQuery>,
|
||||
user: AuthenticatedUser,
|
||||
) -> AppResult<Json<Vec<DocumentResponse>>> {
|
||||
let mut conn = state.db()?;
|
||||
|
||||
let mut base_query = documents::table.into_boxed();
|
||||
let DocumentListQuery {
|
||||
folder_id,
|
||||
include_deleted,
|
||||
include_descendants,
|
||||
query,
|
||||
tags,
|
||||
correspondents,
|
||||
} = params;
|
||||
|
||||
if !query.include_deleted {
|
||||
base_query = base_query.filter(documents::deleted_at.is_null());
|
||||
let mut docs_query = documents::table.into_boxed();
|
||||
|
||||
if !include_deleted {
|
||||
docs_query = docs_query.filter(documents::deleted_at.is_null());
|
||||
}
|
||||
|
||||
match query.folder_id {
|
||||
Some(folder_id) => {
|
||||
base_query = base_query.filter(documents::folder_id.eq(Some(folder_id)));
|
||||
let search_text = query
|
||||
.as_ref()
|
||||
.map(|s| s.trim())
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(|s| s.to_owned());
|
||||
let tags_param = tags
|
||||
.as_ref()
|
||||
.map(|s| s.trim())
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(|s| s.to_owned());
|
||||
let correspondents_param = correspondents
|
||||
.as_ref()
|
||||
.map(|s| s.trim())
|
||||
.filter(|s| !s.is_empty())
|
||||
.map(|s| s.to_owned());
|
||||
|
||||
let mut include_descendants = include_descendants.unwrap_or_else(|| folder_id.is_some());
|
||||
if search_text.is_some() || tags_param.is_some() || correspondents_param.is_some() {
|
||||
include_descendants = true;
|
||||
}
|
||||
|
||||
match (folder_id, include_descendants) {
|
||||
(Some(folder_id), true) => {
|
||||
let descendant_ids = gather_descendant_folder_ids(&mut conn, folder_id)?;
|
||||
docs_query = docs_query.filter(documents::folder_id.eq_any(descendant_ids));
|
||||
}
|
||||
None => {
|
||||
base_query = base_query.filter(documents::folder_id.is_null());
|
||||
(Some(folder_id), false) => {
|
||||
docs_query = docs_query.filter(documents::folder_id.eq(Some(folder_id)));
|
||||
}
|
||||
(None, false) => {
|
||||
docs_query = docs_query.filter(documents::folder_id.is_null());
|
||||
}
|
||||
(None, true) => {}
|
||||
}
|
||||
|
||||
let mut filter_ids: Option<HashSet<Uuid>> = None;
|
||||
let mut quickwit_order: Option<Vec<Uuid>> = None;
|
||||
|
||||
if let Some(query_str) = search_text.as_ref() {
|
||||
let endpoint = state
|
||||
.config
|
||||
.quickwit_endpoint
|
||||
.as_ref()
|
||||
.ok_or_else(|| AppError::internal("quickwit endpoint not configured"))?;
|
||||
let index = state
|
||||
.config
|
||||
.quickwit_index
|
||||
.as_ref()
|
||||
.ok_or_else(|| AppError::internal("quickwit index not configured"))?;
|
||||
|
||||
let ids = quickwit_search(endpoint, index, query_str)
|
||||
.await
|
||||
.map_err(|err| AppError::internal(format!("quickwit search failed: {err}")))?;
|
||||
|
||||
if ids.is_empty() {
|
||||
return Ok(Json(vec![]));
|
||||
}
|
||||
|
||||
quickwit_order = Some(ids.clone());
|
||||
let set: HashSet<Uuid> = ids.into_iter().collect();
|
||||
filter_ids = Some(match &filter_ids {
|
||||
Some(existing) => existing.intersection(&set).copied().collect(),
|
||||
None => set,
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(tags_param) = tags_param.as_ref() {
|
||||
let tag_ids: Result<Vec<Uuid>, _> = tags_param
|
||||
.split(',')
|
||||
.map(|s| Uuid::parse_str(s.trim()))
|
||||
.collect();
|
||||
|
||||
if let Ok(ids) = tag_ids {
|
||||
if !ids.is_empty() {
|
||||
let mut doc_id_set: Option<HashSet<Uuid>> = None;
|
||||
for tag_id in &ids {
|
||||
let docs_for_tag: Vec<Uuid> = document_tags::table
|
||||
.filter(document_tags::tag_id.eq(*tag_id))
|
||||
.select(document_tags::document_id)
|
||||
.load(&mut conn)?;
|
||||
let docs_set: HashSet<Uuid> = docs_for_tag.into_iter().collect();
|
||||
doc_id_set = Some(match doc_id_set {
|
||||
Some(existing) => existing.intersection(&docs_set).cloned().collect(),
|
||||
None => docs_set,
|
||||
});
|
||||
|
||||
if let Some(ref set) = doc_id_set {
|
||||
if set.is_empty() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let matching_doc_ids: HashSet<Uuid> = doc_id_set.unwrap_or_default();
|
||||
|
||||
if matching_doc_ids.is_empty() {
|
||||
return Ok(Json(vec![]));
|
||||
}
|
||||
|
||||
let new_filter = match &filter_ids {
|
||||
Some(existing) => existing.intersection(&matching_doc_ids).copied().collect(),
|
||||
None => matching_doc_ids.clone(),
|
||||
};
|
||||
|
||||
filter_ids = Some(new_filter);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let docs: Vec<Document> = base_query
|
||||
.order(documents::uploaded_at.desc())
|
||||
.load(&mut conn)?;
|
||||
if let Some(correspondents_param) = correspondents_param.as_ref() {
|
||||
let correspondent_ids: Result<Vec<Uuid>, _> = correspondents_param
|
||||
.split(',')
|
||||
.map(|s| Uuid::parse_str(s.trim()))
|
||||
.collect();
|
||||
|
||||
if let Ok(ids) = correspondent_ids {
|
||||
if !ids.is_empty() {
|
||||
let mut doc_id_set: Option<HashSet<Uuid>> = None;
|
||||
for correspondent_id in &ids {
|
||||
let docs_for_correspondent: Vec<Uuid> = document_correspondents::table
|
||||
.filter(document_correspondents::correspondent_id.eq(*correspondent_id))
|
||||
.select(document_correspondents::document_id)
|
||||
.load(&mut conn)?;
|
||||
|
||||
let docs_set: HashSet<Uuid> = docs_for_correspondent.into_iter().collect();
|
||||
doc_id_set = Some(match doc_id_set {
|
||||
Some(existing) => existing.intersection(&docs_set).cloned().collect(),
|
||||
None => docs_set,
|
||||
});
|
||||
|
||||
if let Some(ref set) = doc_id_set {
|
||||
if set.is_empty() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let matching_doc_ids: HashSet<Uuid> = doc_id_set.unwrap_or_default();
|
||||
|
||||
if matching_doc_ids.is_empty() {
|
||||
return Ok(Json(vec![]));
|
||||
}
|
||||
|
||||
let new_filter = match &filter_ids {
|
||||
Some(existing) => existing.intersection(&matching_doc_ids).copied().collect(),
|
||||
None => matching_doc_ids.clone(),
|
||||
};
|
||||
|
||||
filter_ids = Some(new_filter);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(ref set) = filter_ids {
|
||||
if set.is_empty() {
|
||||
return Ok(Json(vec![]));
|
||||
}
|
||||
|
||||
let ids_vec: Vec<Uuid> = set.iter().copied().collect();
|
||||
docs_query = docs_query.filter(documents::id.eq_any(ids_vec));
|
||||
}
|
||||
|
||||
let docs: Vec<Document> = if let Some(order_ids) = quickwit_order.as_ref() {
|
||||
let relevant_ids: Vec<Uuid> = if let Some(filter_set) = filter_ids.as_ref() {
|
||||
order_ids
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|id| filter_set.contains(id))
|
||||
.collect()
|
||||
} else {
|
||||
order_ids.clone()
|
||||
};
|
||||
|
||||
if relevant_ids.is_empty() {
|
||||
return Ok(Json(vec![]));
|
||||
}
|
||||
|
||||
let fetched: Vec<Document> = docs_query.load(&mut conn)?;
|
||||
let mut by_id: HashMap<Uuid, Document> =
|
||||
fetched.into_iter().map(|doc| (doc.id, doc)).collect();
|
||||
|
||||
let mut ordered = Vec::with_capacity(by_id.len());
|
||||
for id in relevant_ids {
|
||||
if let Some(doc) = by_id.remove(&id) {
|
||||
ordered.push(doc);
|
||||
}
|
||||
}
|
||||
|
||||
if !by_id.is_empty() {
|
||||
let mut remaining: Vec<Document> = by_id.into_values().collect();
|
||||
remaining.sort_by(|a, b| b.uploaded_at.cmp(&a.uploaded_at));
|
||||
ordered.extend(remaining);
|
||||
}
|
||||
|
||||
ordered
|
||||
} else {
|
||||
docs_query
|
||||
.order(documents::uploaded_at.desc())
|
||||
.load(&mut conn)?
|
||||
};
|
||||
|
||||
let doc_ids: Vec<Uuid> = docs.iter().map(|doc| doc.id).collect();
|
||||
let tags_map = load_tags_for_documents(&mut conn, &doc_ids)?;
|
||||
@@ -358,7 +563,6 @@ pub async fn list_documents(
|
||||
drop(conn);
|
||||
|
||||
let primary_versions = load_primary_assets(&state, &docs).await?;
|
||||
|
||||
let mut response = Vec::with_capacity(doc_ids.len());
|
||||
for doc in docs {
|
||||
let tags = tags_map.get(&doc.id).cloned();
|
||||
@@ -1738,3 +1942,136 @@ async fn load_asset_responses(
|
||||
pub(crate) fn to_iso(dt: NaiveDateTime) -> String {
|
||||
DateTime::<Utc>::from_naive_utc_and_offset(dt, Utc).to_rfc3339()
|
||||
}
|
||||
|
||||
async fn quickwit_search(endpoint: &str, index: &str, query: &str) -> anyhow::Result<Vec<Uuid>> {
|
||||
let quickwit_query = match build_quickwit_query(query) {
|
||||
Some(q) => q,
|
||||
None => return Ok(vec![]),
|
||||
};
|
||||
|
||||
let client = Client::new();
|
||||
let url = format!("{}/api/v1/{}/search", endpoint.trim_end_matches('/'), index);
|
||||
|
||||
let payload = json!({
|
||||
"query": quickwit_query,
|
||||
"max_hits": QUICKWIT_MAX_HITS,
|
||||
});
|
||||
|
||||
let response = client.post(url).json(&payload).send().await?;
|
||||
if !response.status().is_success() {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(anyhow::anyhow!(
|
||||
"quickwit search failed with status {status}: {body}"
|
||||
));
|
||||
}
|
||||
|
||||
let data: QuickwitSearchResponse = response.json().await?;
|
||||
let mut seen = HashSet::new();
|
||||
let mut doc_ids = Vec::new();
|
||||
|
||||
for hit in data.hits {
|
||||
if let Some(doc_id) = extract_document_id(&hit) {
|
||||
if seen.insert(doc_id) {
|
||||
doc_ids.push(doc_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(doc_ids)
|
||||
}
|
||||
|
||||
fn build_quickwit_query(input: &str) -> Option<String> {
|
||||
let tokens: Vec<String> = input
|
||||
.split_whitespace()
|
||||
.filter(|token| !token.is_empty())
|
||||
.map(|token| {
|
||||
let normalized = token.to_lowercase();
|
||||
escape_quickwit_token(&normalized)
|
||||
})
|
||||
.collect();
|
||||
|
||||
if tokens.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let parts: Vec<String> = tokens
|
||||
.into_iter()
|
||||
.map(|token| format!("(title:{token} OR text:{token})"))
|
||||
.collect();
|
||||
|
||||
Some(parts.join(" AND "))
|
||||
}
|
||||
|
||||
fn escape_quickwit_token(token: &str) -> String {
|
||||
let mut escaped = String::with_capacity(token.len());
|
||||
for ch in token.chars() {
|
||||
match ch {
|
||||
'+' | '-' | '&' | '|' | '!' | '(' | ')' | '{' | '}' | '[' | ']' | '^' | '"' | '~'
|
||||
| '*' | '?' | ':' | '\\' | '/' => {
|
||||
escaped.push('\\');
|
||||
escaped.push(ch);
|
||||
}
|
||||
_ => escaped.push(ch),
|
||||
}
|
||||
}
|
||||
escaped
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct QuickwitSearchResponse {
|
||||
#[serde(default)]
|
||||
hits: Vec<Value>,
|
||||
}
|
||||
|
||||
fn extract_document_id(hit: &Value) -> Option<Uuid> {
|
||||
for key in ["_source", "source", "fields", "stored_fields"] {
|
||||
if let Some(value) = hit.get(key) {
|
||||
if let Some(uuid) = extract_uuid_from_value(value) {
|
||||
return Some(uuid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(value) = hit.get("document_id") {
|
||||
if let Some(uuid) = extract_uuid_from_value(value) {
|
||||
return Some(uuid);
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn extract_uuid_from_value(value: &Value) -> Option<Uuid> {
|
||||
if let Some(obj) = value.as_object() {
|
||||
if let Some(inner) = obj.get("document_id") {
|
||||
return parse_uuid_value(inner);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(arr) = value.as_array() {
|
||||
for item in arr {
|
||||
if let Some(uuid) = extract_uuid_from_value(item) {
|
||||
return Some(uuid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
parse_uuid_value(value)
|
||||
}
|
||||
|
||||
fn parse_uuid_value(value: &Value) -> Option<Uuid> {
|
||||
if let Some(s) = value.as_str() {
|
||||
return Uuid::parse_str(s).ok();
|
||||
}
|
||||
|
||||
if let Some(arr) = value.as_array() {
|
||||
for item in arr {
|
||||
if let Some(uuid) = parse_uuid_value(item) {
|
||||
return Some(uuid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
@@ -1,17 +1,13 @@
|
||||
use anyhow::anyhow;
|
||||
use axum::{
|
||||
extract::{Json, Path, Query, State},
|
||||
extract::{Json, Path, State},
|
||||
http::StatusCode,
|
||||
};
|
||||
use diesel::{dsl::exists, prelude::*, PgConnection};
|
||||
use reqwest::Client;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::{json, Value};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::models::{Document, Folder, NewFolder};
|
||||
use crate::schema::{document_correspondents, document_tags, documents, folders};
|
||||
use crate::schema::{documents, folders};
|
||||
use crate::state::AppState;
|
||||
use crate::{
|
||||
auth::AuthenticatedUser,
|
||||
@@ -23,8 +19,6 @@ use super::documents::{
|
||||
to_document_response, to_iso, DocumentResponse,
|
||||
};
|
||||
|
||||
const QUICKWIT_MAX_HITS: usize = 200;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct CreateFolderRequest {
|
||||
pub name: String,
|
||||
@@ -56,13 +50,6 @@ pub struct FolderContentsResponse {
|
||||
pub documents: Vec<DocumentResponse>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct DocumentSearchQuery {
|
||||
pub query: Option<String>,
|
||||
pub tags: Option<String>,
|
||||
pub correspondents: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct FolderInfo {
|
||||
pub id: Uuid,
|
||||
@@ -244,370 +231,6 @@ pub async fn list_folder_contents(
|
||||
}))
|
||||
}
|
||||
|
||||
pub async fn search_documents(
|
||||
State(state): State<AppState>,
|
||||
Path(folder_identifier): Path<String>,
|
||||
Query(params): Query<DocumentSearchQuery>,
|
||||
user: AuthenticatedUser,
|
||||
) -> AppResult<Json<Vec<DocumentResponse>>> {
|
||||
let mut conn = state.db()?;
|
||||
|
||||
let folder_id = if folder_identifier.eq_ignore_ascii_case("root") {
|
||||
None
|
||||
} else {
|
||||
Some(
|
||||
Uuid::parse_str(&folder_identifier)
|
||||
.map_err(|_| AppError::bad_request("folder identifier must be 'root' or a UUID"))?,
|
||||
)
|
||||
};
|
||||
|
||||
let mut docs_query = documents::table
|
||||
.filter(documents::deleted_at.is_null())
|
||||
.into_boxed();
|
||||
|
||||
if let Some(folder_id) = folder_id {
|
||||
let descendant_ids = gather_descendant_folder_ids(&mut conn, folder_id)?;
|
||||
docs_query = docs_query.filter(documents::folder_id.eq_any(descendant_ids));
|
||||
}
|
||||
|
||||
let mut filter_ids: Option<HashSet<Uuid>> = None;
|
||||
let mut quickwit_order: Option<Vec<Uuid>> = None;
|
||||
|
||||
if let Some(query) = params
|
||||
.query
|
||||
.as_ref()
|
||||
.map(|s| s.trim())
|
||||
.filter(|s| !s.is_empty())
|
||||
{
|
||||
let endpoint = state
|
||||
.config
|
||||
.quickwit_endpoint
|
||||
.as_ref()
|
||||
.ok_or_else(|| AppError::internal("quickwit endpoint not configured"))?;
|
||||
let index = state
|
||||
.config
|
||||
.quickwit_index
|
||||
.as_ref()
|
||||
.ok_or_else(|| AppError::internal("quickwit index not configured"))?;
|
||||
|
||||
let ids = quickwit_search(endpoint, index, query)
|
||||
.await
|
||||
.map_err(|err| AppError::internal(format!("quickwit search failed: {err}")))?;
|
||||
|
||||
if ids.is_empty() {
|
||||
return Ok(Json(vec![]));
|
||||
}
|
||||
|
||||
quickwit_order = Some(ids.clone());
|
||||
let set: HashSet<Uuid> = ids.into_iter().collect();
|
||||
filter_ids = Some(match &filter_ids {
|
||||
Some(existing) => existing.intersection(&set).copied().collect(),
|
||||
None => set,
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(tags_param) = params
|
||||
.tags
|
||||
.as_ref()
|
||||
.map(|s| s.trim())
|
||||
.filter(|s| !s.is_empty())
|
||||
{
|
||||
let tag_ids: Result<Vec<Uuid>, _> = tags_param
|
||||
.split(',')
|
||||
.map(|s| Uuid::parse_str(s.trim()))
|
||||
.collect();
|
||||
|
||||
if let Ok(ids) = tag_ids {
|
||||
if !ids.is_empty() {
|
||||
let mut doc_id_set: Option<HashSet<Uuid>> = None;
|
||||
for tag_id in &ids {
|
||||
let docs_for_tag: Vec<Uuid> = document_tags::table
|
||||
.filter(document_tags::tag_id.eq(*tag_id))
|
||||
.select(document_tags::document_id)
|
||||
.load(&mut conn)?;
|
||||
let docs_set: HashSet<Uuid> = docs_for_tag.into_iter().collect();
|
||||
doc_id_set = Some(match doc_id_set {
|
||||
Some(existing) => existing.intersection(&docs_set).cloned().collect(),
|
||||
None => docs_set,
|
||||
});
|
||||
|
||||
if let Some(ref set) = doc_id_set {
|
||||
if set.is_empty() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let matching_doc_ids: HashSet<Uuid> = doc_id_set.unwrap_or_default();
|
||||
|
||||
if matching_doc_ids.is_empty() {
|
||||
return Ok(Json(vec![]));
|
||||
}
|
||||
|
||||
let new_filter = match &filter_ids {
|
||||
Some(existing) => existing.intersection(&matching_doc_ids).copied().collect(),
|
||||
None => matching_doc_ids.clone(),
|
||||
};
|
||||
|
||||
filter_ids = Some(new_filter);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(correspondents_param) = params
|
||||
.correspondents
|
||||
.as_ref()
|
||||
.map(|s| s.trim())
|
||||
.filter(|s| !s.is_empty())
|
||||
{
|
||||
let correspondent_ids: Result<Vec<Uuid>, _> = correspondents_param
|
||||
.split(',')
|
||||
.map(|s| Uuid::parse_str(s.trim()))
|
||||
.collect();
|
||||
|
||||
if let Ok(ids) = correspondent_ids {
|
||||
if !ids.is_empty() {
|
||||
let mut doc_id_set: Option<HashSet<Uuid>> = None;
|
||||
for correspondent_id in &ids {
|
||||
let docs_for_correspondent: Vec<Uuid> = document_correspondents::table
|
||||
.filter(document_correspondents::correspondent_id.eq(*correspondent_id))
|
||||
.select(document_correspondents::document_id)
|
||||
.load(&mut conn)?;
|
||||
|
||||
let docs_set: HashSet<Uuid> = docs_for_correspondent.into_iter().collect();
|
||||
doc_id_set = Some(match doc_id_set {
|
||||
Some(existing) => existing.intersection(&docs_set).cloned().collect(),
|
||||
None => docs_set,
|
||||
});
|
||||
|
||||
if let Some(ref set) = doc_id_set {
|
||||
if set.is_empty() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let matching_doc_ids: HashSet<Uuid> = doc_id_set.unwrap_or_default();
|
||||
|
||||
if matching_doc_ids.is_empty() {
|
||||
return Ok(Json(vec![]));
|
||||
}
|
||||
|
||||
let new_filter = match &filter_ids {
|
||||
Some(existing) => existing.intersection(&matching_doc_ids).copied().collect(),
|
||||
None => matching_doc_ids.clone(),
|
||||
};
|
||||
|
||||
filter_ids = Some(new_filter);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(ref set) = filter_ids {
|
||||
if set.is_empty() {
|
||||
return Ok(Json(vec![]));
|
||||
}
|
||||
|
||||
let ids_vec: Vec<Uuid> = set.iter().copied().collect();
|
||||
docs_query = docs_query.filter(documents::id.eq_any(ids_vec));
|
||||
}
|
||||
|
||||
let docs: Vec<Document> = if let Some(order_ids) = quickwit_order.as_ref() {
|
||||
let relevant_ids: Vec<Uuid> = if let Some(filter_set) = filter_ids.as_ref() {
|
||||
order_ids
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|id| filter_set.contains(id))
|
||||
.collect()
|
||||
} else {
|
||||
order_ids.clone()
|
||||
};
|
||||
|
||||
if relevant_ids.is_empty() {
|
||||
return Ok(Json(vec![]));
|
||||
}
|
||||
|
||||
let fetched: Vec<Document> = docs_query.load(&mut conn)?;
|
||||
let mut by_id: HashMap<Uuid, Document> =
|
||||
fetched.into_iter().map(|doc| (doc.id, doc)).collect();
|
||||
|
||||
let mut ordered = Vec::with_capacity(by_id.len());
|
||||
for id in relevant_ids {
|
||||
if let Some(doc) = by_id.remove(&id) {
|
||||
ordered.push(doc);
|
||||
}
|
||||
}
|
||||
|
||||
if !by_id.is_empty() {
|
||||
let mut remaining: Vec<Document> = by_id.into_values().collect();
|
||||
remaining.sort_by(|a, b| b.uploaded_at.cmp(&a.uploaded_at));
|
||||
ordered.extend(remaining);
|
||||
}
|
||||
|
||||
ordered
|
||||
} else {
|
||||
docs_query
|
||||
.order(documents::uploaded_at.desc())
|
||||
.load(&mut conn)?
|
||||
};
|
||||
|
||||
let doc_ids: Vec<Uuid> = docs.iter().map(|doc| doc.id).collect();
|
||||
let tags_map = load_tags_for_documents(&mut conn, &doc_ids)?;
|
||||
let mut correspondents_map = load_correspondents_for_documents(&mut conn, &doc_ids)?;
|
||||
drop(conn);
|
||||
|
||||
let primary_versions = load_primary_assets(&state, &docs).await?;
|
||||
let mut response = Vec::with_capacity(doc_ids.len());
|
||||
for doc in docs {
|
||||
let tags = tags_map.get(&doc.id).cloned();
|
||||
let correspondents = correspondents_map.remove(&doc.id).unwrap_or_default();
|
||||
let current_version = primary_versions.get(&doc.id).cloned();
|
||||
response.push(to_document_response(
|
||||
&state,
|
||||
user.user_id,
|
||||
doc,
|
||||
tags,
|
||||
correspondents,
|
||||
current_version,
|
||||
)?);
|
||||
}
|
||||
|
||||
Ok(Json(response))
|
||||
}
|
||||
|
||||
async fn quickwit_search(endpoint: &str, index: &str, query: &str) -> anyhow::Result<Vec<Uuid>> {
|
||||
let quickwit_query = match build_quickwit_query(query) {
|
||||
Some(q) => q,
|
||||
None => return Ok(vec![]),
|
||||
};
|
||||
|
||||
let client = Client::new();
|
||||
let url = format!("{}/api/v1/{}/search", endpoint.trim_end_matches('/'), index);
|
||||
|
||||
let payload = json!({
|
||||
"query": quickwit_query,
|
||||
"max_hits": QUICKWIT_MAX_HITS,
|
||||
});
|
||||
|
||||
let response = client.post(url).json(&payload).send().await?;
|
||||
if !response.status().is_success() {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(anyhow!(
|
||||
"quickwit search failed with status {status}: {body}"
|
||||
));
|
||||
}
|
||||
|
||||
let data: QuickwitSearchResponse = response.json().await?;
|
||||
let mut seen = HashSet::new();
|
||||
let mut doc_ids = Vec::new();
|
||||
|
||||
for hit in data.hits {
|
||||
if let Some(doc_id) = extract_document_id(&hit) {
|
||||
if seen.insert(doc_id) {
|
||||
doc_ids.push(doc_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(doc_ids)
|
||||
}
|
||||
|
||||
fn build_quickwit_query(input: &str) -> Option<String> {
|
||||
let tokens: Vec<String> = input
|
||||
.split_whitespace()
|
||||
.filter(|token| !token.is_empty())
|
||||
.map(|token| {
|
||||
let normalized = token.to_lowercase();
|
||||
escape_quickwit_token(&normalized)
|
||||
})
|
||||
.collect();
|
||||
|
||||
if tokens.is_empty() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let parts: Vec<String> = tokens
|
||||
.into_iter()
|
||||
.map(|token| format!("(title:{token} OR text:{token})"))
|
||||
.collect();
|
||||
|
||||
Some(parts.join(" AND "))
|
||||
}
|
||||
|
||||
fn escape_quickwit_token(token: &str) -> String {
|
||||
let mut escaped = String::with_capacity(token.len());
|
||||
for ch in token.chars() {
|
||||
match ch {
|
||||
'+' | '-' | '&' | '|' | '!' | '(' | ')' | '{' | '}' | '[' | ']' | '^' | '"' | '~'
|
||||
| '*' | '?' | ':' | '\\' | '/' => {
|
||||
escaped.push('\\');
|
||||
escaped.push(ch);
|
||||
}
|
||||
_ => escaped.push(ch),
|
||||
}
|
||||
}
|
||||
escaped
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct QuickwitSearchResponse {
|
||||
#[serde(default)]
|
||||
hits: Vec<Value>,
|
||||
}
|
||||
|
||||
fn extract_document_id(hit: &Value) -> Option<Uuid> {
|
||||
for key in ["_source", "source", "fields", "stored_fields"] {
|
||||
if let Some(value) = hit.get(key) {
|
||||
if let Some(uuid) = extract_uuid_from_value(value) {
|
||||
return Some(uuid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(value) = hit.get("document_id") {
|
||||
if let Some(uuid) = extract_uuid_from_value(value) {
|
||||
return Some(uuid);
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
fn extract_uuid_from_value(value: &Value) -> Option<Uuid> {
|
||||
if let Some(obj) = value.as_object() {
|
||||
if let Some(inner) = obj.get("document_id") {
|
||||
return parse_uuid_value(inner);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(arr) = value.as_array() {
|
||||
for item in arr {
|
||||
if let Some(uuid) = extract_uuid_from_value(item) {
|
||||
return Some(uuid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
parse_uuid_value(value)
|
||||
}
|
||||
|
||||
fn parse_uuid_value(value: &Value) -> Option<Uuid> {
|
||||
if let Some(s) = value.as_str() {
|
||||
return Uuid::parse_str(s).ok();
|
||||
}
|
||||
|
||||
if let Some(arr) = value.as_array() {
|
||||
for item in arr {
|
||||
if let Some(uuid) = parse_uuid_value(item) {
|
||||
return Some(uuid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
None
|
||||
}
|
||||
|
||||
pub async fn delete_folder(
|
||||
State(state): State<AppState>,
|
||||
Path(folder_id): Path<Uuid>,
|
||||
@@ -771,7 +394,10 @@ fn folder_to_info(folder: Folder) -> FolderInfo {
|
||||
}
|
||||
}
|
||||
|
||||
fn gather_descendant_folder_ids(conn: &mut PgConnection, folder_id: Uuid) -> AppResult<Vec<Uuid>> {
|
||||
pub(super) fn gather_descendant_folder_ids(
|
||||
conn: &mut PgConnection,
|
||||
folder_id: Uuid,
|
||||
) -> AppResult<Vec<Uuid>> {
|
||||
let mut ids = vec![folder_id];
|
||||
let mut queue = vec![folder_id];
|
||||
|
||||
|
||||
@@ -102,8 +102,7 @@ pub fn create_router(state: AppState) -> Router<()> {
|
||||
"/:id",
|
||||
delete(folders::delete_folder).patch(folders::update_folder),
|
||||
)
|
||||
.route("/:id/contents", get(folders::list_folder_contents))
|
||||
.route("/:id/documents", get(folders::search_documents));
|
||||
.route("/:id/contents", get(folders::list_folder_contents));
|
||||
|
||||
let tags_routes = Router::new()
|
||||
.route("/", get(tags::list_tags).post(tags::create_tag))
|
||||
|
||||
Reference in New Issue
Block a user