basic search

This commit is contained in:
2025-10-11 17:05:12 +02:00
parent 0a669064dd
commit 7233e3a534
2 changed files with 270 additions and 82 deletions
+221 -9
View File
@@ -1,10 +1,13 @@
use anyhow::anyhow;
use axum::{ use axum::{
extract::{Json, Path, Query, State}, extract::{Json, Path, Query, State},
http::StatusCode, http::StatusCode,
}; };
use diesel::{dsl::exists, prelude::*, PgConnection}; use diesel::{dsl::exists, prelude::*, PgConnection};
use reqwest::Client;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use std::collections::HashSet; use serde_json::{json, Value};
use std::collections::{HashMap, HashSet};
use uuid::Uuid; use uuid::Uuid;
use crate::error::{AppError, AppResult}; use crate::error::{AppError, AppResult};
@@ -17,6 +20,8 @@ use super::documents::{
DocumentResponse, DocumentResponse,
}; };
const QUICKWIT_MAX_HITS: usize = 200;
#[derive(Deserialize)] #[derive(Deserialize)]
pub struct CreateFolderRequest { pub struct CreateFolderRequest {
pub name: String, pub name: String,
@@ -250,14 +255,40 @@ pub async fn search_documents(
docs_query = docs_query.filter(documents::folder_id.eq_any(descendant_ids)); docs_query = docs_query.filter(documents::folder_id.eq_any(descendant_ids));
} }
let mut filter_ids: Option<HashSet<Uuid>> = None;
let mut quickwit_order: Option<Vec<Uuid>> = None;
if let Some(query) = params if let Some(query) = params
.query .query
.as_ref() .as_ref()
.map(|s| s.trim()) .map(|s| s.trim())
.filter(|s| !s.is_empty()) .filter(|s| !s.is_empty())
{ {
let pattern = format!("%{}%", query); let endpoint = state
docs_query = docs_query.filter(documents::original_name.ilike(pattern)); .config
.quickwit_endpoint
.as_ref()
.ok_or_else(|| AppError::internal("quickwit endpoint not configured"))?;
let index = state
.config
.quickwit_index
.as_ref()
.ok_or_else(|| AppError::internal("quickwit index not configured"))?;
let ids = quickwit_search(endpoint, index, query)
.await
.map_err(|err| AppError::internal(format!("quickwit search failed: {err}")))?;
if ids.is_empty() {
return Ok(Json(vec![]));
}
quickwit_order = Some(ids.clone());
let set: HashSet<Uuid> = ids.into_iter().collect();
filter_ids = Some(match &filter_ids {
Some(existing) => existing.intersection(&set).copied().collect(),
None => set,
});
} }
if let Some(tags_param) = params if let Some(tags_param) = params
@@ -292,21 +323,69 @@ pub async fn search_documents(
} }
} }
let matching_doc_ids: Vec<Uuid> = let matching_doc_ids: HashSet<Uuid> = doc_id_set.unwrap_or_default();
doc_id_set.unwrap_or_default().into_iter().collect();
if matching_doc_ids.is_empty() { if matching_doc_ids.is_empty() {
return Ok(Json(vec![])); return Ok(Json(vec![]));
} }
docs_query = docs_query.filter(documents::id.eq_any(matching_doc_ids)); let new_filter = match &filter_ids {
Some(existing) => existing.intersection(&matching_doc_ids).copied().collect(),
None => matching_doc_ids.clone(),
};
filter_ids = Some(new_filter);
} }
} }
} }
let docs: Vec<Document> = docs_query if let Some(ref set) = filter_ids {
.order(documents::uploaded_at.desc()) if set.is_empty() {
.load(&mut conn)?; return Ok(Json(vec![]));
}
let ids_vec: Vec<Uuid> = set.iter().copied().collect();
docs_query = docs_query.filter(documents::id.eq_any(ids_vec));
}
let docs: Vec<Document> = if let Some(order_ids) = quickwit_order.as_ref() {
let relevant_ids: Vec<Uuid> = if let Some(filter_set) = filter_ids.as_ref() {
order_ids
.iter()
.copied()
.filter(|id| filter_set.contains(id))
.collect()
} else {
order_ids.clone()
};
if relevant_ids.is_empty() {
return Ok(Json(vec![]));
}
let fetched: Vec<Document> = docs_query.load(&mut conn)?;
let mut by_id: HashMap<Uuid, Document> =
fetched.into_iter().map(|doc| (doc.id, doc)).collect();
let mut ordered = Vec::with_capacity(by_id.len());
for id in relevant_ids {
if let Some(doc) = by_id.remove(&id) {
ordered.push(doc);
}
}
if !by_id.is_empty() {
let mut remaining: Vec<Document> = by_id.into_values().collect();
remaining.sort_by(|a, b| b.uploaded_at.cmp(&a.uploaded_at));
ordered.extend(remaining);
}
ordered
} else {
docs_query
.order(documents::uploaded_at.desc())
.load(&mut conn)?
};
let doc_ids: Vec<Uuid> = docs.iter().map(|doc| doc.id).collect(); let doc_ids: Vec<Uuid> = docs.iter().map(|doc| doc.id).collect();
let tags_map = load_tags_for_documents(&mut conn, &doc_ids)?; let tags_map = load_tags_for_documents(&mut conn, &doc_ids)?;
@@ -325,6 +404,139 @@ pub async fn search_documents(
Ok(Json(response)) Ok(Json(response))
} }
async fn quickwit_search(endpoint: &str, index: &str, query: &str) -> anyhow::Result<Vec<Uuid>> {
let quickwit_query = match build_quickwit_query(query) {
Some(q) => q,
None => return Ok(vec![]),
};
let client = Client::new();
let url = format!("{}/api/v1/{}/search", endpoint.trim_end_matches('/'), index);
let payload = json!({
"query": quickwit_query,
"max_hits": QUICKWIT_MAX_HITS,
});
let response = client.post(url).json(&payload).send().await?;
if !response.status().is_success() {
let status = response.status();
let body = response.text().await.unwrap_or_default();
return Err(anyhow!(
"quickwit search failed with status {status}: {body}"
));
}
let data: QuickwitSearchResponse = response.json().await?;
let mut seen = HashSet::new();
let mut doc_ids = Vec::new();
for hit in data.hits {
if let Some(doc_id) = extract_document_id(&hit) {
if seen.insert(doc_id) {
doc_ids.push(doc_id);
}
}
}
Ok(doc_ids)
}
fn build_quickwit_query(input: &str) -> Option<String> {
let tokens: Vec<String> = input
.split_whitespace()
.filter(|token| !token.is_empty())
.map(|token| {
let normalized = token.to_lowercase();
escape_quickwit_token(&normalized)
})
.collect();
if tokens.is_empty() {
return None;
}
let parts: Vec<String> = tokens
.into_iter()
.map(|token| format!("(title:{token} OR text:{token})"))
.collect();
Some(parts.join(" AND "))
}
fn escape_quickwit_token(token: &str) -> String {
let mut escaped = String::with_capacity(token.len());
for ch in token.chars() {
match ch {
'+' | '-' | '&' | '|' | '!' | '(' | ')' | '{' | '}' | '[' | ']' | '^' | '"' | '~'
| '*' | '?' | ':' | '\\' | '/' => {
escaped.push('\\');
escaped.push(ch);
}
_ => escaped.push(ch),
}
}
escaped
}
#[derive(Deserialize)]
struct QuickwitSearchResponse {
#[serde(default)]
hits: Vec<Value>,
}
fn extract_document_id(hit: &Value) -> Option<Uuid> {
for key in ["_source", "source", "fields", "stored_fields"] {
if let Some(value) = hit.get(key) {
if let Some(uuid) = extract_uuid_from_value(value) {
return Some(uuid);
}
}
}
if let Some(value) = hit.get("document_id") {
if let Some(uuid) = extract_uuid_from_value(value) {
return Some(uuid);
}
}
None
}
fn extract_uuid_from_value(value: &Value) -> Option<Uuid> {
if let Some(obj) = value.as_object() {
if let Some(inner) = obj.get("document_id") {
return parse_uuid_value(inner);
}
}
if let Some(arr) = value.as_array() {
for item in arr {
if let Some(uuid) = extract_uuid_from_value(item) {
return Some(uuid);
}
}
}
parse_uuid_value(value)
}
fn parse_uuid_value(value: &Value) -> Option<Uuid> {
if let Some(s) = value.as_str() {
return Uuid::parse_str(s).ok();
}
if let Some(arr) = value.as_array() {
for item in arr {
if let Some(uuid) = parse_uuid_value(item) {
return Some(uuid);
}
}
}
None
}
pub async fn delete_folder( pub async fn delete_folder(
State(state): State<AppState>, State(state): State<AppState>,
Path(folder_id): Path<Uuid>, Path(folder_id): Path<Uuid>,
+49 -73
View File
@@ -167,7 +167,7 @@ const DropOverlay = ({ active, folderName }) => (
</div> </div>
); );
const LoginView = ({ onSubmit, loading, status }) => ( const LoginView = ({ onSubmit, status }) => (
<div className="login-screen"> <div className="login-screen">
<div className="login-card"> <div className="login-card">
<h1>Paperless-NEO</h1> <h1>Paperless-NEO</h1>
@@ -190,9 +190,7 @@ const LoginView = ({ onSubmit, loading, status }) => (
autoComplete="current-password" autoComplete="current-password"
required required
/> />
<button type="submit" disabled={loading}> <button type="submit">Sign in</button>
{loading ? 'Signing in…' : 'Sign in'}
</button>
</form> </form>
<StatusBanner status={status} /> <StatusBanner status={status} />
</div> </div>
@@ -209,7 +207,6 @@ const FolderNode = ({
onDragOver, onDragOver,
onDragLeave, onDragLeave,
onDelete, onDelete,
disableActions,
renderChildren, renderChildren,
onFolderDragStart, onFolderDragStart,
onFolderDragEnd, onFolderDragEnd,
@@ -257,7 +254,6 @@ const FolderNode = ({
<button <button
type="button" type="button"
className="icon-button ghost" className="icon-button ghost"
disabled={disableActions}
onClick={(event) => { onClick={(event) => {
event.stopPropagation(); event.stopPropagation();
onDelete(node.id); onDelete(node.id);
@@ -323,7 +319,6 @@ const FilterBar = ({
const DocumentsTable = ({ const DocumentsTable = ({
currentFolderName, currentFolderName,
breadcrumbs, breadcrumbs,
loading,
onRefresh, onRefresh,
subfolders, subfolders,
documents, documents,
@@ -390,8 +385,8 @@ const DocumentsTable = ({
)} )}
</div> </div>
<div className="header-actions"> <div className="header-actions">
<button className="secondary" onClick={onRefresh} disabled={loading}> <button className="secondary" onClick={onRefresh}>
{loading ? 'Refreshing…' : 'Refresh'} Refresh
</button> </button>
</div> </div>
</div> </div>
@@ -648,7 +643,6 @@ const DetailPanel = ({
onTagAdd, onTagAdd,
onTagRemove, onTagRemove,
onRegenerateThumbnails, onRegenerateThumbnails,
loading,
previewEntry, previewEntry,
onOpenPreview, onOpenPreview,
onBulkTagAdd, onBulkTagAdd,
@@ -802,7 +796,6 @@ const DetailPanel = ({
type="button" type="button"
className="secondary" className="secondary"
onClick={() => onOpenPreview(singleDoc.id)} onClick={() => onOpenPreview(singleDoc.id)}
disabled={loading}
> >
Open preview Open preview
</button> </button>
@@ -810,7 +803,6 @@ const DetailPanel = ({
type="button" type="button"
className="secondary" className="secondary"
onClick={() => onRegenerateThumbnails(singleDoc.id)} onClick={() => onRegenerateThumbnails(singleDoc.id)}
disabled={loading}
> >
Re-run analysis Re-run analysis
</button> </button>
@@ -911,9 +903,7 @@ const DetailPanel = ({
}} }}
> >
<input name="tag" placeholder="Add tag to selection" list="tag-catalog" /> <input name="tag" placeholder="Add tag to selection" list="tag-catalog" />
<button type="submit" disabled={loading}> <button type="submit">Add tag</button>
Add tag
</button>
</form> </form>
<form <form
className="inline" className="inline"
@@ -926,7 +916,7 @@ const DetailPanel = ({
}} }}
> >
<input name="tag" placeholder="Remove tag from selection" list="tag-catalog" /> <input name="tag" placeholder="Remove tag from selection" list="tag-catalog" />
<button type="submit" className="secondary" disabled={loading}> <button type="submit" className="secondary">
Remove tag Remove tag
</button> </button>
</form> </form>
@@ -945,7 +935,6 @@ const DetailPanel = ({
name="target" name="target"
defaultValue={defaultMoveTarget || 'root'} defaultValue={defaultMoveTarget || 'root'}
onChange={(event) => onBulkMove?.({ target: event.target.value })} onChange={(event) => onBulkMove?.({ target: event.target.value })}
disabled={loading}
> >
{folderOptions.map((option) => ( {folderOptions.map((option) => (
<option key={option.id} value={option.id}> <option key={option.id} value={option.id}>
@@ -958,7 +947,6 @@ const DetailPanel = ({
type="button" type="button"
className="secondary" className="secondary"
onClick={() => onBulkReanalyze?.()} onClick={() => onBulkReanalyze?.()}
disabled={loading}
> >
Re-analyze selection Re-analyze selection
</button> </button>
@@ -982,7 +970,6 @@ const PreviewWorkspace = ({
document, document,
detail, detail,
previewEntry, previewEntry,
loading,
onClose, onClose,
onDownload, onDownload,
onRegenerateThumbnails, onRegenerateThumbnails,
@@ -1028,14 +1015,13 @@ const PreviewWorkspace = ({
type="button" type="button"
className="secondary" className="secondary"
onClick={() => onRegenerateThumbnails(document.id)} onClick={() => onRegenerateThumbnails(document.id)}
disabled={loading}
> >
Regenerate thumbnails Re-run analysis
</button> </button>
</div> </div>
</header> </header>
<div className="preview-workspace__body"> <div className="preview-workspace__body">
{loading || !previewEntry?.url ? ( {!previewEntry?.url ? (
<div className="preview-workspace__message">Loading preview</div> <div className="preview-workspace__message">Loading preview</div>
) : ( ) : (
<iframe <iframe
@@ -1064,7 +1050,6 @@ const Sidebar = ({
onDragLeave, onDragLeave,
onCreateFolder, onCreateFolder,
onDeleteFolder, onDeleteFolder,
disableActions,
selectedFolder, selectedFolder,
onFolderDragStart, onFolderDragStart,
onFolderDragEnd, onFolderDragEnd,
@@ -1087,7 +1072,6 @@ const Sidebar = ({
onDragOver={onDragOver} onDragOver={onDragOver}
onDragLeave={onDragLeave} onDragLeave={onDragLeave}
onDelete={onDeleteFolder} onDelete={onDeleteFolder}
disableActions={disableActions}
renderChildren={renderNodes} renderChildren={renderNodes}
onFolderDragStart={onFolderDragStart} onFolderDragStart={onFolderDragStart}
onFolderDragEnd={onFolderDragEnd} onFolderDragEnd={onFolderDragEnd}
@@ -1104,7 +1088,6 @@ const Sidebar = ({
onDragOver, onDragOver,
onDragLeave, onDragLeave,
onDeleteFolder, onDeleteFolder,
disableActions,
onFolderDragStart, onFolderDragStart,
onFolderDragEnd, onFolderDragEnd,
draggedFolderId, draggedFolderId,
@@ -1132,9 +1115,7 @@ const Sidebar = ({
}} }}
> >
<input name="folder-name" placeholder="New folder name" required /> <input name="folder-name" placeholder="New folder name" required />
<button type="submit" disabled={disableActions}> <button type="submit">Create</button>
Create
</button>
</form> </form>
<div className="folder-tree-scroller"> <div className="folder-tree-scroller">
<ul className="folder-tree"> <ul className="folder-tree">
@@ -3334,12 +3315,11 @@ const handleDownload = useCallback(
className="secondary" className="secondary"
type="button" type="button"
onClick={handleBulkReanalyze} onClick={handleBulkReanalyze}
disabled={loading}
> >
{loading ? 'Working…' : 'Re-analyze All'} Re-analyze All
</button> </button>
<button className="secondary" onClick={handleLogout} disabled={loading}> <button className="secondary" onClick={handleLogout}>
{loading ? 'Working…' : 'Log out'} Log out
</button> </button>
</div> </div>
</div> </div>
@@ -3350,51 +3330,48 @@ const handleDownload = useCallback(
document={previewWorkspaceDocument} document={previewWorkspaceDocument}
detail={previewWorkspaceDetail} detail={previewWorkspaceDetail}
previewEntry={previewWorkspaceEntry} previewEntry={previewWorkspaceEntry}
loading={previewDocumentLoading}
onClose={closeDocumentPreview} onClose={closeDocumentPreview}
onDownload={handleDownload} onDownload={handleDownload}
onRegenerateThumbnails={handleThumbnailRegeneration} onRegenerateThumbnails={handleThumbnailRegeneration}
/> />
) : ( ) : (
<> <>
<Sidebar <Sidebar
folderNodes={folderNodes} folderNodes={folderNodes}
onToggle={folderClickHandlers.onToggle} onToggle={folderClickHandlers.onToggle}
onSelect={folderClickHandlers.onSelect} onSelect={folderClickHandlers.onSelect}
onDrop={folderClickHandlers.onDrop} onDrop={folderClickHandlers.onDrop}
onDragOver={folderClickHandlers.onDragOver} onDragOver={folderClickHandlers.onDragOver}
onDragLeave={folderClickHandlers.onDragLeave} onDragLeave={folderClickHandlers.onDragLeave}
onCreateFolder={handleFolderCreate} onCreateFolder={handleFolderCreate}
onDeleteFolder={handleFolderDelete} onDeleteFolder={handleFolderDelete}
disableActions={loading} selectedFolder={selectedFolder}
selectedFolder={selectedFolder} onFolderDragStart={handleFolderDragStart}
onFolderDragStart={handleFolderDragStart} onFolderDragEnd={handleFolderDragEnd}
onFolderDragEnd={handleFolderDragEnd} draggedFolderId={draggedFolderId}
draggedFolderId={draggedFolderId} />
/> <DocumentsTable
<DocumentsTable currentFolderName={currentFolderName}
currentFolderName={currentFolderName} breadcrumbs={breadcrumbs}
breadcrumbs={breadcrumbs} onRefresh={refreshCurrentFolder}
loading={loading} subfolders={currentSubfolders}
onRefresh={refreshCurrentFolder} documents={documents}
subfolders={currentSubfolders} searchResults={searchResults}
documents={documents} isFilterActive={isFilterActive}
searchResults={searchResults} onFolderSelect={selectFolder}
isFilterActive={isFilterActive} onFolderDrop={folderClickHandlers.onDrop}
onFolderSelect={selectFolder} onFolderDragOver={folderClickHandlers.onDragOver}
onFolderDrop={folderClickHandlers.onDrop} onFolderDragLeave={folderClickHandlers.onDragLeave}
onFolderDragOver={folderClickHandlers.onDragOver} onFolderDragStart={handleFolderDragStart}
onFolderDragLeave={folderClickHandlers.onDragLeave} onFolderDragEnd={handleFolderDragEnd}
onFolderDragStart={handleFolderDragStart} draggedFolderId={draggedFolderId}
onFolderDragEnd={handleFolderDragEnd} onFolderDelete={handleFolderDelete}
draggedFolderId={draggedFolderId} onDocumentRowClick={handleDocumentRowClick}
onFolderDelete={handleFolderDelete} onDocumentOpen={openDocumentPreview}
onDocumentRowClick={handleDocumentRowClick} selectedDocumentIds={selectedDocumentIds}
onDocumentOpen={openDocumentPreview} focusedDocumentId={focusedDocumentId}
selectedDocumentIds={selectedDocumentIds} draggingDocumentIds={draggedDocumentIds}
focusedDocumentId={focusedDocumentId} onDocumentDragStart={(event, documentId) => {
draggingDocumentIds={draggedDocumentIds}
onDocumentDragStart={(event, documentId) => {
const selection = selectedDocumentIds.includes(documentId) const selection = selectedDocumentIds.includes(documentId)
? selectedDocumentIds ? selectedDocumentIds
: [documentId]; : [documentId];
@@ -3432,7 +3409,6 @@ const handleDownload = useCallback(
onTagAdd={handleTagAdd} onTagAdd={handleTagAdd}
onTagRemove={handleTagRemove} onTagRemove={handleTagRemove}
onRegenerateThumbnails={handleThumbnailRegeneration} onRegenerateThumbnails={handleThumbnailRegeneration}
loading={loading}
previewEntry={selectedPreviewEntry} previewEntry={selectedPreviewEntry}
onOpenPreview={openDocumentPreview} onOpenPreview={openDocumentPreview}
onBulkTagAdd={handleBulkTagAddFromDetail} onBulkTagAdd={handleBulkTagAddFromDetail}
@@ -3449,7 +3425,7 @@ const handleDownload = useCallback(
</main> </main>
</> </>
) : ( ) : (
<LoginView onSubmit={handleLogin} loading={loading} status={status} /> <LoginView onSubmit={handleLogin} status={status} />
)} )}
</div> </div>
); );