basic search

This commit is contained in:
2025-10-11 17:05:12 +02:00
parent 0a669064dd
commit 7233e3a534
2 changed files with 270 additions and 82 deletions
+221 -9
View File
@@ -1,10 +1,13 @@
use anyhow::anyhow;
use axum::{
extract::{Json, Path, Query, State},
http::StatusCode,
};
use diesel::{dsl::exists, prelude::*, PgConnection};
use reqwest::Client;
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
use serde_json::{json, Value};
use std::collections::{HashMap, HashSet};
use uuid::Uuid;
use crate::error::{AppError, AppResult};
@@ -17,6 +20,8 @@ use super::documents::{
DocumentResponse,
};
const QUICKWIT_MAX_HITS: usize = 200;
#[derive(Deserialize)]
pub struct CreateFolderRequest {
pub name: String,
@@ -250,14 +255,40 @@ pub async fn search_documents(
docs_query = docs_query.filter(documents::folder_id.eq_any(descendant_ids));
}
let mut filter_ids: Option<HashSet<Uuid>> = None;
let mut quickwit_order: Option<Vec<Uuid>> = None;
if let Some(query) = params
.query
.as_ref()
.map(|s| s.trim())
.filter(|s| !s.is_empty())
{
let pattern = format!("%{}%", query);
docs_query = docs_query.filter(documents::original_name.ilike(pattern));
let endpoint = state
.config
.quickwit_endpoint
.as_ref()
.ok_or_else(|| AppError::internal("quickwit endpoint not configured"))?;
let index = state
.config
.quickwit_index
.as_ref()
.ok_or_else(|| AppError::internal("quickwit index not configured"))?;
let ids = quickwit_search(endpoint, index, query)
.await
.map_err(|err| AppError::internal(format!("quickwit search failed: {err}")))?;
if ids.is_empty() {
return Ok(Json(vec![]));
}
quickwit_order = Some(ids.clone());
let set: HashSet<Uuid> = ids.into_iter().collect();
filter_ids = Some(match &filter_ids {
Some(existing) => existing.intersection(&set).copied().collect(),
None => set,
});
}
if let Some(tags_param) = params
@@ -292,21 +323,69 @@ pub async fn search_documents(
}
}
let matching_doc_ids: Vec<Uuid> =
doc_id_set.unwrap_or_default().into_iter().collect();
let matching_doc_ids: HashSet<Uuid> = doc_id_set.unwrap_or_default();
if matching_doc_ids.is_empty() {
return Ok(Json(vec![]));
}
docs_query = docs_query.filter(documents::id.eq_any(matching_doc_ids));
let new_filter = match &filter_ids {
Some(existing) => existing.intersection(&matching_doc_ids).copied().collect(),
None => matching_doc_ids.clone(),
};
filter_ids = Some(new_filter);
}
}
}
let docs: Vec<Document> = docs_query
.order(documents::uploaded_at.desc())
.load(&mut conn)?;
if let Some(ref set) = filter_ids {
if set.is_empty() {
return Ok(Json(vec![]));
}
let ids_vec: Vec<Uuid> = set.iter().copied().collect();
docs_query = docs_query.filter(documents::id.eq_any(ids_vec));
}
let docs: Vec<Document> = if let Some(order_ids) = quickwit_order.as_ref() {
let relevant_ids: Vec<Uuid> = if let Some(filter_set) = filter_ids.as_ref() {
order_ids
.iter()
.copied()
.filter(|id| filter_set.contains(id))
.collect()
} else {
order_ids.clone()
};
if relevant_ids.is_empty() {
return Ok(Json(vec![]));
}
let fetched: Vec<Document> = docs_query.load(&mut conn)?;
let mut by_id: HashMap<Uuid, Document> =
fetched.into_iter().map(|doc| (doc.id, doc)).collect();
let mut ordered = Vec::with_capacity(by_id.len());
for id in relevant_ids {
if let Some(doc) = by_id.remove(&id) {
ordered.push(doc);
}
}
if !by_id.is_empty() {
let mut remaining: Vec<Document> = by_id.into_values().collect();
remaining.sort_by(|a, b| b.uploaded_at.cmp(&a.uploaded_at));
ordered.extend(remaining);
}
ordered
} else {
docs_query
.order(documents::uploaded_at.desc())
.load(&mut conn)?
};
let doc_ids: Vec<Uuid> = docs.iter().map(|doc| doc.id).collect();
let tags_map = load_tags_for_documents(&mut conn, &doc_ids)?;
@@ -325,6 +404,139 @@ pub async fn search_documents(
Ok(Json(response))
}
async fn quickwit_search(endpoint: &str, index: &str, query: &str) -> anyhow::Result<Vec<Uuid>> {
let quickwit_query = match build_quickwit_query(query) {
Some(q) => q,
None => return Ok(vec![]),
};
let client = Client::new();
let url = format!("{}/api/v1/{}/search", endpoint.trim_end_matches('/'), index);
let payload = json!({
"query": quickwit_query,
"max_hits": QUICKWIT_MAX_HITS,
});
let response = client.post(url).json(&payload).send().await?;
if !response.status().is_success() {
let status = response.status();
let body = response.text().await.unwrap_or_default();
return Err(anyhow!(
"quickwit search failed with status {status}: {body}"
));
}
let data: QuickwitSearchResponse = response.json().await?;
let mut seen = HashSet::new();
let mut doc_ids = Vec::new();
for hit in data.hits {
if let Some(doc_id) = extract_document_id(&hit) {
if seen.insert(doc_id) {
doc_ids.push(doc_id);
}
}
}
Ok(doc_ids)
}
fn build_quickwit_query(input: &str) -> Option<String> {
let tokens: Vec<String> = input
.split_whitespace()
.filter(|token| !token.is_empty())
.map(|token| {
let normalized = token.to_lowercase();
escape_quickwit_token(&normalized)
})
.collect();
if tokens.is_empty() {
return None;
}
let parts: Vec<String> = tokens
.into_iter()
.map(|token| format!("(title:{token} OR text:{token})"))
.collect();
Some(parts.join(" AND "))
}
fn escape_quickwit_token(token: &str) -> String {
let mut escaped = String::with_capacity(token.len());
for ch in token.chars() {
match ch {
'+' | '-' | '&' | '|' | '!' | '(' | ')' | '{' | '}' | '[' | ']' | '^' | '"' | '~'
| '*' | '?' | ':' | '\\' | '/' => {
escaped.push('\\');
escaped.push(ch);
}
_ => escaped.push(ch),
}
}
escaped
}
#[derive(Deserialize)]
struct QuickwitSearchResponse {
#[serde(default)]
hits: Vec<Value>,
}
fn extract_document_id(hit: &Value) -> Option<Uuid> {
for key in ["_source", "source", "fields", "stored_fields"] {
if let Some(value) = hit.get(key) {
if let Some(uuid) = extract_uuid_from_value(value) {
return Some(uuid);
}
}
}
if let Some(value) = hit.get("document_id") {
if let Some(uuid) = extract_uuid_from_value(value) {
return Some(uuid);
}
}
None
}
fn extract_uuid_from_value(value: &Value) -> Option<Uuid> {
if let Some(obj) = value.as_object() {
if let Some(inner) = obj.get("document_id") {
return parse_uuid_value(inner);
}
}
if let Some(arr) = value.as_array() {
for item in arr {
if let Some(uuid) = extract_uuid_from_value(item) {
return Some(uuid);
}
}
}
parse_uuid_value(value)
}
fn parse_uuid_value(value: &Value) -> Option<Uuid> {
if let Some(s) = value.as_str() {
return Uuid::parse_str(s).ok();
}
if let Some(arr) = value.as_array() {
for item in arr {
if let Some(uuid) = parse_uuid_value(item) {
return Some(uuid);
}
}
}
None
}
pub async fn delete_folder(
State(state): State<AppState>,
Path(folder_id): Path<Uuid>,
+49 -73
View File
@@ -167,7 +167,7 @@ const DropOverlay = ({ active, folderName }) => (
</div>
);
const LoginView = ({ onSubmit, loading, status }) => (
const LoginView = ({ onSubmit, status }) => (
<div className="login-screen">
<div className="login-card">
<h1>Paperless-NEO</h1>
@@ -190,9 +190,7 @@ const LoginView = ({ onSubmit, loading, status }) => (
autoComplete="current-password"
required
/>
<button type="submit" disabled={loading}>
{loading ? 'Signing in…' : 'Sign in'}
</button>
<button type="submit">Sign in</button>
</form>
<StatusBanner status={status} />
</div>
@@ -209,7 +207,6 @@ const FolderNode = ({
onDragOver,
onDragLeave,
onDelete,
disableActions,
renderChildren,
onFolderDragStart,
onFolderDragEnd,
@@ -257,7 +254,6 @@ const FolderNode = ({
<button
type="button"
className="icon-button ghost"
disabled={disableActions}
onClick={(event) => {
event.stopPropagation();
onDelete(node.id);
@@ -323,7 +319,6 @@ const FilterBar = ({
const DocumentsTable = ({
currentFolderName,
breadcrumbs,
loading,
onRefresh,
subfolders,
documents,
@@ -390,8 +385,8 @@ const DocumentsTable = ({
)}
</div>
<div className="header-actions">
<button className="secondary" onClick={onRefresh} disabled={loading}>
{loading ? 'Refreshing…' : 'Refresh'}
<button className="secondary" onClick={onRefresh}>
Refresh
</button>
</div>
</div>
@@ -648,7 +643,6 @@ const DetailPanel = ({
onTagAdd,
onTagRemove,
onRegenerateThumbnails,
loading,
previewEntry,
onOpenPreview,
onBulkTagAdd,
@@ -802,7 +796,6 @@ const DetailPanel = ({
type="button"
className="secondary"
onClick={() => onOpenPreview(singleDoc.id)}
disabled={loading}
>
Open preview
</button>
@@ -810,7 +803,6 @@ const DetailPanel = ({
type="button"
className="secondary"
onClick={() => onRegenerateThumbnails(singleDoc.id)}
disabled={loading}
>
Re-run analysis
</button>
@@ -911,9 +903,7 @@ const DetailPanel = ({
}}
>
<input name="tag" placeholder="Add tag to selection" list="tag-catalog" />
<button type="submit" disabled={loading}>
Add tag
</button>
<button type="submit">Add tag</button>
</form>
<form
className="inline"
@@ -926,7 +916,7 @@ const DetailPanel = ({
}}
>
<input name="tag" placeholder="Remove tag from selection" list="tag-catalog" />
<button type="submit" className="secondary" disabled={loading}>
<button type="submit" className="secondary">
Remove tag
</button>
</form>
@@ -945,7 +935,6 @@ const DetailPanel = ({
name="target"
defaultValue={defaultMoveTarget || 'root'}
onChange={(event) => onBulkMove?.({ target: event.target.value })}
disabled={loading}
>
{folderOptions.map((option) => (
<option key={option.id} value={option.id}>
@@ -958,7 +947,6 @@ const DetailPanel = ({
type="button"
className="secondary"
onClick={() => onBulkReanalyze?.()}
disabled={loading}
>
Re-analyze selection
</button>
@@ -982,7 +970,6 @@ const PreviewWorkspace = ({
document,
detail,
previewEntry,
loading,
onClose,
onDownload,
onRegenerateThumbnails,
@@ -1028,14 +1015,13 @@ const PreviewWorkspace = ({
type="button"
className="secondary"
onClick={() => onRegenerateThumbnails(document.id)}
disabled={loading}
>
Regenerate thumbnails
Re-run analysis
</button>
</div>
</header>
<div className="preview-workspace__body">
{loading || !previewEntry?.url ? (
{!previewEntry?.url ? (
<div className="preview-workspace__message">Loading preview</div>
) : (
<iframe
@@ -1064,7 +1050,6 @@ const Sidebar = ({
onDragLeave,
onCreateFolder,
onDeleteFolder,
disableActions,
selectedFolder,
onFolderDragStart,
onFolderDragEnd,
@@ -1087,7 +1072,6 @@ const Sidebar = ({
onDragOver={onDragOver}
onDragLeave={onDragLeave}
onDelete={onDeleteFolder}
disableActions={disableActions}
renderChildren={renderNodes}
onFolderDragStart={onFolderDragStart}
onFolderDragEnd={onFolderDragEnd}
@@ -1104,7 +1088,6 @@ const Sidebar = ({
onDragOver,
onDragLeave,
onDeleteFolder,
disableActions,
onFolderDragStart,
onFolderDragEnd,
draggedFolderId,
@@ -1132,9 +1115,7 @@ const Sidebar = ({
}}
>
<input name="folder-name" placeholder="New folder name" required />
<button type="submit" disabled={disableActions}>
Create
</button>
<button type="submit">Create</button>
</form>
<div className="folder-tree-scroller">
<ul className="folder-tree">
@@ -3334,12 +3315,11 @@ const handleDownload = useCallback(
className="secondary"
type="button"
onClick={handleBulkReanalyze}
disabled={loading}
>
{loading ? 'Working…' : 'Re-analyze All'}
Re-analyze All
</button>
<button className="secondary" onClick={handleLogout} disabled={loading}>
{loading ? 'Working…' : 'Log out'}
<button className="secondary" onClick={handleLogout}>
Log out
</button>
</div>
</div>
@@ -3350,51 +3330,48 @@ const handleDownload = useCallback(
document={previewWorkspaceDocument}
detail={previewWorkspaceDetail}
previewEntry={previewWorkspaceEntry}
loading={previewDocumentLoading}
onClose={closeDocumentPreview}
onDownload={handleDownload}
onRegenerateThumbnails={handleThumbnailRegeneration}
/>
) : (
<>
<Sidebar
folderNodes={folderNodes}
onToggle={folderClickHandlers.onToggle}
onSelect={folderClickHandlers.onSelect}
onDrop={folderClickHandlers.onDrop}
onDragOver={folderClickHandlers.onDragOver}
onDragLeave={folderClickHandlers.onDragLeave}
onCreateFolder={handleFolderCreate}
onDeleteFolder={handleFolderDelete}
disableActions={loading}
selectedFolder={selectedFolder}
onFolderDragStart={handleFolderDragStart}
onFolderDragEnd={handleFolderDragEnd}
draggedFolderId={draggedFolderId}
/>
<DocumentsTable
currentFolderName={currentFolderName}
breadcrumbs={breadcrumbs}
loading={loading}
onRefresh={refreshCurrentFolder}
subfolders={currentSubfolders}
documents={documents}
searchResults={searchResults}
isFilterActive={isFilterActive}
onFolderSelect={selectFolder}
onFolderDrop={folderClickHandlers.onDrop}
onFolderDragOver={folderClickHandlers.onDragOver}
onFolderDragLeave={folderClickHandlers.onDragLeave}
onFolderDragStart={handleFolderDragStart}
onFolderDragEnd={handleFolderDragEnd}
draggedFolderId={draggedFolderId}
onFolderDelete={handleFolderDelete}
onDocumentRowClick={handleDocumentRowClick}
onDocumentOpen={openDocumentPreview}
selectedDocumentIds={selectedDocumentIds}
focusedDocumentId={focusedDocumentId}
draggingDocumentIds={draggedDocumentIds}
onDocumentDragStart={(event, documentId) => {
<Sidebar
folderNodes={folderNodes}
onToggle={folderClickHandlers.onToggle}
onSelect={folderClickHandlers.onSelect}
onDrop={folderClickHandlers.onDrop}
onDragOver={folderClickHandlers.onDragOver}
onDragLeave={folderClickHandlers.onDragLeave}
onCreateFolder={handleFolderCreate}
onDeleteFolder={handleFolderDelete}
selectedFolder={selectedFolder}
onFolderDragStart={handleFolderDragStart}
onFolderDragEnd={handleFolderDragEnd}
draggedFolderId={draggedFolderId}
/>
<DocumentsTable
currentFolderName={currentFolderName}
breadcrumbs={breadcrumbs}
onRefresh={refreshCurrentFolder}
subfolders={currentSubfolders}
documents={documents}
searchResults={searchResults}
isFilterActive={isFilterActive}
onFolderSelect={selectFolder}
onFolderDrop={folderClickHandlers.onDrop}
onFolderDragOver={folderClickHandlers.onDragOver}
onFolderDragLeave={folderClickHandlers.onDragLeave}
onFolderDragStart={handleFolderDragStart}
onFolderDragEnd={handleFolderDragEnd}
draggedFolderId={draggedFolderId}
onFolderDelete={handleFolderDelete}
onDocumentRowClick={handleDocumentRowClick}
onDocumentOpen={openDocumentPreview}
selectedDocumentIds={selectedDocumentIds}
focusedDocumentId={focusedDocumentId}
draggingDocumentIds={draggedDocumentIds}
onDocumentDragStart={(event, documentId) => {
const selection = selectedDocumentIds.includes(documentId)
? selectedDocumentIds
: [documentId];
@@ -3432,7 +3409,6 @@ const handleDownload = useCallback(
onTagAdd={handleTagAdd}
onTagRemove={handleTagRemove}
onRegenerateThumbnails={handleThumbnailRegeneration}
loading={loading}
previewEntry={selectedPreviewEntry}
onOpenPreview={openDocumentPreview}
onBulkTagAdd={handleBulkTagAddFromDetail}
@@ -3449,7 +3425,7 @@ const handleDownload = useCallback(
</main>
</>
) : (
<LoginView onSubmit={handleLogin} loading={loading} status={status} />
<LoginView onSubmit={handleLogin} status={status} />
)}
</div>
);