reanalyze removal
This commit is contained in:
@@ -15,9 +15,12 @@ use uuid::Uuid;
|
|||||||
use backend::{
|
use backend::{
|
||||||
config::AppConfig,
|
config::AppConfig,
|
||||||
db::{self, PgPool},
|
db::{self, PgPool},
|
||||||
|
jobs::{enqueue_job, JOB_ANALYZE_DOCUMENT},
|
||||||
models::{DocumentAsset, DocumentAssetObject, NewUser, NewUserMembership, Tenant, User},
|
models::{DocumentAsset, DocumentAssetObject, NewUser, NewUserMembership, Tenant, User},
|
||||||
s3,
|
s3,
|
||||||
schema::{document_asset_objects, document_assets, tenants, user_memberships, users},
|
schema::{
|
||||||
|
document_asset_objects, document_assets, documents, tenants, user_memberships, users,
|
||||||
|
},
|
||||||
storage::{ObjectStorage, S3Storage, TenantStorage},
|
storage::{ObjectStorage, S3Storage, TenantStorage},
|
||||||
utils::tracing::init_tracing,
|
utils::tracing::init_tracing,
|
||||||
};
|
};
|
||||||
@@ -83,6 +86,9 @@ enum Command {
|
|||||||
username: String,
|
username: String,
|
||||||
slug: String,
|
slug: String,
|
||||||
},
|
},
|
||||||
|
ReanalyzeDocuments {
|
||||||
|
slug: String,
|
||||||
|
},
|
||||||
ListTenants,
|
ListTenants,
|
||||||
DeleteAssets(String),
|
DeleteAssets(String),
|
||||||
QuickwitCreate(String),
|
QuickwitCreate(String),
|
||||||
@@ -100,6 +106,7 @@ impl Command {
|
|||||||
delete-tenant <slug>\n\
|
delete-tenant <slug>\n\
|
||||||
add-user-to-tenant <username> <slug> [role]\n\
|
add-user-to-tenant <username> <slug> [role]\n\
|
||||||
remove-user-from-tenant <username> <slug>\n\
|
remove-user-from-tenant <username> <slug>\n\
|
||||||
|
reanalyze-documents <slug>\n\
|
||||||
list-tenants\n\
|
list-tenants\n\
|
||||||
delete-assets <slug>\n\
|
delete-assets <slug>\n\
|
||||||
quickwit-create-index <slug>\n\
|
quickwit-create-index <slug>\n\
|
||||||
@@ -138,6 +145,9 @@ impl Command {
|
|||||||
username: args.next().ok_or_else(|| anyhow!("username required"))?,
|
username: args.next().ok_or_else(|| anyhow!("username required"))?,
|
||||||
slug: args.next().ok_or_else(|| anyhow!("tenant slug required"))?,
|
slug: args.next().ok_or_else(|| anyhow!("tenant slug required"))?,
|
||||||
}),
|
}),
|
||||||
|
Some("reanalyze-documents") => Ok(Self::ReanalyzeDocuments {
|
||||||
|
slug: args.next().ok_or_else(|| anyhow!("tenant slug required"))?,
|
||||||
|
}),
|
||||||
Some("list-tenants") => Ok(Self::ListTenants),
|
Some("list-tenants") => Ok(Self::ListTenants),
|
||||||
Some("delete-assets") => Ok(Self::DeleteAssets(
|
Some("delete-assets") => Ok(Self::DeleteAssets(
|
||||||
args.next().ok_or_else(|| anyhow!("tenant slug required"))?,
|
args.next().ok_or_else(|| anyhow!("tenant slug required"))?,
|
||||||
@@ -179,6 +189,7 @@ async fn main() -> Result<()> {
|
|||||||
Command::RemoveUserFromTenant { username, slug } => {
|
Command::RemoveUserFromTenant { username, slug } => {
|
||||||
remove_user_from_tenant(&pool, &username, &slug)?
|
remove_user_from_tenant(&pool, &username, &slug)?
|
||||||
}
|
}
|
||||||
|
Command::ReanalyzeDocuments { slug } => reanalyze_documents(&pool, &slug)?,
|
||||||
Command::ListTenants => list_tenants(&pool)?,
|
Command::ListTenants => list_tenants(&pool)?,
|
||||||
Command::DeleteAssets(slug) => delete_assets_for_tenant(&config, &pool, &slug).await?,
|
Command::DeleteAssets(slug) => delete_assets_for_tenant(&config, &pool, &slug).await?,
|
||||||
Command::QuickwitCreate(slug) => {
|
Command::QuickwitCreate(slug) => {
|
||||||
@@ -436,6 +447,50 @@ fn remove_user_from_tenant(pool: &PgPool, username: &str, slug: &str) -> Result<
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn reanalyze_documents(pool: &PgPool, slug: &str) -> Result<()> {
|
||||||
|
let mut conn = pool.get().context("failed to get database connection")?;
|
||||||
|
|
||||||
|
let tenant: Tenant = tenants::table
|
||||||
|
.filter(tenants::slug.eq(slug))
|
||||||
|
.first(&mut conn)
|
||||||
|
.optional()?
|
||||||
|
.ok_or_else(|| anyhow!("tenant '{}' not found", slug))?;
|
||||||
|
|
||||||
|
let targets: Vec<(Uuid, Uuid)> = documents::table
|
||||||
|
.filter(documents::tenant_id.eq(tenant.id))
|
||||||
|
.filter(documents::deleted_at.is_null())
|
||||||
|
.select((documents::id, documents::current_version_id))
|
||||||
|
.load(&mut conn)?;
|
||||||
|
|
||||||
|
if targets.is_empty() {
|
||||||
|
println!("tenant '{}' has no active documents", slug);
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut queued = 0usize;
|
||||||
|
for (document_id, version_id) in targets {
|
||||||
|
enqueue_job(
|
||||||
|
&mut conn,
|
||||||
|
tenant.id,
|
||||||
|
JOB_ANALYZE_DOCUMENT,
|
||||||
|
serde_json::json!({
|
||||||
|
"document_id": document_id,
|
||||||
|
"document_version_id": version_id,
|
||||||
|
"force": true,
|
||||||
|
}),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.map_err(|err| anyhow!("failed to enqueue analyze job: {}", err))?;
|
||||||
|
queued += 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
println!(
|
||||||
|
"queued {} documents for re-analysis in tenant '{}'",
|
||||||
|
queued, slug
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn list_tenants(pool: &PgPool) -> Result<()> {
|
fn list_tenants(pool: &PgPool) -> Result<()> {
|
||||||
let mut conn = pool.get().context("failed to get database connection")?;
|
let mut conn = pool.get().context("failed to get database connection")?;
|
||||||
let tenants: Vec<Tenant> = tenants::table
|
let tenants: Vec<Tenant> = tenants::table
|
||||||
|
|||||||
@@ -789,39 +789,6 @@ pub async fn request_document_assets(
|
|||||||
Ok(StatusCode::ACCEPTED)
|
Ok(StatusCode::ACCEPTED)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn reanalyze_all_documents(
|
|
||||||
TenantScopedConn {
|
|
||||||
mut conn,
|
|
||||||
tenant_id,
|
|
||||||
..
|
|
||||||
}: TenantScopedConn,
|
|
||||||
) -> AppResult<(StatusCode, Json<BulkReanalyzeResponse>)> {
|
|
||||||
let targets: Vec<(Uuid, Uuid)> = documents::table
|
|
||||||
.filter(documents::deleted_at.is_null())
|
|
||||||
.filter(documents::tenant_id.eq(tenant_id))
|
|
||||||
.select((documents::id, documents::current_version_id))
|
|
||||||
.load(&mut conn)?;
|
|
||||||
|
|
||||||
let mut queued = 0usize;
|
|
||||||
for (document_id, version_id) in targets {
|
|
||||||
enqueue_job(
|
|
||||||
&mut conn,
|
|
||||||
tenant_id,
|
|
||||||
JOB_ANALYZE_DOCUMENT,
|
|
||||||
json!({
|
|
||||||
"document_id": document_id,
|
|
||||||
"document_version_id": version_id,
|
|
||||||
"force": true,
|
|
||||||
}),
|
|
||||||
None,
|
|
||||||
)
|
|
||||||
.map_err(|err| AppError::internal(format!("failed to enqueue analyze job: {err}")))?;
|
|
||||||
queued += 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok((StatusCode::ACCEPTED, Json(BulkReanalyzeResponse { queued })))
|
|
||||||
}
|
|
||||||
|
|
||||||
pub async fn reanalyze_selected_documents(
|
pub async fn reanalyze_selected_documents(
|
||||||
TenantScopedConn {
|
TenantScopedConn {
|
||||||
mut conn,
|
mut conn,
|
||||||
|
|||||||
@@ -58,7 +58,6 @@ pub fn create_router(state: AppState) -> Router<()> {
|
|||||||
"/",
|
"/",
|
||||||
get(documents::list_documents).post(documents::upload_document),
|
get(documents::list_documents).post(documents::upload_document),
|
||||||
)
|
)
|
||||||
.route("/reanalyze", post(documents::reanalyze_all_documents))
|
|
||||||
.route("/bulk/move", post(documents::bulk_move_documents))
|
.route("/bulk/move", post(documents::bulk_move_documents))
|
||||||
.route("/bulk/tags", post(documents::bulk_update_tags))
|
.route("/bulk/tags", post(documents::bulk_update_tags))
|
||||||
.route(
|
.route(
|
||||||
|
|||||||
@@ -56,12 +56,12 @@ struct DocumentDownload {
|
|||||||
filename: String,
|
filename: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
struct BulkReanalyze {
|
struct BulkReanalyze {
|
||||||
queued: usize,
|
queued: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
|
||||||
struct BulkMoveResult {
|
struct BulkMoveResult {
|
||||||
updated: usize,
|
updated: usize,
|
||||||
}
|
}
|
||||||
@@ -318,97 +318,6 @@ async fn duplicate_and_restore_document() -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
|
||||||
async fn bulk_reanalyze_documents() -> Result<()> {
|
|
||||||
let _lock = acquire_db_lock().await;
|
|
||||||
let app = TestApp::new().await?;
|
|
||||||
|
|
||||||
let password = "bulkpass";
|
|
||||||
app.insert_user("alex", password, "admin").await?;
|
|
||||||
let token = app.login_token("alex", password).await?;
|
|
||||||
|
|
||||||
app.clear_jobs().await?;
|
|
||||||
|
|
||||||
let first_bytes = b"first doc";
|
|
||||||
let first = app
|
|
||||||
.upload_document(
|
|
||||||
"/api/documents",
|
|
||||||
"first.txt",
|
|
||||||
"text/plain",
|
|
||||||
first_bytes,
|
|
||||||
None,
|
|
||||||
&token,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
assert_eq!(first.status(), StatusCode::CREATED);
|
|
||||||
let first_body = body_to_vec(first.into_body()).await?;
|
|
||||||
let first_detail: DocumentDetail = serde_json::from_slice(&first_body)?;
|
|
||||||
|
|
||||||
let second_bytes = b"second doc";
|
|
||||||
let second = app
|
|
||||||
.upload_document(
|
|
||||||
"/api/documents",
|
|
||||||
"second.txt",
|
|
||||||
"text/plain",
|
|
||||||
second_bytes,
|
|
||||||
None,
|
|
||||||
&token,
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
assert_eq!(second.status(), StatusCode::CREATED);
|
|
||||||
let second_body = body_to_vec(second.into_body()).await?;
|
|
||||||
let second_detail: DocumentDetail = serde_json::from_slice(&second_body)?;
|
|
||||||
|
|
||||||
app.clear_jobs().await?;
|
|
||||||
|
|
||||||
let response = app
|
|
||||||
.post_json(
|
|
||||||
"/api/documents/reanalyze",
|
|
||||||
&serde_json::json!({}),
|
|
||||||
Some(&token),
|
|
||||||
)
|
|
||||||
.await?;
|
|
||||||
assert_eq!(response.status(), StatusCode::ACCEPTED);
|
|
||||||
let body = body_to_vec(response.into_body()).await?;
|
|
||||||
let bulk: BulkReanalyze = serde_json::from_slice(&body)?;
|
|
||||||
assert_eq!(bulk.queued, 2);
|
|
||||||
|
|
||||||
let jobs = app.jobs_by_type("analyze-document").await?;
|
|
||||||
assert_eq!(jobs.len(), 2);
|
|
||||||
let mut payload_docs = Vec::new();
|
|
||||||
for job in jobs {
|
|
||||||
let payload: AnalyzeJobPayload = serde_json::from_value(job.payload)?;
|
|
||||||
assert!(payload.force);
|
|
||||||
payload_docs.push((payload.document_id, payload.document_version_id));
|
|
||||||
}
|
|
||||||
|
|
||||||
let mut expected = vec![
|
|
||||||
(
|
|
||||||
first_detail.document.id,
|
|
||||||
first_detail
|
|
||||||
.document
|
|
||||||
.current_version
|
|
||||||
.as_ref()
|
|
||||||
.expect("first current version")
|
|
||||||
.id,
|
|
||||||
),
|
|
||||||
(
|
|
||||||
second_detail.document.id,
|
|
||||||
second_detail
|
|
||||||
.document
|
|
||||||
.current_version
|
|
||||||
.as_ref()
|
|
||||||
.expect("second current version")
|
|
||||||
.id,
|
|
||||||
),
|
|
||||||
];
|
|
||||||
payload_docs.sort();
|
|
||||||
expected.sort();
|
|
||||||
assert_eq!(payload_docs, expected);
|
|
||||||
|
|
||||||
app.cleanup().await?;
|
|
||||||
Ok(())
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn bulk_move_documents_to_folder() -> Result<()> {
|
async fn bulk_move_documents_to_folder() -> Result<()> {
|
||||||
|
|||||||
@@ -18,7 +18,6 @@ Documents
|
|||||||
---------
|
---------
|
||||||
- GET /api/documents - List or search documents. Optional filters: `folder_id` (defaults to root when omitted), `include_deleted`, `include_descendants` (defaults to true when a `folder_id` is provided and no other override is supplied), `query` (Quickwit full-text), `tags` (comma-separated tag UUIDs), and `correspondents` (comma-separated correspondent UUIDs). Each entry includes tags, correspondent assignments, and current version info.
|
- GET /api/documents - List or search documents. Optional filters: `folder_id` (defaults to root when omitted), `include_deleted`, `include_descendants` (defaults to true when a `folder_id` is provided and no other override is supplied), `query` (Quickwit full-text), `tags` (comma-separated tag UUIDs), and `correspondents` (comma-separated correspondent UUIDs). Each entry includes tags, correspondent assignments, and current version info.
|
||||||
- POST /api/documents - Upload a document via multipart form-data (`file`, optional metadata/folder fields).
|
- POST /api/documents - Upload a document via multipart form-data (`file`, optional metadata/folder fields).
|
||||||
- POST /api/documents/reanalyze - Queue re-analysis for every non-deleted document.
|
|
||||||
- POST /api/documents/bulk/move - Move multiple documents to a target folder.
|
- POST /api/documents/bulk/move - Move multiple documents to a target folder.
|
||||||
- POST /api/documents/bulk/tags - Add or remove tags across multiple documents.
|
- POST /api/documents/bulk/tags - Add or remove tags across multiple documents.
|
||||||
- POST /api/documents/bulk/correspondents - Bulk correspondent actions. Default `action=add` replaces existing assignments for the provided roles before adding the supplied correspondents; `action=remove` drops the specified correspondent/role pairs.
|
- POST /api/documents/bulk/correspondents - Bulk correspondent actions. Default `action=add` replaces existing assignments for the provided roles before adding the supplied correspondents; `action=remove` drops the specified correspondent/role pairs.
|
||||||
|
|||||||
@@ -4070,26 +4070,6 @@ const AppLayout = () => {
|
|||||||
}
|
}
|
||||||
}, [appDispatch, setStatusMessage]);
|
}, [appDispatch, setStatusMessage]);
|
||||||
|
|
||||||
const handleBulkReanalyze = useCallback(async () => {
|
|
||||||
try {
|
|
||||||
setLoading(true);
|
|
||||||
const { data } = await api.post('/documents/reanalyze');
|
|
||||||
const total = data?.queued ?? 0;
|
|
||||||
const suffix = total === 1 ? '' : 's';
|
|
||||||
setStatusMessage(
|
|
||||||
`Queued re-analysis for ${total} document${suffix}.`,
|
|
||||||
'success',
|
|
||||||
);
|
|
||||||
} catch (error) {
|
|
||||||
const message =
|
|
||||||
error.response?.data?.error || 'Failed to queue document re-analysis.';
|
|
||||||
notifyApiError(error, message);
|
|
||||||
} finally {
|
|
||||||
setLoading(false);
|
|
||||||
}
|
|
||||||
}, [notifyApiError, setStatusMessage]);
|
|
||||||
|
|
||||||
|
|
||||||
const folderClickHandlers = {
|
const folderClickHandlers = {
|
||||||
onToggle: async (folderId) => {
|
onToggle: async (folderId) => {
|
||||||
const node = folderNodes.get(folderId);
|
const node = folderNodes.get(folderId);
|
||||||
@@ -4600,7 +4580,6 @@ const AppLayout = () => {
|
|||||||
status,
|
status,
|
||||||
setStatusMessage,
|
setStatusMessage,
|
||||||
dropOverlayState,
|
dropOverlayState,
|
||||||
handleBulkReanalyze,
|
|
||||||
handleLogout,
|
handleLogout,
|
||||||
sidebarProps,
|
sidebarProps,
|
||||||
tags,
|
tags,
|
||||||
@@ -4638,7 +4617,6 @@ const AppLayout = () => {
|
|||||||
status,
|
status,
|
||||||
setStatusMessage,
|
setStatusMessage,
|
||||||
dropOverlayState,
|
dropOverlayState,
|
||||||
handleBulkReanalyze,
|
|
||||||
handleLogout,
|
handleLogout,
|
||||||
sidebarProps,
|
sidebarProps,
|
||||||
tags,
|
tags,
|
||||||
@@ -4757,13 +4735,6 @@ const AppLayout = () => {
|
|||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
<div className="app-bar__actions">
|
<div className="app-bar__actions">
|
||||||
<button
|
|
||||||
className="secondary"
|
|
||||||
type="button"
|
|
||||||
onClick={handleBulkReanalyze}
|
|
||||||
>
|
|
||||||
Re-analyze All
|
|
||||||
</button>
|
|
||||||
<button className="secondary" onClick={handleLogout}>
|
<button className="secondary" onClick={handleLogout}>
|
||||||
Log out
|
Log out
|
||||||
</button>
|
</button>
|
||||||
|
|||||||
Reference in New Issue
Block a user