feat: implement hybrid document search combining Quickwit results with PostgreSQL pg_trgm title search, supported by a new index.
This commit is contained in:
@@ -0,0 +1 @@
|
||||
DROP INDEX IF EXISTS tenant.idx_documents_title_trgm;
|
||||
@@ -0,0 +1,6 @@
|
||||
CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
||||
|
||||
CREATE INDEX idx_documents_title_trgm
|
||||
ON tenant.documents
|
||||
USING gin (title gin_trgm_ops)
|
||||
WHERE deleted_at IS NULL;
|
||||
@@ -476,7 +476,9 @@ impl<'a> DocumentsService<'a> {
|
||||
let mut quickwit_order: Option<Vec<Uuid>> = None;
|
||||
|
||||
if let Some(query_str) = search_text.as_ref() {
|
||||
debug!(query = %query_str, "performing quickwit document search");
|
||||
debug!(query = %query_str, "performing hybrid document search");
|
||||
|
||||
// 1. Quickwit Search
|
||||
let endpoint = self
|
||||
.state
|
||||
.config
|
||||
@@ -489,19 +491,37 @@ impl<'a> DocumentsService<'a> {
|
||||
.as_ref()
|
||||
.ok_or_else(|| AppError::internal("quickwit index not configured for tenant"))?;
|
||||
|
||||
let ids = quickwit_search(endpoint, index, tenant_id, query_str)
|
||||
let quickwit_ids = quickwit_search(endpoint, index, tenant_id, query_str)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
error!(error = ?err, "quickwit search failed");
|
||||
AppError::internal("quickwit search failed")
|
||||
})?;
|
||||
|
||||
if ids.is_empty() {
|
||||
// 2. Postgres Title Search
|
||||
let postgres_ids: Vec<Uuid> = documents::table
|
||||
.filter(documents::tenant_id.eq(tenant_id))
|
||||
.filter(documents::deleted_at.is_null())
|
||||
.filter(documents::title.ilike(format!("%{}%", query_str)))
|
||||
.select(documents::id)
|
||||
.load(conn)?;
|
||||
|
||||
// 3. Combine Results
|
||||
let mut combined_ids = quickwit_ids.clone();
|
||||
let quickwit_set: HashSet<Uuid> = quickwit_ids.iter().cloned().collect();
|
||||
|
||||
for id in postgres_ids {
|
||||
if !quickwit_set.contains(&id) {
|
||||
combined_ids.push(id);
|
||||
}
|
||||
}
|
||||
|
||||
if combined_ids.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
quickwit_order = Some(ids.clone());
|
||||
let set: HashSet<Uuid> = ids.into_iter().collect();
|
||||
quickwit_order = Some(combined_ids.clone());
|
||||
let set: HashSet<Uuid> = combined_ids.into_iter().collect();
|
||||
filter_ids = intersect_option_sets(filter_ids, set);
|
||||
}
|
||||
|
||||
@@ -593,7 +613,7 @@ impl<'a> DocumentsService<'a> {
|
||||
let mut responses = self.hydrate_documents(conn, tenant_id, user_id, docs)?;
|
||||
|
||||
if let Some(order) = quickwit_order {
|
||||
let order_map: HashMap<Uuid, usize> = order
|
||||
let order_map: HashMap<Uuid, usize> = order
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(idx, id)| (id, idx))
|
||||
|
||||
Reference in New Issue
Block a user