more multi-tenancy

2025-10-23 16:06:56 +02:00
parent e175d28c2c
commit 0ad79c9bc1
35 changed files with 1229 additions and 534 deletions
@@ -1,83 +1,314 @@
 use std::env;
+use std::sync::Arc;

-use anyhow::{Context, Result};
+use anyhow::{anyhow, Context, Result};
 use diesel::prelude::*;
+use once_cell::sync::Lazy;
+use reqwest::{Client, Method, StatusCode};
+use serde_json::json;
 use uuid::Uuid;

 use backend::{
    config::AppConfig,
-    db,
-    models::{DocumentAsset, DocumentAssetObject},
+    db::{self, PgPool},
+    models::{DocumentAsset, DocumentAssetObject, Tenant},
    s3,
-    schema::{document_asset_objects, document_assets},
-    storage::{ObjectStorage, S3Storage},
+    schema::{document_asset_objects, document_assets, tenants},
+    storage::{ObjectStorage, S3Storage, TenantStorage},
+    utils::tracing::init_tracing,
 };

+static QUICKWIT_INDEX_TEMPLATE: Lazy<serde_json::Value> = Lazy::new(|| {
+    json!({
+      "version": 0.8,
+      "index_id": "documents",
+      "doc_mapping": {
+        "tokenizers": [
+          {
+            "name": "substring",
+            "type": "ngram",
+            "min_gram": 2,
+            "max_gram": 20,
+            "prefix_only": false
+          }
+        ],
+        "field_mappings": [
+          { "name": "tenant_id", "type": "text", "stored": true },
+          { "name": "document_id", "type": "text", "stored": true },
+          { "name": "version_id", "type": "text", "stored": true },
+          { "name": "title", "type": "text", "tokenizer": "substring", "stored": true },
+          { "name": "text", "type": "text", "tokenizer": "substring", "record": "position" }
+        ]
+      },
+      "search_settings": {
+        "default_search_fields": ["title", "text"]
+      }
+    })
+});
+
+#[derive(Debug)]
+enum Command {
+    ListTenants,
+    DeleteAssets(String),
+    QuickwitCreate(String),
+    QuickwitDelete(String),
+}
+
+impl Command {
+    fn usage() -> &'static str {
+        "Usage: maintenance <list-tenants|delete-assets <slug>|quickwit-create-index <slug>|quickwit-delete-index <slug>>"
+    }
+
+    fn parse() -> Result<Self> {
+        let mut args = env::args().skip(1);
+        match args.next().as_deref() {
+            Some("list-tenants") => Ok(Self::ListTenants),
+            Some("delete-assets") => Ok(Self::DeleteAssets(
+                args.next().ok_or_else(|| anyhow!("tenant slug required"))?,
+            )),
+            Some("quickwit-create-index") => Ok(Self::QuickwitCreate(
+                args.next().ok_or_else(|| anyhow!("tenant slug required"))?,
+            )),
+            Some("quickwit-delete-index") => Ok(Self::QuickwitDelete(
+                args.next().ok_or_else(|| anyhow!("tenant slug required"))?,
+            )),
+            _ => Err(anyhow!(Self::usage())),
+        }
+    }
+}
+
 #[tokio::main]
 async fn main() -> Result<()> {
-    let mut args = env::args().skip(1);
-    match args.next().as_deref() {
-        Some("delete-assets") => delete_all_assets().await?,
-        Some(cmd) => {
-            eprintln!("Unknown command: {cmd}\nUsage: maintenance delete-assets");
-            std::process::exit(1);
+    init_tracing("info");
+    let command = Command::parse()?;
+    let config = AppConfig::load_and_log("maintenance")?;
+    let pool = db::init_pool_with_size(&config.database_url, config.database_max_pool_size)?;
+
+    match command {
+        Command::ListTenants => list_tenants(&pool)?,
+        Command::DeleteAssets(slug) => delete_assets_for_tenant(&config, &pool, &slug).await?,
+        Command::QuickwitCreate(slug) => {
+            quickwit_index(&config, &pool, &slug, Method::POST).await?
        }
-        None => {
-            eprintln!("Usage: maintenance delete-assets");
-            std::process::exit(1);
+        Command::QuickwitDelete(slug) => {
+            quickwit_index(&config, &pool, &slug, Method::DELETE).await?
        }
    }

    Ok(())
 }

-async fn delete_all_assets() -> Result<()> {
-    let config = AppConfig::from_env()?;
-    tracing::info!(
-        component = "maintenance",
-        database_url = %config.redacted_database_url(),
-        pool_size = config.database_max_pool_size,
-        s3_bucket = %config.s3_bucket,
-        "loaded backend configuration"
-    );
-    let pool = db::init_pool_with_size(&config.database_url, config.database_max_pool_size)?;
-
-    let s3_client = s3::build_client(&config).await?;
-    let storage = S3Storage::new(s3_client, config.s3_bucket.clone());
-
+fn list_tenants(pool: &PgPool) -> Result<()> {
    let mut conn = pool.get().context("failed to get database connection")?;
-
-    let assets: Vec<DocumentAsset> = document_assets::table
+    let tenants: Vec<Tenant> = tenants::table
+        .order(tenants::slug.asc())
        .load(&mut conn)
-        .context("failed to load document assets")?;
+        .context("failed to load tenants")?;

-    if assets.is_empty() {
-        println!("No assets found.");
+    if tenants.is_empty() {
+        println!("No tenants found.");
        return Ok(());
    }

-    println!("Deleting {} assets…", assets.len());
+    for tenant in tenants {
+        println!("{} ({})", tenant.slug, tenant.id);
+    }
+
+    Ok(())
+}
+
+async fn delete_assets_for_tenant(
+    config: &AppConfig,
+    pool: &PgPool,
+    tenant_slug: &str,
+) -> Result<()> {
+    let s3_client = s3::build_client(config).await?;
+    let storage: Arc<dyn ObjectStorage> =
+        Arc::new(S3Storage::new(s3_client, config.s3_bucket.clone()));
+
+    let mut conn = pool.get().context("failed to get database connection")?;
+    let tenant: Tenant = tenants::table
+        .filter(tenants::slug.eq(tenant_slug))
+        .first(&mut conn)
+        .optional()
+        .context("failed to load tenant")?
+        .ok_or_else(|| anyhow!("tenant '{}' not found", tenant_slug))?;
+
+    let tenant_storage = TenantStorage::new(Arc::clone(&storage), &tenant)
+        .with_context(|| format!("missing storage root for tenant {}", tenant.slug))?;
+
+    let assets: Vec<DocumentAsset> = document_assets::table
+        .filter(document_assets::tenant_id.eq(tenant.id))
+        .load(&mut conn)
+        .with_context(|| format!("failed to load assets for tenant {}", tenant.slug))?;
+
+    if assets.is_empty() {
+        println!("Tenant {}: no assets", tenant.slug);
+        return Ok(());
+    }
+
+    println!(
+        "Tenant {} ({}): deleting {} assets…",
+        tenant.slug,
+        tenant.id,
+        assets.len()
+    );

    let asset_ids: Vec<Uuid> = assets.iter().map(|asset| asset.id).collect();

    let objects: Vec<DocumentAssetObject> = document_asset_objects::table
+        .filter(document_asset_objects::tenant_id.eq(tenant.id))
        .filter(document_asset_objects::asset_id.eq_any(&asset_ids))
        .load(&mut conn)
-        .context("failed to load document asset objects")?;
+        .with_context(|| format!("failed to load asset objects for tenant {}", tenant.slug))?;

    for object in &objects {
-        if let Err(err) = storage.delete_object(&object.s3_key).await {
+        if let Err(err) = tenant_storage.delete_object(&object.s3_key).await {
            eprintln!(
-                "Failed to delete object {} from storage: {err}",
-                object.s3_key
+                "Failed to delete object {} (tenant {}): {err}",
+                object.s3_key, tenant.slug
            );
        }
    }

-    diesel::delete(document_assets::table)
-        .execute(&mut conn)
-        .context("failed to remove asset records")?;
+    diesel::delete(
+        document_asset_objects::table
+            .filter(document_asset_objects::tenant_id.eq(tenant.id))
+            .filter(document_asset_objects::asset_id.eq_any(&asset_ids)),
+    )
+    .execute(&mut conn)
+    .with_context(|| format!("failed to remove asset objects for tenant {}", tenant.slug))?;

-    println!("Asset records deleted.");
+    diesel::delete(document_assets::table.filter(document_assets::tenant_id.eq(tenant.id)))
+        .execute(&mut conn)
+        .with_context(|| format!("failed to remove asset records for tenant {}", tenant.slug))?;
+
+    println!("Tenant {}: asset records deleted.", tenant.slug);
    Ok(())
 }
+
+async fn quickwit_index(
+    config: &AppConfig,
+    pool: &PgPool,
+    slug: &str,
+    method: Method,
+) -> Result<()> {
+    let endpoint = config
+        .quickwit_endpoint
+        .as_ref()
+        .ok_or_else(|| anyhow!("quickwit endpoint not configured"))?;
+
+    let mut conn = pool.get().context("failed to get database connection")?;
+    let tenant: Tenant = tenants::table
+        .filter(tenants::slug.eq(slug))
+        .first(&mut conn)
+        .optional()
+        .context("failed to query tenants")?
+        .ok_or_else(|| anyhow!("tenant '{}' not found", slug))?;
+
+    let client = Client::new();
+    let index_id = format!("documents-{}", tenant.id);
+    let base_endpoint = endpoint.trim_end_matches('/');
+
+    match method {
+        Method::POST => {
+            let payload = render_index_template(&index_id);
+            let response = client
+                .post(format!("{}/api/v1/indexes", base_endpoint))
+                .header("content-type", "application/json")
+                .body(payload)
+                .send()
+                .await
+                .context("failed to send create index request")?;
+
+            match response.status() {
+                status if status.is_success() => {
+                    diesel::update(tenants::table.filter(tenants::id.eq(tenant.id)))
+                        .set(tenants::quickwit_index.eq(Some(index_id.clone())))
+                        .execute(&mut conn)
+                        .context("failed to update tenant quickwit_index")?;
+
+                    println!(
+                        "Tenant '{}' quickwit index set to '{}'.",
+                        tenant.slug, index_id
+                    );
+                }
+                StatusCode::CONFLICT => {
+                    let lookup = client
+                        .get(format!("{}/api/v1/indexes/{}", base_endpoint, index_id))
+                        .send()
+                        .await
+                        .context("failed to verify existing quickwit index")?;
+
+                    let lookup_status = lookup.status();
+                    if !lookup_status.is_success() {
+                        let body = lookup.text().await.unwrap_or_default();
+                        return Err(anyhow!(
+                            "quickwit reported conflict but index lookup failed with status {}: {}",
+                            lookup_status,
+                            body
+                        ));
+                    }
+
+                    diesel::update(tenants::table.filter(tenants::id.eq(tenant.id)))
+                        .set(tenants::quickwit_index.eq(Some(index_id.clone())))
+                        .execute(&mut conn)
+                        .context("failed to update tenant quickwit_index")?;
+
+                    println!(
+                        "Tenant '{}' quickwit index set to '{}'.",
+                        tenant.slug, index_id
+                    );
+                }
+                status => {
+                    let body = response.text().await.unwrap_or_default();
+                    return Err(anyhow!(
+                        "quickwit create index failed with status {}: {}",
+                        status,
+                        body
+                    ));
+                }
+            }
+        }
+        Method::DELETE => {
+            let response = client
+                .delete(format!("{}/api/v1/indexes/{}", base_endpoint, index_id))
+                .send()
+                .await
+                .context("failed to send delete index request")?;
+
+            match response.status() {
+                status if status.is_success() || status == StatusCode::NOT_FOUND => {
+                    diesel::update(tenants::table.filter(tenants::id.eq(tenant.id)))
+                        .set(tenants::quickwit_index.eq::<Option<String>>(None))
+                        .execute(&mut conn)
+                        .context("failed to clear tenant quickwit_index")?;
+
+                    println!("Tenant '{}' quickwit index cleared.", tenant.slug);
+                }
+                status => {
+                    let body = response.text().await.unwrap_or_default();
+                    return Err(anyhow!(
+                        "quickwit delete index failed with status {}: {}",
+                        status,
+                        body
+                    ));
+                }
+            }
+        }
+        _ => unreachable!(),
+    }
+
+    Ok(())
+}
+
+fn render_index_template(index_id: &str) -> String {
+    let mut template = QUICKWIT_INDEX_TEMPLATE.clone();
+    if let Some(obj) = template.as_object_mut() {
+        obj.insert(
+            "index_id".to_string(),
+            serde_json::Value::String(index_id.to_string()),
+        );
+    }
+    template.to_string()
+}
@@ -1,47 +1,24 @@
 use std::net::SocketAddr;
-use std::sync::Arc;

 use tokio::net::TcpListener;
 use tower::make::Shared;
-use tracing_subscriber::EnvFilter;

-use backend::auth::jwt::JwtService;
-use backend::config::AppConfig;
-use backend::db;
-use backend::routes::webdav;
-use backend::s3::build_client;
-use backend::state::AppState;
-use backend::storage::S3Storage;
+use backend::{routes::webdav, utils::bootstrap::init_component};

 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
-    dotenv::dotenv().ok();
-    init_tracing();
-
-    let config = AppConfig::from_env()?;
+    let state = init_component("webdav", None).await?;
+    let webdav_host = state.config.webdav_host.clone();
+    let webdav_port = state.config.webdav_port;
    tracing::info!(
        component = "webdav",
-        database_url = %config.redacted_database_url(),
-        pool_size = config.database_max_pool_size,
-        server_host = %config.server_host,
-        server_port = config.server_port,
-        webdav_host = %config.webdav_host,
-        webdav_port = config.webdav_port,
-        quickwit_enabled = config.quickwit_endpoint.is_some(),
-        s3_bucket = %config.s3_bucket,
-        "loaded backend configuration"
+        webdav_host = %webdav_host,
+        webdav_port,
+        "starting webdav server"
    );
-    let pool = db::init_pool_with_size(&config.database_url, config.database_max_pool_size)?;
-    let s3_client = build_client(&config).await?;
-    let storage = Arc::new(S3Storage::new(s3_client, config.s3_bucket.clone()));
-    let jwt = JwtService::from_config(&config)?;

-    let state = AppState::new(pool, config, storage, jwt);
-    let listen_addr: SocketAddr = {
-        let config = state.config.clone();
-        format!("{}:{}", config.webdav_host, config.webdav_port).parse()?
-    };
-    let router = webdav::create_router().with_state(state);
+    let listen_addr: SocketAddr = format!("{}:{}", webdav_host, webdav_port).parse()?;
+    let router = webdav::create_router().with_state(state.as_ref().clone());

    let listener = TcpListener::bind(listen_addr).await?;
    tracing::info!("listening for WebDAV on {}", listen_addr);
@@ -49,12 +26,3 @@ async fn main() -> anyhow::Result<()> {
    axum::serve(listener, Shared::new(router)).await?;
    Ok(())
 }
-
-fn init_tracing() {
-    let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
-    tracing_subscriber::fmt()
-        .with_env_filter(filter)
-        .with_target(false)
-        .compact()
-        .init();
-}
@@ -1,33 +1,13 @@
-use std::{sync::Arc, time::Duration};
+use std::time::Duration;

 use tokio::signal;
-use tracing_subscriber::EnvFilter;

-use backend::{
-    auth::jwt::JwtService, config::AppConfig, db, default_handlers, s3::build_client,
-    state::AppState, storage::S3Storage, Worker,
-};
+use backend::{default_handlers, utils::bootstrap::init_component, Worker};

 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
-    dotenv::dotenv().ok();
-    init_tracing();
-
-    let config = AppConfig::from_env()?;
-    tracing::info!(
-        component = "worker",
-        database_url = %config.redacted_database_url(),
-        pool_size = 1,
-        quickwit_enabled = config.quickwit_endpoint.is_some(),
-        s3_bucket = %config.s3_bucket,
-        "loaded backend configuration"
-    );
-    let pool = db::init_pool_with_size(&config.database_url, 1)?;
-    let s3_client = build_client(&config).await?;
-    let storage = Arc::new(S3Storage::new(s3_client, config.s3_bucket.clone()));
-    let jwt = JwtService::from_config(&config)?;
-
-    let state = Arc::new(AppState::new(pool, config, storage, jwt));
+    let state = init_component("worker", Some(1)).await?;
+    tracing::info!(component = "worker", "starting worker process");
    let worker = Worker::new(state, default_handlers(), Duration::from_secs(2));

    tokio::select! {
@@ -39,12 +19,3 @@ async fn main() -> anyhow::Result<()> {

    Ok(())
 }
-
-fn init_tracing() {
-    let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
-    tracing_subscriber::fmt()
-        .with_env_filter(filter)
-        .with_target(false)
-        .compact()
-        .init();
-}