Content dedup on both sides, stored checksums, spool-free zips, upload stats

- sha256+crc32 hashed during upload streaming; unique index per album
- duplicate content returns the existing photo (race-safe via 23505)
- client hashes locally (WebCrypto) and skips the transfer entirely for
  content the album already has
- zip downloads stream S3->response directly using the stored crc32;
  pre-hash photos spool once and self-heal (crc via zip, sha via reprocess)
- upload UI: overall progress bar, bytes, live speed, ETA

src/routes/photos.rs

@@ -5,6 +5,7 @@ use axum::http::HeaderMap;
 use axum::Json;
 use futures::StreamExt;
 use serde::Deserialize;
+use sha2::Digest;
 use tokio::io::AsyncWriteExt;
 use uuid::Uuid;
 
@@ -60,17 +61,23 @@ pub async fn upload(
         .unwrap_or("application/octet-stream")
         .to_string();
 
-    // Stream the request body to a temp file so large raws never sit in memory.
+    // Stream the request body to a temp file so large raws never sit in
+    // memory, hashing as it flows: sha256 for duplicate detection, crc32 for
+    // spool-free zip downloads.
     let dir = tempfile::tempdir().map_err(anyhow::Error::from)?;
     let path = dir.path().join("upload.bin");
     let mut file = tokio::fs::File::create(&path)
         .await
         .map_err(anyhow::Error::from)?;
     let mut stream = body.into_data_stream();
+    let mut sha = sha2::Sha256::new();
+    let mut crc = crc32fast::Hasher::new();
     let mut size: i64 = 0;
     while let Some(chunk) = stream.next().await {
         let chunk = chunk.map_err(|e| ApiError::bad_request(format!("upload aborted: {e}")))?;
         size += chunk.len() as i64;
+        sha.update(&chunk);
+        crc.update(&chunk);
         file.write_all(&chunk).await.map_err(anyhow::Error::from)?;
     }
     file.flush().await.map_err(anyhow::Error::from)?;
@@ -78,6 +85,20 @@ pub async fn upload(
     if size == 0 {
         return Err(ApiError::bad_request("empty upload"));
     }
+    let sha256 = hex::encode(sha.finalize());
+    let crc32 = i64::from(crc.finalize());
+
+    // Same content already in this album? Return it — re-dragging a folder
+    // after a partial upload just fills the gaps instead of duplicating.
+    let existing: Option<Photo> =
+        sqlx::query_as("select * from photos where album_id = $1 and sha256 = $2")
+            .bind(album_id)
+            .bind(&sha256)
+            .fetch_optional(&state.db)
+            .await?;
+    if let Some(existing) = existing {
+        return Ok(Json(existing));
+    }
 
     // Upload to S3 first, then create the row and enqueue processing in one
     // transaction — a photo row can never exist without its job, and a failed
@@ -86,30 +107,33 @@ pub async fn upload(
     let key = s3::original_key(photo_id, &filename);
     s3::put_file(&state, &key, &path, &content_type).await?;
 
-    let result: ApiResult<(sqlx::Transaction<'static, sqlx::Postgres>, Photo)> = async {
-        let mut tx = state.db.begin().await?;
-        let photo: Photo = sqlx::query_as(
-            "insert into photos (id, album_id, filename, content_type, size_bytes, status)
-             values ($1, $2, $3, $4, $5, $6)
-             returning *",
-        )
-        .bind(photo_id)
-        .bind(album_id)
-        .bind(&filename)
-        .bind(&content_type)
-        .bind(size)
-        .bind(PhotoStatus::Uploaded.as_str())
-        .fetch_one(&mut *tx)
-        .await?;
-        jobs::enqueue(
-            &mut *tx,
-            JobKind::ProcessPhoto,
-            serde_json::json!({ "photo_id": photo_id }),
-        )
-        .await?;
-        Ok((tx, photo))
-    }
-    .await;
+    let result: Result<(sqlx::Transaction<'static, sqlx::Postgres>, Photo), sqlx::Error> =
+        async {
+            let mut tx = state.db.begin().await?;
+            let photo: Photo = sqlx::query_as(
+                "insert into photos (id, album_id, filename, content_type, size_bytes, status, sha256, crc32)
+                 values ($1, $2, $3, $4, $5, $6, $7, $8)
+                 returning *",
+            )
+            .bind(photo_id)
+            .bind(album_id)
+            .bind(&filename)
+            .bind(&content_type)
+            .bind(size)
+            .bind(PhotoStatus::Uploaded.as_str())
+            .bind(&sha256)
+            .bind(crc32)
+            .fetch_one(&mut *tx)
+            .await?;
+            jobs::enqueue(
+                &mut *tx,
+                JobKind::ProcessPhoto,
+                serde_json::json!({ "photo_id": photo_id }),
+            )
+            .await?;
+            Ok((tx, photo))
+        }
+        .await;
 
     let (tx, photo) = match result {
         Ok(pair) => pair,
@@ -118,7 +142,24 @@ pub async fn upload(
             if let Err(cleanup) = s3::delete_prefix(&state, &s3::photo_prefix(photo_id)).await {
                 tracing::error!("failed to clean up s3 after aborted upload: {cleanup:#}");
             }
-            return Err(e);
+            // Concurrent identical upload beat us to the unique index — hand
+            // back the winner instead of an error.
+            let unique_violation = matches!(
+                &e,
+                sqlx::Error::Database(db) if db.code().as_deref() == Some("23505")
+            );
+            if unique_violation {
+                let winner: Option<Photo> =
+                    sqlx::query_as("select * from photos where album_id = $1 and sha256 = $2")
+                        .bind(album_id)
+                        .bind(&sha256)
+                        .fetch_optional(&state.db)
+                        .await?;
+                if let Some(winner) = winner {
+                    return Ok(Json(winner));
+                }
+            }
+            return Err(e.into());
         }
     };
     if let Err(e) = tx.commit().await {
@@ -133,6 +174,21 @@ pub async fn upload(
     Ok(Json(photo))
 }
 
+/// Client-side dedup support: lets the uploader skip transferring files whose
+/// content already exists in the album.
+pub async fn by_hash(
+    State(state): State<AppState>,
+    Path((album_id, sha256)): Path<(Uuid, String)>,
+) -> ApiResult<Json<Photo>> {
+    let photo: Option<Photo> =
+        sqlx::query_as("select * from photos where album_id = $1 and sha256 = $2")
+            .bind(album_id)
+            .bind(&sha256)
+            .fetch_optional(&state.db)
+            .await?;
+    photo.map(Json).ok_or_else(ApiError::not_found)
+}
+
 pub async fn delete(
     State(state): State<AppState>,
     Path(photo_id): Path<Uuid>,