rust-s3
This commit is contained in:
Generated
+457
-928
File diff suppressed because it is too large
Load Diff
+1
-3
@@ -18,9 +18,7 @@ uuid = { version = "1.6", features = ["v4", "serde"] }
|
||||
chrono = { version = "0.4", features = ["serde"] }
|
||||
|
||||
# S3
|
||||
aws-config = "1.1"
|
||||
aws-sdk-s3 = "1.14"
|
||||
aws-credential-types = "1.2"
|
||||
rust-s3 = { version = "0.37", features = ["with-tokio", "tokio-rustls-tls"] }
|
||||
|
||||
# Serialization
|
||||
serde = { version = "1.0", features = ["derive"] }
|
||||
|
||||
@@ -492,9 +492,8 @@ async fn delete_assets_for_tenant(
|
||||
pool: &PgPool,
|
||||
tenant_id: Uuid,
|
||||
) -> Result<()> {
|
||||
let s3_client = s3::build_client(config).await?;
|
||||
let storage: Arc<dyn ObjectStorage> =
|
||||
Arc::new(S3Storage::new(s3_client, config.s3_bucket.clone()));
|
||||
let bucket = s3::build_bucket(config)?;
|
||||
let storage: Arc<dyn ObjectStorage> = Arc::new(S3Storage::new(bucket));
|
||||
|
||||
let mut conn = pool.get().context("failed to get database connection")?;
|
||||
let tenant: Tenant = tenants::table
|
||||
|
||||
+26
-30
@@ -1,38 +1,34 @@
|
||||
use anyhow::Result;
|
||||
use aws_config::meta::region::RegionProviderChain;
|
||||
use aws_credential_types::Credentials;
|
||||
use aws_sdk_s3::{
|
||||
config::{Builder as S3ConfigBuilder, Region},
|
||||
Client as S3Client,
|
||||
};
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use s3::{bucket::Bucket, creds::Credentials, region::Region};
|
||||
|
||||
use crate::config::AppConfig;
|
||||
|
||||
pub async fn build_client(config: &AppConfig) -> Result<S3Client> {
|
||||
let region = Region::new(config.aws_region.clone());
|
||||
let region_provider = RegionProviderChain::first_try(Some(region))
|
||||
.or_default_provider()
|
||||
.or_else("us-east-1");
|
||||
pub fn build_bucket(config: &AppConfig) -> Result<Bucket> {
|
||||
let region = if let Some(endpoint) = &config.aws_endpoint_url {
|
||||
Region::Custom {
|
||||
region: config.aws_region.clone(),
|
||||
endpoint: endpoint.clone(),
|
||||
}
|
||||
} else {
|
||||
config
|
||||
.aws_region
|
||||
.parse::<Region>()
|
||||
.context("invalid AWS region")?
|
||||
};
|
||||
|
||||
#[allow(deprecated)]
|
||||
let mut loader = aws_config::from_env().region(region_provider);
|
||||
|
||||
if let Some(endpoint) = &config.aws_endpoint_url {
|
||||
loader = loader.endpoint_url(endpoint);
|
||||
}
|
||||
|
||||
if let (Some(access_key), Some(secret_key)) = (
|
||||
config.aws_access_key_id.clone(),
|
||||
config.aws_secret_access_key.clone(),
|
||||
let credentials = if let (Some(access_key), Some(secret_key)) = (
|
||||
config.aws_access_key_id.as_deref(),
|
||||
config.aws_secret_access_key.as_deref(),
|
||||
) {
|
||||
let credentials = Credentials::new(access_key, secret_key, None, None, "static");
|
||||
loader = loader.credentials_provider(credentials);
|
||||
}
|
||||
Credentials::new(Some(access_key), Some(secret_key), None, None, None)
|
||||
.context("failed to create static AWS credentials")?
|
||||
} else {
|
||||
Credentials::default().context("failed to load AWS credentials")?
|
||||
};
|
||||
|
||||
let base_config = loader.load().await;
|
||||
let s3_config = S3ConfigBuilder::from(&base_config)
|
||||
.force_path_style(true)
|
||||
.build();
|
||||
let bucket = Bucket::new(&config.s3_bucket, region, credentials)
|
||||
.map_err(|err| anyhow!("failed to create S3 bucket client: {err}"))?;
|
||||
let bucket = bucket.with_path_style();
|
||||
|
||||
Ok(S3Client::from_conf(s3_config))
|
||||
Ok(*bucket)
|
||||
}
|
||||
|
||||
@@ -34,11 +34,8 @@ impl AppState {
|
||||
) -> anyhow::Result<Self> {
|
||||
let pool_size = pool_size_override.unwrap_or(config.database_max_pool_size);
|
||||
let pool = crate::db::init_pool_with_size(&config.database_url, pool_size)?;
|
||||
let s3_client = crate::s3::build_client(&config).await?;
|
||||
let storage = Arc::new(crate::storage::S3Storage::new(
|
||||
s3_client,
|
||||
config.s3_bucket.clone(),
|
||||
));
|
||||
let bucket = crate::s3::build_bucket(&config)?;
|
||||
let storage = Arc::new(crate::storage::S3Storage::new(bucket));
|
||||
let jwt = crate::auth::jwt::JwtService::from_config(&config)?;
|
||||
|
||||
Ok(Self::new(pool, config, storage, jwt))
|
||||
|
||||
+36
-58
@@ -1,12 +1,10 @@
|
||||
use std::time::Duration;
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::{anyhow, Context, Result};
|
||||
use async_trait::async_trait;
|
||||
use aws_sdk_s3::presigning::PresigningConfig;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::Client as S3Client;
|
||||
use s3::bucket::Bucket;
|
||||
|
||||
use crate::models::Tenant;
|
||||
|
||||
@@ -33,16 +31,16 @@ pub trait ObjectStorage: Send + Sync + 'static {
|
||||
}
|
||||
|
||||
pub struct S3Storage {
|
||||
client: S3Client,
|
||||
bucket: String,
|
||||
bucket: Bucket,
|
||||
}
|
||||
|
||||
impl S3Storage {
|
||||
pub fn new(client: S3Client, bucket: impl Into<String>) -> Self {
|
||||
Self {
|
||||
client,
|
||||
bucket: bucket.into(),
|
||||
}
|
||||
pub fn new(bucket: Bucket) -> Self {
|
||||
Self { bucket }
|
||||
}
|
||||
|
||||
fn default_content_type(content_type: Option<String>) -> String {
|
||||
content_type.unwrap_or_else(|| "application/octet-stream".to_string())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,23 +53,19 @@ impl ObjectStorage for S3Storage {
|
||||
content_type: Option<String>,
|
||||
content_disposition: Option<String>,
|
||||
) -> Result<()> {
|
||||
let mut request = self
|
||||
.client
|
||||
.put_object()
|
||||
.bucket(&self.bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(bytes));
|
||||
let mut builder = self
|
||||
.bucket
|
||||
.put_object_builder(key, &bytes)
|
||||
.with_content_type(Self::default_content_type(content_type));
|
||||
|
||||
if let Some(content_type) = content_type {
|
||||
request = request.content_type(content_type);
|
||||
if let Some(disposition) = content_disposition {
|
||||
builder = builder
|
||||
.with_content_disposition(disposition)
|
||||
.context("invalid content disposition header")?;
|
||||
}
|
||||
|
||||
if let Some(content_disposition) = content_disposition {
|
||||
request = request.content_disposition(content_disposition);
|
||||
}
|
||||
|
||||
request
|
||||
.send()
|
||||
builder
|
||||
.execute()
|
||||
.await
|
||||
.context("failed to upload object to S3")?;
|
||||
|
||||
@@ -84,51 +78,35 @@ impl ObjectStorage for S3Storage {
|
||||
expires_in: Duration,
|
||||
response_content_disposition: Option<&str>,
|
||||
) -> Result<String> {
|
||||
let presign_config = PresigningConfig::builder()
|
||||
.expires_in(expires_in)
|
||||
.build()
|
||||
.context("failed to build S3 presigning config")?;
|
||||
let expiry_secs = u32::try_from(expires_in.as_secs())
|
||||
.context("presign expiry exceeds u32 range")?;
|
||||
|
||||
let mut request = self.client.get_object().bucket(&self.bucket).key(key);
|
||||
let mut queries = HashMap::new();
|
||||
if let Some(value) = response_content_disposition {
|
||||
request = request.response_content_disposition(value);
|
||||
queries.insert(
|
||||
"response-content-disposition".to_string(),
|
||||
value.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
let presigned = request
|
||||
.presigned(presign_config)
|
||||
self.bucket
|
||||
.presign_get(key, expiry_secs, (!queries.is_empty()).then_some(queries))
|
||||
.await
|
||||
.context("failed to generate presigned download URL")?;
|
||||
|
||||
Ok(presigned.uri().to_string())
|
||||
.context("failed to generate presigned download URL")
|
||||
}
|
||||
|
||||
async fn get_object(&self, key: &str) -> Result<Vec<u8>> {
|
||||
let response = self
|
||||
.client
|
||||
.get_object()
|
||||
.bucket(&self.bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
let data = self
|
||||
.bucket
|
||||
.get_object(key)
|
||||
.await
|
||||
.context("failed to download object from S3")?;
|
||||
|
||||
let bytes = response
|
||||
.body
|
||||
.collect()
|
||||
.await
|
||||
.context("failed to read object stream")?
|
||||
.into_bytes()
|
||||
.to_vec();
|
||||
|
||||
Ok(bytes)
|
||||
Ok(data.into_bytes().to_vec())
|
||||
}
|
||||
|
||||
async fn delete_object(&self, key: &str) -> Result<()> {
|
||||
self.client
|
||||
.delete_object()
|
||||
.bucket(&self.bucket)
|
||||
.key(key)
|
||||
.send()
|
||||
self.bucket
|
||||
.delete_object(key)
|
||||
.await
|
||||
.context("failed to delete object from S3")?;
|
||||
Ok(())
|
||||
|
||||
Reference in New Issue
Block a user