Rust : comment lister les objets d'un bucket S3 / MinIO avec un préfixe

//! Exemple : lister tous les objets d'un bucket S3 qui commencent par un préfixe donné.
//!
//! Utilisation (identifiants d'environnement ou profil de configuration requis) :
//! RUST_LOG=info cargo run --example list_s3_prefix -- <bucket> <prefix>
//!
//! Vous pouvez aussi définir AWS_REGION ou AWS_DEFAULT_REGION ; sinon, la chaîne
//! de résolution de région par défaut s'applique (env, profil, IMDS, etc.).

use anyhow::{Context, Result};
use aws_sdk_s3::types::Object;
use aws_sdk_s3::Client;
use std::io::{self, Write};

#[tokio::main]
async fn main() -> Result<()> {
    // Analyse basique des arguments
    let mut args = std::env::args().skip(1);
    let bucket = args
        .next()
        .context("Argument <bucket> manquant. Utilisation : cargo run --example list_s3_prefix -- <bucket> <prefix>")?;
    let prefix = args
        .next()
        .context("Argument <prefix> manquant. Utilisation : cargo run --example list_s3_prefix -- <bucket> <prefix>")?;

    // Charger la configuration partagée AWS (résout la région, les identifiants, etc.)
    let config = aws_config::from_env().behavior_version(aws_config::BehaviorVersion::latest()).load().await;

    // Configurer le client S3 avec un endpoint personnalisé si fourni
    let mut s3_config_builder = aws_sdk_s3::config::Builder::from(&config);
    if let Ok(endpoint) = std::env::var("S3_ENDPOINT") {
        s3_config_builder = s3_config_builder.endpoint_url(endpoint);
    }
    if let Ok(force_path_style) = std::env::var("S3_FORCE_PATH_STYLE") {
        if force_path_style == "1" {
            s3_config_builder = s3_config_builder.force_path_style(true);
        }
    }
    let s3_config = s3_config_builder.build();
    let client = Client::from_conf(s3_config);

    list_prefix(&client, &bucket, &prefix).await
}

async fn list_prefix(client: &Client, bucket: &str, prefix: &str) -> Result<()> {
    let mut continuation_token: Option<String> = None;
    let mut total_objects: u64 = 0;
    let mut total_bytes: u128 = 0;

    let mut stdout = io::stdout();

    loop {
        let mut req = client
            .list_objects_v2()
            .bucket(bucket)
            .prefix(prefix)
            .max_keys(1000);
        if let Some(token) = &continuation_token {
            req = req.continuation_token(token);
        }

        let resp = req
            .send()
            .await
            .with_context(|| format!("Listage des objets (continuation_token={:?})", continuation_token))?;

        for obj in resp.contents() {
            output_object(&mut stdout, obj)?;
            total_objects += 1;
            if let Some(sz) = obj.size() { total_bytes += sz as u128; }
        }

        match resp.is_truncated() {
            Some(true) => {
                continuation_token = resp.next_continuation_token().map(|s| s.to_string());
            }
            _ => break,
        }
    }

    writeln!(stdout, "# RÉSUMÉ objects={} total_bytes={}", total_objects, total_bytes)?;
    Ok(())
}

fn output_object(writer: &mut impl Write, obj: &Object) -> Result<()> {
    let key = obj.key().unwrap_or("<no-key>");
    let size = obj.size().unwrap_or_default();
    let last_modified = obj.last_modified().map(|dt| dt.to_string()).unwrap_or_else(|| "".into());
    writeln!(writer, "{}\t{}\t{}", key, size, last_modified)?;
    Ok(())
}

[package]
name = "rust_parse_status_logs"
version = "0.1.0"
edition = "2021"

[dependencies]
anyhow = "1"
tokio = { version = "1", features = ["rt-multi-thread", "macros", "io-std"] }
aws-config = { version = "1", features = ["behavior-version-latest"] }
aws-sdk-s3 = { version = "1", features = ["behavior-version-latest"] }

[profile.release]
opt-level = 3
lto = "thin"
codegen-units = 1

#!/bin/bash
export AWS_ACCESS_KEY_ID=my-access-key
export AWS_SECRET_ACCESS_KEY=Quoo6Eis1eiphabahB9quohs6eic9B
export AWS_REGION=eu-central-1
export S3_ENDPOINT=https://minio.mydomain.com
export S3_FORCE_PATH_STYLE=1
cargo run -- mybucket Prefix

export AWS_ACCESS_KEY_ID=your-access-key
export AWS_SECRET_ACCESS_KEY=your-secret-key
export AWS_REGION=eu-central-1
cargo run -- mybucket Prefix