Rust example: How to list objects in S3 / MinIO bucket with prefix
This example program lists all objects in an S3 or MinIO bucket with a specified prefix. It uses the aws-sdk-s3
crate to interact with the S3-compatible service.
main.rs
//! Example: List all objects in an S3 bucket that start with a given prefix.
//!
//! Usage (environment credentials or config profile required):
//! RUST_LOG=info cargo run --example list_s3_prefix -- <bucket> <prefix>
//!
//! You can also set AWS_REGION or AWS_DEFAULT_REGION; otherwise, the default
//! region resolution chain applies (env, profile, IMDS, etc.).
use anyhow::{Context, Result};
use aws_sdk_s3::types::Object;
use aws_sdk_s3::Client;
use std::io::{self, Write};
#[tokio::main]
async fn main() -> Result<()> {
// Basic argument parsing
let mut args = std::env::args().skip(1);
let bucket = args
.next()
.context("Missing <bucket> argument. Usage: cargo run --example list_s3_prefix -- <bucket> <prefix>")?;
let prefix = args
.next()
.context("Missing <prefix> argument. Usage: cargo run --example list_s3_prefix -- <bucket> <prefix>")?;
// Load AWS shared config (resolves region, credentials, etc.)
let config = aws_config::from_env().behavior_version(aws_config::BehaviorVersion::latest()).load().await;
// Configure S3 client with custom endpoint if provided
let mut s3_config_builder = aws_sdk_s3::config::Builder::from(&config);
if let Ok(endpoint) = std::env::var("S3_ENDPOINT") {
s3_config_builder = s3_config_builder.endpoint_url(endpoint);
}
if let Ok(force_path_style) = std::env::var("S3_FORCE_PATH_STYLE") {
if force_path_style == "1" {
s3_config_builder = s3_config_builder.force_path_style(true);
}
}
let s3_config = s3_config_builder.build();
let client = Client::from_conf(s3_config);
list_prefix(&client, &bucket, &prefix).await
}
async fn list_prefix(client: &Client, bucket: &str, prefix: &str) -> Result<()> {
let mut continuation_token: Option<String> = None;
let mut total_objects: u64 = 0;
let mut total_bytes: u128 = 0;
let mut stdout = io::stdout();
loop {
let mut req = client
.list_objects_v2()
.bucket(bucket)
.prefix(prefix)
.max_keys(1000);
if let Some(token) = &continuation_token {
req = req.continuation_token(token);
}
let resp = req
.send()
.await
.with_context(|| format!("Listing objects (continuation_token={:?})", continuation_token))?;
for obj in resp.contents() {
output_object(&mut stdout, obj)?;
total_objects += 1;
if let Some(sz) = obj.size() { total_bytes += sz as u128; }
}
match resp.is_truncated() {
Some(true) => {
continuation_token = resp.next_continuation_token().map(|s| s.to_string());
}
_ => break,
}
}
writeln!(stdout, "# SUMMARY objects={} total_bytes={}", total_objects, total_bytes)?;
Ok(())
}
fn output_object(writer: &mut impl Write, obj: &Object) -> Result<()> {
let key = obj.key().unwrap_or("<no-key>");
let size = obj.size().unwrap_or_default();
let last_modified = obj.last_modified().map(|dt| dt.to_string()).unwrap_or_else(|| "".into());
writeln!(writer, "{}\t{}\t{}", key, size, last_modified)?;
Ok(())
}
Cargo.toml
[package]
name = "rust_parse_status_logs"
version = "0.1.0"
edition = "2021"
[dependencies]
anyhow = "1"
tokio = { version = "1", features = ["rt-multi-thread", "macros", "io-std"] }
aws-config = { version = "1", features = ["behavior-version-latest"] }
aws-sdk-s3 = { version = "1", features = ["behavior-version-latest"] }
[profile.release]
opt-level = 3
lto = "thin"
codegen-units = 1
Example usage (MinIO)
#!/bin/bash
export AWS_ACCESS_KEY_ID=my-access-key
export AWS_SECRET_ACCESS_KEY=Quoo6Eis1eiphabahB9quohs6eic9B
export AWS_REGION=eu-central-1
export S3_ENDPOINT=https://minio.mydomain.com
export S3_FORCE_PATH_STYLE=1
cargo run -- mybucket Prefix
Example usage (AWS S3)
export AWS_ACCESS_KEY_ID=your-access-key
export AWS_SECRET_ACCESS_KEY=your-secret-key
export AWS_REGION=eu-central-1
cargo run -- mybucket Prefix
If this post helped you, please consider buying me a coffee or donating via PayPal to support research & publishing of new posts on TechOverflow