There is a way in Python that uses botos'3 delete_objects, tqdm to give a bit of feedback, and multiple threads to give it a little bit of parallelism (in spite of the GIL, multiple requests to S3 should run at the same time)
This is at https://gist.github.com/michalc/20b79c9028c342ef7c38df8693f8715b but to copy it here:
# From some light testing deletes at between 2000 to 6000 objects per second, and works best if you
# have objects distributed into folders/CommonPrefixes as specified by the delimiter.
from concurrent.futures import FIRST_EXCEPTION, ThreadPoolExecutor, wait
from dataclasses import dataclass, field
from functools import partial
from typing import Callable, Optional, Tuple
from queue import PriorityQueue
import boto3
from botocore.config import Config
def bulk_delete(
bucket, prefix,
workers=4, page_size=1000, delimiter='/',
get_s3_client=lambda: boto3.client('s3', config=Config(retries={'max_attempts': 16, 'mode': 'adaptive'})),
on_delete=lambda num: None,
):
s3 = get_s3_client()
queue = PriorityQueue()
@dataclass(order=True)
class Work:
# A tuple that determines the priority of the bit of work in "func". This is a sort of
# "coordinate" in the paginated node tree that prioritises a depth-first search.
priority: Tuple[Tuple[int,int], ...]
# The work function itself that fetches a page of Key/CommonPrefixes, or deletes
func: Optional[Callable[[], None]] = field(compare=False)
# A sentinal "stop" Work instance with priority chosen to be before all work. So when it's
# queued the workers will stop at the very next opportunity
stop = Work(((-1,-1),), None)
def do_work():
while (work := queue.get()) is not stop:
work.func()
queue.task_done()
with queue.mutex:
unfinished_tasks = queue.unfinished_tasks
if unfinished_tasks == 0:
for _ in range(0, workers):
queue.put(stop)
def list_iter(prefix):
return iter(s3.get_paginator('list_objects_v2').paginate(
Bucket=bucket, Prefix=prefix,
Delimiter=delimiter, MaxKeys=page_size, PaginationConfig={'PageSize': page_size},
))
def delete_page(page):
s3.delete_objects(Bucket=bucket, Delete={'Objects': page})
on_delete(len(page))
def process_page(page_iter, priority):
try:
page = next(page_iter)
except StopIteration:
return
# Deleting a page is done at the same priority as this function. It will often be done
# straight after this call because this call must have been the highest priority for it to
# have run, but there could be unfinished nodes earlier in the depth-first search that have
# since submitted work, and so would be prioritised over the deletion
if contents := page.get('Contents'):
delete_priority = priority
queue.put(Work(
priority=delete_priority,
func=partial(delete_page, [{'Key': obj['Key']} for obj in contents]),
))
# Processing child prefixes are done after deletion and in order. Importantly anything each
# child prefix itself enqueues should be done before the work of any later child prefixes
# to make it a depth-first search. Appending the index of the child prefix to the priority
# tuple of this function does this, because the work inside each child prefix will only
# ever enqueue work at its priority or greater, but always less than the priority of
# subsequent child prefixes or the next page.
for prefix_index, obj in enumerate(page.get('CommonPrefixes', [])):
prefix_priority = priority + ((prefix_index,0),)
queue.put(Work(
priority=prefix_priority,
func=partial(process_page,
page_iter=list_iter(obj['Prefix']), priority=prefix_priority)
))
# The next page in pagination for this prefix is processed after delete for this page,
# after all the child prefixes are processed, and after anything the child prefixes
# themselves enqueue.
next_page_priority = priority[:-1] + ((priority[-1][0], priority[-1][1] + 1),)
queue.put(Work(
priority=next_page_priority,
func=partial(process_page, page_iter=page_iter, priority=next_page_priority),
))
with ThreadPoolExecutor(max_workers=workers) as worker_pool:
# Bootstrap with the first page
priority = ((0,0),)
queue.put(Work(
priority=priority,
func=partial(process_page, page_iter=list_iter(prefix), priority=priority),
))
try:
# Run workers, waiting for the first exception, if any, raised by them
done, _ = wait(
tuple(worker_pool.submit(do_work) for _ in range(0, workers)),
return_when=FIRST_EXCEPTION,
)
finally:
# The workers might have all stopped cleanly, or just one since it raised an exception,
# or none of them have stopped because the above raised an exception (e.g. on
# KeyboardInterrupt). By putting enough stop sentinels in the queue, we can make sure
# to eventually stop any remaining workers in any of these cases.
for _ in range(0, workers):
queue.put(stop)
# If an exception raised by a worker, re-raise it
if e := next(iter(done)).exception():
raise e from None
used as:
from tqdm import tqdm
print('Deleting...')
with tqdm(unit=' objects') as pbar:
bulk_delete(
bucket='my-bucket', prefix='my-prefix/',
on_delete=lambda num: pbar.update(num),
)
print('Done')
As the gist suggests, it seems to delete at a rate between 2000 and 6000 objects per second, which from my testing is better than plain delete_objects without multiple threads. But your milage may vary.
It works by walking the tree of CommonPrefixes (folder-like structures in S3), and paginating through objects in them, but walking and deleting multiple branches of the tree at the same time. So it should give a performance benefit over plain pagination + delete_objects with no threads if there is a good amount of such CommonPrefixes, and there is a good amount of objects per CommonPrefix.
