Auto Cleanup Azure Blob Storage

This article gives you a snippet to clean up your blob storage frequent, to keep only a specific time of data.

Azure Blob Storage offers a brilliant and straightforward solution for storing vast amounts of data. However, when it’s unnecessary to retain all data indefinitely, such as data only needed for a few days, it becomes essential to periodically clean up the storage. This ensures optimal resource management and cost-effectiveness within your Azure environment.

using Microsoft.Azure.Functions.Worker;
using Microsoft.Extensions.Logging;

namespace OliverSamples
{
    public class CleanupFunction(ILoggerFactory loggerFactory)
    {
        private readonly ILoggerFactory _loggerFactory = loggerFactory;
        private readonly ILogger _logger = loggerFactory.CreateLogger<CleanupFunction>();

        [Function("StorageCleanup")]
        public async Task Run([TimerTrigger("0 */2 * * * *")] TimerInfo myTimer)
        {
            _logger.LogInformation($"C# Timer trigger function executed at: {DateTime.Now}");

            StorageService storageService = new(_loggerFactory);
            await storageService.DeleteOldData();

            if (myTimer.ScheduleStatus is not null)
            {
                _logger.LogInformation($"Next timer schedule at: {myTimer.ScheduleStatus.Next}");
            }
        }
    }
}

The logic for cleaning up the storage resides within a small service helper that I’ve personally developed.

using Azure.Storage.Blobs;
using Azure.Storage.Blobs.Models;
using Microsoft.Extensions.Logging;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
using System.Text.Json;

public class OliverSamples
{
    private readonly string _blogStorageConnectionString;
    private readonly ILogger<StorageService> _logger;
    private CloudStorageAccount? _storageAccount;
    private CloudBlobClient? _blobClient;
    private int _maxHoursToKeep = 24;

    public StorageService(ILoggerFactory loggerFactory)
    {
        _logger = loggerFactory.CreateLogger<StorageService>();

        string blogStorageConnectionString = Environment.GetEnvironmentVariable(Const.AppSettings.STORAGE_ACCOUNT_CONNECTION_STRING) ?? "";
        if (string.IsNullOrEmpty(blogStorageConnectionString))
        {
            throw new Exception($"Configuration '{Const.AppSettings.STORAGE_ACCOUNT_CONNECTION_STRING}' is not set.");
        }
        _blogStorageConnectionString = blogStorageConnectionString;
    }

    private CloudBlobClient GetBlobClient()
    {
        if (_blobClient != null)
        {
            return _blobClient;
        }
        _storageAccount ??= CloudStorageAccount.Parse(_blogStorageConnectionString);
        _blobClient = _storageAccount.CreateCloudBlobClient();
        return _blobClient;
    }

    public async Task DeleteOldData()
    {
        List<string> containerToClean =
        [
            "MyContainer1", 
            "MyContainer2", 
            "MyContainer3"
        ];

        foreach(var container in containerToClean)
        {
            await CleanContainer(container);
        }
    }

    private async Task CleanContainer(string containerName)
    {
        CloudBlobClient blobClient = GetBlobClient();
        CloudBlobContainer container = blobClient.GetContainerReference(containerName);
        BlobContinuationToken continuationToken = null;
        do
        {
            var resultSegment = await container.ListBlobsSegmentedAsync(null, true, BlobListingDetails.Metadata, null, continuationToken, null, null);
            continuationToken = resultSegment.ContinuationToken;
            foreach (IListBlobItem item in resultSegment.Results)
            {
                if (item is CloudBlockBlob blockBlob)
                {
                    DateTimeOffset? created = blockBlob.Properties.Created;
                    if (created.HasValue && DateTimeOffset.UtcNow.Subtract(created.Value).TotalHours > _maxHoursToKeep)
                    {
                        await blockBlob.DeleteAsync();
                    }
                }
            }
        } while (continuationToken != null);
    }

}

Conclusion

With this Azure Function you clean containers in your blob storage ever 5 minutes. Files that are older than 24 hours, will be removed.