Batch Processor

Process items in batches with configurable batch size and error handling

python (3.9) 2025-11-12 batch processing async error-handling

Description

A utility for processing items in batches. Handles errors gracefully, supports progress tracking, and allows custom batch processing functions.

Features

  • Configurable batch size
  • Error handling per batch
  • Progress callbacks
  • Custom processing functions

Code

RAW
from typing import List, Callable, Optional, Anyfrom functools import partialclass BatchProcessor:    def __init__(self, batch_size: int = 10, continue_on_error: bool = true):        self.batch_size = batch_size        self.continue_on_error = continue_on_error        def process(        self,        items: List[Any],        processor: Callable[[List[Any]], Any],        on_progress: Optional[Callable[[int, int], None]] = None    ) -> List[Any]:        """        Process items in batches.                Args:            items: List of items to process            processor: Function to process each batch            on_progress: Optional callback(current_batch, total_batches)                Returns:            List of results from each batch        """        results = []        total_batches = (len(items) + self.batch_size - 1) // self.batch_size                for i in range(0, len(items), self.batch_size):            batch = items[i:i + self.batch_size]            batch_num = (i // self.batch_size) + 1                        if on_progress:                on_progress(batch_num, total_batches)                        try:                result = processor(batch)                results.append(result)            except Exception as e:                if self.continue_on_error:                    results.append({'error': str(e), 'batch': batch})                else:                    raise                return results
RAW
from batch_processor import BatchProcessorprocessor = BatchProcessor(batch_size=10, continue_on_error=true)items = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]def process_batch(batch):    return [item * 2 for item in batch]def on_progress(current, total):    print(f'Processing batch {current}/{total}')results = processor.process(items, process_batch, on_progress)print(f'Processed {len(results)} batches')

Comments

No comments yet. Be the first to comment!