Batch Processor
Process items in batches with configurable batch size and error handling
Description
A utility for processing items in batches. Handles errors gracefully, supports progress tracking, and allows custom batch processing functions.
Features
- Configurable batch size
- Error handling per batch
- Progress callbacks
- Custom processing functions
Code
from typing import List, Callable, Optional, Anyfrom functools import partialclass BatchProcessor: def __init__(self, batch_size: int = 10, continue_on_error: bool = true): self.batch_size = batch_size self.continue_on_error = continue_on_error def process( self, items: List[Any], processor: Callable[[List[Any]], Any], on_progress: Optional[Callable[[int, int], None]] = None ) -> List[Any]: """ Process items in batches. Args: items: List of items to process processor: Function to process each batch on_progress: Optional callback(current_batch, total_batches) Returns: List of results from each batch """ results = [] total_batches = (len(items) + self.batch_size - 1) // self.batch_size for i in range(0, len(items), self.batch_size): batch = items[i:i + self.batch_size] batch_num = (i // self.batch_size) + 1 if on_progress: on_progress(batch_num, total_batches) try: result = processor(batch) results.append(result) except Exception as e: if self.continue_on_error: results.append({'error': str(e), 'batch': batch}) else: raise return results
from batch_processor import BatchProcessorprocessor = BatchProcessor(batch_size=10, continue_on_error=true)items = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15]def process_batch(batch): return [item * 2 for item in batch]def on_progress(current, total): print(f'Processing batch {current}/{total}')results = processor.process(items, process_batch, on_progress)print(f'Processed {len(results)} batches')
Comments
No comments yet. Be the first to comment!
Please login to leave a comment.