from typing import List, Callable, Optional, Any
from functools import partial
class BatchProcessor:
def __init__(self, batch_size: int = 10, continue_on_error: bool = true):
self.batch_size = batch_size
self.continue_on_error = continue_on_error
def process(
self,
items: List[Any],
processor: Callable[[List[Any]], Any],
on_progress: Optional[Callable[[int, int], None]] = None
) -> List[Any]:
"""
Process items in batches.
Args:
items: List of items to process
processor: Function to process each batch
on_progress: Optional callback(current_batch, total_batches)
Returns:
List of results from each batch
"""
results = []
total_batches = (len(items) + self.batch_size - 1) // self.batch_size
for i in range(0, len(items), self.batch_size):
batch = items[i:i + self.batch_size]
batch_num = (i // self.batch_size) + 1
if on_progress:
on_progress(batch_num, total_batches)
try:
result = processor(batch)
results.append(result)
except Exception as e:
if self.continue_on_error:
results.append({'error': str(e), 'batch': batch})
else:
raise
return results