firecrawl/apps/api/src/lib/batch-process.ts

export async function batchProcess<T>(
  array: T[],
  batchSize: number,
  asyncFunction: (item: T, index: number) => Promise<void>
): Promise<void> {
  const batches: T[][] = [];
  for (let i = 0; i < array.length; i += batchSize) {
    const batch = array.slice(i, i + batchSize);
    batches.push(batch);
  }

  for (const batch of batches) {
    await Promise.all(batch.map((item, i) => asyncFunction(item, i)));
  }
}