Batch Processing

import { Nawa } from '@nawalabs/sdk'

const nawa = new Nawa({
  apiKey: process.env.NAWA_API_KEY,
  maxRetries: 3
})

interface Comment {
  id: string
  text: string
  platform: string
}

async function processBatch(comments: Comment[], concurrency = 5) {
  const results = []
  const queue = [...comments]

  // Process with controlled concurrency
  const workers = Array.from({ length: concurrency }, async () => {
    while (queue.length > 0) {
      const comment = queue.shift()!

      const { data, error } = await nawa.classify({
        text: comment.text,
        platform: comment.platform as any
      })

      if (error?.type === 'rate_limit_error') {
        // Put it back and wait
        queue.unshift(comment)
        const delay = error.retryAfter ?? 2000
        await new Promise(r => setTimeout(r, delay))
        continue
      }

      results.push({
        id: comment.id,
        classification: data,
        error: error?.message
      })
    }
  })

  await Promise.all(workers)
  return results
}

// Usage
const comments = [
  { id: '1', text: 'متى الجزء الثاني؟', platform: 'youtube' },
  { id: '2', text: 'ما شاء الله عليك', platform: 'youtube' },
  { id: '3', text: 'الصوت وحش أوي', platform: 'youtube' },
  // ... thousands more
]

const results = await processBatch(comments, 5)
console.log(`Processed ${results.length} comments`)

import asyncio
from nawa import AsyncNawa

async def process_batch(comments: list[dict], concurrency: int = 5):
    async with AsyncNawa(api_key="nawa_live_sk_xxx") as nawa:
        semaphore = asyncio.Semaphore(concurrency)
        results = []

        async def classify_one(comment):
            async with semaphore:
                result = await nawa.classify(
                    text=comment["text"],
                    platform=comment["platform"]
                )

                if result.error and result.error.type == "rate_limit_error":
                    delay = result.error.retry_after or 2
                    await asyncio.sleep(delay)
                    # Retry once
                    result = await nawa.classify(
                        text=comment["text"],
                        platform=comment["platform"]
                    )

                return {
                    "id": comment["id"],
                    "classification": result.data,
                    "error": result.error.message if result.error else None,
                }

        tasks = [classify_one(c) for c in comments]
        results = await asyncio.gather(*tasks)
        return results

# Usage
comments = [
    {"id": "1", "text": "متى الجزء الثاني؟", "platform": "youtube"},
    {"id": "2", "text": "ما شاء الله عليك", "platform": "youtube"},
    {"id": "3", "text": "الصوت وحش أوي", "platform": "youtube"},
    # ... thousands more
]

results = asyncio.run(process_batch(comments, concurrency=5))
print(f"Processed {len(results)} comments")

const { data } = await nawa.classify({ text: 'متى الجزء الثاني؟', platform: 'youtube' })
console.log(data.cached) // true on subsequent calls

const unique = [...new Map(comments.map(c => [c.text, c])).values()]
console.log(`Deduplicated ${comments.length} → ${unique.length} comments`)