Crawl

Multi-page crawl

Follow links across a site and extract clean content from every page.

# 1) Submit the crawl job -> returns { "jobId": "..." }
JOB_ID=$(curl -s -X POST https://api.anakin.io/v1/crawl \
  -H "X-API-Key: $ANAKIN_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{
  "url": "https://example.com",
  "maxPages": 10,
  "depth": 1,
  "country": "us"
}' | jq -r '.jobId')

# 2) Poll until the crawl finishes
for i in $(seq 1 120); do
  RESP=$(curl -s https://api.anakin.io/v1/crawl/$JOB_ID -H "X-API-Key: $ANAKIN_API_KEY")
  STATUS=$(echo "$RESP" | jq -r '.status')
  [ "$STATUS" = "completed" ] || [ "$STATUS" = "failed" ] && break
  sleep 2
done

echo "$RESP" | jq '.results[].url'

POST /v1/crawl → poll GET /v1/crawl/{id}