Batch Scrape

Use Batch Scraping to submit up to 10,000 URLs in a single API call. This is useful when you need to extract the same type of data from a large number of pages — for example, scraping product pages or company profiles at scale. Batch Scrape works by sending an array of requests under the requests field.

When to Use Batch Scraping

Extracting titles, authors, and publish dates from a list of blog or news article URLs
Running aiExtractRules across a set of company websites to collect structured data like founding year, services, and contact info
Gathering legal notices or disclaimers from the footer pages of 5,000+ policy URLs

Submit a Batch Scrape Job

curl --request POST \
  --url 'https://api.hasdata.com/scrape/batch/web' \
  --header 'Content-Type: application/json' \
  --header 'x-api-key: <your-api-key>' \
  --data '{"requests":[{"url":"https://hasdata.com","outputFormat":["text","html"],"aiExtractRules":{"company":{"type":"string"},"email":{"type":"string"},"yearFounded":{"type":"number"},"isHiring":{"type":"boolean"}}},{"url":"https://example.com","outputFormat":["text","html"],"aiExtractRules":{"company":{"type":"string"},"email":{"type":"string"},"yearFounded":{"type":"number"},"isHiring":{"type":"boolean"}}}]}'

const axios = require('axios').default;

const options = {
  method: 'POST',
  url: 'https://api.hasdata.com/scrape/batch/web',
  headers: {'Content-Type': 'application/json', 'x-api-key': '<your-api-key>'},
  data: {
    requests: [
      {
        url: 'https://hasdata.com',
        outputFormat: ['text', 'html'],
        aiExtractRules: {
          company: {type: 'string'},
          email: {type: 'string'},
          yearFounded: {type: 'number'},
          isHiring: {type: 'boolean'}
        }
      },
      {
        url: 'https://example.com',
        outputFormat: ['text', 'html'],
        aiExtractRules: {
          company: {type: 'string'},
          email: {type: 'string'},
          yearFounded: {type: 'number'},
          isHiring: {type: 'boolean'}
        }
      }
    ]
  }
};

try {
  const { data } = await axios.request(options);
  console.log(data);
} catch (error) {
  console.error(error);
}

import requests

url = "https://api.hasdata.com/scrape/batch/web"

payload = { "requests": [
        {
            "url": "https://hasdata.com",
            "outputFormat": ["text", "html"],
            "aiExtractRules": {
                "company": { "type": "string" },
                "email": { "type": "string" },
                "yearFounded": { "type": "number" },
                "isHiring": { "type": "boolean" }
            }
        },
        {
            "url": "https://example.com",
            "outputFormat": ["text", "html"],
            "aiExtractRules": {
                "company": { "type": "string" },
                "email": { "type": "string" },
                "yearFounded": { "type": "number" },
                "isHiring": { "type": "boolean" }
            }
        }
    ] }
headers = {
    "Content-Type": "application/json",
    "x-api-key": "<your-api-key>"
}

response = requests.post(url, json=payload, headers=headers)

print(response.json())

<?php

$payload = [
    "requests" => [["url" => "https://hasdata.com", "outputFormat" => ["text", "html"], "aiExtractRules" => ["company" => ["type" => "string"], "email" => ["type" => "string"], "yearFounded" => ["type" => "number"], "isHiring" => ["type" => "boolean"]]], ["url" => "https://example.com", "outputFormat" => ["text", "html"], "aiExtractRules" => ["company" => ["type" => "string"], "email" => ["type" => "string"], "yearFounded" => ["type" => "number"], "isHiring" => ["type" => "boolean"]]]],
];

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.hasdata.com/scrape/batch/web",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode($payload),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json",
    "x-api-key: <your-api-key>",
  ],
]);

$response = curl_exec($curl);
curl_close($curl);

echo $response;

OkHttpClient client = new OkHttpClient();

MediaType mediaType = MediaType.parse("application/json");
String json = """
  {
    "requests": [
      {
        "url": "https://hasdata.com",
        "outputFormat": [
          "text",
          "html"
        ],
        "aiExtractRules": {
          "company": {
            "type": "string"
          },
          "email": {
            "type": "string"
          },
          "yearFounded": {
            "type": "number"
          },
          "isHiring": {
            "type": "boolean"
          }
        }
      },
      {
        "url": "https://example.com",
        "outputFormat": [
          "text",
          "html"
        ],
        "aiExtractRules": {
          "company": {
            "type": "string"
          },
          "email": {
            "type": "string"
          },
          "yearFounded": {
            "type": "number"
          },
          "isHiring": {
            "type": "boolean"
          }
        }
      }
    ]
  }
""";
RequestBody requestBody = RequestBody.create(json, mediaType);

Request request = new Request.Builder()
  .url("https://api.hasdata.com/scrape/batch/web")
  .post(requestBody)
  .addHeader("Content-Type", "application/json")
  .addHeader("x-api-key", "<your-api-key>")
  .build();

Response response = client.newCall(request).execute();

using System.Net.Http;
using System.Text;

var client = new HttpClient();

var json = """
{
  "requests": [
    {
      "url": "https://hasdata.com",
      "outputFormat": [
        "text",
        "html"
      ],
      "aiExtractRules": {
        "company": {
          "type": "string"
        },
        "email": {
          "type": "string"
        },
        "yearFounded": {
          "type": "number"
        },
        "isHiring": {
          "type": "boolean"
        }
      }
    },
    {
      "url": "https://example.com",
      "outputFormat": [
        "text",
        "html"
      ],
      "aiExtractRules": {
        "company": {
          "type": "string"
        },
        "email": {
          "type": "string"
        },
        "yearFounded": {
          "type": "number"
        },
        "isHiring": {
          "type": "boolean"
        }
      }
    }
  ]
}
""";
var content = new StringContent(json, Encoding.UTF8, "application/json");

var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.hasdata.com/scrape/batch/web")
{
    Content = content,
};
request.Headers.Add("x-api-key", "<your-api-key>");

using var response = await client.SendAsync(request);
response.EnsureSuccessStatusCode();
var body = await response.Content.ReadAsStringAsync();
Console.WriteLine(body);

require 'net/http'
require 'uri'
require 'json'

uri = URI("https://api.hasdata.com/scrape/batch/web")
payload = {
  "requests" => [{"url" => "https://hasdata.com", "outputFormat" => ["text", "html"], "aiExtractRules" => {"company" => {"type" => "string"}, "email" => {"type" => "string"}, "yearFounded" => {"type" => "number"}, "isHiring" => {"type" => "boolean"}}}, {"url" => "https://example.com", "outputFormat" => ["text", "html"], "aiExtractRules" => {"company" => {"type" => "string"}, "email" => {"type" => "string"}, "yearFounded" => {"type" => "number"}, "isHiring" => {"type" => "boolean"}}}],
}

http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = true

request = Net::HTTP::Post.new(uri)
request["Content-Type"] = 'application/json'
request["x-api-key"] = '<your-api-key>'
request.body = payload.to_json

response = http.request(request)
puts response.read_body

use reqwest::blocking::Client;
use serde_json::json;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let client = Client::new();
    let payload = json!({
        "requests": [
            {
                "url": "https://hasdata.com",
                "outputFormat": [
                    "text",
                    "html"
                ],
                "aiExtractRules": {
                    "company": {
                        "type": "string"
                    },
                    "email": {
                        "type": "string"
                    },
                    "yearFounded": {
                        "type": "number"
                    },
                    "isHiring": {
                        "type": "boolean"
                    }
                }
            },
            {
                "url": "https://example.com",
                "outputFormat": [
                    "text",
                    "html"
                ],
                "aiExtractRules": {
                    "company": {
                        "type": "string"
                    },
                    "email": {
                        "type": "string"
                    },
                    "yearFounded": {
                        "type": "number"
                    },
                    "isHiring": {
                        "type": "boolean"
                    }
                }
            }
        ]
    });
    let res = client
        .post("https://api.hasdata.com/scrape/batch/web")
        .header("Content-Type", "application/json")
        .header("x-api-key", "<your-api-key>")
        .json(&payload)
        .send()?
        .text()?;
    println!("{}", res);
    Ok(())
}

package main

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
)

func main() {
	payload := []byte(`{
  "requests": [
    {
      "url": "https://hasdata.com",
      "outputFormat": [
        "text",
        "html"
      ],
      "aiExtractRules": {
        "company": {
          "type": "string"
        },
        "email": {
          "type": "string"
        },
        "yearFounded": {
          "type": "number"
        },
        "isHiring": {
          "type": "boolean"
        }
      }
    },
    {
      "url": "https://example.com",
      "outputFormat": [
        "text",
        "html"
      ],
      "aiExtractRules": {
        "company": {
          "type": "string"
        },
        "email": {
          "type": "string"
        },
        "yearFounded": {
          "type": "number"
        },
        "isHiring": {
          "type": "boolean"
        }
      }
    }
  ]
}`)

	req, _ := http.NewRequest("POST", "https://api.hasdata.com/scrape/batch/web", bytes.NewBuffer(payload))
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("x-api-key", "<your-api-key>")

	res, _ := http.DefaultClient.Do(req)
	defer res.Body.Close()

	responseBody, _ := io.ReadAll(res.Body)
	fmt.Println(string(responseBody))
}

Response

{
  "jobId": "9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0",
  "status": "ok"
}

This means the batch job was accepted and is being processed asynchronously.

Get Job Status & Results

To check the status of your batch job:

curl --request GET \
  --url 'https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0' \
  --header 'Content-Type: application/json' \
  --header 'x-api-key: <your-api-key>'

const axios = require('axios').default;

const options = {
  method: 'GET',
  url: 'https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0',
  headers: {'Content-Type': 'application/json', 'x-api-key': '<your-api-key>'}
};

try {
  const { data } = await axios.request(options);
  console.log(data);
} catch (error) {
  console.error(error);
}

import requests

url = "https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0"

headers = {
    "Content-Type": "application/json",
    "x-api-key": "<your-api-key>"
}

response = requests.get(url, headers=headers)

print(response.json())

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json",
    "x-api-key: <your-api-key>",
  ],
]);

$response = curl_exec($curl);
curl_close($curl);

echo $response;

OkHttpClient client = new OkHttpClient();

Request request = new Request.Builder()
  .url("https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0")
  .get()
  .addHeader("Content-Type", "application/json")
  .addHeader("x-api-key", "<your-api-key>")
  .build();

Response response = client.newCall(request).execute();

using System.Net.Http;

var client = new HttpClient();

var request = new HttpRequestMessage(new HttpMethod("GET"), "https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0");
request.Headers.Add("x-api-key", "<your-api-key>");

using var response = await client.SendAsync(request);
response.EnsureSuccessStatusCode();
var content = await response.Content.ReadAsStringAsync();
Console.WriteLine(content);

require 'net/http'
require 'uri'

uri = URI("https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0")

http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = true

request = Net::HTTP::Get.new(uri)
request["Content-Type"] = 'application/json'
request["x-api-key"] = '<your-api-key>'

response = http.request(request)
puts response.read_body

use reqwest::blocking::Client;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let client = Client::new();
    let res = client
        .get("https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0")
        .header("Content-Type", "application/json")
        .header("x-api-key", "<your-api-key>")
        .send()?
        .text()?;
    println!("{}", res);
    Ok(())
}

package main

import (
	"fmt"
	"io"
	"net/http"
)

func main() {
	req, _ := http.NewRequest("GET", "https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0", nil)
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("x-api-key", "<your-api-key>")

	res, _ := http.DefaultClient.Do(req)
	defer res.Body.Close()

	body, _ := io.ReadAll(res.Body)
	fmt.Println(string(body))
}

To retrieve results once ready (supports pagination):

curl --request GET -G \
  --url 'https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results' \
  --data-urlencode 'page=1' \
  --data-urlencode 'limit=100' \
  --header 'Content-Type: application/json' \
  --header 'x-api-key: <your-api-key>'

const axios = require('axios').default;

const options = {
  method: 'GET',
  url: 'https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results',
  params: {page: '1', limit: '100'},
  headers: {'Content-Type': 'application/json', 'x-api-key': '<your-api-key>'}
};

try {
  const { data } = await axios.request(options);
  console.log(data);
} catch (error) {
  console.error(error);
}

import requests

url = "https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results"

querystring = {"page":"1","limit":"100"}

headers = {
    "Content-Type": "application/json",
    "x-api-key": "<your-api-key>"
}

response = requests.get(url, headers=headers, params=querystring)

print(response.json())

<?php

$params = [
    "page" => "1",
    "limit" => "100",
];

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results?" . http_build_query($params),
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json",
    "x-api-key: <your-api-key>",
  ],
]);

$response = curl_exec($curl);
curl_close($curl);

echo $response;

OkHttpClient client = new OkHttpClient();

HttpUrl url = HttpUrl.parse("https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results")
  .newBuilder()
  .addQueryParameter("page", "1")
  .addQueryParameter("limit", "100")
  .build();

Request request = new Request.Builder()
  .url(url)
  .get()
  .addHeader("Content-Type", "application/json")
  .addHeader("x-api-key", "<your-api-key>")
  .build();

Response response = client.newCall(request).execute();

using System.Net.Http;
using System.Web;

var client = new HttpClient();

var query = HttpUtility.ParseQueryString(string.Empty);
query["page"] = "1";
query["limit"] = "100";

var url = $"https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results?{query}";

var request = new HttpRequestMessage(new HttpMethod("GET"), url);
request.Headers.Add("x-api-key", "<your-api-key>");

using var response = await client.SendAsync(request);
response.EnsureSuccessStatusCode();
var content = await response.Content.ReadAsStringAsync();
Console.WriteLine(content);

require 'net/http'
require 'uri'

uri = URI("https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results")
params = {
  "page" => "1",
  "limit" => "100",
}
uri.query = URI.encode_www_form(params)

http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = true

request = Net::HTTP::Get.new(uri)
request["Content-Type"] = 'application/json'
request["x-api-key"] = '<your-api-key>'

response = http.request(request)
puts response.read_body

use reqwest::blocking::Client;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let client = Client::new();
    let res = client
        .get("https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results")
        .query(&[("page", "1")])
        .query(&[("limit", "100")])
        .header("Content-Type", "application/json")
        .header("x-api-key", "<your-api-key>")
        .send()?
        .text()?;
    println!("{}", res);
    Ok(())
}

package main

import (
	"fmt"
	"io"
	"net/http"
	"net/url"
)

func main() {
	params := url.Values{}
	params.Set("page", "1")
	params.Set("limit", "100")

	u := "https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results?" + params.Encode()

	req, _ := http.NewRequest("GET", u, nil)
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("x-api-key", "<your-api-key>")

	res, _ := http.DefaultClient.Do(req)
	defer res.Body.Close()

	body, _ := io.ReadAll(res.Body)
	fmt.Println(string(body))
}

Example Result

{
  "page": 0,
  "limit": 100,
  "total": 2,
  "results": [
    {
      "query": {
        "url": "https://example.com",
        "outputFormat": [
          "text",
          "html"
        ],
        "aiExtractRules": {
          "company": {
            "type": "string"
          },
          "email": {
            "type": "string"
          },
          "yearFounded": {
            "type": "number"
          },
          "isHiring": {
            "type": "boolean"
          }
        }
      },
      "result": {
        "id": "ca061bb6-64d8-462d-8ee9-16dd5aaf5efe",
        "status": "ok",
        "json": "https://files-dev.hasdata.com/ca061bb6-64d8-462d-8ee9-16dd5aaf5efe.json"
      }
    },
    {
      "query": {
        "url": "https://hasdata.com",
        "outputFormat": [
          "text",
          "html"
        ],
        "aiExtractRules": {
          "company": {
            "type": "string"
          },
          "email": {
            "type": "string"
          },
          "yearFounded": {
            "type": "number"
          },
          "isHiring": {
            "type": "boolean"
          }
        }
      },
      "result": {
        "id": "e6a75ddb-2d3f-40a5-9610-748ec3bd34a2",
        "status": "ok",
        "json": "https://files-dev.hasdata.com/e6a75ddb-2d3f-40a5-9610-748ec3bd34a2.json"
      }
    }
  ]
}

Notes

Maximum batch size: 10,000 URLs
Failed URLs do not consume credits

Get Started

Features

When to Use Batch Scraping

Submit a Batch Scrape Job

Response

Get Job Status & Results

Example Result

Notes

​When to Use Batch Scraping

​Submit a Batch Scrape Job

​Response

​Get Job Status & Results

​Example Result

​Notes

When to Use Batch Scraping

Submit a Batch Scrape Job

Response

Get Job Status & Results

Example Result

Notes