Skip to main content
Use Batch Scraping to submit up to 10,000 URLs in a single API call. This is useful when you need to extract the same type of data from a large number of pages — for example, scraping product pages or company profiles at scale. Batch Scrape works by sending an array of requests under the requests field.

​
When to Use Batch Scraping

  • Extracting titles, authors, and publish dates from a list of blog or news article URLs
  • Running aiExtractRules across a set of company websites to collect structured data like founding year, services, and contact info
  • Gathering legal notices or disclaimers from the footer pages of 5,000+ policy URLs

​
Submit a Batch Scrape Job

curl --request POST \
  --url 'https://api.hasdata.com/scrape/batch/web' \
  --header 'Content-Type: application/json' \
  --header 'x-api-key: <your-api-key>' \
  --data '{"requests":[{"url":"https://hasdata.com","outputFormat":["text","html"],"aiExtractRules":{"company":{"type":"string"},"email":{"type":"string"},"yearFounded":{"type":"number"},"isHiring":{"type":"boolean"}}},{"url":"https://example.com","outputFormat":["text","html"],"aiExtractRules":{"company":{"type":"string"},"email":{"type":"string"},"yearFounded":{"type":"number"},"isHiring":{"type":"boolean"}}}]}'

const axios = require('axios').default;

const options = {
  method: 'POST',
  url: 'https://api.hasdata.com/scrape/batch/web',
  headers: {'Content-Type': 'application/json', 'x-api-key': '<your-api-key>'},
  data: {
    requests: [
      {
        url: 'https://hasdata.com',
        outputFormat: ['text', 'html'],
        aiExtractRules: {
          company: {type: 'string'},
          email: {type: 'string'},
          yearFounded: {type: 'number'},
          isHiring: {type: 'boolean'}
        }
      },
      {
        url: 'https://example.com',
        outputFormat: ['text', 'html'],
        aiExtractRules: {
          company: {type: 'string'},
          email: {type: 'string'},
          yearFounded: {type: 'number'},
          isHiring: {type: 'boolean'}
        }
      }
    ]
  }
};

try {
  const { data } = await axios.request(options);
  console.log(data);
} catch (error) {
  console.error(error);
}

import requests

url = "https://api.hasdata.com/scrape/batch/web"

payload = { "requests": [
        {
            "url": "https://hasdata.com",
            "outputFormat": ["text", "html"],
            "aiExtractRules": {
                "company": { "type": "string" },
                "email": { "type": "string" },
                "yearFounded": { "type": "number" },
                "isHiring": { "type": "boolean" }
            }
        },
        {
            "url": "https://example.com",
            "outputFormat": ["text", "html"],
            "aiExtractRules": {
                "company": { "type": "string" },
                "email": { "type": "string" },
                "yearFounded": { "type": "number" },
                "isHiring": { "type": "boolean" }
            }
        }
    ] }
headers = {
    "Content-Type": "application/json",
    "x-api-key": "<your-api-key>"
}

response = requests.post(url, json=payload, headers=headers)

print(response.json())

<?php

$payload = [
    "requests" => [["url" => "https://hasdata.com", "outputFormat" => ["text", "html"], "aiExtractRules" => ["company" => ["type" => "string"], "email" => ["type" => "string"], "yearFounded" => ["type" => "number"], "isHiring" => ["type" => "boolean"]]], ["url" => "https://example.com", "outputFormat" => ["text", "html"], "aiExtractRules" => ["company" => ["type" => "string"], "email" => ["type" => "string"], "yearFounded" => ["type" => "number"], "isHiring" => ["type" => "boolean"]]]],
];

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.hasdata.com/scrape/batch/web",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode($payload),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json",
    "x-api-key: <your-api-key>",
  ],
]);

$response = curl_exec($curl);
curl_close($curl);

echo $response;

OkHttpClient client = new OkHttpClient();

MediaType mediaType = MediaType.parse("application/json");
String json = """
  {
    "requests": [
      {
        "url": "https://hasdata.com",
        "outputFormat": [
          "text",
          "html"
        ],
        "aiExtractRules": {
          "company": {
            "type": "string"
          },
          "email": {
            "type": "string"
          },
          "yearFounded": {
            "type": "number"
          },
          "isHiring": {
            "type": "boolean"
          }
        }
      },
      {
        "url": "https://example.com",
        "outputFormat": [
          "text",
          "html"
        ],
        "aiExtractRules": {
          "company": {
            "type": "string"
          },
          "email": {
            "type": "string"
          },
          "yearFounded": {
            "type": "number"
          },
          "isHiring": {
            "type": "boolean"
          }
        }
      }
    ]
  }
""";
RequestBody requestBody = RequestBody.create(json, mediaType);

Request request = new Request.Builder()
  .url("https://api.hasdata.com/scrape/batch/web")
  .post(requestBody)
  .addHeader("Content-Type", "application/json")
  .addHeader("x-api-key", "<your-api-key>")
  .build();

Response response = client.newCall(request).execute();

using System.Net.Http;
using System.Text;

var client = new HttpClient();

var json = """
{
  "requests": [
    {
      "url": "https://hasdata.com",
      "outputFormat": [
        "text",
        "html"
      ],
      "aiExtractRules": {
        "company": {
          "type": "string"
        },
        "email": {
          "type": "string"
        },
        "yearFounded": {
          "type": "number"
        },
        "isHiring": {
          "type": "boolean"
        }
      }
    },
    {
      "url": "https://example.com",
      "outputFormat": [
        "text",
        "html"
      ],
      "aiExtractRules": {
        "company": {
          "type": "string"
        },
        "email": {
          "type": "string"
        },
        "yearFounded": {
          "type": "number"
        },
        "isHiring": {
          "type": "boolean"
        }
      }
    }
  ]
}
""";
var content = new StringContent(json, Encoding.UTF8, "application/json");

var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.hasdata.com/scrape/batch/web")
{
    Content = content,
};
request.Headers.Add("x-api-key", "<your-api-key>");

using var response = await client.SendAsync(request);
response.EnsureSuccessStatusCode();
var body = await response.Content.ReadAsStringAsync();
Console.WriteLine(body);

require 'net/http'
require 'uri'
require 'json'

uri = URI("https://api.hasdata.com/scrape/batch/web")
payload = {
  "requests" => [{"url" => "https://hasdata.com", "outputFormat" => ["text", "html"], "aiExtractRules" => {"company" => {"type" => "string"}, "email" => {"type" => "string"}, "yearFounded" => {"type" => "number"}, "isHiring" => {"type" => "boolean"}}}, {"url" => "https://example.com", "outputFormat" => ["text", "html"], "aiExtractRules" => {"company" => {"type" => "string"}, "email" => {"type" => "string"}, "yearFounded" => {"type" => "number"}, "isHiring" => {"type" => "boolean"}}}],
}

http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = true

request = Net::HTTP::Post.new(uri)
request["Content-Type"] = 'application/json'
request["x-api-key"] = '<your-api-key>'
request.body = payload.to_json

response = http.request(request)
puts response.read_body

use reqwest::blocking::Client;
use serde_json::json;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let client = Client::new();
    let payload = json!({
        "requests": [
            {
                "url": "https://hasdata.com",
                "outputFormat": [
                    "text",
                    "html"
                ],
                "aiExtractRules": {
                    "company": {
                        "type": "string"
                    },
                    "email": {
                        "type": "string"
                    },
                    "yearFounded": {
                        "type": "number"
                    },
                    "isHiring": {
                        "type": "boolean"
                    }
                }
            },
            {
                "url": "https://example.com",
                "outputFormat": [
                    "text",
                    "html"
                ],
                "aiExtractRules": {
                    "company": {
                        "type": "string"
                    },
                    "email": {
                        "type": "string"
                    },
                    "yearFounded": {
                        "type": "number"
                    },
                    "isHiring": {
                        "type": "boolean"
                    }
                }
            }
        ]
    });
    let res = client
        .post("https://api.hasdata.com/scrape/batch/web")
        .header("Content-Type", "application/json")
        .header("x-api-key", "<your-api-key>")
        .json(&payload)
        .send()?
        .text()?;
    println!("{}", res);
    Ok(())
}

package main

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
)

func main() {
	payload := []byte(`{
  "requests": [
    {
      "url": "https://hasdata.com",
      "outputFormat": [
        "text",
        "html"
      ],
      "aiExtractRules": {
        "company": {
          "type": "string"
        },
        "email": {
          "type": "string"
        },
        "yearFounded": {
          "type": "number"
        },
        "isHiring": {
          "type": "boolean"
        }
      }
    },
    {
      "url": "https://example.com",
      "outputFormat": [
        "text",
        "html"
      ],
      "aiExtractRules": {
        "company": {
          "type": "string"
        },
        "email": {
          "type": "string"
        },
        "yearFounded": {
          "type": "number"
        },
        "isHiring": {
          "type": "boolean"
        }
      }
    }
  ]
}`)

	req, _ := http.NewRequest("POST", "https://api.hasdata.com/scrape/batch/web", bytes.NewBuffer(payload))
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("x-api-key", "<your-api-key>")

	res, _ := http.DefaultClient.Do(req)
	defer res.Body.Close()

	responseBody, _ := io.ReadAll(res.Body)
	fmt.Println(string(responseBody))
}

​
Response

{
  "jobId": "9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0",
  "status": "ok"
}
This means the batch job was accepted and is being processed asynchronously.

​
Get Job Status & Results

To check the status of your batch job: 
curl --request GET \
  --url 'https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0' \
  --header 'Content-Type: application/json' \
  --header 'x-api-key: <your-api-key>'

const axios = require('axios').default;

const options = {
  method: 'GET',
  url: 'https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0',
  headers: {'Content-Type': 'application/json', 'x-api-key': '<your-api-key>'}
};

try {
  const { data } = await axios.request(options);
  console.log(data);
} catch (error) {
  console.error(error);
}

import requests

url = "https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0"

headers = {
    "Content-Type": "application/json",
    "x-api-key": "<your-api-key>"
}

response = requests.get(url, headers=headers)

print(response.json())

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json",
    "x-api-key: <your-api-key>",
  ],
]);

$response = curl_exec($curl);
curl_close($curl);

echo $response;

OkHttpClient client = new OkHttpClient();

Request request = new Request.Builder()
  .url("https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0")
  .get()
  .addHeader("Content-Type", "application/json")
  .addHeader("x-api-key", "<your-api-key>")
  .build();

Response response = client.newCall(request).execute();

using System.Net.Http;

var client = new HttpClient();

var request = new HttpRequestMessage(new HttpMethod("GET"), "https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0");
request.Headers.Add("x-api-key", "<your-api-key>");

using var response = await client.SendAsync(request);
response.EnsureSuccessStatusCode();
var content = await response.Content.ReadAsStringAsync();
Console.WriteLine(content);

require 'net/http'
require 'uri'

uri = URI("https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0")

http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = true

request = Net::HTTP::Get.new(uri)
request["Content-Type"] = 'application/json'
request["x-api-key"] = '<your-api-key>'

response = http.request(request)
puts response.read_body

use reqwest::blocking::Client;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let client = Client::new();
    let res = client
        .get("https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0")
        .header("Content-Type", "application/json")
        .header("x-api-key", "<your-api-key>")
        .send()?
        .text()?;
    println!("{}", res);
    Ok(())
}

package main

import (
	"fmt"
	"io"
	"net/http"
)

func main() {
	req, _ := http.NewRequest("GET", "https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0", nil)
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("x-api-key", "<your-api-key>")

	res, _ := http.DefaultClient.Do(req)
	defer res.Body.Close()

	body, _ := io.ReadAll(res.Body)
	fmt.Println(string(body))
}
To retrieve results once ready (supports pagination): 
curl --request GET -G \
  --url 'https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results' \
  --data-urlencode 'page=1' \
  --data-urlencode 'limit=100' \
  --header 'Content-Type: application/json' \
  --header 'x-api-key: <your-api-key>'

const axios = require('axios').default;

const options = {
  method: 'GET',
  url: 'https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results',
  params: {page: '1', limit: '100'},
  headers: {'Content-Type': 'application/json', 'x-api-key': '<your-api-key>'}
};

try {
  const { data } = await axios.request(options);
  console.log(data);
} catch (error) {
  console.error(error);
}

import requests

url = "https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results"

querystring = {"page":"1","limit":"100"}

headers = {
    "Content-Type": "application/json",
    "x-api-key": "<your-api-key>"
}

response = requests.get(url, headers=headers, params=querystring)

print(response.json())

<?php

$params = [
    "page" => "1",
    "limit" => "100",
];

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results?" . http_build_query($params),
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json",
    "x-api-key: <your-api-key>",
  ],
]);

$response = curl_exec($curl);
curl_close($curl);

echo $response;

OkHttpClient client = new OkHttpClient();

HttpUrl url = HttpUrl.parse("https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results")
  .newBuilder()
  .addQueryParameter("page", "1")
  .addQueryParameter("limit", "100")
  .build();

Request request = new Request.Builder()
  .url(url)
  .get()
  .addHeader("Content-Type", "application/json")
  .addHeader("x-api-key", "<your-api-key>")
  .build();

Response response = client.newCall(request).execute();

using System.Net.Http;
using System.Web;

var client = new HttpClient();

var query = HttpUtility.ParseQueryString(string.Empty);
query["page"] = "1";
query["limit"] = "100";

var url = $"https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results?{query}";

var request = new HttpRequestMessage(new HttpMethod("GET"), url);
request.Headers.Add("x-api-key", "<your-api-key>");

using var response = await client.SendAsync(request);
response.EnsureSuccessStatusCode();
var content = await response.Content.ReadAsStringAsync();
Console.WriteLine(content);

require 'net/http'
require 'uri'

uri = URI("https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results")
params = {
  "page" => "1",
  "limit" => "100",
}
uri.query = URI.encode_www_form(params)

http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = true

request = Net::HTTP::Get.new(uri)
request["Content-Type"] = 'application/json'
request["x-api-key"] = '<your-api-key>'

response = http.request(request)
puts response.read_body

use reqwest::blocking::Client;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let client = Client::new();
    let res = client
        .get("https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results")
        .query(&[("page", "1")])
        .query(&[("limit", "100")])
        .header("Content-Type", "application/json")
        .header("x-api-key", "<your-api-key>")
        .send()?
        .text()?;
    println!("{}", res);
    Ok(())
}

package main

import (
	"fmt"
	"io"
	"net/http"
	"net/url"
)

func main() {
	params := url.Values{}
	params.Set("page", "1")
	params.Set("limit", "100")

	u := "https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results?" + params.Encode()

	req, _ := http.NewRequest("GET", u, nil)
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("x-api-key", "<your-api-key>")

	res, _ := http.DefaultClient.Do(req)
	defer res.Body.Close()

	body, _ := io.ReadAll(res.Body)
	fmt.Println(string(body))
}

​
Example Result

{
  "page": 0,
  "limit": 100,
  "total": 2,
  "results": [
    {
      "query": {
        "url": "https://example.com",
        "outputFormat": [
          "text",
          "html"
        ],
        "aiExtractRules": {
          "company": {
            "type": "string"
          },
          "email": {
            "type": "string"
          },
          "yearFounded": {
            "type": "number"
          },
          "isHiring": {
            "type": "boolean"
          }
        }
      },
      "result": {
        "id": "ca061bb6-64d8-462d-8ee9-16dd5aaf5efe",
        "status": "ok",
        "json": "https://files-dev.hasdata.com/ca061bb6-64d8-462d-8ee9-16dd5aaf5efe.json"
      }
    },
    {
      "query": {
        "url": "https://hasdata.com",
        "outputFormat": [
          "text",
          "html"
        ],
        "aiExtractRules": {
          "company": {
            "type": "string"
          },
          "email": {
            "type": "string"
          },
          "yearFounded": {
            "type": "number"
          },
          "isHiring": {
            "type": "boolean"
          }
        }
      },
      "result": {
        "id": "e6a75ddb-2d3f-40a5-9610-748ec3bd34a2",
        "status": "ok",
        "json": "https://files-dev.hasdata.com/e6a75ddb-2d3f-40a5-9610-748ec3bd34a2.json"
      }
    }
  ]
}

​
Notes

  • Maximum batch size: 10,000 URLs
  • Failed URLs do not consume credits