> ## Documentation Index
> Fetch the complete documentation index at: https://docs.hasdata.com/llms.txt
> Use this file to discover all available pages before exploring further.

# Batch Scrape

Use **Batch Scraping** to submit up to **10,000 URLs** in a single API call. This is useful when you need to extract the **same type of data** from a large number of pages — for example, scraping product pages or company profiles at scale.

Batch Scrape works by sending an array of requests under the `requests` field.

## When to Use Batch Scraping

* Extracting titles, authors, and publish dates from a list of blog or news article URLs
* Running `aiExtractRules` across a set of company websites to collect structured data like founding year, services, and contact info
* Gathering legal notices or disclaimers from the footer pages of 5,000+ policy URLs

## Submit a Batch Scrape Job

<CodeGroup>
  ```bash cURL theme={null}
  curl --request POST \
    --url 'https://api.hasdata.com/scrape/batch/web' \
    --header 'Content-Type: application/json' \
    --header 'x-api-key: <your-api-key>' \
    --data '{"requests":[{"url":"https://hasdata.com","outputFormat":["text","html"],"aiExtractRules":{"company":{"type":"string"},"email":{"type":"string"},"yearFounded":{"type":"number"},"isHiring":{"type":"boolean"}}},{"url":"https://example.com","outputFormat":["text","html"],"aiExtractRules":{"company":{"type":"string"},"email":{"type":"string"},"yearFounded":{"type":"number"},"isHiring":{"type":"boolean"}}}]}'
  ```

  ```javascript Node.js theme={null}
  const axios = require('axios').default;

  const options = {
    method: 'POST',
    url: 'https://api.hasdata.com/scrape/batch/web',
    headers: {'Content-Type': 'application/json', 'x-api-key': '<your-api-key>'},
    data: {
      requests: [
        {
          url: 'https://hasdata.com',
          outputFormat: ['text', 'html'],
          aiExtractRules: {
            company: {type: 'string'},
            email: {type: 'string'},
            yearFounded: {type: 'number'},
            isHiring: {type: 'boolean'}
          }
        },
        {
          url: 'https://example.com',
          outputFormat: ['text', 'html'],
          aiExtractRules: {
            company: {type: 'string'},
            email: {type: 'string'},
            yearFounded: {type: 'number'},
            isHiring: {type: 'boolean'}
          }
        }
      ]
    }
  };

  try {
    const { data } = await axios.request(options);
    console.log(data);
  } catch (error) {
    console.error(error);
  }
  ```

  ```python Python theme={null}
  import requests

  url = "https://api.hasdata.com/scrape/batch/web"

  payload = { "requests": [
          {
              "url": "https://hasdata.com",
              "outputFormat": ["text", "html"],
              "aiExtractRules": {
                  "company": { "type": "string" },
                  "email": { "type": "string" },
                  "yearFounded": { "type": "number" },
                  "isHiring": { "type": "boolean" }
              }
          },
          {
              "url": "https://example.com",
              "outputFormat": ["text", "html"],
              "aiExtractRules": {
                  "company": { "type": "string" },
                  "email": { "type": "string" },
                  "yearFounded": { "type": "number" },
                  "isHiring": { "type": "boolean" }
              }
          }
      ] }
  headers = {
      "Content-Type": "application/json",
      "x-api-key": "<your-api-key>"
  }

  response = requests.post(url, json=payload, headers=headers)

  print(response.json())
  ```

  ```php PHP theme={null}
  <?php

  $payload = [
      "requests" => [["url" => "https://hasdata.com", "outputFormat" => ["text", "html"], "aiExtractRules" => ["company" => ["type" => "string"], "email" => ["type" => "string"], "yearFounded" => ["type" => "number"], "isHiring" => ["type" => "boolean"]]], ["url" => "https://example.com", "outputFormat" => ["text", "html"], "aiExtractRules" => ["company" => ["type" => "string"], "email" => ["type" => "string"], "yearFounded" => ["type" => "number"], "isHiring" => ["type" => "boolean"]]]],
  ];

  $curl = curl_init();

  curl_setopt_array($curl, [
    CURLOPT_URL => "https://api.hasdata.com/scrape/batch/web",
    CURLOPT_RETURNTRANSFER => true,
    CURLOPT_CUSTOMREQUEST => "POST",
    CURLOPT_POSTFIELDS => json_encode($payload),
    CURLOPT_HTTPHEADER => [
      "Content-Type: application/json",
      "x-api-key: <your-api-key>",
    ],
  ]);

  $response = curl_exec($curl);
  curl_close($curl);

  echo $response;
  ```

  ```java Java theme={null}
  OkHttpClient client = new OkHttpClient();

  MediaType mediaType = MediaType.parse("application/json");
  String json = """
    {
      "requests": [
        {
          "url": "https://hasdata.com",
          "outputFormat": [
            "text",
            "html"
          ],
          "aiExtractRules": {
            "company": {
              "type": "string"
            },
            "email": {
              "type": "string"
            },
            "yearFounded": {
              "type": "number"
            },
            "isHiring": {
              "type": "boolean"
            }
          }
        },
        {
          "url": "https://example.com",
          "outputFormat": [
            "text",
            "html"
          ],
          "aiExtractRules": {
            "company": {
              "type": "string"
            },
            "email": {
              "type": "string"
            },
            "yearFounded": {
              "type": "number"
            },
            "isHiring": {
              "type": "boolean"
            }
          }
        }
      ]
    }
  """;
  RequestBody requestBody = RequestBody.create(json, mediaType);

  Request request = new Request.Builder()
    .url("https://api.hasdata.com/scrape/batch/web")
    .post(requestBody)
    .addHeader("Content-Type", "application/json")
    .addHeader("x-api-key", "<your-api-key>")
    .build();

  Response response = client.newCall(request).execute();
  ```

  ```csharp C# theme={null}
  using System.Net.Http;
  using System.Text;

  var client = new HttpClient();

  var json = """
  {
    "requests": [
      {
        "url": "https://hasdata.com",
        "outputFormat": [
          "text",
          "html"
        ],
        "aiExtractRules": {
          "company": {
            "type": "string"
          },
          "email": {
            "type": "string"
          },
          "yearFounded": {
            "type": "number"
          },
          "isHiring": {
            "type": "boolean"
          }
        }
      },
      {
        "url": "https://example.com",
        "outputFormat": [
          "text",
          "html"
        ],
        "aiExtractRules": {
          "company": {
            "type": "string"
          },
          "email": {
            "type": "string"
          },
          "yearFounded": {
            "type": "number"
          },
          "isHiring": {
            "type": "boolean"
          }
        }
      }
    ]
  }
  """;
  var content = new StringContent(json, Encoding.UTF8, "application/json");

  var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.hasdata.com/scrape/batch/web")
  {
      Content = content,
  };
  request.Headers.Add("x-api-key", "<your-api-key>");

  using var response = await client.SendAsync(request);
  response.EnsureSuccessStatusCode();
  var body = await response.Content.ReadAsStringAsync();
  Console.WriteLine(body);
  ```

  ```ruby Ruby theme={null}
  require 'net/http'
  require 'uri'
  require 'json'

  uri = URI("https://api.hasdata.com/scrape/batch/web")
  payload = {
    "requests" => [{"url" => "https://hasdata.com", "outputFormat" => ["text", "html"], "aiExtractRules" => {"company" => {"type" => "string"}, "email" => {"type" => "string"}, "yearFounded" => {"type" => "number"}, "isHiring" => {"type" => "boolean"}}}, {"url" => "https://example.com", "outputFormat" => ["text", "html"], "aiExtractRules" => {"company" => {"type" => "string"}, "email" => {"type" => "string"}, "yearFounded" => {"type" => "number"}, "isHiring" => {"type" => "boolean"}}}],
  }

  http = Net::HTTP.new(uri.host, uri.port)
  http.use_ssl = true

  request = Net::HTTP::Post.new(uri)
  request["Content-Type"] = 'application/json'
  request["x-api-key"] = '<your-api-key>'
  request.body = payload.to_json

  response = http.request(request)
  puts response.read_body
  ```

  ```rust Rust theme={null}
  use reqwest::blocking::Client;
  use serde_json::json;

  fn main() -> Result<(), Box<dyn std::error::Error>> {
      let client = Client::new();
      let payload = json!({
          "requests": [
              {
                  "url": "https://hasdata.com",
                  "outputFormat": [
                      "text",
                      "html"
                  ],
                  "aiExtractRules": {
                      "company": {
                          "type": "string"
                      },
                      "email": {
                          "type": "string"
                      },
                      "yearFounded": {
                          "type": "number"
                      },
                      "isHiring": {
                          "type": "boolean"
                      }
                  }
              },
              {
                  "url": "https://example.com",
                  "outputFormat": [
                      "text",
                      "html"
                  ],
                  "aiExtractRules": {
                      "company": {
                          "type": "string"
                      },
                      "email": {
                          "type": "string"
                      },
                      "yearFounded": {
                          "type": "number"
                      },
                      "isHiring": {
                          "type": "boolean"
                      }
                  }
              }
          ]
      });
      let res = client
          .post("https://api.hasdata.com/scrape/batch/web")
          .header("Content-Type", "application/json")
          .header("x-api-key", "<your-api-key>")
          .json(&payload)
          .send()?
          .text()?;
      println!("{}", res);
      Ok(())
  }
  ```

  ```go Go theme={null}
  package main

  import (
  	"bytes"
  	"fmt"
  	"io"
  	"net/http"
  )

  func main() {
  	payload := []byte(`{
    "requests": [
      {
        "url": "https://hasdata.com",
        "outputFormat": [
          "text",
          "html"
        ],
        "aiExtractRules": {
          "company": {
            "type": "string"
          },
          "email": {
            "type": "string"
          },
          "yearFounded": {
            "type": "number"
          },
          "isHiring": {
            "type": "boolean"
          }
        }
      },
      {
        "url": "https://example.com",
        "outputFormat": [
          "text",
          "html"
        ],
        "aiExtractRules": {
          "company": {
            "type": "string"
          },
          "email": {
            "type": "string"
          },
          "yearFounded": {
            "type": "number"
          },
          "isHiring": {
            "type": "boolean"
          }
        }
      }
    ]
  }`)

  	req, _ := http.NewRequest("POST", "https://api.hasdata.com/scrape/batch/web", bytes.NewBuffer(payload))
  	req.Header.Add("Content-Type", "application/json")
  	req.Header.Add("x-api-key", "<your-api-key>")

  	res, _ := http.DefaultClient.Do(req)
  	defer res.Body.Close()

  	responseBody, _ := io.ReadAll(res.Body)
  	fmt.Println(string(responseBody))
  }
  ```
</CodeGroup>

## Response

```json theme={null}
{
  "jobId": "9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0",
  "status": "ok"
}
```

This means the batch job was accepted and is being processed asynchronously.

## Get Job Status & Results

To check the status of your batch job:

<CodeGroup>
  ```bash cURL theme={null}
  curl --request GET \
    --url 'https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0' \
    --header 'Content-Type: application/json' \
    --header 'x-api-key: <your-api-key>'
  ```

  ```javascript Node.js theme={null}
  const axios = require('axios').default;

  const options = {
    method: 'GET',
    url: 'https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0',
    headers: {'Content-Type': 'application/json', 'x-api-key': '<your-api-key>'}
  };

  try {
    const { data } = await axios.request(options);
    console.log(data);
  } catch (error) {
    console.error(error);
  }
  ```

  ```python Python theme={null}
  import requests

  url = "https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0"

  headers = {
      "Content-Type": "application/json",
      "x-api-key": "<your-api-key>"
  }

  response = requests.get(url, headers=headers)

  print(response.json())
  ```

  ```php PHP theme={null}
  <?php

  $curl = curl_init();

  curl_setopt_array($curl, [
    CURLOPT_URL => "https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0",
    CURLOPT_RETURNTRANSFER => true,
    CURLOPT_CUSTOMREQUEST => "GET",
    CURLOPT_HTTPHEADER => [
      "Content-Type: application/json",
      "x-api-key: <your-api-key>",
    ],
  ]);

  $response = curl_exec($curl);
  curl_close($curl);

  echo $response;
  ```

  ```java Java theme={null}
  OkHttpClient client = new OkHttpClient();

  Request request = new Request.Builder()
    .url("https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0")
    .get()
    .addHeader("Content-Type", "application/json")
    .addHeader("x-api-key", "<your-api-key>")
    .build();

  Response response = client.newCall(request).execute();
  ```

  ```csharp C# theme={null}
  using System.Net.Http;

  var client = new HttpClient();

  var request = new HttpRequestMessage(new HttpMethod("GET"), "https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0");
  request.Headers.Add("x-api-key", "<your-api-key>");

  using var response = await client.SendAsync(request);
  response.EnsureSuccessStatusCode();
  var content = await response.Content.ReadAsStringAsync();
  Console.WriteLine(content);
  ```

  ```ruby Ruby theme={null}
  require 'net/http'
  require 'uri'

  uri = URI("https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0")

  http = Net::HTTP.new(uri.host, uri.port)
  http.use_ssl = true

  request = Net::HTTP::Get.new(uri)
  request["Content-Type"] = 'application/json'
  request["x-api-key"] = '<your-api-key>'

  response = http.request(request)
  puts response.read_body
  ```

  ```rust Rust theme={null}
  use reqwest::blocking::Client;

  fn main() -> Result<(), Box<dyn std::error::Error>> {
      let client = Client::new();
      let res = client
          .get("https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0")
          .header("Content-Type", "application/json")
          .header("x-api-key", "<your-api-key>")
          .send()?
          .text()?;
      println!("{}", res);
      Ok(())
  }
  ```

  ```go Go theme={null}
  package main

  import (
  	"fmt"
  	"io"
  	"net/http"
  )

  func main() {
  	req, _ := http.NewRequest("GET", "https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0", nil)
  	req.Header.Add("Content-Type", "application/json")
  	req.Header.Add("x-api-key", "<your-api-key>")

  	res, _ := http.DefaultClient.Do(req)
  	defer res.Body.Close()

  	body, _ := io.ReadAll(res.Body)
  	fmt.Println(string(body))
  }
  ```
</CodeGroup>

To retrieve results once ready (supports pagination):

<CodeGroup>
  ```bash cURL theme={null}
  curl --request GET -G \
    --url 'https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results' \
    --data-urlencode 'page=1' \
    --data-urlencode 'limit=100' \
    --header 'Content-Type: application/json' \
    --header 'x-api-key: <your-api-key>'
  ```

  ```javascript Node.js theme={null}
  const axios = require('axios').default;

  const options = {
    method: 'GET',
    url: 'https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results',
    params: {page: '1', limit: '100'},
    headers: {'Content-Type': 'application/json', 'x-api-key': '<your-api-key>'}
  };

  try {
    const { data } = await axios.request(options);
    console.log(data);
  } catch (error) {
    console.error(error);
  }
  ```

  ```python Python theme={null}
  import requests

  url = "https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results"

  querystring = {"page":"1","limit":"100"}

  headers = {
      "Content-Type": "application/json",
      "x-api-key": "<your-api-key>"
  }

  response = requests.get(url, headers=headers, params=querystring)

  print(response.json())
  ```

  ```php PHP theme={null}
  <?php

  $params = [
      "page" => "1",
      "limit" => "100",
  ];

  $curl = curl_init();

  curl_setopt_array($curl, [
    CURLOPT_URL => "https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results?" . http_build_query($params),
    CURLOPT_RETURNTRANSFER => true,
    CURLOPT_CUSTOMREQUEST => "GET",
    CURLOPT_HTTPHEADER => [
      "Content-Type: application/json",
      "x-api-key: <your-api-key>",
    ],
  ]);

  $response = curl_exec($curl);
  curl_close($curl);

  echo $response;
  ```

  ```java Java theme={null}
  OkHttpClient client = new OkHttpClient();

  HttpUrl url = HttpUrl.parse("https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results")
    .newBuilder()
    .addQueryParameter("page", "1")
    .addQueryParameter("limit", "100")
    .build();

  Request request = new Request.Builder()
    .url(url)
    .get()
    .addHeader("Content-Type", "application/json")
    .addHeader("x-api-key", "<your-api-key>")
    .build();

  Response response = client.newCall(request).execute();
  ```

  ```csharp C# theme={null}
  using System.Net.Http;
  using System.Web;

  var client = new HttpClient();

  var query = HttpUtility.ParseQueryString(string.Empty);
  query["page"] = "1";
  query["limit"] = "100";

  var url = $"https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results?{query}";

  var request = new HttpRequestMessage(new HttpMethod("GET"), url);
  request.Headers.Add("x-api-key", "<your-api-key>");

  using var response = await client.SendAsync(request);
  response.EnsureSuccessStatusCode();
  var content = await response.Content.ReadAsStringAsync();
  Console.WriteLine(content);
  ```

  ```ruby Ruby theme={null}
  require 'net/http'
  require 'uri'

  uri = URI("https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results")
  params = {
    "page" => "1",
    "limit" => "100",
  }
  uri.query = URI.encode_www_form(params)

  http = Net::HTTP.new(uri.host, uri.port)
  http.use_ssl = true

  request = Net::HTTP::Get.new(uri)
  request["Content-Type"] = 'application/json'
  request["x-api-key"] = '<your-api-key>'

  response = http.request(request)
  puts response.read_body
  ```

  ```rust Rust theme={null}
  use reqwest::blocking::Client;

  fn main() -> Result<(), Box<dyn std::error::Error>> {
      let client = Client::new();
      let res = client
          .get("https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results")
          .query(&[("page", "1")])
          .query(&[("limit", "100")])
          .header("Content-Type", "application/json")
          .header("x-api-key", "<your-api-key>")
          .send()?
          .text()?;
      println!("{}", res);
      Ok(())
  }
  ```

  ```go Go theme={null}
  package main

  import (
  	"fmt"
  	"io"
  	"net/http"
  	"net/url"
  )

  func main() {
  	params := url.Values{}
  	params.Set("page", "1")
  	params.Set("limit", "100")

  	u := "https://api.hasdata.com/scrape/batch/web/9a35f32e-4f9c-4d49-9c6e-7c4de4a091e0/results?" + params.Encode()

  	req, _ := http.NewRequest("GET", u, nil)
  	req.Header.Add("Content-Type", "application/json")
  	req.Header.Add("x-api-key", "<your-api-key>")

  	res, _ := http.DefaultClient.Do(req)
  	defer res.Body.Close()

  	body, _ := io.ReadAll(res.Body)
  	fmt.Println(string(body))
  }
  ```
</CodeGroup>

## Example Result

```json theme={null}
{
  "page": 0,
  "limit": 100,
  "total": 2,
  "results": [
    {
      "query": {
        "url": "https://example.com",
        "outputFormat": [
          "text",
          "html"
        ],
        "aiExtractRules": {
          "company": {
            "type": "string"
          },
          "email": {
            "type": "string"
          },
          "yearFounded": {
            "type": "number"
          },
          "isHiring": {
            "type": "boolean"
          }
        }
      },
      "result": {
        "id": "ca061bb6-64d8-462d-8ee9-16dd5aaf5efe",
        "status": "ok",
        "json": "https://files-dev.hasdata.com/ca061bb6-64d8-462d-8ee9-16dd5aaf5efe.json"
      }
    },
    {
      "query": {
        "url": "https://hasdata.com",
        "outputFormat": [
          "text",
          "html"
        ],
        "aiExtractRules": {
          "company": {
            "type": "string"
          },
          "email": {
            "type": "string"
          },
          "yearFounded": {
            "type": "number"
          },
          "isHiring": {
            "type": "boolean"
          }
        }
      },
      "result": {
        "id": "e6a75ddb-2d3f-40a5-9610-748ec3bd34a2",
        "status": "ok",
        "json": "https://files-dev.hasdata.com/e6a75ddb-2d3f-40a5-9610-748ec3bd34a2.json"
      }
    }
  ]
}
```

## Notes

* Maximum batch size: **10,000 URLs**
* Failed URLs do **not** consume credits