Quickstart - Websites Crawler - HasData Documentation

The Websites Crawler lets you crawl and extract content from multiple pages of a website by following internal links. You submit one or more starting URLs and define how deep the crawler should go using maxDepth. You can also limit which paths should be followed using regex with includePaths. This scraper job is asynchronous. You’ll receive a jobId, and results can be fetched via polling or delivered to a webhook.

Example Request

curl --request POST \
  --url 'https://api.hasdata.com/scrapers/crawler/jobs' \
  --header 'Content-Type: application/json' \
  --header 'x-api-key: <your-api-key>' \
  --data '{"urls":["https://example.com"],"maxDepth":3,"includePaths":"(blog/.+|articles/.+)","outputFormat":["text","json"],"webhook":{"url":"https://yourdomain.com/webhook","events":["scraper.job.started","scraper.job.finished","scraper.data.scraped"]}}'

const axios = require('axios').default;

const options = {
  method: 'POST',
  url: 'https://api.hasdata.com/scrapers/crawler/jobs',
  headers: {'Content-Type': 'application/json', 'x-api-key': '<your-api-key>'},
  data: {
    urls: ['https://example.com'],
    maxDepth: 3,
    includePaths: '(blog/.+|articles/.+)',
    outputFormat: ['text', 'json'],
    webhook: {
      url: 'https://yourdomain.com/webhook',
      events: ['scraper.job.started', 'scraper.job.finished', 'scraper.data.scraped']
    }
  }
};

try {
  const { data } = await axios.request(options);
  console.log(data);
} catch (error) {
  console.error(error);
}

import requests

url = "https://api.hasdata.com/scrapers/crawler/jobs"

payload = {
    "urls": ["https://example.com"],
    "maxDepth": 3,
    "includePaths": "(blog/.+|articles/.+)",
    "outputFormat": ["text", "json"],
    "webhook": {
        "url": "https://yourdomain.com/webhook",
        "events": ["scraper.job.started", "scraper.job.finished", "scraper.data.scraped"]
    }
}
headers = {
    "Content-Type": "application/json",
    "x-api-key": "<your-api-key>"
}

response = requests.post(url, json=payload, headers=headers)

print(response.json())

<?php

$payload = [
    "urls" => ["https://example.com"],
    "maxDepth" => 3,
    "includePaths" => "(blog/.+|articles/.+)",
    "outputFormat" => ["text", "json"],
    "webhook" => ["url" => "https://yourdomain.com/webhook", "events" => ["scraper.job.started", "scraper.job.finished", "scraper.data.scraped"]],
];

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.hasdata.com/scrapers/crawler/jobs",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode($payload),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json",
    "x-api-key: <your-api-key>",
  ],
]);

$response = curl_exec($curl);
curl_close($curl);

echo $response;

OkHttpClient client = new OkHttpClient();

MediaType mediaType = MediaType.parse("application/json");
String json = """
  {
    "urls": [
      "https://example.com"
    ],
    "maxDepth": 3,
    "includePaths": "(blog/.+|articles/.+)",
    "outputFormat": [
      "text",
      "json"
    ],
    "webhook": {
      "url": "https://yourdomain.com/webhook",
      "events": [
        "scraper.job.started",
        "scraper.job.finished",
        "scraper.data.scraped"
      ]
    }
  }
""";
RequestBody requestBody = RequestBody.create(json, mediaType);

Request request = new Request.Builder()
  .url("https://api.hasdata.com/scrapers/crawler/jobs")
  .post(requestBody)
  .addHeader("Content-Type", "application/json")
  .addHeader("x-api-key", "<your-api-key>")
  .build();

Response response = client.newCall(request).execute();

using System.Net.Http;
using System.Text;

var client = new HttpClient();

var json = """
{
  "urls": [
    "https://example.com"
  ],
  "maxDepth": 3,
  "includePaths": "(blog/.+|articles/.+)",
  "outputFormat": [
    "text",
    "json"
  ],
  "webhook": {
    "url": "https://yourdomain.com/webhook",
    "events": [
      "scraper.job.started",
      "scraper.job.finished",
      "scraper.data.scraped"
    ]
  }
}
""";
var content = new StringContent(json, Encoding.UTF8, "application/json");

var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.hasdata.com/scrapers/crawler/jobs")
{
    Content = content,
};
request.Headers.Add("x-api-key", "<your-api-key>");

using var response = await client.SendAsync(request);
response.EnsureSuccessStatusCode();
var body = await response.Content.ReadAsStringAsync();
Console.WriteLine(body);

require 'net/http'
require 'uri'
require 'json'

uri = URI("https://api.hasdata.com/scrapers/crawler/jobs")
payload = {
  "urls" => ["https://example.com"],
  "maxDepth" => 3,
  "includePaths" => "(blog/.+|articles/.+)",
  "outputFormat" => ["text", "json"],
  "webhook" => {"url" => "https://yourdomain.com/webhook", "events" => ["scraper.job.started", "scraper.job.finished", "scraper.data.scraped"]},
}

http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = true

request = Net::HTTP::Post.new(uri)
request["Content-Type"] = 'application/json'
request["x-api-key"] = '<your-api-key>'
request.body = payload.to_json

response = http.request(request)
puts response.read_body

use reqwest::blocking::Client;
use serde_json::json;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let client = Client::new();
    let payload = json!({
        "urls": [
            "https://example.com"
        ],
        "maxDepth": 3,
        "includePaths": "(blog/.+|articles/.+)",
        "outputFormat": [
            "text",
            "json"
        ],
        "webhook": {
            "url": "https://yourdomain.com/webhook",
            "events": [
                "scraper.job.started",
                "scraper.job.finished",
                "scraper.data.scraped"
            ]
        }
    });
    let res = client
        .post("https://api.hasdata.com/scrapers/crawler/jobs")
        .header("Content-Type", "application/json")
        .header("x-api-key", "<your-api-key>")
        .json(&payload)
        .send()?
        .text()?;
    println!("{}", res);
    Ok(())
}

package main

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
)

func main() {
	payload := []byte(`{
  "urls": [
    "https://example.com"
  ],
  "maxDepth": 3,
  "includePaths": "(blog/.+|articles/.+)",
  "outputFormat": [
    "text",
    "json"
  ],
  "webhook": {
    "url": "https://yourdomain.com/webhook",
    "events": [
      "scraper.job.started",
      "scraper.job.finished",
      "scraper.data.scraped"
    ]
  }
}`)

	req, _ := http.NewRequest("POST", "https://api.hasdata.com/scrapers/crawler/jobs", bytes.NewBuffer(payload))
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("x-api-key", "<your-api-key>")

	res, _ := http.DefaultClient.Do(req)
	defer res.Body.Close()

	responseBody, _ := io.ReadAll(res.Body)
	fmt.Println(string(responseBody))
}

Use Web Scraping API Params

You can use any parameters from the Web Scraping API inside a Websites Crawler job — including:

extractRules
aiExtractRules
headers
proxyType / proxyCountry
blockResources, jsScenario, outputFormat, and more

All parameters are applied to each crawled page individually.

Get Scraper Job Status

To get the status of an existing scraper job, make a GET request to the endpoint /scrapers/jobs/:jobId:

curl --request GET \
  --url 'https://api.hasdata.com/scrapers/jobs/:jobId' \
  --header 'Content-Type: application/json' \
  --header 'x-api-key: <your-api-key>'

const axios = require('axios').default;

const options = {
  method: 'GET',
  url: 'https://api.hasdata.com/scrapers/jobs/:jobId',
  headers: {'Content-Type': 'application/json', 'x-api-key': '<your-api-key>'}
};

try {
  const { data } = await axios.request(options);
  console.log(data);
} catch (error) {
  console.error(error);
}

import requests

url = "https://api.hasdata.com/scrapers/jobs/:jobId"

headers = {
    "Content-Type": "application/json",
    "x-api-key": "<your-api-key>"
}

response = requests.get(url, headers=headers)

print(response.json())

<?php

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.hasdata.com/scrapers/jobs/:jobId",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_CUSTOMREQUEST => "GET",
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json",
    "x-api-key: <your-api-key>",
  ],
]);

$response = curl_exec($curl);
curl_close($curl);

echo $response;

OkHttpClient client = new OkHttpClient();

Request request = new Request.Builder()
  .url("https://api.hasdata.com/scrapers/jobs/:jobId")
  .get()
  .addHeader("Content-Type", "application/json")
  .addHeader("x-api-key", "<your-api-key>")
  .build();

Response response = client.newCall(request).execute();

using System.Net.Http;

var client = new HttpClient();

var request = new HttpRequestMessage(new HttpMethod("GET"), "https://api.hasdata.com/scrapers/jobs/:jobId");
request.Headers.Add("x-api-key", "<your-api-key>");

using var response = await client.SendAsync(request);
response.EnsureSuccessStatusCode();
var content = await response.Content.ReadAsStringAsync();
Console.WriteLine(content);

require 'net/http'
require 'uri'

uri = URI("https://api.hasdata.com/scrapers/jobs/:jobId")

http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = true

request = Net::HTTP::Get.new(uri)
request["Content-Type"] = 'application/json'
request["x-api-key"] = '<your-api-key>'

response = http.request(request)
puts response.read_body

use reqwest::blocking::Client;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let client = Client::new();
    let res = client
        .get("https://api.hasdata.com/scrapers/jobs/:jobId")
        .header("Content-Type", "application/json")
        .header("x-api-key", "<your-api-key>")
        .send()?
        .text()?;
    println!("{}", res);
    Ok(())
}

package main

import (
	"fmt"
	"io"
	"net/http"
)

func main() {
	req, _ := http.NewRequest("GET", "https://api.hasdata.com/scrapers/jobs/:jobId", nil)
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("x-api-key", "<your-api-key>")

	res, _ := http.DefaultClient.Do(req)
	defer res.Body.Close()

	body, _ := io.ReadAll(res.Body)
	fmt.Println(string(body))
}

Response

{
  "id": "dd1a8c53-2d47-4444-977d-8d653a6a3c82",
  "status": "finished",
  "creditsSpent": 200,
  "dataRowsCount": 20,
  "data": {
    "csv": "https://api.hasdata.com/scrapers/jobs/dd1a8c53-2d47-4444-977d-8d653a6a3c82/results/b6cc6733-6d0e-4e44-9e94-38688aad3884.csv",
    "json": "https://api.hasdata.com/scrapers/jobs/dd1a8c53-2d47-4444-977d-8d653a6a3c82/results/9cb592e3-6700-42ff-b58c-e7da3f478f28.json",
    "xlsx": "https://api.hasdata.com/scrapers/jobs/dd1a8c53-2d47-4444-977d-8d653a6a3c82/results/ecea853c-e0ca-4a23-ae74-eea0588e54b6.xlsx"
  },
  "input": {
    "limit": 25,
    "urls": ["https://hasdata.com", "https://example.com"],
    "maxDepth": 5,
    "includePaths": "(blog/.+|articles/.+)",
    "webhook": {
      "url": "https://example.com/webhook",
      "events": ["scraper.job.started", "scraper.job.finished", "scraper.data.scraped"]
    }
  }
}

Webhook

The webhook will notify you of events related to the scraper job. Here is an example webhook payload for the scraper.data.scraped event:

{
  "event": "scraper.data.scraped",
  "timestamp": "2025-04-11T14:30:00Z",
  "jobId": "dd1a8c53-2d47-4444-977d-8d653a6a3c82",
  "jobStatus": "in_progress",
  "data": [
    {
      "text": "Extracted text here...",
      "statusCode": 200,
      "statusText": "OK",
      "url": "https://hasdata.com/blog",
      "depth": 1,
      "title": "Blog | HasData"
    }
  ]
}

Overview

Getting Results

Job Control

Quickstart - Websites Crawler

Example Request

Use Web Scraping API Params

Get Scraper Job Status

Webhook

​Example Request

​Use Web Scraping API Params

​Get Scraper Job Status

​Webhook

Example Request

Use Web Scraping API Params

Get Scraper Job Status

Webhook