Custom Headers and Cookies

Use the headers parameter to send custom HTTP headers and cookies with your request. This is useful when the target website requires specific headers (like User-Agent, Referer, or Cookie) to return the expected content.

Invalid or mismatched headers may cause the request to be blocked

If you override critical headers (e.g. Accept-Encoding), it may impact response parsing

When to Use

Set a User-Agent to request the desktop or mobile version of a page
Force a language or locale-specific version of a page (e.g. Accept-Language: fr-FR)
Pre-set a consent Cookie so cookie-banner or region interstitials don’t block the page content
Keep a consistent Cookie-based session across multiple requests, e.g. to preserve a currency, search filters, or cart state you set on an earlier request
Pass an Authorization header or API key required by an endpoint you have your own legitimate credentials for, such as your own site, a sandbox environment, or a partner API

Format

The headers parameter is a JSON object with header names and values.

{
  "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)",
  "Referer": "https://example.com/",
  "Cookie": "session_id=abc123"
}

Example Request

curl --request POST \
  --url 'https://api.hasdata.com/scrape/web' \
  --header 'Content-Type: application/json' \
  --header 'x-api-key: <your-api-key>' \
  --data '{"url":"https://example.com/products","headers":{"User-Agent":"Mozilla/5.0","Cookie":"consent=accepted; currency=USD"},"outputFormat":["html"]}'

hasdata web-scraping \
  --url 'https://example.com/products' \
  --headers-json '{"User-Agent":"Mozilla/5.0","Cookie":"consent=accepted; currency=USD"}' \
  --output-format html

const axios = require('axios').default;

const options = {
  method: 'POST',
  url: 'https://api.hasdata.com/scrape/web',
  headers: {'Content-Type': 'application/json', 'x-api-key': '<your-api-key>'},
  data: {
    url: 'https://example.com/products',
    headers: {'User-Agent': 'Mozilla/5.0', Cookie: 'consent=accepted; currency=USD'},
    outputFormat: ['html']
  }
};

try {
  const { data } = await axios.request(options);
  console.log(data);
} catch (error) {
  console.error(error);
}

import requests

url = "https://api.hasdata.com/scrape/web"

payload = {
    "url": "https://example.com/products",
    "headers": {
        "User-Agent": "Mozilla/5.0",
        "Cookie": "consent=accepted; currency=USD"
    },
    "outputFormat": ["html"]
}
headers = {
    "Content-Type": "application/json",
    "x-api-key": "<your-api-key>"
}

response = requests.post(url, json=payload, headers=headers)

print(response.json())

<?php

$payload = [
    "url" => "https://example.com/products",
    "headers" => ["User-Agent" => "Mozilla/5.0", "Cookie" => "consent=accepted; currency=USD"],
    "outputFormat" => ["html"],
];

$curl = curl_init();

curl_setopt_array($curl, [
  CURLOPT_URL => "https://api.hasdata.com/scrape/web",
  CURLOPT_RETURNTRANSFER => true,
  CURLOPT_CUSTOMREQUEST => "POST",
  CURLOPT_POSTFIELDS => json_encode($payload),
  CURLOPT_HTTPHEADER => [
    "Content-Type: application/json",
    "x-api-key: <your-api-key>",
  ],
]);

$response = curl_exec($curl);
curl_close($curl);

echo $response;

OkHttpClient client = new OkHttpClient();

MediaType mediaType = MediaType.parse("application/json");
String json = """
  {
    "url": "https://example.com/products",
    "headers": {
      "User-Agent": "Mozilla/5.0",
      "Cookie": "consent=accepted; currency=USD"
    },
    "outputFormat": [
      "html"
    ]
  }
""";
RequestBody requestBody = RequestBody.create(json, mediaType);

Request request = new Request.Builder()
  .url("https://api.hasdata.com/scrape/web")
  .post(requestBody)
  .addHeader("Content-Type", "application/json")
  .addHeader("x-api-key", "<your-api-key>")
  .build();

Response response = client.newCall(request).execute();

using System.Net.Http;
using System.Text;

var client = new HttpClient();

var json = """
{
  "url": "https://example.com/products",
  "headers": {
    "User-Agent": "Mozilla/5.0",
    "Cookie": "consent=accepted; currency=USD"
  },
  "outputFormat": [
    "html"
  ]
}
""";
var content = new StringContent(json, Encoding.UTF8, "application/json");

var request = new HttpRequestMessage(new HttpMethod("POST"), "https://api.hasdata.com/scrape/web")
{
    Content = content,
};
request.Headers.Add("x-api-key", "<your-api-key>");

using var response = await client.SendAsync(request);
response.EnsureSuccessStatusCode();
var body = await response.Content.ReadAsStringAsync();
Console.WriteLine(body);

require 'net/http'
require 'uri'
require 'json'

uri = URI("https://api.hasdata.com/scrape/web")
payload = {
  "url" => "https://example.com/products",
  "headers" => {"User-Agent" => "Mozilla/5.0", "Cookie" => "consent=accepted; currency=USD"},
  "outputFormat" => ["html"],
}

http = Net::HTTP.new(uri.host, uri.port)
http.use_ssl = true

request = Net::HTTP::Post.new(uri)
request["Content-Type"] = 'application/json'
request["x-api-key"] = '<your-api-key>'
request.body = payload.to_json

response = http.request(request)
puts response.read_body

use reqwest::blocking::Client;
use serde_json::json;

fn main() -> Result<(), Box<dyn std::error::Error>> {
    let client = Client::new();
    let payload = json!({
        "url": "https://example.com/products",
        "headers": {
            "User-Agent": "Mozilla/5.0",
            "Cookie": "consent=accepted; currency=USD"
        },
        "outputFormat": [
            "html"
        ]
    });
    let res = client
        .post("https://api.hasdata.com/scrape/web")
        .header("Content-Type", "application/json")
        .header("x-api-key", "<your-api-key>")
        .json(&payload)
        .send()?
        .text()?;
    println!("{}", res);
    Ok(())
}

package main

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
)

func main() {
	payload := []byte(`{
  "url": "https://example.com/products",
  "headers": {
    "User-Agent": "Mozilla/5.0",
    "Cookie": "consent=accepted; currency=USD"
  },
  "outputFormat": [
    "html"
  ]
}`)

	req, _ := http.NewRequest("POST", "https://api.hasdata.com/scrape/web", bytes.NewBuffer(payload))
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("x-api-key", "<your-api-key>")

	res, _ := http.DefaultClient.Do(req)
	defer res.Body.Close()

	responseBody, _ := io.ReadAll(res.Body)
	fmt.Println(string(responseBody))
}

Notes

All custom headers are passed as-is to the target page
To send cookies, use the Cookie header — multiple cookies should be in standard format: "key=value; key2=value2"

Get Started

Features

When to Use

Format

Example Request

Notes

​When to Use

​Format

​Example Request

​Notes

When to Use

Format

Example Request

Notes