Bankworks

Browser sessions

A session holds one Chrome browser context open, keeps its cookies and storage, and runs same-origin fetch() calls inside the loaded page. Use it when state between requests matters.

Most work does not need a session. Fetching many independent pages is what crawl jobs are for, and they parallelise across the fleet. A session occupies one entire worker for its lifetime — while it is open, that worker does no other work.

Jobs or sessions?

Crawl jobBrowser session
Good forMany independent pagesA sequence inside one site
CookiesFresh per pageKept for the session
ParallelismAcross the whole fleetOne worker, one at a time
ReturnsRendered HTMLResponse bodies from in-page fetch
POST requestsNo — a job opens pages (GET)Yes, to the entry page's origin

Lifecycle

POST /v1/browser-sessionsjobs:create
Python
import os
import time

import requests

API = os.environ.get("API", "https://bankworks.info")
POLL_SECONDS = float(os.environ.get("POLL_SECONDS", "2"))
ENTRY_URL = "https://www.example.com/"  # must be a hostname we have enabled for you

http = requests.Session()
http.headers["Authorization"] = f"Bearer {os.environ['API_KEY']}"


def wait_for(path, done, failed):
    while True:
        body = http.get(f"{API}{path}", timeout=30).json()
        if body["status"] in done:
            return body
        if body["status"] in failed:
            raise RuntimeError(f"{path} ended {body['status']}: {body.get('error')}")
        time.sleep(POLL_SECONDS)


created = http.post(
    f"{API}/v1/browser-sessions",
    json={"entryUrl": ENTRY_URL, "ttlSeconds": 600, "idleTimeoutSeconds": 120},
    timeout=30,
)
created.raise_for_status()
session_id = created.json()["sessionId"]

try:
    # The worker has loaded the entry page once the session is READY.
    wait_for(f"/v1/browser-sessions/{session_id}", {"READY"},
             {"FAILED", "BLOCKED", "EXPIRED", "CLOSED"})

    # An in-page fetch(): same origin as the entry page, cookies kept.
    queued = http.post(
        f"{API}/v1/browser-sessions/{session_id}/fetches",
        json={"url": f"{ENTRY_URL}robots.txt", "method": "GET"},
        timeout=30,
    )
    queued.raise_for_status()
    fetch_id = queued.json()["fetchId"]

    fetch_path = f"/v1/browser-sessions/{session_id}/fetches/{fetch_id}"
    result = wait_for(fetch_path, {"DONE"}, {"FAILED", "BLOCKED", "CANCELLED"})
    print("upstream status:", result["httpStatus"])

    body = http.get(f"{API}{fetch_path}/body", timeout=60)
    body.raise_for_status()
    print(body.text[:200])
finally:
    # A session holds a whole worker. Close it as soon as you are done.
    http.delete(f"{API}/v1/browser-sessions/{session_id}", timeout=30)
Node.js
const API = process.env.API ?? "https://bankworks.info";
const API_KEY = process.env.API_KEY;
const POLL_SECONDS = Number(process.env.POLL_SECONDS ?? 2);
const ENTRY_URL = "https://www.example.com/"; // must be a hostname we have enabled for you
if (!API_KEY) throw new Error("Set API_KEY first");

const sleep = (seconds) => new Promise((done) => setTimeout(done, seconds * 1000));

async function call(path, options = {}) {
  const response = await fetch(`${API}${path}`, {
    ...options,
    headers: {
      Authorization: `Bearer ${API_KEY}`,
      "Content-Type": "application/json",
    },
  });
  if (!response.ok) throw new Error(`HTTP ${response.status} for ${path}`);
  return response;
}

async function waitFor(path, done, failed) {
  while (true) {
    const body = await (await call(path)).json();
    if (done.includes(body.status)) return body;
    if (failed.includes(body.status)) {
      throw new Error(`${path} ended ${body.status}: ${body.error}`);
    }
    await sleep(POLL_SECONDS);
  }
}

const created = await call("/v1/browser-sessions", {
  method: "POST",
  body: JSON.stringify({ entryUrl: ENTRY_URL, ttlSeconds: 600, idleTimeoutSeconds: 120 }),
});
const { sessionId } = await created.json();

try {
  // The worker has loaded the entry page once the session is READY.
  await waitFor(`/v1/browser-sessions/${sessionId}`, ["READY"],
    ["FAILED", "BLOCKED", "EXPIRED", "CLOSED"]);

  // An in-page fetch(): same origin as the entry page, cookies kept.
  const queued = await call(`/v1/browser-sessions/${sessionId}/fetches`, {
    method: "POST",
    body: JSON.stringify({ url: `${ENTRY_URL}robots.txt`, method: "GET" }),
  });
  const { fetchId } = await queued.json();

  const fetchPath = `/v1/browser-sessions/${sessionId}/fetches/${fetchId}`;
  const result = await waitFor(fetchPath, ["DONE"], ["FAILED", "BLOCKED", "CANCELLED"]);
  console.log("upstream status:", result.httpStatus);

  const body = await call(`${fetchPath}/body`);
  console.log((await body.text()).slice(0, 200));
} finally {
  // A session holds a whole worker. Close it as soon as you are done.
  await call(`/v1/browser-sessions/${sessionId}`, { method: "DELETE" });
}
cURL
curl -sS -X POST "$API/v1/browser-sessions" \
  -H "Authorization: Bearer $API_KEY" \
  -H 'Content-Type: application/json' \
  -d '{"entryUrl": "https://www.example.com/", "ttlSeconds": 600, "idleTimeoutSeconds": 120}'

# …poll GET /v1/browser-sessions/$SESSION_ID until "status" is "READY", then:

curl -sS -X POST "$API/v1/browser-sessions/$SESSION_ID/fetches" \
  -H "Authorization: Bearer $API_KEY" \
  -H 'Content-Type: application/json' \
  -d '{"url": "https://www.example.com/robots.txt", "method": "GET"}'

curl -sS -X DELETE "$API/v1/browser-sessions/$SESSION_ID" \
  -H "Authorization: Bearer $API_KEY"

The session starts PENDING, becomes READY once a worker has loaded the entry page, and ends when you close it, its TTL expires, or it goes idle past idleTimeoutSeconds.

Regular or Advance

A session has a package too, chosen when it is opened, and every fetch in it is billed at that package's price. Use "package": "advance" when the entry page needs additional browser handling, including a supported CAPTCHA challenge; leave it out for Regular. See Regular or Advance for what each one does.

JSON
{"entryUrl": "https://www.example.com/",
 "ttlSeconds": 600,
 "idleTimeoutSeconds": 120,
 "package": "advance"}
GET /v1/browser-sessions/{sessionId}jobs:read
POST /v1/browser-sessions/{sessionId}/fetchesjobs:create
GET /v1/browser-sessions/{sessionId}/fetches/{fetchId}jobs:read
GET /v1/browser-sessions/{sessionId}/fetches/{fetchId}/bodyresults:read
DELETE /v1/browser-sessions/{sessionId}jobs:create

Fetch request

POST /v1/browser-sessions/{sessionId}/fetchesjobs:create

A fetch runs fetch() inside the loaded page, so it goes out with the session's cookies and from the page's own origin — the same way the site's own scripts call their API.

FieldTypeDefaultNotes
urlstringrequiredSame origin as the session's entryUrl (scheme, host and port)
methodstring"GET""GET" or "POST"
headersobject—Only accept, content-type and x-requested-with. Cookies, origin and user agent are the browser's own
bodystring—POST only, up to 1 MiB. Send JSON or form data as a string, with the matching content-type
timeoutMsinteger300001000–120000

Send a POST

Open the session on the page that makes the request, so its cookies are set, then send the same request the page sends. Copy the URL, the content-type and the body from your browser's network panel; leave cookies and other headers out — the session supplies them.

Python
import json
import os
import time

import requests

API = os.environ.get("API", "https://bankworks.info")
POLL_SECONDS = float(os.environ.get("POLL_SECONDS", "2"))
SITE = "https://www.example.com"  # the session's entry hostname, enabled for you

http = requests.Session()
http.headers["Authorization"] = f"Bearer {os.environ['API_KEY']}"


def wait_for(path, done, failed):
    while True:
        body = http.get(f"{API}{path}", timeout=30).json()
        if body["status"] in done:
            return body
        if body["status"] in failed:
            raise RuntimeError(f"{path} ended {body['status']}: {body.get('error')}")
        time.sleep(POLL_SECONDS)


# Open the page that makes the request, so its cookies are set first.
created = http.post(f"{API}/v1/browser-sessions",
                    json={"entryUrl": f"{SITE}/jobs/search", "ttlSeconds": 600}, timeout=30)
created.raise_for_status()
session_id = created.json()["sessionId"]

try:
    wait_for(f"/v1/browser-sessions/{session_id}", {"READY"},
             {"FAILED", "BLOCKED", "EXPIRED", "CLOSED"})

    # A POST from inside the page: same origin, the session's cookies sent along.
    # The body is a string -- here, the JSON the site's own search sends.
    search = {"location": ["75"], "pagination": {"range": 20, "startIndex": 20}}
    queued = http.post(f"{API}/v1/browser-sessions/{session_id}/fetches", json={
        "url": f"{SITE}/api/search",
        "method": "POST",
        "headers": {"content-type": "application/json", "accept": "application/json"},
        "body": json.dumps(search),
    }, timeout=30)
    queued.raise_for_status()
    fetch_id = queued.json()["fetchId"]

    fetch_path = f"/v1/browser-sessions/{session_id}/fetches/{fetch_id}"
    result = wait_for(fetch_path, {"DONE"}, {"FAILED", "BLOCKED", "CANCELLED"})
    print("upstream status:", result["httpStatus"])

    answer = http.get(f"{API}{fetch_path}/body", timeout=60)
    answer.raise_for_status()
    print(f"{len(answer.json()['results'])} result(s)")
finally:
    # A session holds a whole worker. Close it as soon as you are done.
    http.delete(f"{API}/v1/browser-sessions/{session_id}", timeout=30)
Node.js
const API = process.env.API ?? "https://bankworks.info";
const API_KEY = process.env.API_KEY;
const POLL_SECONDS = Number(process.env.POLL_SECONDS ?? 2);
const SITE = "https://www.example.com"; // the session's entry hostname, enabled for you
if (!API_KEY) throw new Error("Set API_KEY first");

const sleep = (seconds) => new Promise((done) => setTimeout(done, seconds * 1000));

async function call(path, options = {}) {
  const response = await fetch(`${API}${path}`, {
    ...options,
    headers: { Authorization: `Bearer ${API_KEY}`, "Content-Type": "application/json" },
  });
  if (!response.ok) throw new Error(`HTTP ${response.status} for ${path}`);
  return response;
}

async function waitFor(path, done, failed) {
  while (true) {
    const body = await (await call(path)).json();
    if (done.includes(body.status)) return body;
    if (failed.includes(body.status)) throw new Error(`${path} ended ${body.status}: ${body.error}`);
    await sleep(POLL_SECONDS);
  }
}

// Open the page that makes the request, so its cookies are set first.
const created = await call("/v1/browser-sessions", {
  method: "POST",
  body: JSON.stringify({ entryUrl: `${SITE}/jobs/search`, ttlSeconds: 600 }),
});
const { sessionId } = await created.json();

try {
  await waitFor(`/v1/browser-sessions/${sessionId}`, ["READY"], ["FAILED", "BLOCKED", "EXPIRED", "CLOSED"]);

  // A POST from inside the page: same origin, the session's cookies sent along.
  // The body is a string -- here, the JSON the site's own search sends.
  const search = { location: ["75"], pagination: { range: 20, startIndex: 20 } };
  const queued = await call(`/v1/browser-sessions/${sessionId}/fetches`, {
    method: "POST",
    body: JSON.stringify({
      url: `${SITE}/api/search`,
      method: "POST",
      headers: { "content-type": "application/json", accept: "application/json" },
      body: JSON.stringify(search),
    }),
  });
  const { fetchId } = await queued.json();

  const fetchPath = `/v1/browser-sessions/${sessionId}/fetches/${fetchId}`;
  const result = await waitFor(fetchPath, ["DONE"], ["FAILED", "BLOCKED", "CANCELLED"]);
  console.log("upstream status:", result.httpStatus);

  const answer = await (await call(`${fetchPath}/body`)).json();
  console.log(`${answer.results.length} result(s)`);
} finally {
  // A session holds a whole worker. Close it as soon as you are done.
  await call(`/v1/browser-sessions/${sessionId}`, { method: "DELETE" });
}
cURL
API="${API:-https://bankworks.info}"  # your key is read from $API_KEY
# SESSION_ID: a session opened on https://www.example.com/jobs/search, now READY.

# A POST from inside the page. "body" is a string, so JSON is sent escaped.
curl -sS -X POST "$API/v1/browser-sessions/$SESSION_ID/fetches" \
  -H "Authorization: Bearer $API_KEY" \
  -H 'Content-Type: application/json' \
  -d "$(jq -n --arg body '{"location":["75"],"pagination":{"range":20,"startIndex":20}}' '{
    url: "https://www.example.com/api/search",
    method: "POST",
    headers: {"content-type": "application/json", "accept": "application/json"},
    body: $body
  }')"

The fetch ends DONE with the site's own status in httpStatus, so check it: a 400, 404 or 422 from the site (a body it did not accept) is still DONE. A 403, a 429 or a block page ends BLOCKED instead. Read the response from …/fetches/{fetchId}/body. For a form post, send content-type: application/x-www-form-urlencoded and a body such as q=shoes&page=2.

Entry URLs are allowlisted

Sessions may only be opened against hostnames an operator has enabled, and the match is exact — www.example.com does not cover example.com. An entry URL outside the list returns 400. Crawl jobs are not restricted this way.

Ask us to add a hostname before you build against it.

Limits worth knowing