Quickstart
Four requests: submit, poll, read, download. The script at the bottom does all four and is safe to copy as-is.
1. Get an API key
Register, then wait for your account to be reviewed.
On approval you receive a key of the form
bce_live_<id>_<secret>.
Every sample on this site reads the key from API_KEY, so set
it once in your shell:
export API_KEY='bce_live_…'
export API='https://bankworks.info'
2. Submit a job
import os
import requests
API = os.environ.get("API", "https://bankworks.info")
API_KEY = os.environ["API_KEY"]
response = requests.post(
f"{API}/v1/jobs",
headers={"Authorization": f"Bearer {API_KEY}"},
json={
"urls": ["https://example.com/"],
"resultTtlSeconds": 3600,
"options": {
"enableJavaScript": True,
"postLoadDelayMs": 5000,
},
},
timeout=30,
)
response.raise_for_status()
job = response.json()
print(f"job {job['jobId']} with {job['taskCount']} task(s)")
const API = process.env.API ?? "https://bankworks.info";
const API_KEY = process.env.API_KEY;
if (!API_KEY) throw new Error("Set API_KEY first");
const response = await fetch(`${API}/v1/jobs`, {
method: "POST",
headers: {
Authorization: `Bearer ${API_KEY}`,
"Content-Type": "application/json",
},
body: JSON.stringify({
urls: ["https://example.com/"],
resultTtlSeconds: 3600,
options: { enableJavaScript: true, postLoadDelayMs: 5000 },
}),
});
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${await response.text()}`);
}
const job = await response.json();
console.log(`job ${job.jobId} with ${job.taskCount} task(s)`);
API="${API:-https://bankworks.info}" # your key is read from $API_KEY
curl -sS -X POST "$API/v1/jobs" \
-H "Authorization: Bearer $API_KEY" \
-H 'Content-Type: application/json' \
-d '{
"urls": ["https://example.com/"],
"resultTtlSeconds": 3600,
"options": {
"enableJavaScript": true,
"postLoadDelayMs": 5000
}
}'
{"ok": true, "jobId": "0199…", "taskCount": 1, "deliveryDeferred": 0}
https://example.com/ is a placeholder. Replace it with a page
you have the right to collect.
3. Poll the job
import os
import time
import requests
API = os.environ.get("API", "https://bankworks.info")
API_KEY = os.environ["API_KEY"]
JOB_ID = os.environ["JOB_ID"]
POLL_SECONDS = float(os.environ.get("POLL_SECONDS", "5"))
while True:
response = requests.get(
f"{API}/v1/jobs/{JOB_ID}",
headers={"Authorization": f"Bearer {API_KEY}"},
params={"limit": 100, "offset": 0},
timeout=30,
)
response.raise_for_status()
body = response.json()
job = body["job"]
finished = job["doneTasks"] + job["failedTasks"]
print(f"{job['doneTasks']} done, {job['failedTasks']} failed of {job['totalTasks']}")
if finished >= job["totalTasks"]:
break
time.sleep(POLL_SECONDS)
for task in body["tasks"]:
print(task["taskId"], task["status"], task.get("error") or "")
const API = process.env.API ?? "https://bankworks.info";
const API_KEY = process.env.API_KEY;
const JOB_ID = process.env.JOB_ID;
const POLL_SECONDS = Number(process.env.POLL_SECONDS ?? 5);
if (!API_KEY || !JOB_ID) throw new Error("Set API_KEY and JOB_ID first");
const sleep = (seconds) => new Promise((done) => setTimeout(done, seconds * 1000));
let body;
while (true) {
const response = await fetch(`${API}/v1/jobs/${JOB_ID}?limit=100&offset=0`, {
headers: { Authorization: `Bearer ${API_KEY}` },
});
if (!response.ok) throw new Error(`HTTP ${response.status}`);
body = await response.json();
const { job } = body;
const finished = job.doneTasks + job.failedTasks;
console.log(`${job.doneTasks} done, ${job.failedTasks} failed of ${job.totalTasks}`);
if (finished >= job.totalTasks) break;
await sleep(POLL_SECONDS);
}
for (const task of body.tasks) {
console.log(task.taskId, task.status, task.error ?? "");
}
while :; do
state=$(curl -sS -H "Authorization: Bearer $API_KEY" \
"$API/v1/jobs/$JOB_ID?limit=100&offset=0")
done_n=$(printf '%s' "$state" | jq -r '.job.doneTasks')
fail_n=$(printf '%s' "$state" | jq -r '.job.failedTasks')
total=$(printf '%s' "$state" | jq -r '.job.totalTasks')
echo "$done_n done, $fail_n failed of $total"
[ "$((done_n + fail_n))" -ge "$total" ] && break
sleep "${POLL_SECONDS:-5}"
done
The response carries job.totalTasks,
job.doneTasks and job.failedTasks, plus the task
rows. A job is finished when
doneTasks + failedTasks === totalTasks.
4. Download the HTML
import hashlib
import os
import requests
API = os.environ.get("API", "https://bankworks.info")
API_KEY = os.environ["API_KEY"]
TASK_ID = os.environ["TASK_ID"]
response = requests.get(
f"{API}/v1/tasks/{TASK_ID}/html",
headers={"Authorization": f"Bearer {API_KEY}"},
timeout=60,
)
response.raise_for_status()
# X-HTML-SHA256 is computed over the stored bytes, so you can prove what you got.
expected = response.headers["X-HTML-SHA256"].lower()
actual = hashlib.sha256(response.content).hexdigest()
if actual != expected:
raise SystemExit(f"checksum mismatch: {actual} != {expected}")
with open(f"{TASK_ID}.html", "wb") as out:
out.write(response.content)
print(f"wrote {TASK_ID}.html ({len(response.content)} bytes, sha256 verified)")
import { createHash } from "node:crypto";
import { writeFile } from "node:fs/promises";
const API = process.env.API ?? "https://bankworks.info";
const API_KEY = process.env.API_KEY;
const TASK_ID = process.env.TASK_ID;
if (!API_KEY || !TASK_ID) throw new Error("Set API_KEY and TASK_ID first");
const response = await fetch(`${API}/v1/tasks/${TASK_ID}/html`, {
headers: { Authorization: `Bearer ${API_KEY}` },
});
if (!response.ok) throw new Error(`HTTP ${response.status}`);
const html = Buffer.from(await response.arrayBuffer());
// X-HTML-SHA256 is computed over the stored bytes, so you can prove what you got.
const expected = response.headers.get("x-html-sha256").toLowerCase();
const actual = createHash("sha256").update(html).digest("hex");
if (actual !== expected) throw new Error(`checksum mismatch: ${actual} != ${expected}`);
await writeFile(`${TASK_ID}.html`, html);
console.log(`wrote ${TASK_ID}.html (${html.length} bytes, sha256 verified)`);
curl -sS -H "Authorization: Bearer $API_KEY" \
-o page.html -D headers.txt \
"$API/v1/tasks/$TASK_ID/html"
grep -i x-html-sha256 headers.txt
X-HTML-SHA256 is computed over the stored bytes, so you can
verify what you received. The response is an attachment by default; add
?inline=1 to render it in a browser.
The whole thing as one script
#!/usr/bin/env python3
"""Submit URLs as one job, wait for it, and save every rendered page."""
import hashlib
import os
import sys
import time
import requests
API = os.environ.get("API", "https://bankworks.info")
POLL_SECONDS = float(os.environ.get("POLL_SECONDS", "5"))
http = requests.Session()
http.headers["Authorization"] = f"Bearer {os.environ['API_KEY']}"
def get(path, **kwargs):
response = http.get(f"{API}{path}", timeout=60, **kwargs)
response.raise_for_status()
return response
def main(urls):
created = http.post(
f"{API}/v1/jobs",
json={"urls": urls, "options": {"postLoadDelayMs": 5000}},
timeout=30,
)
created.raise_for_status()
job_id = created.json()["jobId"]
print("job", job_id)
# Poll the job, not every task: one request reports all of them.
while True:
job = get(f"/v1/jobs/{job_id}", params={"limit": 1}).json()["job"]
print(f" {job['doneTasks']} done, {job['failedTasks']} failed of {job['totalTasks']}")
if job["doneTasks"] + job["failedTasks"] >= job["totalTasks"]:
break
time.sleep(POLL_SECONDS)
# Tasks come back 100 at a time.
tasks, offset = [], 0
while True:
page = get(f"/v1/jobs/{job_id}", params={"limit": 100, "offset": offset}).json()
tasks += page["tasks"]
if len(page["tasks"]) < 100:
break
offset += 100
for task in tasks:
if task["status"] != "DONE":
print(" failed:", task["url"], task.get("error"))
continue
html = get(f"/v1/tasks/{task['taskId']}/html")
digest = hashlib.sha256(html.content).hexdigest()
assert digest == html.headers["X-HTML-SHA256"].lower(), "checksum mismatch"
with open(f"{task['taskId']}.html", "wb") as out:
out.write(html.content)
print(f" wrote {task['taskId']}.html for {task['url']}")
if __name__ == "__main__":
if len(sys.argv) < 2:
sys.exit("usage: crawl.py URL [URL ...]")
main(sys.argv[1:])
import { createHash } from "node:crypto";
import { writeFile } from "node:fs/promises";
const API = process.env.API ?? "https://bankworks.info";
const API_KEY = process.env.API_KEY;
const POLL_SECONDS = Number(process.env.POLL_SECONDS ?? 5);
const urls = process.argv.slice(2);
if (!API_KEY || urls.length === 0) {
console.error("usage: API_KEY=… node crawl.mjs URL [URL ...]");
process.exit(1);
}
const sleep = (seconds) => new Promise((done) => setTimeout(done, seconds * 1000));
async function call(path, options = {}) {
const response = await fetch(`${API}${path}`, {
...options,
headers: {
Authorization: `Bearer ${API_KEY}`,
"Content-Type": "application/json",
},
});
if (!response.ok) throw new Error(`HTTP ${response.status} for ${path}`);
return response;
}
const created = await call("/v1/jobs", {
method: "POST",
body: JSON.stringify({ urls, options: { postLoadDelayMs: 5000 } }),
});
const { jobId } = await created.json();
console.log("job", jobId);
// Poll the job, not every task: one request reports all of them.
while (true) {
const { job } = await (await call(`/v1/jobs/${jobId}?limit=1`)).json();
console.log(` ${job.doneTasks} done, ${job.failedTasks} failed of ${job.totalTasks}`);
if (job.doneTasks + job.failedTasks >= job.totalTasks) break;
await sleep(POLL_SECONDS);
}
// Tasks come back 100 at a time.
const tasks = [];
for (let offset = 0; ; offset += 100) {
const page = await (await call(`/v1/jobs/${jobId}?limit=100&offset=${offset}`)).json();
tasks.push(...page.tasks);
if (page.tasks.length < 100) break;
}
for (const task of tasks) {
if (task.status !== "DONE") {
console.log(" failed:", task.url, task.error);
continue;
}
const response = await call(`/v1/tasks/${task.taskId}/html`);
const html = Buffer.from(await response.arrayBuffer());
const digest = createHash("sha256").update(html).digest("hex");
if (digest !== response.headers.get("x-html-sha256").toLowerCase()) {
throw new Error("checksum mismatch");
}
await writeFile(`${task.taskId}.html`, html);
console.log(` wrote ${task.taskId}.html for ${task.url}`);
}
#!/usr/bin/env bash
set -euo pipefail
API="${API:-https://bankworks.info}"
: "${API_KEY:?set API_KEY first}"
[ "$#" -ge 1 ] || { echo "usage: crawl.sh URL [URL ...]" >&2; exit 1; }
# jq builds the JSON, so a URL containing a quote cannot break the request.
body=$(jq -n '{urls: $ARGS.positional, options: {postLoadDelayMs: 5000}}' --args "$@")
job=$(curl -sS -X POST "$API/v1/jobs" \
-H "Authorization: Bearer $API_KEY" \
-H 'Content-Type: application/json' \
-d "$body")
job_id=$(printf '%s' "$job" | jq -r '.jobId')
echo "job $job_id"
# Poll the job, not every task: one request reports all of them. The interval
# is chosen against a ~13s render; polling faster only spends the rate limit.
while :; do
state=$(curl -sS -H "Authorization: Bearer $API_KEY" \
"$API/v1/jobs/$job_id?limit=100&offset=0")
done_n=$(printf '%s' "$state" | jq -r '.job.doneTasks')
fail_n=$(printf '%s' "$state" | jq -r '.job.failedTasks')
total=$(printf '%s' "$state" | jq -r '.job.totalTasks')
echo " $done_n done, $fail_n failed of $total"
[ "$((done_n + fail_n))" -ge "$total" ] && break
sleep "${POLL_SECONDS:-5}"
done
# This reads the first 100 tasks; the Python and Node.js samples page through
# larger jobs.
printf '%s' "$state" | jq -r '.tasks[] | select(.status != "DONE") |
" failed: \(.url) \(.error)"'
printf '%s' "$state" | jq -r '.tasks[] | select(.status == "DONE") | .taskId' |
while read -r task; do
curl -sS -H "Authorization: Bearer $API_KEY" \
-o "$task.html" "$API/v1/tasks/$task/html"
echo " wrote $task.html"
done