Pay for and read sealed articles (AI agents)

// agent.mjs: pay for and read one CrawlerToll sealed article over plain HTTP.
// Node 22+, plus viem for signing:  npm i viem
// Dry run (no payment):  node agent.mjs https://www.crawlertoll.com/demo/micro.html
// Pay:                   PAY=1 AGENT_PRIVATE_KEY=0x... node agent.mjs <url>
import { privateKeyToAccount } from "viem/accounts";
 
const URL_ = process.argv[2];
const MAX_MICROS = 50_000; // your spending cap: never pay more than 0.05 USDC
const UA = "MyAgent/1.0 (+https://example.com/agent)"; // your real crawler user agent
const KEYS_URL = "https://registry.crawlertoll.com/.well-known/crawlertoll-keys.json";
const USDC = { "eip155:8453": "0x833589fCD6eDb6E08f4c7C32D4f71b54bdA02913", "eip155:84532": "0x036CbD53842c5426634e7929541eC2318f3dCF7e" };
 
const b64 = (s) => Uint8Array.from(atob(s), (c) => c.charCodeAt(0));
const hex = (u) => [...u].map((x) => x.toString(16).padStart(2, "0")).join("");
const site = (h) => h.toLowerCase().replace(/^www\./, "");
const canonical = (v) => v === null || typeof v !== "object" ? JSON.stringify(v)
  : Array.isArray(v) ? "[" + v.map(canonical).join(",") + "]"
  : "{" + Object.keys(v).filter((k) => v[k] !== undefined).sort().map((k) => JSON.stringify(k) + ":" + canonical(v[k])).join(",") + "}";
const fail = (msg) => { throw new Error(msg + ": not paying."); };
 
// 1. Fetch the article: the encrypted text is in the page, the key URL in the Link header.
const page = await fetch(URL_, { headers: { "User-Agent": UA } });
const host = new URL(page.url).host;
if (!page.url.startsWith("https://") || site(host) !== site(new URL(URL_).host)) fail("redirected off https or to another site");
const html = await page.text();
const blob = JSON.parse(html.match(/<script type="application\/ct-sealed\+json"(?![^>]*data-part)[^>]*>([\s\S]*?)<\/script>/)?.[1] ?? fail("no sealed article"));
if (site(blob.content_id.split("/")[0]) !== site(host)) fail("the sealed article belongs to another site");
const keyUrl = (page.headers.get("Link") || "").split(/,(?=\s*<)/)
  .map((l) => l.match(/<([^>]+)>(.*)/)).find((m) => m && /rel="?payment"?/.test(m[2]) && /part="?full"?/.test(m[2]) && !/tier=/.test(m[2]))?.[1] ?? fail("no payment link");
if (new URL(keyUrl).host !== "registry.crawlertoll.com") fail("the payment link is not the CrawlerToll registry");
 
// 2. Ask for the key with no payment: the answer is a 402 with a signed offer.
const q = await fetch(keyUrl, { method: "POST", redirect: "error", headers: { "Content-Type": "application/json", "User-Agent": UA }, body: "{}" });
if (q.status !== 402) fail(`expected a 402 offer, got ${q.status}`);
const offer = await q.json();
 
// 3. Check the offer before signing anything.
const { keys } = await (await fetch(KEYS_URL, { redirect: "error" })).json();
const k = keys.find((x) => x.kid === offer.kid && x.alg === "Ed25519" && x.status === "active") ?? fail("unknown signing key");
const signed = Object.fromEntries(Object.entries(offer).filter(([n]) => !["signature", "meter", "pass_expired", "pack_exhausted"].includes(n)));
const pub = await crypto.subtle.importKey("raw", b64(k.public_key), { name: "Ed25519" }, false, ["verify"]);
if (!(await crypto.subtle.verify({ name: "Ed25519" }, pub, b64(offer.signature), new TextEncoder().encode("ct_offer_v1:" + canonical(signed))))) fail("bad offer signature");
if (offer.content_id !== blob.content_id || site(offer.publisher) !== site(host)) fail("the offer is for another article or publisher");
if (Date.now() > Date.parse(offer.expires_at)) fail("the offer expired");
const x = offer.x402 ?? fail("no USDC offer");
if (USDC[x.networkCaip2]?.toLowerCase() !== x.asset.toLowerCase() || x.eip712.verifyingContract.toLowerCase() !== x.asset.toLowerCase()) fail("not USDC on Base");
const micros = Number(x.priceMicros);
console.log(`${offer.content_id}: ${micros / 1e6} USDC to ${x.payTo} on ${x.networkCaip2}`);
if (!(micros > 0 && micros <= MAX_MICROS)) fail(`price is above your cap of ${MAX_MICROS / 1e6} USDC`);
if (!process.env.PAY) { console.log("Dry run: set PAY=1 to pay."); process.exit(0); }
 
// 4. Sign a USDC transfer for exactly the offered amount. The nonce is the
//    SHA-256 of a secret you keep, so a lost response can be sent again.
const account = privateKeyToAccount(process.env.AGENT_PRIVATE_KEY);
const secret = crypto.getRandomValues(new Uint8Array(32));
const nonce = "0x" + hex(new Uint8Array(await crypto.subtle.digest("SHA-256", secret)));
const authorization = { from: account.address, to: x.payTo, value: String(micros), validAfter: "0", validBefore: String(Math.floor(Date.now() / 1000) + 600), nonce };
const signature = await account.signTypedData({
  domain: x.eip712,
  types: { TransferWithAuthorization: [
    { name: "from", type: "address" }, { name: "to", type: "address" }, { name: "value", type: "uint256" },
    { name: "validAfter", type: "uint256" }, { name: "validBefore", type: "uint256" }, { name: "nonce", type: "bytes32" },
  ] },
  primaryType: "TransferWithAuthorization",
  message: authorization,
});
const paymentHeader = btoa(JSON.stringify({
  x402Version: 2, scheme: "exact", network: x.networkCaip2,
  accepted: { scheme: "exact", network: x.networkCaip2, amount: String(micros), asset: x.asset, payTo: x.payTo, maxTimeoutSeconds: 300, extra: { name: x.eip712.name, version: x.eip712.version } },
  payload: { signature, authorization },
  extensions: {},
}));
const body = JSON.stringify({ replay_secret: "0x" + hex(secret) });
 
// 5. Send the payment. Keep paymentHeader and body until you have the key:
//    sending the same two again never charges twice.
const r = await fetch(keyUrl, { method: "POST", redirect: "error", headers: { "Content-Type": "application/json", "User-Agent": UA, "PAYMENT-SIGNATURE": paymentHeader }, body });
if (r.status !== 200) throw new Error(`HTTP ${r.status}: see "If something goes wrong"`);
const { cek } = await r.json(); // keep this key: it opens the article again without paying
 
// 6. Decrypt on your side: AES-256-GCM, with the content id as associated data.
const key = await crypto.subtle.importKey("raw", b64(cek), "AES-GCM", false, ["decrypt"]);
const plain = await crypto.subtle.decrypt({ name: "AES-GCM", iv: b64(blob.nonce), additionalData: new TextEncoder().encode(blob.content_id) }, key, b64(blob.ciphertext));
console.log(new TextDecoder().decode(plain));

node agent.mjs https://www.crawlertoll.com/demo/micro.html

PAY=1 AGENT_PRIVATE_KEY=0x... node agent.mjs https://www.crawlertoll.com/demo/micro.html