Recipes

#!/usr/bin/env bash
# Fail a deploy when a PerfBee crawl finds broken links that the previous crawl did not have.
#
# Needs bash, curl and jq. Put the secrets in your CI settings, never in the repository:
#   PERFBEE_API_KEY   a key with the `crawl` scope (Settings > API keys)
#   PERFBEE_SITE_ID   the id of the site in My Sites (GET /sites lists them)
#   IDEMPOTENCY_KEY   one value per intended crawl, for example the commit hash (default: $GITHUB_SHA)
# Optional:
#   PERFBEE_MODE (fast or full, default fast), PERFBEE_MAX_PAGES (default 500), PERFBEE_TIMEOUT (seconds, default 1800),
#   PERFBEE_API (default https://dashboard.perfbee.com/api/v1)
#
# Exit 0: no new broken links. Exit 1: new broken links. Exit 2: the check could not finish.
set -euo pipefail

API="${PERFBEE_API:-https://dashboard.perfbee.com/api/v1}"
MODE="${PERFBEE_MODE:-fast}"
MAX_PAGES="${PERFBEE_MAX_PAGES:-500}"
TIMEOUT="${PERFBEE_TIMEOUT:-1800}"
WAIT_SECONDS=25          # the longest ?wait the API accepts
SHOW_NEW_BROKEN=20       # how many new broken URLs to print
DEFAULT_PAUSE=5          # seconds to pause when the API gives no Retry-After

WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT

fail() {
  echo "perfbee: $2" >&2
  exit "$1"
}

for tool in curl jq; do
  command -v "$tool" >/dev/null 2>&1 || fail 2 "$tool is required"
done
[ -n "${PERFBEE_API_KEY:-}" ] || fail 2 "PERFBEE_API_KEY is not set"
[ -n "${PERFBEE_SITE_ID:-}" ] || fail 2 "PERFBEE_SITE_ID is not set"
IDEMPOTENCY_KEY="${IDEMPOTENCY_KEY:-${GITHUB_SHA:-}}"
[ -n "$IDEMPOTENCY_KEY" ] || fail 2 "set IDEMPOTENCY_KEY (the commit hash works) so a retry never starts a second crawl"

# api METHOD PATH [curl options]: sets $status; the answer is in $WORK/body, its headers in $WORK/headers.
api() {
  local method="$1" path="$2"
  shift 2
  status="$(curl --silent --show-error --max-time $((WAIT_SECONDS + 15)) --request "$method" \
    --header "Authorization: Bearer $PERFBEE_API_KEY" --header "Accept: application/json" \
    --output "$WORK/body" --dump-header "$WORK/headers" --write-out '%{http_code}' "$@" "$API$path")" \
    || fail 2 "could not reach $API"
}

# header NAME: the value of a response header, empty when it is not there.
header() {
  tr -d '\r' < "$WORK/headers" | awk -F': *' -v name="$1" 'tolower($1) == tolower(name) { print $2; exit }'
}

# problem: the error the API answered, in one line.
problem() {
  local text retry
  text="$(jq -r '"\(.title // "Error"): \(.detail // "no detail") [\(.code // "?"), request \(.request_id // "?")]"' \
    "$WORK/body" 2>/dev/null)" || text="HTTP $status"
  retry="$(header Retry-After)"
  [ -z "$retry" ] || text="$text Retry after ${retry}s."
  echo "$text"
}

# 1. Start the crawl. The same Idempotency-Key and body return the crawl already started, so a retry costs nothing.
payload="$(jq -n --argjson site "$PERFBEE_SITE_ID" --arg mode "$MODE" --argjson pages "$MAX_PAGES" \
  '{site_id: $site, mode: $mode, max_pages: $pages}')"
api POST /crawls --header "Content-Type: application/json" --header "Idempotency-Key: $IDEMPOTENCY_KEY" --data "$payload"
case "$status" in
  200 | 202) ;;
  *) fail 2 "could not start a crawl: $(problem)" ;;
esac
crawl_id="$(jq -r .id "$WORK/body")" || fail 2 "unexpected answer: $(head -c 200 "$WORK/body")"
echo "perfbee: crawl $crawl_id started"

# 2. Wait. Each call holds for up to 25 seconds and returns as soon as the crawl is over.
deadline=$(($(date +%s) + TIMEOUT))
state=unknown
while :; do
  pause=0
  api GET "/crawls/$crawl_id?wait=$WAIT_SECONDS"
  if [ "$status" = 200 ]; then
    state="$(jq -r .status "$WORK/body")" || fail 2 "unexpected answer: $(head -c 200 "$WORK/body")"
    case "$state" in completed | failed | cancelled | partial) break ;; esac
    pause="$(header Retry-After)"   # set when the server had no free wait: ask again after that many seconds
  elif [ "$status" = 429 ]; then
    pause="$(header Retry-After)"
    pause="${pause:-$DEFAULT_PAUSE}"
  else
    fail 2 "could not read crawl $crawl_id: $(problem)"
  fi
  case "$pause" in '' | *[!0-9]*) pause=0 ;; esac
  [ "$(date +%s)" -lt "$deadline" ] || fail 2 "gave up after ${TIMEOUT}s: crawl $crawl_id is still $state"
  [ "$pause" -eq 0 ] || sleep "$pause"
done
report="$(jq -r .report_url "$WORK/body")"
[ "$state" = completed ] || fail 2 "crawl $crawl_id ended as $state. Report: $report"

# 3. Compare with the previous crawl of this site in the same mode.
api GET "/crawls/$crawl_id/diff"
[ "$status" = 200 ] || fail 2 "could not read the comparison: $(problem)"
previous="$(jq -r .has_previous "$WORK/body")" || fail 2 "unexpected answer: $(head -c 200 "$WORK/body")"
if [ "$previous" != true ]; then
  echo "perfbee: the first $MODE crawl of this site, so there is nothing to compare. Report: $report"
  exit 0
fi
if jq -e '.summary.new_broken == 0' "$WORK/body" > /dev/null; then
  echo "perfbee: no new broken links ($(jq -r .summary.fixed_broken "$WORK/body") fixed). Report: $report"
  exit 0
fi
echo "perfbee: $(jq -r .summary.new_broken "$WORK/body") new broken links. Report: $report" >&2
api GET "/crawls/$crawl_id/diff?include=lists&limit=$SHOW_NEW_BROKEN"
if [ "$status" = 200 ]; then
  jq -r '.lists.new_broken[] | "  " + .' "$WORK/body" >&2
fi
exit 1

# Fail a workflow when a PerfBee crawl finds broken links that the previous crawl did not have.
#
# Set once in the repository settings:
#   secret   PERFBEE_API_KEY   a key with the `crawl` scope (Settings > API keys in PerfBee)
#   variable PERFBEE_SITE_ID   the site's id: GET /sites lists them
# Run this job after the job that deploys: add `needs: deploy` below.
name: Check for new broken links

on:
  workflow_dispatch:
  push:
    branches: [main]

jobs:
  broken-links:
    runs-on: ubuntu-latest
    timeout-minutes: 45
    env:
      PERFBEE_API: https://dashboard.perfbee.com/api/v1
      PERFBEE_SITE_ID: ${{ vars.PERFBEE_SITE_ID }}
    steps:
      - name: Start a crawl
        id: start
        env:
          PERFBEE_API_KEY: ${{ secrets.PERFBEE_API_KEY }}
        run: |
          status=$(curl --silent --show-error --output start.json --write-out '%{http_code}' \
            --request POST "$PERFBEE_API/crawls" \
            --header "Authorization: Bearer $PERFBEE_API_KEY" \
            --header "Content-Type: application/json" \
            --header "Idempotency-Key: ${{ github.sha }}-${{ github.run_attempt }}" \
            --data "{\"site_id\": $PERFBEE_SITE_ID, \"mode\": \"fast\", \"max_pages\": 500}")
          if [ "$status" != 200 ] && [ "$status" != 202 ]; then
            echo "::error::PerfBee refused the crawl: $(jq -r '.detail' start.json)"
            exit 1
          fi
          echo "crawl_id=$(jq -r .id start.json)" >> "$GITHUB_OUTPUT"

      - name: Wait for the crawl
        env:
          PERFBEE_API_KEY: ${{ secrets.PERFBEE_API_KEY }}
          CRAWL_ID: ${{ steps.start.outputs.crawl_id }}
        run: |
          deadline=$(( $(date +%s) + 1800 ))
          while :; do
            curl --silent --show-error --fail --retry 3 --output crawl.json \
              --header "Authorization: Bearer $PERFBEE_API_KEY" \
              "$PERFBEE_API/crawls/$CRAWL_ID?wait=25"
            state=$(jq -r .status crawl.json)
            case "$state" in
              completed) break ;;
              failed | cancelled | partial)
                echo "::error::The crawl ended as $state. Report: $(jq -r .report_url crawl.json)"
                exit 1 ;;
            esac
            [ "$(date +%s)" -lt "$deadline" ] || { echo "::error::The crawl is still $state after 30 minutes."; exit 1; }
          done

      - name: Fail on new broken links
        env:
          PERFBEE_API_KEY: ${{ secrets.PERFBEE_API_KEY }}
          CRAWL_ID: ${{ steps.start.outputs.crawl_id }}
        run: |
          curl --silent --show-error --fail --retry 3 --output diff.json \
            --header "Authorization: Bearer $PERFBEE_API_KEY" \
            "$PERFBEE_API/crawls/$CRAWL_ID/diff"
          if [ "$(jq -r .has_previous diff.json)" != true ]; then
            echo "The first crawl of this site in this mode: nothing to compare."
            exit 0
          fi
          if ! jq -e '.summary.new_broken == 0' diff.json > /dev/null; then
            echo "::error::$(jq -r .summary.new_broken diff.json) new broken links. Report: $(jq -r .report_url crawl.json)"
            exit 1
          fi
          echo "No new broken links."

PerfBee-Signature: t=1700000000,v1=3f5c...e9a1

// Verify a PerfBee webhook with Node's standard library only (Node 18 or later).
//
//   PERFBEE_WEBHOOK_SECRET=whsec_... node verify-webhook.mjs      # listens on PORT (default 3000)
//
// Two secrets may be set, comma separated, while you rotate one: a delivery is accepted when any v1 value in the
// header matches any secret you hold.
import { createHmac, timingSafeEqual } from 'node:crypto';
import { createServer } from 'node:http';
import { pathToFileURL } from 'node:url';

const TOLERANCE_SECONDS = 300;

// header: the PerfBee-Signature value. rawBody: the bytes of the request body, exactly as received (a Buffer).
export function verifySignature(header, rawBody, secrets, nowSeconds = Date.now() / 1000) {
  let timestamp = null;
  const signatures = [];
  for (const part of String(header || '').split(',')) {
    const [name, ...rest] = part.trim().split('=');
    const value = rest.join('=');
    if (name === 't' && /^[0-9]+$/.test(value)) timestamp = Number(value);
    else if (name === 'v1' && value) signatures.push(value);
  }
  if (timestamp === null || signatures.length === 0) return false;
  if (Math.abs(nowSeconds - timestamp) > TOLERANCE_SECONDS) return false;
  for (const secret of secrets) {
    const expected = createHmac('sha256', secret).update(`${timestamp}.`).update(rawBody).digest();
    for (const candidate of signatures) {
      const given = Buffer.from(candidate, 'hex');
      if (given.length === expected.length && timingSafeEqual(given, expected)) return true;
    }
  }
  return false;
}

function serve(secrets, port) {
  const seen = new Set();
  return createServer((request, response) => {
    const chunks = [];
    request.on('data', (chunk) => chunks.push(chunk));
    request.on('end', () => {
      const rawBody = Buffer.concat(chunks);
      if (!verifySignature(request.headers['perfbee-signature'], rawBody, secrets)) {
        response.writeHead(400).end('bad signature');
        return;
      }
      const event = JSON.parse(rawBody.toString('utf8'));
      if (!seen.has(event.id)) {   // an event can arrive more than once: its id is the same every time
        seen.add(event.id);
        console.log(`${event.id} ${event.event}`);
      }
      response.writeHead(200).end('ok');
    });
  }).listen(port);
}

if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
  const secrets = (process.env.PERFBEE_WEBHOOK_SECRET || '').split(',').filter(Boolean);
  if (secrets.length === 0) throw new Error('Set PERFBEE_WEBHOOK_SECRET to the whsec_ secret of the endpoint.');
  serve(secrets, Number(process.env.PORT || 3000));
}

"""Verify a PerfBee webhook with Python's standard library only (Python 3.9 or later).

    PERFBEE_WEBHOOK_SECRET=whsec_... python verify_webhook.py     # listens on PORT (default 3000)

Two secrets may be set, comma separated, while you rotate one: a delivery is accepted when any v1 value in the header
matches any secret you hold.
"""
import hashlib
import hmac
import json
import os
import time
from http.server import BaseHTTPRequestHandler, HTTPServer

TOLERANCE_SECONDS = 300


def verify_signature(header, raw_body, secrets, now=None):
    """header: the PerfBee-Signature value. raw_body: the bytes of the request body, exactly as received."""
    timestamp, signatures = None, []
    for part in (header or "").split(","):
        name, _, value = part.strip().partition("=")
        if name == "t" and value.isascii() and value.isdigit():
            timestamp = int(value)
        elif name == "v1" and value:
            signatures.append(value.encode())
    if timestamp is None or not signatures:
        return False
    if abs((time.time() if now is None else now) - timestamp) > TOLERANCE_SECONDS:
        return False
    for secret in secrets:
        signed = str(timestamp).encode() + b"." + raw_body
        expected = hmac.new(secret.encode(), signed, hashlib.sha256).hexdigest().encode()
        if any(hmac.compare_digest(expected, candidate) for candidate in signatures):
            return True
    return False


def make_handler(secrets):
    seen = set()

    class Handler(BaseHTTPRequestHandler):
        def do_POST(self):
            raw_body = self.rfile.read(int(self.headers.get("Content-Length") or 0))
            if not verify_signature(self.headers.get("PerfBee-Signature"), raw_body, secrets):
                self.send_response(400)
                self.end_headers()
                self.wfile.write(b"bad signature")
                return
            event = json.loads(raw_body)
            if event["id"] not in seen:   # an event can arrive more than once: its id is the same every time
                seen.add(event["id"])
                print(event["id"], event["event"], flush=True)
            self.send_response(200)
            self.end_headers()
            self.wfile.write(b"ok")

    return Handler


if __name__ == "__main__":
    held = [secret for secret in os.environ.get("PERFBEE_WEBHOOK_SECRET", "").split(",") if secret]
    if not held:
        raise SystemExit("Set PERFBEE_WEBHOOK_SECRET to the whsec_ secret of the endpoint.")
    HTTPServer(("", int(os.environ.get("PORT", "3000"))), make_handler(held)).serve_forever()

<?php
// Verify a PerfBee webhook with PHP's standard functions only (PHP 7.4 or later).
//
// Put this file where your web server can run it, and set PERFBEE_WEBHOOK_SECRET to the whsec_ secret of the endpoint.
// Two secrets may be set, comma separated, while you rotate one: a delivery is accepted when any v1 value in the header
// matches any secret you hold.
declare(strict_types=1);

const PERFBEE_TOLERANCE_SECONDS = 300;

// $header: the PerfBee-Signature value. $rawBody: the request body exactly as received.
function perfbee_verify_signature(string $header, string $rawBody, array $secrets, ?int $now = null): bool
{
    $timestamp = null;
    $signatures = [];
    foreach (explode(',', $header) as $part) {
        $pair = explode('=', trim($part), 2);
        if (count($pair) !== 2) {
            continue;
        }
        if ($pair[0] === 't' && ctype_digit($pair[1])) {
            $timestamp = (int) $pair[1];
        } elseif ($pair[0] === 'v1' && $pair[1] !== '') {
            $signatures[] = $pair[1];
        }
    }
    if ($timestamp === null || count($signatures) === 0) {
        return false;
    }
    if (abs(($now ?? time()) - $timestamp) > PERFBEE_TOLERANCE_SECONDS) {
        return false;
    }
    foreach ($secrets as $secret) {
        $expected = hash_hmac('sha256', $timestamp . '.' . $rawBody, $secret);
        foreach ($signatures as $candidate) {
            if (hash_equals($expected, $candidate)) {
                return true;
            }
        }
    }
    return false;
}

if (PHP_SAPI !== 'cli') {
    $secrets = array_filter(explode(',', (string) getenv('PERFBEE_WEBHOOK_SECRET')));
    $rawBody = (string) file_get_contents('php://input');
    $header = $_SERVER['HTTP_PERFBEE_SIGNATURE'] ?? '';
    if (!perfbee_verify_signature($header, $rawBody, $secrets)) {
        http_response_code(400);
        echo 'bad signature';
        exit;
    }
    $event = json_decode($rawBody, true);
    error_log($event['id'] . ' ' . $event['event']);
    http_response_code(200);
    echo 'ok';
}

{
  "name": "PerfBee crawl summary to Slack",
  "nodes": [
    {
      "parameters": {
        "content": "## PerfBee crawl summary to Slack\n\n1. Copy the Production URL of the Webhook node.\n2. In PerfBee, Settings > Webhooks, add it as an endpoint for the event **Crawl data ready** (scan_data_ready).\n3. Make a Header Auth credential named PerfBee API key: name Authorization, value Bearer pbk_... (a key with the read scope). Choose it in the node Read the crawl.\n4. Replace REPLACE_ME in the node Post to Slack with your Slack incoming webhook URL.\n\nThis workflow does not check the PerfBee-Signature header. The Production URL of an n8n webhook is long and random: keep it private.",
        "height": 300,
        "width": 460
      },
      "id": "7f2c1a10-0000-4000-8000-000000000001",
      "name": "Read me",
      "type": "n8n-nodes-base.stickyNote",
      "typeVersion": 1,
      "position": [
        -120,
        -40
      ]
    },
    {
      "parameters": {
        "httpMethod": "POST",
        "path": "perfbee",
        "options": {}
      },
      "id": "7f2c1a10-0000-4000-8000-000000000002",
      "name": "PerfBee webhook",
      "type": "n8n-nodes-base.webhook",
      "typeVersion": 2,
      "position": [
        400,
        160
      ],
      "webhookId": "7f2c1a10-0000-4000-8000-0000000000a1"
    },
    {
      "parameters": {
        "conditions": {
          "options": {
            "caseSensitive": true,
            "leftValue": "",
            "typeValidation": "strict"
          },
          "conditions": [
            {
              "id": "7f2c1a10-0000-4000-8000-0000000000b1",
              "leftValue": "={{ $json.body.event }}",
              "rightValue": "scan_data_ready",
              "operator": {
                "type": "string",
                "operation": "equals"
              }
            }
          ],
          "combinator": "and"
        },
        "options": {}
      },
      "id": "7f2c1a10-0000-4000-8000-000000000003",
      "name": "Crawl data ready?",
      "type": "n8n-nodes-base.if",
      "typeVersion": 2,
      "position": [
        620,
        160
      ]
    },
    {
      "parameters": {
        "url": "=https://dashboard.perfbee.com/api/v1/crawls/{{ $json.body.data.crawl_id }}",
        "authentication": "genericCredentialType",
        "genericAuthType": "httpHeaderAuth",
        "options": {}
      },
      "id": "7f2c1a10-0000-4000-8000-000000000004",
      "name": "Read the crawl",
      "type": "n8n-nodes-base.httpRequest",
      "typeVersion": 4.2,
      "position": [
        840,
        140
      ],
      "credentials": {
        "httpHeaderAuth": {
          "id": "REPLACE_ME",
          "name": "PerfBee API key"
        }
      }
    },
    {
      "parameters": {
        "assignments": {
          "assignments": [
            {
              "id": "7f2c1a10-0000-4000-8000-0000000000c1",
              "name": "text",
              "value": "=PerfBee crawled {{ $json.url }} ({{ $json.mode }}): {{ $json.pages_checked }} pages, {{ $json.broken_count }} broken links. {{ $json.report_url }}",
              "type": "string"
            }
          ]
        },
        "options": {}
      },
      "id": "7f2c1a10-0000-4000-8000-000000000005",
      "name": "Write the message",
      "type": "n8n-nodes-base.set",
      "typeVersion": 3.4,
      "position": [
        1060,
        140
      ]
    },
    {
      "parameters": {
        "method": "POST",
        "url": "https://hooks.slack.com/services/REPLACE_ME",
        "sendBody": true,
        "specifyBody": "json",
        "jsonBody": "={{ JSON.stringify({ text: $json.text }) }}",
        "options": {}
      },
      "id": "7f2c1a10-0000-4000-8000-000000000006",
      "name": "Post to Slack",
      "type": "n8n-nodes-base.httpRequest",
      "typeVersion": 4.2,
      "position": [
        1280,
        140
      ]
    }
  ],
  "connections": {
    "PerfBee webhook": {
      "main": [
        [
          {
            "node": "Crawl data ready?",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Crawl data ready?": {
      "main": [
        [
          {
            "node": "Read the crawl",
            "type": "main",
            "index": 0
          }
        ],
        []
      ]
    },
    "Read the crawl": {
      "main": [
        [
          {
            "node": "Write the message",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Write the message": {
      "main": [
        [
          {
            "node": "Post to Slack",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  },
  "pinData": {},
  "settings": {
    "executionOrder": "v1"
  },
  "active": false,
  "tags": []
}

curl -sf -X POST "$PERFBEE_API/webhook-endpoints" \
  -H "Authorization: Bearer $PERFBEE_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"url": "https://hooks.zapier.com/hooks/catch/123456/abcdef/", "events": ["scan_data_ready"], "description": "Zapier"}'

curl -sf -X POST "$PERFBEE_API/feeds" \
  -H "Authorization: Bearer $PERFBEE_API_KEY" \
  -H "Content-Type: application/json" \
  -d '{"dataset": "crawl-history", "site_id": 12}'

/**
 * PerfBee pages -> one tab of a Google Sheet (Apps Script, nothing to install).
 *
 * 1. In PerfBee, Settings > API keys: make a key with the `read` scope.
 * 2. In your sheet: Extensions > Apps Script. Paste this file over the editor's content and save.
 * 3. Project Settings (the gear) > Script properties. Add:
 *      PERFBEE_API_KEY       the key. It is a secret: keep it here, never in the code or the sheet.
 *      PERFBEE_SITE_ID       the site's id (GET /sites lists them).
 *      PERFBEE_PAGE_FILTERS  optional, as a query string, for example: view=pages&status=broken
 * 4. Reload the sheet. The PerfBee menu has Refresh now and Refresh every hour.
 *
 * Google asks you to allow the script once. "Unverified app" is normal for a script you wrote yourself.
 * A run that needs more than 5 minutes stops and leaves the tab as it was: Apps Script ends a run at 6 minutes.
 * Narrow the list with PERFBEE_PAGE_FILTERS (for example `prefix=https://www.example.com/shop/`) when that happens.
 */
const API_BASE = 'https://dashboard.perfbee.com/api/v1';
const TAB_NAME = 'PerfBee pages';
const PAGE_SIZE = 200;                      // the most a page of the list holds
const RUN_BUDGET_MS = 5 * 60 * 1000;        // stop before Apps Script's own 6 minute limit
const MAX_TRIES = 4;                        // tries per request when PerfBee says "slow down" or "busy"
const MAX_WALKS = 3;                        // walks of the whole list when a cursor stops being valid
const DEFAULT_PAUSE_SECONDS = 5;
const MAX_PAUSE_SECONDS = 60;

// [header, how to read it from a page row, text or number]
const COLUMNS = [
  ['id', (p) => p.id, 'number'],
  ['url', (p) => p.url, 'text'],
  ['status', (p) => p.status_class, 'text'],
  ['status code', (p) => p.status_code, 'number'],
  ['indexability', (p) => p.indexability, 'text'],
  ['title', (p) => p.title, 'text'],
  ['h1', (p) => p.h1, 'text'],
  ['words', (p) => p.word_count, 'number'],
  ['response ms', (p) => p.response_time_ms, 'number'],
  ['inbound links', (p) => p.inbound_links, 'number'],
  ['depth', (p) => p.crawl_depth, 'number'],
  ['issues', (p) => (p.issues || []).join(', '), 'text'],
  ['last seen', (p) => p.last_seen_at, 'text'],
];

function onOpen() {
  SpreadsheetApp.getUi()
    .createMenu('PerfBee')
    .addItem('Refresh now', 'refreshPages')
    .addItem('Refresh every hour', 'startHourlyRefresh')
    .addItem('Stop refreshing', 'stopRefresh')
    .addToUi();
}

/** Read every page of the site's list and replace the tab with it. On any problem the tab is left as it was. */
function refreshPages() {
  const started = Date.now();
  const properties = PropertiesService.getScriptProperties();
  const key = properties.getProperty('PERFBEE_API_KEY');
  const siteId = properties.getProperty('PERFBEE_SITE_ID');
  if (!key || !siteId) {
    throw new Error('Set PERFBEE_API_KEY and PERFBEE_SITE_ID in Project Settings > Script properties.');
  }
  const filters = properties.getProperty('PERFBEE_PAGE_FILTERS') || '';
  let pages = null;
  for (let walk = 1; pages === null; walk++) {
    pages = walkPages_(key, siteId, filters, started, walk < MAX_WALKS);
  }
  writeTab_(pages);
}

/** One walk of the list. Returns the rows, or null when the cursor stopped being valid and a new walk is allowed. */
function walkPages_(key, siteId, filters, started, mayRestart) {
  const rows = [];
  let cursor = null;
  do {
    if (Date.now() - started > RUN_BUDGET_MS) {
      throw new Error('Stopped after 5 minutes with ' + rows.length + ' pages read. The tab was not changed. Narrow the list with PERFBEE_PAGE_FILTERS.');
    }
    let page;
    try {
      page = getJson_(listUrl_(siteId, filters, cursor), key, started);
    } catch (error) {
      if (mayRestart && error.problem && error.problem.code === 'invalid_cursor') return null;   // the site changed under the walk
      throw error;
    }
    page.data.forEach((row) => rows.push(COLUMNS.map((column) => column[1](row))));
    cursor = page.has_more ? page.next_cursor : null;
  } while (cursor);
  return rows;
}

function listUrl_(siteId, filters, cursor) {
  // Keep every filter the same on every request: a cursor only fits the filters it was made for.
  let url = API_BASE + '/sites/' + encodeURIComponent(siteId) + '/pages?limit=' + PAGE_SIZE;
  if (filters) url += '&' + filters;
  if (cursor) url += '&cursor=' + encodeURIComponent(cursor);
  return url;
}

/** GET a JSON answer. A 429 or 503 is waited out (Retry-After) up to MAX_TRIES times; anything else stops the run. */
function getJson_(url, key, started) {
  for (let tries = 1; ; tries++) {
    const response = UrlFetchApp.fetch(url, {
      method: 'get', muteHttpExceptions: true,
      headers: { Authorization: 'Bearer ' + key, Accept: 'application/json' },
    });
    const code = response.getResponseCode();
    if (code === 200) return JSON.parse(response.getContentText());
    if ((code === 429 || code === 503) && tries < MAX_TRIES) {
      const pause = pauseSeconds_(response);
      if (Date.now() - started + pause * 1000 > RUN_BUDGET_MS) {
        throw new Error('PerfBee asked for a pause of ' + pause + ' seconds, which would pass the 5 minute budget.');
      }
      Utilities.sleep(pause * 1000);
      continue;
    }
    throw problemError_(code, response);
  }
}

function pauseSeconds_(response) {
  const headers = response.getHeaders();
  const name = Object.keys(headers).filter((header) => header.toLowerCase() === 'retry-after')[0];
  const seconds = name ? parseInt(headers[name], 10) : NaN;
  return isNaN(seconds) || seconds < 1 ? DEFAULT_PAUSE_SECONDS : Math.min(seconds, MAX_PAUSE_SECONDS);
}

function problemError_(code, response) {
  let problem = null;
  try {
    problem = JSON.parse(response.getContentText());
  } catch (ignored) {
    problem = null;
  }
  const error = new Error('PerfBee answered ' + code + (problem && problem.detail ? ': ' + problem.detail : '.'));
  error.problem = problem;
  return error;
}

function writeTab_(rows) {
  const spreadsheet = SpreadsheetApp.getActive();
  const sheet = spreadsheet.getSheetByName(TAB_NAME) || spreadsheet.insertSheet(TAB_NAME);
  const header = COLUMNS.map((column) => column[0]);
  sheet.clearContents();
  sheet.getRange(1, 1, 1, header.length).setValues([header]);
  sheet.setFrozenRows(1);
  if (!rows.length) return;
  COLUMNS.forEach((column, index) => {
    // Text columns are formatted as plain text first, so a crawled title such as "=1+1" stays text.
    if (column[2] === 'text') sheet.getRange(2, index + 1, rows.length, 1).setNumberFormat('@');
  });
  sheet.getRange(2, 1, rows.length, header.length).setValues(rows.map((row) => row.map((value) => (value === null || value === undefined ? '' : value))));
}

function startHourlyRefresh() {
  stopRefresh();
  ScriptApp.newTrigger('refreshPages').timeBased().everyHours(1).create();
  SpreadsheetApp.getActive().toast('Refreshing every hour.', 'PerfBee');
}

function stopRefresh() {
  ScriptApp.getProjectTriggers().forEach((trigger) => {
    if (trigger.getHandlerFunction() === 'refreshPages') ScriptApp.deleteTrigger(trigger);
  });
}

{"data": [], "next_cursor": "eyJ2IjoxfQ.opaque", "has_more": true}

#!/usr/bin/env bash
# Print every item of a PerfBee list, one JSON object per line. Needs bash, curl and jq.
#
#   PERFBEE_API_KEY=pbk_... ./paginate.sh '/sites/12/pages?status=broken'
#
# PERFBEE_API sets another base URL (default https://dashboard.perfbee.com/api/v1).
set -euo pipefail

API="${PERFBEE_API:-https://dashboard.perfbee.com/api/v1}"
PATH_AND_QUERY="${1:?usage: paginate.sh '/path?filters'}"
[ -n "${PERFBEE_API_KEY:-}" ] || { echo "PERFBEE_API_KEY is not set" >&2; exit 2; }
LIMIT=200          # the most a page holds
MAX_TRIES=4        # tries per request when PerfBee says "slow down" or "busy"
MAX_WALKS=3        # walks of the whole list when a cursor stops being valid
DEFAULT_PAUSE=5    # seconds to pause when there is no Retry-After
MAX_PAUSE=60

WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT

fail() {
  echo "perfbee: $1" >&2
  exit 1
}

# pause: seconds to wait, from the Retry-After header of the last answer.
pause() {
  local value
  value="$(tr -d '\r' < "$WORK/headers" | awk -F': *' 'tolower($1) == "retry-after" { print $2; exit }')"
  case "$value" in '' | *[!0-9]*) value=$DEFAULT_PAUSE ;; esac
  [ "$value" -ge 1 ] || value=$DEFAULT_PAUSE
  [ "$value" -le "$MAX_PAUSE" ] || value=$MAX_PAUSE
  echo "$value"
}

# walk: one walk of the list into $WORK/rows. Returns 3 when the cursor stopped being valid (start again).
walk() {
  local cursor="" url separator tries more
  : > "$WORK/rows"
  case "$PATH_AND_QUERY" in *\?*) separator="&" ;; *) separator="?" ;; esac
  while :; do
    # Every filter stays the same on every request: a cursor only fits the filters it was made for.
    url="$API$PATH_AND_QUERY${separator}limit=$LIMIT"
    [ -z "$cursor" ] || url="$url&cursor=$(jq -rn --arg c "$cursor" '$c | @uri')"
    tries=1
    while :; do
      status="$(curl --silent --show-error --max-time 60 --header "Authorization: Bearer $PERFBEE_API_KEY" \
        --output "$WORK/body" --dump-header "$WORK/headers" --write-out '%{http_code}' "$url")" || fail "could not reach $API"
      case "$status" in
        200) break ;;
        429 | 503)
          [ "$tries" -lt "$MAX_TRIES" ] || fail "still answering $status after $MAX_TRIES tries"
          sleep "$(pause)"
          tries=$((tries + 1)) ;;
        400)
          [ "$(jq -r .code "$WORK/body" 2>/dev/null)" != invalid_cursor ] || return 3
          fail "$(jq -r .detail "$WORK/body")" ;;
        *) fail "HTTP $status: $(jq -r '.detail // empty' "$WORK/body" 2>/dev/null)" ;;
      esac
    done
    jq -c '.data[]' "$WORK/body" >> "$WORK/rows" || fail "unexpected answer: $(head -c 200 "$WORK/body")"
    more="$(jq -r .has_more "$WORK/body")" || fail "unexpected answer: $(head -c 200 "$WORK/body")"
    [ "$more" = true ] || return 0
    cursor="$(jq -r .next_cursor "$WORK/body")"
  done
}

walks=0
while :; do
  walks=$((walks + 1))
  result=0
  walk || result=$?
  [ "$result" -eq 0 ] && break
  [ "$result" -eq 3 ] && [ "$walks" -lt "$MAX_WALKS" ] || fail "the list kept changing while it was read"
done
cat "$WORK/rows"

"""Print every item of a PerfBee list, one JSON object per line (Python 3.9 or later, standard library only).

    PERFBEE_API_KEY=pbk_... python paginate.py '/sites/12/pages?status=broken'

PERFBEE_API sets another base URL (default https://dashboard.perfbee.com/api/v1).
"""
import json
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request

API = os.environ.get("PERFBEE_API", "https://dashboard.perfbee.com/api/v1")
LIMIT = 200          # the most a page holds
MAX_TRIES = 4        # tries per request when PerfBee says "slow down" or "busy"
MAX_WALKS = 3        # walks of the whole list when a cursor stops being valid
DEFAULT_PAUSE = 5    # seconds to pause when there is no Retry-After
MAX_PAUSE = 60


class ApiProblem(Exception):
    def __init__(self, status, problem):
        super().__init__(f"HTTP {status}: {(problem or {}).get('detail', '')}")
        self.status, self.problem = status, problem or {}


def pause_seconds(retry_after):
    try:
        seconds = int(retry_after)
    except (TypeError, ValueError):
        return DEFAULT_PAUSE
    return min(seconds, MAX_PAUSE) if seconds >= 1 else DEFAULT_PAUSE


def get_json(url, key):
    """GET one answer. A 429 or 503 is waited out (Retry-After); any other error raises ApiProblem."""
    for tries in range(1, MAX_TRIES + 1):
        request = urllib.request.Request(url, headers={"Authorization": f"Bearer {key}", "Accept": "application/json"})
        try:
            with urllib.request.urlopen(request, timeout=60) as response:
                return json.load(response)
        except urllib.error.HTTPError as error:
            try:
                problem = json.loads(error.read())
            except ValueError:
                problem = None
            if error.code in (429, 503) and tries < MAX_TRIES:
                time.sleep(pause_seconds(error.headers.get("Retry-After")))
                continue
            raise ApiProblem(error.code, problem) from None


def list_url(path, cursor):
    # Every filter stays the same on every request: a cursor only fits the filters it was made for.
    separator = "&" if "?" in path else "?"
    url = f"{API}{path}{separator}limit={LIMIT}"
    return url + (f"&cursor={urllib.parse.quote(cursor, safe='')}" if cursor else "")


def list_all(path, key):
    """Every item of the list. The items are kept until the walk ends, so a restart never repeats one."""
    for walk in range(1, MAX_WALKS + 1):
        items, cursor = [], None
        try:
            while True:
                page = get_json(list_url(path, cursor), key)
                items.extend(page["data"])
                if not page["has_more"]:
                    return items
                cursor = page["next_cursor"]
        except ApiProblem as problem:
            if problem.problem.get("code") != "invalid_cursor" or walk == MAX_WALKS:
                raise   # invalid_cursor: the list changed under the walk, so walk it again from the start


if __name__ == "__main__":
    if len(sys.argv) != 2 or not os.environ.get("PERFBEE_API_KEY"):
        raise SystemExit("usage: PERFBEE_API_KEY=pbk_... python paginate.py '/path?filters'")
    try:
        for item in list_all(sys.argv[1], os.environ["PERFBEE_API_KEY"]):
            print(json.dumps(item))
    except ApiProblem as error:
        raise SystemExit(f"perfbee: {error}")

// Print every item of a PerfBee list, one JSON object per line (Node 18 or later, no packages).
//
//   PERFBEE_API_KEY=pbk_... node paginate.mjs '/sites/12/pages?status=broken'
//
// PERFBEE_API sets another base URL (default https://dashboard.perfbee.com/api/v1).
import { pathToFileURL } from 'node:url';

const LIMIT = 200;          // the most a page holds
const MAX_TRIES = 4;        // tries per request when PerfBee says "slow down" or "busy"
const MAX_WALKS = 3;        // walks of the whole list when a cursor stops being valid
const DEFAULT_PAUSE = 5;    // seconds to pause when there is no Retry-After
const MAX_PAUSE = 60;

export class ApiProblem extends Error {
  constructor(status, problem) {
    super(`HTTP ${status}: ${(problem && problem.detail) || ''}`);
    this.status = status;
    this.problem = problem || {};
  }
}

const wait = (seconds) => new Promise((done) => setTimeout(done, seconds * 1000));

function pauseSeconds(retryAfter) {
  const seconds = parseInt(retryAfter, 10);
  return Number.isNaN(seconds) || seconds < 1 ? DEFAULT_PAUSE : Math.min(seconds, MAX_PAUSE);
}

async function getJson(url, key, sleep) {
  for (let tries = 1; ; tries++) {
    const response = await fetch(url, { headers: { Authorization: `Bearer ${key}`, Accept: 'application/json' } });
    if (response.ok) return response.json();
    const problem = await response.json().catch(() => null);
    if ((response.status === 429 || response.status === 503) && tries < MAX_TRIES) {
      await sleep(pauseSeconds(response.headers.get('retry-after')));
      continue;
    }
    throw new ApiProblem(response.status, problem);
  }
}

function listUrl(api, path, cursor) {
  // Every filter stays the same on every request: a cursor only fits the filters it was made for.
  const url = `${api}${path}${path.includes('?') ? '&' : '?'}limit=${LIMIT}`;
  return cursor ? `${url}&cursor=${encodeURIComponent(cursor)}` : url;
}

// Every item of the list. The items are kept until the walk ends, so a restart never repeats one.
export async function listAll(path, key, { api = process.env.PERFBEE_API || 'https://dashboard.perfbee.com/api/v1', sleep = wait } = {}) {
  for (let walk = 1; ; walk++) {
    const items = [];
    let cursor = null;
    try {
      for (;;) {
        const page = await getJson(listUrl(api, path, cursor), key, sleep);
        items.push(...page.data);
        if (!page.has_more) return items;
        cursor = page.next_cursor;
      }
    } catch (error) {
      // invalid_cursor: the list changed under the walk, so walk it again from the start
      if (!(error instanceof ApiProblem) || error.problem.code !== 'invalid_cursor' || walk === MAX_WALKS) throw error;
    }
  }
}

if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
  const [path] = process.argv.slice(2);
  if (!path || !process.env.PERFBEE_API_KEY) {
    console.error("usage: PERFBEE_API_KEY=pbk_... node paginate.mjs '/path?filters'");
    process.exit(2);
  }
  listAll(path, process.env.PERFBEE_API_KEY)
    .then((items) => items.forEach((item) => console.log(JSON.stringify(item))))
    .catch((error) => {
      console.error(`perfbee: ${error.message}`);
      process.exit(1);
    });
}

#!/usr/bin/env bash
# Fail a deploy when a PerfBee crawl finds broken links that the previous crawl did not have.
#
# Needs bash, curl and jq. Put the secrets in your CI settings, never in the repository:
#   PERFBEE_API_KEY   a key with the `crawl` scope (Settings > API keys)
#   PERFBEE_SITE_ID   the id of the site in My Sites (GET /sites lists them)
#   IDEMPOTENCY_KEY   one value per intended crawl, for example the commit hash (default: $GITHUB_SHA)
# Optional:
#   PERFBEE_MODE (fast or full, default fast), PERFBEE_MAX_PAGES (default 500), PERFBEE_TIMEOUT (seconds, default 1800),
#   PERFBEE_API (default https://dashboard.perfbee.com/api/v1)
#
# Exit 0: no new broken links. Exit 1: new broken links. Exit 2: the check could not finish.
set -euo pipefail

API="${PERFBEE_API:-https://dashboard.perfbee.com/api/v1}"
MODE="${PERFBEE_MODE:-fast}"
MAX_PAGES="${PERFBEE_MAX_PAGES:-500}"
TIMEOUT="${PERFBEE_TIMEOUT:-1800}"
WAIT_SECONDS=25          # the longest ?wait the API accepts
SHOW_NEW_BROKEN=20       # how many new broken URLs to print
DEFAULT_PAUSE=5          # seconds to pause when the API gives no Retry-After

WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT

fail() {
  echo "perfbee: $2" >&2
  exit "$1"
}

for tool in curl jq; do
  command -v "$tool" >/dev/null 2>&1 || fail 2 "$tool is required"
done
[ -n "${PERFBEE_API_KEY:-}" ] || fail 2 "PERFBEE_API_KEY is not set"
[ -n "${PERFBEE_SITE_ID:-}" ] || fail 2 "PERFBEE_SITE_ID is not set"
IDEMPOTENCY_KEY="${IDEMPOTENCY_KEY:-${GITHUB_SHA:-}}"
[ -n "$IDEMPOTENCY_KEY" ] || fail 2 "set IDEMPOTENCY_KEY (the commit hash works) so a retry never starts a second crawl"

# api METHOD PATH [curl options]: sets $status; the answer is in $WORK/body, its headers in $WORK/headers.
api() {
  local method="$1" path="$2"
  shift 2
  status="$(curl --silent --show-error --max-time $((WAIT_SECONDS + 15)) --request "$method" \
    --header "Authorization: Bearer $PERFBEE_API_KEY" --header "Accept: application/json" \
    --output "$WORK/body" --dump-header "$WORK/headers" --write-out '%{http_code}' "$@" "$API$path")" \
    || fail 2 "could not reach $API"
}

# header NAME: the value of a response header, empty when it is not there.
header() {
  tr -d '\r' < "$WORK/headers" | awk -F': *' -v name="$1" 'tolower($1) == tolower(name) { print $2; exit }'
}

# problem: the error the API answered, in one line.
problem() {
  local text retry
  text="$(jq -r '"\(.title // "Error"): \(.detail // "no detail") [\(.code // "?"), request \(.request_id // "?")]"' \
    "$WORK/body" 2>/dev/null)" || text="HTTP $status"
  retry="$(header Retry-After)"
  [ -z "$retry" ] || text="$text Retry after ${retry}s."
  echo "$text"
}

# 1. Start the crawl. The same Idempotency-Key and body return the crawl already started, so a retry costs nothing.
payload="$(jq -n --argjson site "$PERFBEE_SITE_ID" --arg mode "$MODE" --argjson pages "$MAX_PAGES" \
  '{site_id: $site, mode: $mode, max_pages: $pages}')"
api POST /crawls --header "Content-Type: application/json" --header "Idempotency-Key: $IDEMPOTENCY_KEY" --data "$payload"
case "$status" in
  200 | 202) ;;
  *) fail 2 "could not start a crawl: $(problem)" ;;
esac
crawl_id="$(jq -r .id "$WORK/body")" || fail 2 "unexpected answer: $(head -c 200 "$WORK/body")"
echo "perfbee: crawl $crawl_id started"

# 2. Wait. Each call holds for up to 25 seconds and returns as soon as the crawl is over.
deadline=$(($(date +%s) + TIMEOUT))
state=unknown
while :; do
  pause=0
  api GET "/crawls/$crawl_id?wait=$WAIT_SECONDS"
  if [ "$status" = 200 ]; then
    state="$(jq -r .status "$WORK/body")" || fail 2 "unexpected answer: $(head -c 200 "$WORK/body")"
    case "$state" in completed | failed | cancelled | partial) break ;; esac
    pause="$(header Retry-After)"   # set when the server had no free wait: ask again after that many seconds
  elif [ "$status" = 429 ]; then
    pause="$(header Retry-After)"
    pause="${pause:-$DEFAULT_PAUSE}"
  else
    fail 2 "could not read crawl $crawl_id: $(problem)"
  fi
  case "$pause" in '' | *[!0-9]*) pause=0 ;; esac
  [ "$(date +%s)" -lt "$deadline" ] || fail 2 "gave up after ${TIMEOUT}s: crawl $crawl_id is still $state"
  [ "$pause" -eq 0 ] || sleep "$pause"
done
report="$(jq -r .report_url "$WORK/body")"
[ "$state" = completed ] || fail 2 "crawl $crawl_id ended as $state. Report: $report"

# 3. Compare with the previous crawl of this site in the same mode.
api GET "/crawls/$crawl_id/diff"
[ "$status" = 200 ] || fail 2 "could not read the comparison: $(problem)"
previous="$(jq -r .has_previous "$WORK/body")" || fail 2 "unexpected answer: $(head -c 200 "$WORK/body")"
if [ "$previous" != true ]; then
  echo "perfbee: the first $MODE crawl of this site, so there is nothing to compare. Report: $report"
  exit 0
fi
if jq -e '.summary.new_broken == 0' "$WORK/body" > /dev/null; then
  echo "perfbee: no new broken links ($(jq -r .summary.fixed_broken "$WORK/body") fixed). Report: $report"
  exit 0
fi
echo "perfbee: $(jq -r .summary.new_broken "$WORK/body") new broken links. Report: $report" >&2
api GET "/crawls/$crawl_id/diff?include=lists&limit=$SHOW_NEW_BROKEN"
if [ "$status" = 200 ]; then
  jq -r '.lists.new_broken[] | "  " + .' "$WORK/body" >&2
fi
exit 1

# Fail a workflow when a PerfBee crawl finds broken links that the previous crawl did not have.
#
# Set once in the repository settings:
#   secret   PERFBEE_API_KEY   a key with the `crawl` scope (Settings > API keys in PerfBee)
#   variable PERFBEE_SITE_ID   the site's id: GET /sites lists them
# Run this job after the job that deploys: add `needs: deploy` below.
name: Check for new broken links

on:
  workflow_dispatch:
  push:
    branches: [main]

jobs:
  broken-links:
    runs-on: ubuntu-latest
    timeout-minutes: 45
    env:
      PERFBEE_API: https://dashboard.perfbee.com/api/v1
      PERFBEE_SITE_ID: ${{ vars.PERFBEE_SITE_ID }}
    steps:
      - name: Start a crawl
        id: start
        env:
          PERFBEE_API_KEY: ${{ secrets.PERFBEE_API_KEY }}
        run: |
          status=$(curl --silent --show-error --output start.json --write-out '%{http_code}' \
            --request POST "$PERFBEE_API/crawls" \
            --header "Authorization: Bearer $PERFBEE_API_KEY" \
            --header "Content-Type: application/json" \
            --header "Idempotency-Key: ${{ github.sha }}-${{ github.run_attempt }}" \
            --data "{\"site_id\": $PERFBEE_SITE_ID, \"mode\": \"fast\", \"max_pages\": 500}")
          if [ "$status" != 200 ] && [ "$status" != 202 ]; then
            echo "::error::PerfBee refused the crawl: $(jq -r '.detail' start.json)"
            exit 1
          fi
          echo "crawl_id=$(jq -r .id start.json)" >> "$GITHUB_OUTPUT"

      - name: Wait for the crawl
        env:
          PERFBEE_API_KEY: ${{ secrets.PERFBEE_API_KEY }}
          CRAWL_ID: ${{ steps.start.outputs.crawl_id }}
        run: |
          deadline=$(( $(date +%s) + 1800 ))
          while :; do
            curl --silent --show-error --fail --retry 3 --output crawl.json \
              --header "Authorization: Bearer $PERFBEE_API_KEY" \
              "$PERFBEE_API/crawls/$CRAWL_ID?wait=25"
            state=$(jq -r .status crawl.json)
            case "$state" in
              completed) break ;;
              failed | cancelled | partial)
                echo "::error::The crawl ended as $state. Report: $(jq -r .report_url crawl.json)"
                exit 1 ;;
            esac
            [ "$(date +%s)" -lt "$deadline" ] || { echo "::error::The crawl is still $state after 30 minutes."; exit 1; }
          done

      - name: Fail on new broken links
        env:
          PERFBEE_API_KEY: ${{ secrets.PERFBEE_API_KEY }}
          CRAWL_ID: ${{ steps.start.outputs.crawl_id }}
        run: |
          curl --silent --show-error --fail --retry 3 --output diff.json \
            --header "Authorization: Bearer $PERFBEE_API_KEY" \
            "$PERFBEE_API/crawls/$CRAWL_ID/diff"
          if [ "$(jq -r .has_previous diff.json)" != true ]; then
            echo "The first crawl of this site in this mode: nothing to compare."
            exit 0
          fi
          if ! jq -e '.summary.new_broken == 0' diff.json > /dev/null; then
            echo "::error::$(jq -r .summary.new_broken diff.json) new broken links. Report: $(jq -r .report_url crawl.json)"
            exit 1
          fi
          echo "No new broken links."

{
  "name": "PerfBee crawl summary to Slack",
  "nodes": [
    {
      "parameters": {
        "content": "## PerfBee crawl summary to Slack\n\n1. Copy the Production URL of the Webhook node.\n2. In PerfBee, Settings > Webhooks, add it as an endpoint for the event **Crawl data ready** (scan_data_ready).\n3. Make a Header Auth credential named PerfBee API key: name Authorization, value Bearer pbk_... (a key with the read scope). Choose it in the node Read the crawl.\n4. Replace REPLACE_ME in the node Post to Slack with your Slack incoming webhook URL.\n\nThis workflow does not check the PerfBee-Signature header. The Production URL of an n8n webhook is long and random: keep it private.",
        "height": 300,
        "width": 460
      },
      "id": "7f2c1a10-0000-4000-8000-000000000001",
      "name": "Read me",
      "type": "n8n-nodes-base.stickyNote",
      "typeVersion": 1,
      "position": [
        -120,
        -40
      ]
    },
    {
      "parameters": {
        "httpMethod": "POST",
        "path": "perfbee",
        "options": {}
      },
      "id": "7f2c1a10-0000-4000-8000-000000000002",
      "name": "PerfBee webhook",
      "type": "n8n-nodes-base.webhook",
      "typeVersion": 2,
      "position": [
        400,
        160
      ],
      "webhookId": "7f2c1a10-0000-4000-8000-0000000000a1"
    },
    {
      "parameters": {
        "conditions": {
          "options": {
            "caseSensitive": true,
            "leftValue": "",
            "typeValidation": "strict"
          },
          "conditions": [
            {
              "id": "7f2c1a10-0000-4000-8000-0000000000b1",
              "leftValue": "={{ $json.body.event }}",
              "rightValue": "scan_data_ready",
              "operator": {
                "type": "string",
                "operation": "equals"
              }
            }
          ],
          "combinator": "and"
        },
        "options": {}
      },
      "id": "7f2c1a10-0000-4000-8000-000000000003",
      "name": "Crawl data ready?",
      "type": "n8n-nodes-base.if",
      "typeVersion": 2,
      "position": [
        620,
        160
      ]
    },
    {
      "parameters": {
        "url": "=https://dashboard.perfbee.com/api/v1/crawls/{{ $json.body.data.crawl_id }}",
        "authentication": "genericCredentialType",
        "genericAuthType": "httpHeaderAuth",
        "options": {}
      },
      "id": "7f2c1a10-0000-4000-8000-000000000004",
      "name": "Read the crawl",
      "type": "n8n-nodes-base.httpRequest",
      "typeVersion": 4.2,
      "position": [
        840,
        140
      ],
      "credentials": {
        "httpHeaderAuth": {
          "id": "REPLACE_ME",
          "name": "PerfBee API key"
        }
      }
    },
    {
      "parameters": {
        "assignments": {
          "assignments": [
            {
              "id": "7f2c1a10-0000-4000-8000-0000000000c1",
              "name": "text",
              "value": "=PerfBee crawled {{ $json.url }} ({{ $json.mode }}): {{ $json.pages_checked }} pages, {{ $json.broken_count }} broken links. {{ $json.report_url }}",
              "type": "string"
            }
          ]
        },
        "options": {}
      },
      "id": "7f2c1a10-0000-4000-8000-000000000005",
      "name": "Write the message",
      "type": "n8n-nodes-base.set",
      "typeVersion": 3.4,
      "position": [
        1060,
        140
      ]
    },
    {
      "parameters": {
        "method": "POST",
        "url": "https://hooks.slack.com/services/REPLACE_ME",
        "sendBody": true,
        "specifyBody": "json",
        "jsonBody": "={{ JSON.stringify({ text: $json.text }) }}",
        "options": {}
      },
      "id": "7f2c1a10-0000-4000-8000-000000000006",
      "name": "Post to Slack",
      "type": "n8n-nodes-base.httpRequest",
      "typeVersion": 4.2,
      "position": [
        1280,
        140
      ]
    }
  ],
  "connections": {
    "PerfBee webhook": {
      "main": [
        [
          {
            "node": "Crawl data ready?",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Crawl data ready?": {
      "main": [
        [
          {
            "node": "Read the crawl",
            "type": "main",
            "index": 0
          }
        ],
        []
      ]
    },
    "Read the crawl": {
      "main": [
        [
          {
            "node": "Write the message",
            "type": "main",
            "index": 0
          }
        ]
      ]
    },
    "Write the message": {
      "main": [
        [
          {
            "node": "Post to Slack",
            "type": "main",
            "index": 0
          }
        ]
      ]
    }
  },
  "pinData": {},
  "settings": {
    "executionOrder": "v1"
  },
  "active": false,
  "tags": []
}

// Print every item of a PerfBee list, one JSON object per line (Node 18 or later, no packages).
//
//   PERFBEE_API_KEY=pbk_... node paginate.mjs '/sites/12/pages?status=broken'
//
// PERFBEE_API sets another base URL (default https://dashboard.perfbee.com/api/v1).
import { pathToFileURL } from 'node:url';

const LIMIT = 200;          // the most a page holds
const MAX_TRIES = 4;        // tries per request when PerfBee says "slow down" or "busy"
const MAX_WALKS = 3;        // walks of the whole list when a cursor stops being valid
const DEFAULT_PAUSE = 5;    // seconds to pause when there is no Retry-After
const MAX_PAUSE = 60;

export class ApiProblem extends Error {
  constructor(status, problem) {
    super(`HTTP ${status}: ${(problem && problem.detail) || ''}`);
    this.status = status;
    this.problem = problem || {};
  }
}

const wait = (seconds) => new Promise((done) => setTimeout(done, seconds * 1000));

function pauseSeconds(retryAfter) {
  const seconds = parseInt(retryAfter, 10);
  return Number.isNaN(seconds) || seconds < 1 ? DEFAULT_PAUSE : Math.min(seconds, MAX_PAUSE);
}

async function getJson(url, key, sleep) {
  for (let tries = 1; ; tries++) {
    const response = await fetch(url, { headers: { Authorization: `Bearer ${key}`, Accept: 'application/json' } });
    if (response.ok) return response.json();
    const problem = await response.json().catch(() => null);
    if ((response.status === 429 || response.status === 503) && tries < MAX_TRIES) {
      await sleep(pauseSeconds(response.headers.get('retry-after')));
      continue;
    }
    throw new ApiProblem(response.status, problem);
  }
}

function listUrl(api, path, cursor) {
  // Every filter stays the same on every request: a cursor only fits the filters it was made for.
  const url = `${api}${path}${path.includes('?') ? '&' : '?'}limit=${LIMIT}`;
  return cursor ? `${url}&cursor=${encodeURIComponent(cursor)}` : url;
}

// Every item of the list. The items are kept until the walk ends, so a restart never repeats one.
export async function listAll(path, key, { api = process.env.PERFBEE_API || 'https://dashboard.perfbee.com/api/v1', sleep = wait } = {}) {
  for (let walk = 1; ; walk++) {
    const items = [];
    let cursor = null;
    try {
      for (;;) {
        const page = await getJson(listUrl(api, path, cursor), key, sleep);
        items.push(...page.data);
        if (!page.has_more) return items;
        cursor = page.next_cursor;
      }
    } catch (error) {
      // invalid_cursor: the list changed under the walk, so walk it again from the start
      if (!(error instanceof ApiProblem) || error.problem.code !== 'invalid_cursor' || walk === MAX_WALKS) throw error;
    }
  }
}

if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
  const [path] = process.argv.slice(2);
  if (!path || !process.env.PERFBEE_API_KEY) {
    console.error("usage: PERFBEE_API_KEY=pbk_... node paginate.mjs '/path?filters'");
    process.exit(2);
  }
  listAll(path, process.env.PERFBEE_API_KEY)
    .then((items) => items.forEach((item) => console.log(JSON.stringify(item))))
    .catch((error) => {
      console.error(`perfbee: ${error.message}`);
      process.exit(1);
    });
}

"""Print every item of a PerfBee list, one JSON object per line (Python 3.9 or later, standard library only).

    PERFBEE_API_KEY=pbk_... python paginate.py '/sites/12/pages?status=broken'

PERFBEE_API sets another base URL (default https://dashboard.perfbee.com/api/v1).
"""
import json
import os
import sys
import time
import urllib.error
import urllib.parse
import urllib.request

API = os.environ.get("PERFBEE_API", "https://dashboard.perfbee.com/api/v1")
LIMIT = 200          # the most a page holds
MAX_TRIES = 4        # tries per request when PerfBee says "slow down" or "busy"
MAX_WALKS = 3        # walks of the whole list when a cursor stops being valid
DEFAULT_PAUSE = 5    # seconds to pause when there is no Retry-After
MAX_PAUSE = 60


class ApiProblem(Exception):
    def __init__(self, status, problem):
        super().__init__(f"HTTP {status}: {(problem or {}).get('detail', '')}")
        self.status, self.problem = status, problem or {}


def pause_seconds(retry_after):
    try:
        seconds = int(retry_after)
    except (TypeError, ValueError):
        return DEFAULT_PAUSE
    return min(seconds, MAX_PAUSE) if seconds >= 1 else DEFAULT_PAUSE


def get_json(url, key):
    """GET one answer. A 429 or 503 is waited out (Retry-After); any other error raises ApiProblem."""
    for tries in range(1, MAX_TRIES + 1):
        request = urllib.request.Request(url, headers={"Authorization": f"Bearer {key}", "Accept": "application/json"})
        try:
            with urllib.request.urlopen(request, timeout=60) as response:
                return json.load(response)
        except urllib.error.HTTPError as error:
            try:
                problem = json.loads(error.read())
            except ValueError:
                problem = None
            if error.code in (429, 503) and tries < MAX_TRIES:
                time.sleep(pause_seconds(error.headers.get("Retry-After")))
                continue
            raise ApiProblem(error.code, problem) from None


def list_url(path, cursor):
    # Every filter stays the same on every request: a cursor only fits the filters it was made for.
    separator = "&" if "?" in path else "?"
    url = f"{API}{path}{separator}limit={LIMIT}"
    return url + (f"&cursor={urllib.parse.quote(cursor, safe='')}" if cursor else "")


def list_all(path, key):
    """Every item of the list. The items are kept until the walk ends, so a restart never repeats one."""
    for walk in range(1, MAX_WALKS + 1):
        items, cursor = [], None
        try:
            while True:
                page = get_json(list_url(path, cursor), key)
                items.extend(page["data"])
                if not page["has_more"]:
                    return items
                cursor = page["next_cursor"]
        except ApiProblem as problem:
            if problem.problem.get("code") != "invalid_cursor" or walk == MAX_WALKS:
                raise   # invalid_cursor: the list changed under the walk, so walk it again from the start


if __name__ == "__main__":
    if len(sys.argv) != 2 or not os.environ.get("PERFBEE_API_KEY"):
        raise SystemExit("usage: PERFBEE_API_KEY=pbk_... python paginate.py '/path?filters'")
    try:
        for item in list_all(sys.argv[1], os.environ["PERFBEE_API_KEY"]):
            print(json.dumps(item))
    except ApiProblem as error:
        raise SystemExit(f"perfbee: {error}")

#!/usr/bin/env bash
# Print every item of a PerfBee list, one JSON object per line. Needs bash, curl and jq.
#
#   PERFBEE_API_KEY=pbk_... ./paginate.sh '/sites/12/pages?status=broken'
#
# PERFBEE_API sets another base URL (default https://dashboard.perfbee.com/api/v1).
set -euo pipefail

API="${PERFBEE_API:-https://dashboard.perfbee.com/api/v1}"
PATH_AND_QUERY="${1:?usage: paginate.sh '/path?filters'}"
[ -n "${PERFBEE_API_KEY:-}" ] || { echo "PERFBEE_API_KEY is not set" >&2; exit 2; }
LIMIT=200          # the most a page holds
MAX_TRIES=4        # tries per request when PerfBee says "slow down" or "busy"
MAX_WALKS=3        # walks of the whole list when a cursor stops being valid
DEFAULT_PAUSE=5    # seconds to pause when there is no Retry-After
MAX_PAUSE=60

WORK="$(mktemp -d)"
trap 'rm -rf "$WORK"' EXIT

fail() {
  echo "perfbee: $1" >&2
  exit 1
}

# pause: seconds to wait, from the Retry-After header of the last answer.
pause() {
  local value
  value="$(tr -d '\r' < "$WORK/headers" | awk -F': *' 'tolower($1) == "retry-after" { print $2; exit }')"
  case "$value" in '' | *[!0-9]*) value=$DEFAULT_PAUSE ;; esac
  [ "$value" -ge 1 ] || value=$DEFAULT_PAUSE
  [ "$value" -le "$MAX_PAUSE" ] || value=$MAX_PAUSE
  echo "$value"
}

# walk: one walk of the list into $WORK/rows. Returns 3 when the cursor stopped being valid (start again).
walk() {
  local cursor="" url separator tries more
  : > "$WORK/rows"
  case "$PATH_AND_QUERY" in *\?*) separator="&" ;; *) separator="?" ;; esac
  while :; do
    # Every filter stays the same on every request: a cursor only fits the filters it was made for.
    url="$API$PATH_AND_QUERY${separator}limit=$LIMIT"
    [ -z "$cursor" ] || url="$url&cursor=$(jq -rn --arg c "$cursor" '$c | @uri')"
    tries=1
    while :; do
      status="$(curl --silent --show-error --max-time 60 --header "Authorization: Bearer $PERFBEE_API_KEY" \
        --output "$WORK/body" --dump-header "$WORK/headers" --write-out '%{http_code}' "$url")" || fail "could not reach $API"
      case "$status" in
        200) break ;;
        429 | 503)
          [ "$tries" -lt "$MAX_TRIES" ] || fail "still answering $status after $MAX_TRIES tries"
          sleep "$(pause)"
          tries=$((tries + 1)) ;;
        400)
          [ "$(jq -r .code "$WORK/body" 2>/dev/null)" != invalid_cursor ] || return 3
          fail "$(jq -r .detail "$WORK/body")" ;;
        *) fail "HTTP $status: $(jq -r '.detail // empty' "$WORK/body" 2>/dev/null)" ;;
      esac
    done
    jq -c '.data[]' "$WORK/body" >> "$WORK/rows" || fail "unexpected answer: $(head -c 200 "$WORK/body")"
    more="$(jq -r .has_more "$WORK/body")" || fail "unexpected answer: $(head -c 200 "$WORK/body")"
    [ "$more" = true ] || return 0
    cursor="$(jq -r .next_cursor "$WORK/body")"
  done
}

walks=0
while :; do
  walks=$((walks + 1))
  result=0
  walk || result=$?
  [ "$result" -eq 0 ] && break
  [ "$result" -eq 3 ] && [ "$walks" -lt "$MAX_WALKS" ] || fail "the list kept changing while it was read"
done
cat "$WORK/rows"

/**
 * PerfBee pages -> one tab of a Google Sheet (Apps Script, nothing to install).
 *
 * 1. In PerfBee, Settings > API keys: make a key with the `read` scope.
 * 2. In your sheet: Extensions > Apps Script. Paste this file over the editor's content and save.
 * 3. Project Settings (the gear) > Script properties. Add:
 *      PERFBEE_API_KEY       the key. It is a secret: keep it here, never in the code or the sheet.
 *      PERFBEE_SITE_ID       the site's id (GET /sites lists them).
 *      PERFBEE_PAGE_FILTERS  optional, as a query string, for example: view=pages&status=broken
 * 4. Reload the sheet. The PerfBee menu has Refresh now and Refresh every hour.
 *
 * Google asks you to allow the script once. "Unverified app" is normal for a script you wrote yourself.
 * A run that needs more than 5 minutes stops and leaves the tab as it was: Apps Script ends a run at 6 minutes.
 * Narrow the list with PERFBEE_PAGE_FILTERS (for example `prefix=https://www.example.com/shop/`) when that happens.
 */
const API_BASE = 'https://dashboard.perfbee.com/api/v1';
const TAB_NAME = 'PerfBee pages';
const PAGE_SIZE = 200;                      // the most a page of the list holds
const RUN_BUDGET_MS = 5 * 60 * 1000;        // stop before Apps Script's own 6 minute limit
const MAX_TRIES = 4;                        // tries per request when PerfBee says "slow down" or "busy"
const MAX_WALKS = 3;                        // walks of the whole list when a cursor stops being valid
const DEFAULT_PAUSE_SECONDS = 5;
const MAX_PAUSE_SECONDS = 60;

// [header, how to read it from a page row, text or number]
const COLUMNS = [
  ['id', (p) => p.id, 'number'],
  ['url', (p) => p.url, 'text'],
  ['status', (p) => p.status_class, 'text'],
  ['status code', (p) => p.status_code, 'number'],
  ['indexability', (p) => p.indexability, 'text'],
  ['title', (p) => p.title, 'text'],
  ['h1', (p) => p.h1, 'text'],
  ['words', (p) => p.word_count, 'number'],
  ['response ms', (p) => p.response_time_ms, 'number'],
  ['inbound links', (p) => p.inbound_links, 'number'],
  ['depth', (p) => p.crawl_depth, 'number'],
  ['issues', (p) => (p.issues || []).join(', '), 'text'],
  ['last seen', (p) => p.last_seen_at, 'text'],
];

function onOpen() {
  SpreadsheetApp.getUi()
    .createMenu('PerfBee')
    .addItem('Refresh now', 'refreshPages')
    .addItem('Refresh every hour', 'startHourlyRefresh')
    .addItem('Stop refreshing', 'stopRefresh')
    .addToUi();
}

/** Read every page of the site's list and replace the tab with it. On any problem the tab is left as it was. */
function refreshPages() {
  const started = Date.now();
  const properties = PropertiesService.getScriptProperties();
  const key = properties.getProperty('PERFBEE_API_KEY');
  const siteId = properties.getProperty('PERFBEE_SITE_ID');
  if (!key || !siteId) {
    throw new Error('Set PERFBEE_API_KEY and PERFBEE_SITE_ID in Project Settings > Script properties.');
  }
  const filters = properties.getProperty('PERFBEE_PAGE_FILTERS') || '';
  let pages = null;
  for (let walk = 1; pages === null; walk++) {
    pages = walkPages_(key, siteId, filters, started, walk < MAX_WALKS);
  }
  writeTab_(pages);
}

/** One walk of the list. Returns the rows, or null when the cursor stopped being valid and a new walk is allowed. */
function walkPages_(key, siteId, filters, started, mayRestart) {
  const rows = [];
  let cursor = null;
  do {
    if (Date.now() - started > RUN_BUDGET_MS) {
      throw new Error('Stopped after 5 minutes with ' + rows.length + ' pages read. The tab was not changed. Narrow the list with PERFBEE_PAGE_FILTERS.');
    }
    let page;
    try {
      page = getJson_(listUrl_(siteId, filters, cursor), key, started);
    } catch (error) {
      if (mayRestart && error.problem && error.problem.code === 'invalid_cursor') return null;   // the site changed under the walk
      throw error;
    }
    page.data.forEach((row) => rows.push(COLUMNS.map((column) => column[1](row))));
    cursor = page.has_more ? page.next_cursor : null;
  } while (cursor);
  return rows;
}

function listUrl_(siteId, filters, cursor) {
  // Keep every filter the same on every request: a cursor only fits the filters it was made for.
  let url = API_BASE + '/sites/' + encodeURIComponent(siteId) + '/pages?limit=' + PAGE_SIZE;
  if (filters) url += '&' + filters;
  if (cursor) url += '&cursor=' + encodeURIComponent(cursor);
  return url;
}

/** GET a JSON answer. A 429 or 503 is waited out (Retry-After) up to MAX_TRIES times; anything else stops the run. */
function getJson_(url, key, started) {
  for (let tries = 1; ; tries++) {
    const response = UrlFetchApp.fetch(url, {
      method: 'get', muteHttpExceptions: true,
      headers: { Authorization: 'Bearer ' + key, Accept: 'application/json' },
    });
    const code = response.getResponseCode();
    if (code === 200) return JSON.parse(response.getContentText());
    if ((code === 429 || code === 503) && tries < MAX_TRIES) {
      const pause = pauseSeconds_(response);
      if (Date.now() - started + pause * 1000 > RUN_BUDGET_MS) {
        throw new Error('PerfBee asked for a pause of ' + pause + ' seconds, which would pass the 5 minute budget.');
      }
      Utilities.sleep(pause * 1000);
      continue;
    }
    throw problemError_(code, response);
  }
}

function pauseSeconds_(response) {
  const headers = response.getHeaders();
  const name = Object.keys(headers).filter((header) => header.toLowerCase() === 'retry-after')[0];
  const seconds = name ? parseInt(headers[name], 10) : NaN;
  return isNaN(seconds) || seconds < 1 ? DEFAULT_PAUSE_SECONDS : Math.min(seconds, MAX_PAUSE_SECONDS);
}

function problemError_(code, response) {
  let problem = null;
  try {
    problem = JSON.parse(response.getContentText());
  } catch (ignored) {
    problem = null;
  }
  const error = new Error('PerfBee answered ' + code + (problem && problem.detail ? ': ' + problem.detail : '.'));
  error.problem = problem;
  return error;
}

function writeTab_(rows) {
  const spreadsheet = SpreadsheetApp.getActive();
  const sheet = spreadsheet.getSheetByName(TAB_NAME) || spreadsheet.insertSheet(TAB_NAME);
  const header = COLUMNS.map((column) => column[0]);
  sheet.clearContents();
  sheet.getRange(1, 1, 1, header.length).setValues([header]);
  sheet.setFrozenRows(1);
  if (!rows.length) return;
  COLUMNS.forEach((column, index) => {
    // Text columns are formatted as plain text first, so a crawled title such as "=1+1" stays text.
    if (column[2] === 'text') sheet.getRange(2, index + 1, rows.length, 1).setNumberFormat('@');
  });
  sheet.getRange(2, 1, rows.length, header.length).setValues(rows.map((row) => row.map((value) => (value === null || value === undefined ? '' : value))));
}

function startHourlyRefresh() {
  stopRefresh();
  ScriptApp.newTrigger('refreshPages').timeBased().everyHours(1).create();
  SpreadsheetApp.getActive().toast('Refreshing every hour.', 'PerfBee');
}

function stopRefresh() {
  ScriptApp.getProjectTriggers().forEach((trigger) => {
    if (trigger.getHandlerFunction() === 'refreshPages') ScriptApp.deleteTrigger(trigger);
  });
}

// Verify a PerfBee webhook with Node's standard library only (Node 18 or later).
//
//   PERFBEE_WEBHOOK_SECRET=whsec_... node verify-webhook.mjs      # listens on PORT (default 3000)
//
// Two secrets may be set, comma separated, while you rotate one: a delivery is accepted when any v1 value in the
// header matches any secret you hold.
import { createHmac, timingSafeEqual } from 'node:crypto';
import { createServer } from 'node:http';
import { pathToFileURL } from 'node:url';

const TOLERANCE_SECONDS = 300;

// header: the PerfBee-Signature value. rawBody: the bytes of the request body, exactly as received (a Buffer).
export function verifySignature(header, rawBody, secrets, nowSeconds = Date.now() / 1000) {
  let timestamp = null;
  const signatures = [];
  for (const part of String(header || '').split(',')) {
    const [name, ...rest] = part.trim().split('=');
    const value = rest.join('=');
    if (name === 't' && /^[0-9]+$/.test(value)) timestamp = Number(value);
    else if (name === 'v1' && value) signatures.push(value);
  }
  if (timestamp === null || signatures.length === 0) return false;
  if (Math.abs(nowSeconds - timestamp) > TOLERANCE_SECONDS) return false;
  for (const secret of secrets) {
    const expected = createHmac('sha256', secret).update(`${timestamp}.`).update(rawBody).digest();
    for (const candidate of signatures) {
      const given = Buffer.from(candidate, 'hex');
      if (given.length === expected.length && timingSafeEqual(given, expected)) return true;
    }
  }
  return false;
}

function serve(secrets, port) {
  const seen = new Set();
  return createServer((request, response) => {
    const chunks = [];
    request.on('data', (chunk) => chunks.push(chunk));
    request.on('end', () => {
      const rawBody = Buffer.concat(chunks);
      if (!verifySignature(request.headers['perfbee-signature'], rawBody, secrets)) {
        response.writeHead(400).end('bad signature');
        return;
      }
      const event = JSON.parse(rawBody.toString('utf8'));
      if (!seen.has(event.id)) {   // an event can arrive more than once: its id is the same every time
        seen.add(event.id);
        console.log(`${event.id} ${event.event}`);
      }
      response.writeHead(200).end('ok');
    });
  }).listen(port);
}

if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
  const secrets = (process.env.PERFBEE_WEBHOOK_SECRET || '').split(',').filter(Boolean);
  if (secrets.length === 0) throw new Error('Set PERFBEE_WEBHOOK_SECRET to the whsec_ secret of the endpoint.');
  serve(secrets, Number(process.env.PORT || 3000));
}

<?php
// Verify a PerfBee webhook with PHP's standard functions only (PHP 7.4 or later).
//
// Put this file where your web server can run it, and set PERFBEE_WEBHOOK_SECRET to the whsec_ secret of the endpoint.
// Two secrets may be set, comma separated, while you rotate one: a delivery is accepted when any v1 value in the header
// matches any secret you hold.
declare(strict_types=1);

const PERFBEE_TOLERANCE_SECONDS = 300;

// $header: the PerfBee-Signature value. $rawBody: the request body exactly as received.
function perfbee_verify_signature(string $header, string $rawBody, array $secrets, ?int $now = null): bool
{
    $timestamp = null;
    $signatures = [];
    foreach (explode(',', $header) as $part) {
        $pair = explode('=', trim($part), 2);
        if (count($pair) !== 2) {
            continue;
        }
        if ($pair[0] === 't' && ctype_digit($pair[1])) {
            $timestamp = (int) $pair[1];
        } elseif ($pair[0] === 'v1' && $pair[1] !== '') {
            $signatures[] = $pair[1];
        }
    }
    if ($timestamp === null || count($signatures) === 0) {
        return false;
    }
    if (abs(($now ?? time()) - $timestamp) > PERFBEE_TOLERANCE_SECONDS) {
        return false;
    }
    foreach ($secrets as $secret) {
        $expected = hash_hmac('sha256', $timestamp . '.' . $rawBody, $secret);
        foreach ($signatures as $candidate) {
            if (hash_equals($expected, $candidate)) {
                return true;
            }
        }
    }
    return false;
}

if (PHP_SAPI !== 'cli') {
    $secrets = array_filter(explode(',', (string) getenv('PERFBEE_WEBHOOK_SECRET')));
    $rawBody = (string) file_get_contents('php://input');
    $header = $_SERVER['HTTP_PERFBEE_SIGNATURE'] ?? '';
    if (!perfbee_verify_signature($header, $rawBody, $secrets)) {
        http_response_code(400);
        echo 'bad signature';
        exit;
    }
    $event = json_decode($rawBody, true);
    error_log($event['id'] . ' ' . $event['event']);
    http_response_code(200);
    echo 'ok';
}

"""Verify a PerfBee webhook with Python's standard library only (Python 3.9 or later).

    PERFBEE_WEBHOOK_SECRET=whsec_... python verify_webhook.py     # listens on PORT (default 3000)

Two secrets may be set, comma separated, while you rotate one: a delivery is accepted when any v1 value in the header
matches any secret you hold.
"""
import hashlib
import hmac
import json
import os
import time
from http.server import BaseHTTPRequestHandler, HTTPServer

TOLERANCE_SECONDS = 300


def verify_signature(header, raw_body, secrets, now=None):
    """header: the PerfBee-Signature value. raw_body: the bytes of the request body, exactly as received."""
    timestamp, signatures = None, []
    for part in (header or "").split(","):
        name, _, value = part.strip().partition("=")
        if name == "t" and value.isascii() and value.isdigit():
            timestamp = int(value)
        elif name == "v1" and value:
            signatures.append(value.encode())
    if timestamp is None or not signatures:
        return False
    if abs((time.time() if now is None else now) - timestamp) > TOLERANCE_SECONDS:
        return False
    for secret in secrets:
        signed = str(timestamp).encode() + b"." + raw_body
        expected = hmac.new(secret.encode(), signed, hashlib.sha256).hexdigest().encode()
        if any(hmac.compare_digest(expected, candidate) for candidate in signatures):
            return True
    return False


def make_handler(secrets):
    seen = set()

    class Handler(BaseHTTPRequestHandler):
        def do_POST(self):
            raw_body = self.rfile.read(int(self.headers.get("Content-Length") or 0))
            if not verify_signature(self.headers.get("PerfBee-Signature"), raw_body, secrets):
                self.send_response(400)
                self.end_headers()
                self.wfile.write(b"bad signature")
                return
            event = json.loads(raw_body)
            if event["id"] not in seen:   # an event can arrive more than once: its id is the same every time
                seen.add(event["id"])
                print(event["id"], event["event"], flush=True)
            self.send_response(200)
            self.end_headers()
            self.wfile.write(b"ok")

    return Handler


if __name__ == "__main__":
    held = [secret for secret in os.environ.get("PERFBEE_WEBHOOK_SECRET", "").split(",") if secret]
    if not held:
        raise SystemExit("Set PERFBEE_WEBHOOK_SECRET to the whsec_ secret of the endpoint.")
    HTTPServer(("", int(os.environ.get("PORT", "3000"))), make_handler(held)).serve_forever()