clarizen-lib/scripts/scrape-types.py

#!/usr/bin/env python3
"""Discover all Clarizen API types by crawling services + types docs."""
import json
import os
import re
import subprocess
import sys
from html import unescape
from pathlib import Path
from urllib.parse import urljoin

ROOT = Path(__file__).resolve().parents[1]

BASE = os.environ.get("CLARIZEN_BASE_URL", "https://api.clarizen.com")
TOKEN = os.environ.get("CLARIZEN_API_KEY") or os.environ.get("API_KEY_AW")
if not TOKEN:
    sys.exit("Set CLARIZEN_API_KEY or API_KEY_AW")

TYPE_LINK = re.compile(r'href="(/V2\.0/services/types/([^"]+))"', re.I)
SERVICE_LINK = re.compile(r'href="(/V2\.0/services/(?!types)[^"]+)"', re.I)
VARIANT_NAME = re.compile(r"<dt>Type</dt>\s*<dd>\s*<i>([^<]+)</i>", re.I)
PROP_ROW = re.compile(
    r"<td><i><b>(.*?)</b></i></td>\s*<td>(?:<a href=\"([^\"]+)\">)?([^<]+)",
    re.I | re.S,
)
ENUM_ROW = re.compile(r"<tr><td>([^<]+)</td><td>[^<]*</td><td>", re.I)


def fetch(path: str) -> str:
    url = urljoin(BASE, path)
    r = subprocess.run(
        [
            "curl",
            "-sL",
            "--max-time",
            "20",
            url,
            "-H",
            f"Authorization: ApiKey {TOKEN}",
            "-H",
            "Accept: text/html",
        ],
        capture_output=True,
        text=True,
    )
    return r.stdout


def normalize_type_name(href_tail: str) -> str:
    name = href_tail.split("/")[-1]
    if name.lower() in ("object", "string", "integer", "boolean", "json object"):
        return name
    return name[0].upper() + name[1:] if name else name


def parse_type_page(html: str, type_name: str) -> dict:
    variants = []
    parts = re.split(r"<dt>Type</dt>", html, flags=re.I)
    for part in parts[1:]:
        name_m = re.search(r"<dd>\s*<i>([^<]+)</i>", part, re.I)
        if not name_m:
            continue
        variant = unescape(name_m.group(1)).strip()
        desc_m = re.search(
            r"<dt>Description</dt>\s*<dd>([^<]*)", part, re.I | re.S
        )
        description = unescape(desc_m.group(1)).strip() if desc_m else ""

        properties = []
        prop_section = re.search(r"<dt>Properties</dt>(.*?)<dt>", part, re.I | re.S)
        if not prop_section:
            prop_section = re.search(r"<dt>Properties</dt>(.*)", part, re.I | re.S)
        if prop_section:
            block = prop_section.group(1)
            rows = PROP_ROW.findall(block)
            for pname, href, ptype in rows:
                clean_name = re.sub(r"<[^>]+>", "", pname)
                properties.append(
                    {
                        "name": unescape(clean_name).strip(),
                        "typeHref": href or None,
                        "type": unescape(re.sub(r"<[^>]+>", "", ptype)).strip(),
                    }
                )

        enum_values = []
        if re.search(r"<dt>Possible Values</dt>", part, re.I):
            ev_block = re.search(
                r"<dt>Possible Values</dt>(.*?)<dt>", part, re.I | re.S
            )
            if ev_block:
                enum_values = [
                    unescape(m.group(1)).strip()
                    for m in re.finditer(
                        r"<tr><td>([^<]+)</td>", ev_block.group(1), re.I
                    )
                ]

        variants.append(
            {
                "variant": variant,
                "description": description,
                "properties": properties,
                "enumValues": enum_values,
            }
        )

    return {"name": type_name, "variants": variants}


def main():
    seeds = ["/V2.0/services/"]
    # all operations from index
    index = fetch("/V2.0/services/")
    seeds.extend(m.group(1) for m in SERVICE_LINK.finditer(index))

    discovered_types: set[str] = set()
    queue: list[str] = []
    parsed: dict[str, dict] = {}

    def enqueue_type(href: str):
        tail = href.split("/types/")[-1]
        key = normalize_type_name(tail)
        if key.lower() in ("object", "string", "integer", "boolean", "json object"):
            return
        if key not in discovered_types:
            discovered_types.add(key)
            queue.append(key)

    for seed in seeds:
        html = fetch(seed) if seed != "/V2.0/services/" else index
        if seed != "/V2.0/services/":
            html = fetch(seed)
        for m in TYPE_LINK.finditer(html):
            enqueue_type(m.group(1))

    while queue:
        tname = queue.pop(0)
        if tname in parsed:
            continue
        path = f"/V2.0/services/types/{tname}"
        html = fetch(path)
        if "<dt>Type</dt>" not in html and "<dt>Possible Values</dt>" not in html:
            # try lowercase
            path = f"/V2.0/services/types/{tname[0].lower()}{tname[1:]}"
            html = fetch(path)
        info = parse_type_page(html, tname)
        parsed[tname] = info
        for v in info["variants"]:
            for p in v["properties"]:
                if p.get("typeHref"):
                    enqueue_type(p["typeHref"])

    out = {
        "typeCount": len(parsed),
        "types": sorted(parsed.keys()),
        "parsed": parsed,
    }
    out_path = ROOT / "src" / "generated" / "clarizen-schema.json"
    out_path.parent.mkdir(parents=True, exist_ok=True)
    out_path.write_text(json.dumps(out, indent=2), encoding="utf-8")
    print(f"Wrote {out_path} ({len(parsed)} types)")


if __name__ == "__main__":
    main()