first commit
This commit is contained in:
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Discover all Clarizen API types by crawling services + types docs."""
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from html import unescape
|
||||
from pathlib import Path
|
||||
from urllib.parse import urljoin
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
BASE = os.environ.get("CLARIZEN_BASE_URL", "https://api.clarizen.com")
|
||||
TOKEN = os.environ.get("CLARIZEN_API_KEY") or os.environ.get("API_KEY_AW")
|
||||
if not TOKEN:
|
||||
sys.exit("Set CLARIZEN_API_KEY or API_KEY_AW")
|
||||
|
||||
TYPE_LINK = re.compile(r'href="(/V2\.0/services/types/([^"]+))"', re.I)
|
||||
SERVICE_LINK = re.compile(r'href="(/V2\.0/services/(?!types)[^"]+)"', re.I)
|
||||
VARIANT_NAME = re.compile(r"<dt>Type</dt>\s*<dd>\s*<i>([^<]+)</i>", re.I)
|
||||
PROP_ROW = re.compile(
|
||||
r"<td><i><b>(.*?)</b></i></td>\s*<td>(?:<a href=\"([^\"]+)\">)?([^<]+)",
|
||||
re.I | re.S,
|
||||
)
|
||||
ENUM_ROW = re.compile(r"<tr><td>([^<]+)</td><td>[^<]*</td><td>", re.I)
|
||||
|
||||
|
||||
def fetch(path: str) -> str:
|
||||
url = urljoin(BASE, path)
|
||||
r = subprocess.run(
|
||||
[
|
||||
"curl",
|
||||
"-sL",
|
||||
"--max-time",
|
||||
"20",
|
||||
url,
|
||||
"-H",
|
||||
f"Authorization: ApiKey {TOKEN}",
|
||||
"-H",
|
||||
"Accept: text/html",
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return r.stdout
|
||||
|
||||
|
||||
def normalize_type_name(href_tail: str) -> str:
|
||||
name = href_tail.split("/")[-1]
|
||||
if name.lower() in ("object", "string", "integer", "boolean", "json object"):
|
||||
return name
|
||||
return name[0].upper() + name[1:] if name else name
|
||||
|
||||
|
||||
def parse_type_page(html: str, type_name: str) -> dict:
|
||||
variants = []
|
||||
parts = re.split(r"<dt>Type</dt>", html, flags=re.I)
|
||||
for part in parts[1:]:
|
||||
name_m = re.search(r"<dd>\s*<i>([^<]+)</i>", part, re.I)
|
||||
if not name_m:
|
||||
continue
|
||||
variant = unescape(name_m.group(1)).strip()
|
||||
desc_m = re.search(
|
||||
r"<dt>Description</dt>\s*<dd>([^<]*)", part, re.I | re.S
|
||||
)
|
||||
description = unescape(desc_m.group(1)).strip() if desc_m else ""
|
||||
|
||||
properties = []
|
||||
prop_section = re.search(r"<dt>Properties</dt>(.*?)<dt>", part, re.I | re.S)
|
||||
if not prop_section:
|
||||
prop_section = re.search(r"<dt>Properties</dt>(.*)", part, re.I | re.S)
|
||||
if prop_section:
|
||||
block = prop_section.group(1)
|
||||
rows = PROP_ROW.findall(block)
|
||||
for pname, href, ptype in rows:
|
||||
clean_name = re.sub(r"<[^>]+>", "", pname)
|
||||
properties.append(
|
||||
{
|
||||
"name": unescape(clean_name).strip(),
|
||||
"typeHref": href or None,
|
||||
"type": unescape(re.sub(r"<[^>]+>", "", ptype)).strip(),
|
||||
}
|
||||
)
|
||||
|
||||
enum_values = []
|
||||
if re.search(r"<dt>Possible Values</dt>", part, re.I):
|
||||
ev_block = re.search(
|
||||
r"<dt>Possible Values</dt>(.*?)<dt>", part, re.I | re.S
|
||||
)
|
||||
if ev_block:
|
||||
enum_values = [
|
||||
unescape(m.group(1)).strip()
|
||||
for m in re.finditer(
|
||||
r"<tr><td>([^<]+)</td>", ev_block.group(1), re.I
|
||||
)
|
||||
]
|
||||
|
||||
variants.append(
|
||||
{
|
||||
"variant": variant,
|
||||
"description": description,
|
||||
"properties": properties,
|
||||
"enumValues": enum_values,
|
||||
}
|
||||
)
|
||||
|
||||
return {"name": type_name, "variants": variants}
|
||||
|
||||
|
||||
def main():
|
||||
seeds = ["/V2.0/services/"]
|
||||
# all operations from index
|
||||
index = fetch("/V2.0/services/")
|
||||
seeds.extend(m.group(1) for m in SERVICE_LINK.finditer(index))
|
||||
|
||||
discovered_types: set[str] = set()
|
||||
queue: list[str] = []
|
||||
parsed: dict[str, dict] = {}
|
||||
|
||||
def enqueue_type(href: str):
|
||||
tail = href.split("/types/")[-1]
|
||||
key = normalize_type_name(tail)
|
||||
if key.lower() in ("object", "string", "integer", "boolean", "json object"):
|
||||
return
|
||||
if key not in discovered_types:
|
||||
discovered_types.add(key)
|
||||
queue.append(key)
|
||||
|
||||
for seed in seeds:
|
||||
html = fetch(seed) if seed != "/V2.0/services/" else index
|
||||
if seed != "/V2.0/services/":
|
||||
html = fetch(seed)
|
||||
for m in TYPE_LINK.finditer(html):
|
||||
enqueue_type(m.group(1))
|
||||
|
||||
while queue:
|
||||
tname = queue.pop(0)
|
||||
if tname in parsed:
|
||||
continue
|
||||
path = f"/V2.0/services/types/{tname}"
|
||||
html = fetch(path)
|
||||
if "<dt>Type</dt>" not in html and "<dt>Possible Values</dt>" not in html:
|
||||
# try lowercase
|
||||
path = f"/V2.0/services/types/{tname[0].lower()}{tname[1:]}"
|
||||
html = fetch(path)
|
||||
info = parse_type_page(html, tname)
|
||||
parsed[tname] = info
|
||||
for v in info["variants"]:
|
||||
for p in v["properties"]:
|
||||
if p.get("typeHref"):
|
||||
enqueue_type(p["typeHref"])
|
||||
|
||||
out = {
|
||||
"typeCount": len(parsed),
|
||||
"types": sorted(parsed.keys()),
|
||||
"parsed": parsed,
|
||||
}
|
||||
out_path = ROOT / "src" / "generated" / "clarizen-schema.json"
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_text(json.dumps(out, indent=2), encoding="utf-8")
|
||||
print(f"Wrote {out_path} ({len(parsed)} types)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user