Files
planet/scripts/release_publish.py
T
linkong e6aa8c539b
ci / backend (push) Waiting to run
ci / frontend (push) Waiting to run
fix: verify Gitea release assets before publishing
2026-11-22 14:23:20 +08:00

447 lines
16 KiB
Python

#!/usr/bin/env python3
"""Create or update a Gitea Release and stream binary attachments via Gitea API."""
from __future__ import annotations
import argparse
import base64
import hashlib
import json
import os
import re
import subprocess
import sys
import urllib.error
import urllib.parse
import urllib.request
from pathlib import Path
ASSET_SUFFIXES = (".zip", ".appimage", ".deb", ".exe", ".dmg")
class RejectRedirects(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl) -> None:
return None
API_OPENER = urllib.request.build_opener(RejectRedirects())
def run_git_origin_url() -> str:
try:
output = subprocess.check_output(
["git", "remote", "get-url", "origin"],
text=True,
stderr=subprocess.DEVNULL,
)
return output.strip()
except (subprocess.CalledProcessError, FileNotFoundError):
return ""
def parse_server_and_repo(remote_url: str) -> tuple[str, str]:
"""Derive (server_url, owner/repo) from an HTTPS or SSH git remote URL."""
url = remote_url.strip()
if not url:
return "", ""
parsed = urllib.parse.urlsplit(url)
if parsed.scheme in ("https", "http") and parsed.hostname:
server = f"{parsed.scheme}://{parsed.netloc.rsplit('@', 1)[-1]}"
return server, parsed.path.strip("/").removesuffix(".git")
ssh_match = re.match(r"^(?:ssh://)?[^@]+@([^:/]+)[:/](.+?)(?:\.git)?/?$", url)
if ssh_match:
host = ssh_match.group(1)
repo_path = ssh_match.group(2).strip("/")
return f"https://{host}", repo_path
return "", ""
def git_credential_authorization(origin: str, server: str, repo: str) -> str:
"""Reuse a saved credential only for the matching HTTPS origin."""
if (
urllib.parse.urlsplit(origin).scheme != "https"
or (server, repo) != parse_server_and_repo(origin)
):
raise RuntimeError("--use-git-credential requires the target to match HTTPS origin")
try:
result = subprocess.run(
["git", "-c", "credential.interactive=false", "credential", "fill"],
input=f"url={origin}\n\n",
env={**os.environ, "GIT_TERMINAL_PROMPT": "0", "GCM_INTERACTIVE": "Never"},
text=True,
capture_output=True,
timeout=15,
check=True,
)
except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc:
raise RuntimeError("No saved Git credential available for HTTPS origin") from exc
data = dict(line.split("=", 1) for line in result.stdout.splitlines() if "=" in line)
username = data.get("username")
password = data.get("password")
if not username or not password:
raise RuntimeError("Saved Git credential is incomplete")
encoded = base64.b64encode(f"{username}:{password}".encode()).decode()
return f"Basic {encoded}"
def extract_release_notes(changelog_path: Path, tag: str) -> str:
"""Extract the matching version section from docs/CHANGELOG.md."""
if not changelog_path.is_file():
return f"Release {tag}"
version = tag.lstrip("vV")
text = changelog_path.read_text(encoding="utf-8")
lines = text.splitlines()
heading_pattern = re.compile(
rf"^##\s+(?:\[{re.escape(version)}\]|{re.escape(version)}|{re.escape(tag)})(?:\s|$)"
)
next_heading_pattern = re.compile(r"^##\s+")
start_idx: int | None = None
for idx, line in enumerate(lines):
if heading_pattern.match(line.strip()):
start_idx = idx + 1
break
if start_idx is None:
return f"Release {tag}"
collected: list[str] = []
for line in lines[start_idx:]:
if next_heading_pattern.match(line.strip()):
break
collected.append(line)
while collected and (not collected[-1].strip() or collected[-1].strip() == "---"):
collected.pop()
while collected and not collected[0].strip():
collected.pop(0)
notes = "\n".join(collected).strip()
return notes or f"Release {tag}"
def compute_sha256(file_path: Path) -> str:
digest = hashlib.sha256()
with file_path.open("rb") as handle:
while chunk := handle.read(1024 * 1024):
digest.update(chunk)
return digest.hexdigest()
def collect_release_assets(assets_dir: Path) -> list[tuple[Path, str]]:
"""Collect distributable assets directly under assets_dir and write SHA256SUMS.txt."""
if not assets_dir.is_dir():
return []
distributables = sorted(
(
entry
for entry in assets_dir.iterdir()
if entry.is_file() and entry.name.lower().endswith(ASSET_SUFFIXES)
),
key=lambda p: p.name,
)
if not distributables:
return []
records: list[tuple[Path, str]] = []
checksum_lines: list[str] = []
for artifact in distributables:
sha = compute_sha256(artifact)
records.append((artifact, sha))
checksum_lines.append(f"{sha} {artifact.name}")
sums_path = assets_dir / "SHA256SUMS.txt"
sums_path.write_text("\n".join(checksum_lines) + "\n", encoding="utf-8")
sums_sha = compute_sha256(sums_path)
records.append((sums_path, sums_sha))
return records
def api_request(
method: str,
url: str,
authorization: str,
payload: dict | None = None,
) -> tuple[int, dict | list | str | None]:
headers = {
"Authorization": authorization,
"Accept": "application/json",
}
data = None
if payload is not None:
headers["Content-Type"] = "application/json"
data = json.dumps(payload).encode("utf-8")
request = urllib.request.Request(url, data=data, headers=headers, method=method)
try:
with API_OPENER.open(request, timeout=30) as response:
raw = response.read().decode("utf-8", errors="replace")
return response.status, json.loads(raw) if raw.strip() else None
except urllib.error.HTTPError as exc:
raw = exc.read().decode("utf-8", errors="replace")
try:
body: dict | list | str | None = json.loads(raw) if raw.strip() else None
except json.JSONDecodeError:
body = raw
return exc.code, body
def upload_asset_with_curl(
upload_url: str,
authorization: str,
file_path: Path,
) -> None:
"""Stream multipart file upload using curl to avoid buffering ~200MB assets in memory."""
proc = subprocess.run(
[
"curl",
"--disable",
"-sS",
"--connect-timeout",
"15",
"--speed-time",
"60",
"--speed-limit",
"1024",
"-w",
"\n%{http_code}",
"-X",
"POST",
"--header",
"@-",
"--header",
"Content-Type: application/octet-stream",
"--data-binary",
f"@{file_path}",
upload_url,
],
input=f"Authorization: {authorization}\n",
text=True,
capture_output=True,
)
if proc.returncode != 0:
raise RuntimeError(
f"curl upload failed for {file_path.name} (exit={proc.returncode}): {proc.stderr.strip()}"
)
lines = proc.stdout.splitlines()
status_str = lines[-1].strip() if lines else ""
try:
status_code = int(status_str)
except ValueError as exc:
raise RuntimeError(f"Unexpected curl output for {file_path.name}: {proc.stdout}") from exc
if status_code == 413:
raise RuntimeError(
f"Upload rejected with HTTP 413 Payload Too Large for {file_path.name}. "
"Check Gitea app.ini [attachment] MAX_SIZE and Nginx/Ingress client_max_body_size (>= 300M)."
)
if status_code != 201:
raise RuntimeError(f"Upload failed for {file_path.name} (HTTP {status_code})")
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(description="Publish Gitea Release and client binaries.")
parser.add_argument("--tag", required=True, help="Git release tag, e.g. v0.77.0")
parser.add_argument(
"--assets-dir",
default="client/release",
help="Directory containing packaged client binaries (default: client/release)",
)
parser.add_argument(
"--changelog",
default="docs/CHANGELOG.md",
help="Path to changelog markdown file (default: docs/CHANGELOG.md)",
)
parser.add_argument(
"--server",
default="",
help="Gitea base URL (defaults to GITHUB_SERVER_URL or origin remote host)",
)
parser.add_argument(
"--repo",
default="",
help="Repository slug owner/name (defaults to GITHUB_REPOSITORY or origin remote)",
)
parser.add_argument(
"--use-git-credential",
action="store_true",
help="Use the saved Git credential for the matching HTTPS origin",
)
parser.add_argument(
"--dry-run",
action="store_true",
help="Verify release notes and asset checksums without calling remote Gitea APIs",
)
return parser.parse_args(argv)
def find_draft(api_base: str, tag: str, authorization: str) -> dict | None:
page = 1
while True:
status, data = api_request(
"GET", f"{api_base}/releases?draft=true&limit=50&page={page}", authorization
)
if status != 200 or not isinstance(data, list):
raise RuntimeError(f"Could not list release drafts (HTTP {status})")
if not data:
return None
for release in data:
if release.get("tag_name") == tag and release.get("draft") is True:
return release
page += 1
def get_release(api_base: str, tag: str, notes: str, authorization: str) -> dict:
tag_url = f"{api_base}/releases/tags/{urllib.parse.quote(tag, safe='')}"
status, release = api_request("GET", tag_url, authorization)
if status == 200 and isinstance(release, dict) and release.get("id"):
return release
if status != 404:
raise RuntimeError(f"Could not read release {tag} (HTTP {status})")
draft = find_draft(api_base, tag, authorization)
if draft:
return draft
status, release = api_request(
"POST",
f"{api_base}/releases",
authorization,
{"tag_name": tag, "name": tag, "body": notes, "draft": True, "prerelease": False},
)
if (
status != 201
or not isinstance(release, dict)
or release.get("draft") is not True
or not release.get("id")
):
raise RuntimeError(f"Could not create draft release {tag} (HTTP {status})")
print(f"created draft release {tag} (id={release['id']})", flush=True)
return release
def list_assets(release_url: str, authorization: str) -> dict[str, dict]:
status, data = api_request("GET", f"{release_url}/assets", authorization)
if status != 200 or not isinstance(data, list):
raise RuntimeError(f"Could not list release assets (HTTP {status})")
if any(not isinstance(item, dict) or not isinstance(item.get("name"), str) for item in data):
raise RuntimeError("Gitea returned invalid release assets")
assets = {item["name"]: item for item in data}
if len(assets) != len(data):
raise RuntimeError("Gitea returned duplicate release asset names")
return assets
def verify_existing_asset(
asset: dict, local_path: Path, sha: str, server: str, authorization: str
) -> None:
"""Reuse an attachment only if its contents match the local artifact."""
name = local_path.name
if asset.get("size") != local_path.stat().st_size:
raise RuntimeError(f"Existing asset has a different size: {name}")
download = asset.get("browser_download_url")
if not isinstance(download, str) or not download:
raise RuntimeError(f"Missing download URL for {name}")
url = urllib.parse.urljoin(server + "/", download)
target = urllib.parse.urlsplit(url)
trusted = urllib.parse.urlsplit(server)
if (target.scheme, target.netloc) != (trusted.scheme, trusted.netloc):
raise RuntimeError(f"Unexpected download host for {name}")
request = urllib.request.Request(url, headers={"Authorization": authorization})
digest = hashlib.sha256()
try:
with API_OPENER.open(request, timeout=30) as response:
while chunk := response.read(1024 * 1024):
digest.update(chunk)
except urllib.error.URLError as exc:
raise RuntimeError(f"Could not verify existing asset: {name}") from exc
if digest.hexdigest() != sha:
raise RuntimeError(f"Existing asset has a different SHA-256: {name}")
def publish_release(
server: str, repo: str, tag: str, notes: str, assets: list[tuple[Path, str]], authorization: str
) -> None:
api_base = f"{server}/api/v1/repos/{repo}"
release = get_release(api_base, tag, notes, authorization)
release_url = f"{api_base}/releases/{release['id']}"
existing = list_assets(release_url, authorization)
for path, sha in assets:
if path.name in existing:
verify_existing_asset(existing[path.name], path, sha, server, authorization)
print(f"verified existing {path.name}", flush=True)
for path, _sha in assets:
if path.name in existing:
continue
name = urllib.parse.quote(path.name, safe="")
print(f"uploading {path.name}...", flush=True)
upload_asset_with_curl(f"{release_url}/assets?name={name}", authorization, path)
uploaded = list_assets(release_url, authorization)
for path, _sha in assets:
if uploaded.get(path.name, {}).get("size") != path.stat().st_size:
raise RuntimeError(f"Missing or incomplete attachment: {path.name}")
status, published = api_request(
"PATCH", release_url, authorization, {"name": tag, "body": notes, "draft": False}
)
if status != 200 or not isinstance(published, dict) or published.get("draft") is not False:
raise RuntimeError(f"Could not publish release {tag} (HTTP {status})")
print(f"published release {tag} with {len(assets)} asset(s)", flush=True)
def main(argv: list[str] | None = None) -> int:
args = parse_args(argv)
origin = run_git_origin_url()
derived_server, derived_repo = parse_server_and_repo(origin)
server = (args.server or os.environ.get("GITHUB_SERVER_URL") or derived_server).rstrip("/")
repo = (args.repo or os.environ.get("GITHUB_REPOSITORY") or derived_repo).strip("/")
if not server or not repo:
print("error: unable to resolve Gitea server URL or owner/repo slug", file=sys.stderr)
return 1
notes = extract_release_notes(Path(args.changelog), args.tag)
assets_dir = Path(args.assets_dir)
assets = collect_release_assets(assets_dir)
if not assets:
print(f"error: no distributable assets found in {assets_dir}", file=sys.stderr)
return 1
if args.dry_run:
print(f"server: {server}")
print(f"repo: {repo}")
print(f"tag: {args.tag}")
print(f"notes_length: {len(notes)}")
print(f"assets_count: {len(assets)}")
for path, sha in assets:
size_mb = path.stat().st_size / (1024 * 1024)
print(f" - {path.name} ({size_mb:.2f} MiB) sha256={sha}")
return 0
try:
if args.use_git_credential:
authorization = git_credential_authorization(origin, server, repo)
else:
token = os.environ.get("GITEA_TOKEN") or os.environ.get("GITHUB_TOKEN")
if not token:
raise RuntimeError("Set GITEA_TOKEN or use the saved HTTPS Git credential")
authorization = f"token {token}"
publish_release(server, repo, args.tag, notes, assets, authorization)
except (RuntimeError, urllib.error.URLError, TimeoutError) as exc:
print(f"error: {exc}", file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())