#!/usr/bin/env python3 """Publish the reader or a sanitized Git mirror to the magazine's S3 website.""" import argparse import base64 import functools import hashlib import concurrent.futures import html import mimetypes from pathlib import Path import subprocess import tempfile import time from urllib.request import Request, urlopen from urllib.error import HTTPError, URLError from urllib.parse import quote ROOT = Path(__file__).resolve().parents[1] BUCKET = "digest.kovrizhkin.ru" CODE_PAGE = """ Код журнала · Утренняя подборка Александра ← Утренняя подборка Александра

Исходный код

Версия {revision}. В репозитории находятся сборщик, сайт, настройки публикации и редактируемый список источников.

Скачать исходники в ZIP · Источники · Запуск и журнал

git clone https://digest.kovrizhkin.ru/code/alexander-morning-digest.git

Файлы проекта

""" def git(*args, cwd=ROOT): return subprocess.check_output(["git", *args], cwd=cwd) def upload(path, key): mime = mimetypes.guess_type(key)[0] or "application/octet-stream" if key.startswith("code/files/") or key.endswith((".md", ".yaml", ".jsonc", ".toml", ".ts", ".py", ".sql", ".lock")): mime = "text/plain" if mime.startswith("text/") or mime in {"application/javascript", "application/json"}: mime += "; charset=utf-8" # A cache-busting query in the reader also works with short-lived image caching. cache = "public,max-age=3600" if key.endswith((".webp", ".png", ".jpg", ".woff2", ".svg")) else "no-cache" body = path.read_bytes() checksum = hashlib.md5(body).digest() target = f"https://storage.yandexcloud.net/{BUCKET}/{quote(key)}" request = Request(target, data=body, method="PUT", headers={ "Authorization": "Bearer " + iam_token(), "Content-Type": mime, "Cache-Control": cache, "Content-MD5": base64.b64encode(checksum).decode(), }) for attempt in range(4): try: with urlopen(request, timeout=45) as response: if response.headers.get("ETag", "").strip('"') != checksum.hex(): raise OSError("Object checksum mismatch: " + key) return except HTTPError as error: if error.code not in (408, 429, 500, 502, 503, 504) or attempt == 3: raise except (URLError, TimeoutError, OSError): if attempt == 3: raise time.sleep(2 ** attempt) @functools.lru_cache(maxsize=1) def iam_token(): return subprocess.check_output(["yc", "iam", "create-token"], text=True).strip() def upload_batch(pairs): with concurrent.futures.ThreadPoolExecutor(max_workers=6) as pool: for _ in pool.map(lambda pair: upload(*pair), pairs): pass def reader(): public = ROOT / "public" entries = [(p, p.relative_to(public).as_posix()) for p in public.rglob("*") if p.is_file() and not any(part.startswith(".") for part in p.relative_to(public).parts)] if not (public / "index.html").exists(): raise SystemExit("Missing public/index.html") for _, key in entries: if key in {"manifest.json", "status.json"} or key.startswith(("code/", "editions/")): raise SystemExit(f"Reader must not overwrite pipeline output: {key}") upload_batch([p for p in entries if p[1] != "index.html"]) upload(public / "index.html", "index.html") print(f"Published {len(entries)} reader files") def code(): revision = git("rev-parse", "HEAD").decode().strip() tracked = git("ls-tree", "-rz", "--name-only", revision).decode().split("\0") with tempfile.TemporaryDirectory() as temp: base = Path(temp) bare = base / "alexander-morning-digest.git" git("clone", "--bare", "--no-local", str(ROOT), str(bare)) git("repack", "-adq", cwd=bare) git("update-server-info", cwd=bare) git("archive", "--prefix=alexander-morning-digest/", "--format=zip", "-o", str(base / "archive.zip"), revision) objects = [(p, "code/" + p.relative_to(base).as_posix()) for p in (bare / "objects").rglob("*") if p.is_file()] upload_batch(objects) files = [f for f in tracked if f and (Path(f).suffix in {".md", ".yaml", ".yml", ".js", ".ts", ".css", ".html", ".py", ".sql", ".toml", ".json", ".jsonc"} or f==".gitignore") and f != "package-lock.json"] entries = [] for f in files: path = base / "files" / f path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(git("show", f"{revision}:{f}")) entries.append((path, "code/files/" + f)) upload_batch(entries + [(base / "archive.zip", "code/alexander-morning-digest.zip")]) for f in ["info/refs", "HEAD"]: upload(bare / f, "code/" + bare.name + "/" + f) items = "".join('
  • ' + html.escape(f) + "
  • " for f in files) page = CODE_PAGE.replace("{revision}", revision[:12]).replace("{items}", items) (base / "index.html").write_text(page) upload(base / "index.html", "code/index.html") print("Published source", revision) if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("target", choices=["reader", "code"]) target = parser.parse_args().target iam_token() {"reader": reader, "code": code}[target]()