#!/usr/bin/env python3
"""Publish the reader or a sanitized Git mirror to the magazine's S3 website."""
import argparse
import base64
import functools
import hashlib
import concurrent.futures
import html
import mimetypes
from pathlib import Path
import subprocess
import tempfile
import time
from urllib.request import Request, urlopen
from urllib.error import HTTPError, URLError
from urllib.parse import quote
ROOT = Path(__file__).resolve().parents[1]
BUCKET = "digest.kovrizhkin.ru"
CODE_PAGE = """
Код журнала · Утренняя подборка Александра
← Утренняя подборка АлександраИсходный код
Версия {revision}. В репозитории находятся сборщик, сайт, настройки публикации и редактируемый список источников.
Скачать исходники в ZIP · Источники · Запуск и журнал
git clone https://digest.kovrizhkin.ru/code/alexander-morning-digest.git
Файлы проекта
"""
def git(*args, cwd=ROOT):
return subprocess.check_output(["git", *args], cwd=cwd)
def upload(path, key):
mime = mimetypes.guess_type(key)[0] or "application/octet-stream"
if key.startswith("code/files/") or key.endswith((".md", ".yaml", ".jsonc", ".toml", ".ts", ".py", ".sql", ".lock")):
mime = "text/plain"
if mime.startswith("text/") or mime in {"application/javascript", "application/json"}:
mime += "; charset=utf-8"
# A cache-busting query in the reader also works with short-lived image caching.
cache = "public,max-age=3600" if key.endswith((".webp", ".png", ".jpg", ".woff2", ".svg")) else "no-cache"
body = path.read_bytes()
checksum = hashlib.md5(body).digest()
target = f"https://storage.yandexcloud.net/{BUCKET}/{quote(key)}"
request = Request(target, data=body, method="PUT", headers={
"Authorization": "Bearer " + iam_token(), "Content-Type": mime, "Cache-Control": cache,
"Content-MD5": base64.b64encode(checksum).decode(),
})
for attempt in range(4):
try:
with urlopen(request, timeout=45) as response:
if response.headers.get("ETag", "").strip('"') != checksum.hex():
raise OSError("Object checksum mismatch: " + key)
return
except HTTPError as error:
if error.code not in (408, 429, 500, 502, 503, 504) or attempt == 3:
raise
except (URLError, TimeoutError, OSError):
if attempt == 3:
raise
time.sleep(2 ** attempt)
@functools.lru_cache(maxsize=1)
def iam_token():
return subprocess.check_output(["yc", "iam", "create-token"], text=True).strip()
def upload_batch(pairs):
with concurrent.futures.ThreadPoolExecutor(max_workers=6) as pool:
for _ in pool.map(lambda pair: upload(*pair), pairs):
pass
def reader():
public = ROOT / "public"
entries = [(p, p.relative_to(public).as_posix()) for p in public.rglob("*") if p.is_file()
and not any(part.startswith(".") for part in p.relative_to(public).parts)]
if not (public / "index.html").exists():
raise SystemExit("Missing public/index.html")
for _, key in entries:
if key in {"manifest.json", "status.json"} or key.startswith(("code/", "editions/")):
raise SystemExit(f"Reader must not overwrite pipeline output: {key}")
upload_batch([p for p in entries if p[1] != "index.html"])
upload(public / "index.html", "index.html")
print(f"Published {len(entries)} reader files")
def code():
revision = git("rev-parse", "HEAD").decode().strip()
tracked = git("ls-tree", "-rz", "--name-only", revision).decode().split("\0")
with tempfile.TemporaryDirectory() as temp:
base = Path(temp)
bare = base / "alexander-morning-digest.git"
git("clone", "--bare", "--no-local", str(ROOT), str(bare))
git("repack", "-adq", cwd=bare)
git("update-server-info", cwd=bare)
git("archive", "--prefix=alexander-morning-digest/", "--format=zip", "-o", str(base / "archive.zip"), revision)
objects = [(p, "code/" + p.relative_to(base).as_posix()) for p in (bare / "objects").rglob("*") if p.is_file()]
upload_batch(objects)
files = [f for f in tracked if f and (Path(f).suffix in {".md", ".yaml", ".yml", ".js", ".ts", ".css", ".html", ".py", ".sql", ".toml", ".json", ".jsonc"} or f==".gitignore")
and f != "package-lock.json"]
entries = []
for f in files:
path = base / "files" / f
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(git("show", f"{revision}:{f}"))
entries.append((path, "code/files/" + f))
upload_batch(entries + [(base / "archive.zip", "code/alexander-morning-digest.zip")])
for f in ["info/refs", "HEAD"]:
upload(bare / f, "code/" + bare.name + "/" + f)
items = "".join('' + html.escape(f) + "" for f in files)
page = CODE_PAGE.replace("{revision}", revision[:12]).replace("{items}", items)
(base / "index.html").write_text(page)
upload(base / "index.html", "code/index.html")
print("Published source", revision)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("target", choices=["reader", "code"])
target = parser.parse_args().target
iam_token()
{"reader": reader, "code": code}[target]()