import { verifiedCalendarStories } from "./calendar-editor.js"; import { WorkflowEntrypoint } from "cloudflare:workers"; import { parse as parseYaml } from "yaml"; import config from "../config/sources.yaml"; import { dayMs, gather, moscowDate, sections } from "./feeds.js"; import { edit, interleave, repeated, semanticRepeats } from "./editor.js"; import { put, writeJournal } from "./storage.js"; import { publish, markUnavailable } from "./publication.js"; import { ensureStoryImage } from "./story-media.js"; const registry = parseYaml(config).sources; const sourceNames = Object.fromEntries(registry.map((s) => [s.id, s.name])); const iso = () => new Date().toISOString(); const message = (e) => String(e?.message || e).slice(0, 420); const sourceStep = { retries: { limit: 1, delay: "3 seconds", backoff: "linear" }, timeout: "3 minutes", }; const writeStep = { retries: { limit: 3, delay: "5 seconds", backoff: "exponential" }, timeout: "2 minutes", }; const runDate = (event) => { const date = event.payload?.date || moscowDate(event.timestamp || Date.now()); if (!/^\d{4}-\d{2}-\d{2}$/.test(date) || date > moscowDate()) throw Error("Недопустимая дата выпуска"); return date; }; const sql = (env, q, ...args) => env.DB.prepare(q).bind(...args); async function start(env, id, date, stage) { await sql( env, "INSERT OR IGNORE INTO runs(id,stage,issue_date,started_at,outcome) VALUES(?,?,?,?,?)", id, stage, date, iso(), "running", ).run(); } async function finish(env, id, outcome, details) { await sql( env, "UPDATE runs SET finished_at=?,outcome=?,details=? WHERE id=?", iso(), outcome, details, id, ).run(); try { await writeJournal(env, sourceNames); } catch (e) { console.error("Public journal:", message(e)); } return { outcome, details }; } async function capture(env, id, row) { await sql( env, "INSERT OR REPLACE INTO source_runs(run_id,source_id,outcome,checked_at,discovered,details) VALUES(?,?,?,?,?,?)", id, row.id, row.status, row.checkedAt, row.items.length, row.errors.slice(0, 3).join(" · ").slice(0, 650), ).run(); return row; } function statusFor(rows) { const failed = rows .filter((s) => s.status !== "ok") .map((s) => sourceNames[s.id]); if (!failed.length) return { status: "ok" }; const status = rows.every((s) => s.status === "error") ? "error" : "partial"; return { status, notice: (status === "error" ? "Данные обновить не удалось. Источники: " : "Часть источников не удалось проверить: ") + failed.join(", ") + ".", }; } async function savePart(env, p, item) { await sql( env, "INSERT OR REPLACE INTO work_items(run_id,kind,section,item,body,created_at) VALUES(?,?,?,?,?,?)", p.parent, p.kind, p.section, p.item, JSON.stringify(item), iso(), ).run(); } async function takeSection(env, p) { const found = await sql( env, "SELECT body FROM work_items WHERE run_id=? AND kind='source' AND section=? ORDER BY item", p.parent, p.section, ).all(); const rows = found.results.map((s) => JSON.parse(s.body)); for (const source of registry.filter((s) => s.section === p.section)) { if (!rows.some((s) => s.id === source.id)) rows.push({ id: source.id, status: "error", items: [], errors: ["Нет ответа к сроку подготовки."], }); } const srcState = statusFor(rows); const registered = !p.dryRun && p.date === "2026-11-15" ? await sql( env, "SELECT content FROM reviewed WHERE issue_date=?", p.date, ).first() : null; const review = registered ? JSON.parse(registered.content).sections.find((s) => s.id === p.section) : null; if (!review && srcState.status === "error") return { section: { id: p.section, ...srcState, stories: [] }, checkpoints: {}, lost: true, }; const since = new Date(Date.parse(p.date + "T00:00:00Z") - 7 * dayMs) .toISOString() .slice(0, 10); const history = ( await sql( env, "SELECT story FROM stories WHERE issue_date>=? AND issue_date JSON.parse(v.story)); const candidates = interleave(rows.map((s) => s.items)); let selected; if (review) selected = review.stories.slice(0, 6); else if (p.section === "family") { const calendar = await verifiedCalendarStories(candidates, p.date, history); try { selected = await edit(env, p.section, candidates, p.date, history); } catch (error) { if (!calendar.length) throw error; console.warn("Using verified venue schedules:", message(error)); selected = []; } const selectedUrls = new Set(selected.map((story) => story.url)); selected = [ ...selected, ...calendar.filter((story) => !selectedUrls.has(story.url)), ].slice(0, 6); } else selected = await edit(env, p.section, candidates, p.date, history); const stories = []; for (const story of selected) { if (repeated(story, [...history, ...stories])) continue; stories.push( p.dryRun ? story : await ensureStoryImage(story, { section: p.section, date: p.date, put: (key, body, type) => put(env, key, body, type), }), ); } const checkpoints = Object.fromEntries( rows.filter((r) => r.status === "ok").map((r) => [r.id, r.checkedAt]), ); const status = review && srcState.status === "error" ? "partial" : srcState.status; return { section: { ...(review || {}), id: p.section, ...srcState, status, stories }, checkpoints, lost: false, }; } export class WorkItemWorkflow extends WorkflowEntrypoint { async run(event, step) { const p = event.payload, env = this.env; if (p.kind === "source") { const source = registry.find((s) => s.id === p.item); const row = await step.do("read", sourceStep, async () => { try { return await gather(source, p.since, p.until, env); } catch (e) { return { id: p.item, section: p.section, status: "error", checkedAt: p.until, errors: [message(e)], items: [], }; } }); await step.do("save-source", writeStep, async () => { await capture(env, p.parent, row); await savePart(env, p, row); }); return { status: row.status, count: row.items.length }; } let section; try { section = await step.do( "edit", { retries: { limit: 1, delay: "6 seconds" }, timeout: "4 minutes" }, () => takeSection(env, p), ); } catch (e) { section = { section: { id: p.section, status: "error", notice: "Данные обновить не удалось. " + message(e), stories: [], }, checkpoints: {}, lost: true, }; } await step.do("save-section", writeStep, () => savePart(env, p, section)); return { status: section.section.status, count: section.section.stories.length, }; } } async function runJobs(env, step, kind, parent, tasks) { await step.do("dispatch-" + kind, writeStep, async () => { const batch = tasks.map((p) => ({ id: parent + "-" + p.item, params: { ...p, parent, kind }, })); // Array format works on all supported Workflows binding versions; IDs are idempotent. for (let i = 0; i < batch.length; i += 20) await env.JOBS.createBatch(batch.slice(i, i + 20)); return tasks.length; }); // Long provider calls may use one retry. Leave enough of the 45-minute preparation window for both phases. const maxChecks = kind === "source" ? 12 : 15; for (let i = 0; i < maxChecks; i++) { await step.sleep( kind + "-wait-" + i, i === 0 ? "10 seconds" : i < 4 ? "15 seconds" : "1 minute", ); const found = await step.do(kind + "-count-" + i, writeStep, async () => { const row = await sql( env, "SELECT COUNT(*) AS n FROM work_items WHERE run_id=? AND kind=?", parent, kind, ).first(); return row.n; }); if (found === tasks.length) return; } } async function prepare(env, step, event, id, date) { const dryRun = event.payload?.dryRun === true; const existing = await sql( env, "SELECT state FROM editions WHERE issue_date=?", date, ).first(); if (existing && !dryRun) return finish( env, id, existing.state === "published" ? "already-published" : "already-ready", "Выпуск за эту дату уже подготовлен.", ); const window = await step.do("window", async () => { const until = iso(); return { until, since: new Date(Date.parse(until) - 3 * dayMs).toISOString(), }; }); const cursors = Object.fromEntries( ( await env.DB.prepare("SELECT id,success_at FROM sources").all() ).results.map((s) => [s.id, s.success_at]), ); const tasks = registry.map((s) => ({ item: s.id, section: s.section, since: dryRun ? window.since : cursors[s.id] || window.since, until: window.until, })); await runJobs(env, step, "source", id, tasks); await runJobs( env, step, "section", id, sections.map((s) => ({ item: s, section: s, date, dryRun })), ); const stored = await sql( env, "SELECT item,body FROM work_items WHERE run_id=? AND kind='section'", id, ).all(); const parts = new Map( stored.results.map((r) => [r.item, JSON.parse(r.body)]), ); const report = [], seen = [], checkpoints = {}; for (const section of sections) { const part = parts.get(section); if (!part) { report.push({ id: section, status: "error", notice: "Данные обновить не удалось. Рубрика не ответила к сроку подготовки.", stories: [], }); continue; } const g = part.section; g.stories = g.stories.filter((t) => { if (repeated(t, seen)) return false; seen.push(t); return true; }); report.push(g); Object.assign(checkpoints, part.checkpoints); } if ( seen.some((s) => s.eventKey) && report.filter((section) => section.stories.length).length > 1 ) { const drops = await step.do( "global-duplicate-review", sourceStep, async () => semanticRepeats(env, seen), ); const removed = new Set(drops); for (const section of report) section.stories = section.stories.filter( (story) => !removed.has(story.id), ); } const count = report.reduce( (total, section) => total + section.stories.length, 0, ); if ( report.every((s) => s.status === "error") || (!count && report.some((s) => s.status === "error")) ) throw Error( "Не удалось подготовить выпуск. " + report .filter((s) => s.status === "error") .map((s) => s.id + ": " + s.notice) .join(" ") .slice(0, 300), ); if (dryRun) { const summary = report.map((s) => ({ section: s.id, status: s.status, count: s.stories.length, example: s.stories[0]?.title, error: s.status === "error" ? s.notice : undefined, })); const outcome = await finish( env, id, "dry-run", "Автоматическая редактура без публикации. Материалов: " + count + ".", ); return { ...outcome, summary }; } const content = { date, publishedAt: null, sections: report }; await sql( env, "INSERT OR IGNORE INTO editions(issue_date,content,checkpoints,state) VALUES(?,?,?,?)", date, JSON.stringify(content), JSON.stringify(checkpoints), "ready", ).run(); await sql( env, "DELETE FROM work_items WHERE created_at < datetime('now','-8 days')", ).run(); return finish( env, id, "ready", "Материалов: " + count + ". Источники без ошибок: " + Object.keys(checkpoints).length + " из " + registry.length + ".", ); } export class CollectWorkflow extends WorkflowEntrypoint { async run(event, step) { const date = runDate(event), id = event.instanceId; await start(this.env, id, date, "collect"); try { return await prepare(this.env, step, event, id, date); } catch (e) { await finish(this.env, id, "failed", message(e)); throw e; } } } export class PublishWorkflow extends WorkflowEntrypoint { async run(event, step) { const date = runDate(event), id = event.instanceId, env = this.env; await start(env, id, date, "publish"); for (let attempt = 0; attempt < 68; attempt++) { let result; try { result = await step.do("publish-" + attempt, writeStep, () => publish(env, date), ); } catch (e) { result = { outcome: "retry", details: message(e) }; } if (result.outcome !== "retry") return finish(env, id, result.outcome, result.details); await step.do("record-retry-" + attempt, async () => { await finish( env, id, "retry", result.details + " Следующая попытка через 15 минут.", ); try { await markUnavailable(env, date); } catch (e) { console.error("Manifest warning:", message(e)); } }); if (date < moscowDate()) return finish(env, id, "failed", result.details); await step.sleep("wait-" + attempt, "15 minutes"); if (date < moscowDate()) return finish(env, id, "failed", result.details); const ready = await sql( env, "SELECT state FROM editions WHERE issue_date=?", date, ).first(); if (!ready) { await step.do("retry-collection-" + attempt, async () => { const run = await env.COLLECT.create({ params: { date } }); return run.id; }); await step.sleep("allow-collection-" + attempt, "2 minutes"); } } return finish( env, id, "failed", "Время повторной публикации за эту дату истекло.", ); } } export default { async scheduled(controller, env, ctx) { const date = moscowDate(controller.scheduledTime), collect = controller.cron === "15 3 * * *"; const task = collect ? env.COLLECT : env.PUBLISH; const id = (collect ? "collect-" : "publish-") + date; ctx.waitUntil( task.create({ id, params: { date } }).catch((e) => { if (!/already exists|instance.*exists/i.test(message(e))) throw e; }), ); }, async fetch() { return new Response("Not found", { status: 404 }); }, };