import { XMLParser } from "fast-xml-parser"; import { crocusCalendar, vdnhCalendar, vdnhDetails, organizerDetails, } from "./family-sources.js"; const xml = new XMLParser({ ignoreAttributes: false, attributeNamePrefix: "@", processEntities: true, }); export const sections = ["ai", "safety", "business", "games", "tech", "family"]; const days = 86400000; export const dayMs = days; export function moscowDate(time = Date.now()) { return new Date(new Date(time).getTime() + 3 * 3600000) .toISOString() .slice(0, 10); } export function clean(value = "") { return String(value) .replace(/]*>[\s\S]*?<\/script>/gi, " ") .replace(/]*>[\s\S]*?<\/style>/gi, " ") .replace(/<[^>]+>/g, " ") .replace(/&(?:nbsp|#160);/g, " ") .replace(/&/g, "&") .replace(/"/g, '"') .replace(/&(?:apos|#39);/g, "'") .replace(/</g, "<") .replace(/>/g, ">") .replace(/\s+/g, " ") .trim(); } function str(value) { return typeof value === "object" ? value?.["#text"] || "" : value || ""; } export function urlKey(value) { try { const u = new URL(value); if (!["https:", "http:"].includes(u.protocol)) return ""; u.hash = ""; for (const k of [...u.searchParams.keys()]) if (/^(utm_|ref$|source$|cmpid$)/i.test(k)) u.searchParams.delete(k); return u.toString().replace(/\/$/, ""); } catch { return ""; } } const dateOf = (v) => { const t = Date.parse(str(v)); return Number.isFinite(t) ? new Date(t).toISOString() : null; }; const regionalHosts = new Set([ "www.crocus-expo.ru", "crocus-expo.ru", "www.darwinmuseum.ru", "darwinmuseum.ru", "vdnh.ru", "www.vdnh.ru", "mosmuseum.ru", "www.mosmuseum.ru", "eurasia.rkf.org.ru", ]); async function regionalBody(url, limit, env) { const response = await fetch(env.REGIONAL_FETCH_URL, { method: "POST", headers: { Authorization: "Api-Key " + env.REGIONAL_FETCH_KEY, "Content-Type": "application/json", }, body: JSON.stringify({ url, limit }), signal: AbortSignal.timeout(18000), }); if (!response.ok) throw Error("Regional reader HTTP " + response.status); const result = await response.json(); const bytes = typeof Uint8Array.fromBase64 === "function" ? Uint8Array.fromBase64(result.gz) : Uint8Array.from(atob(result.gz), (c) => c.charCodeAt(0)); const stream = new Blob([bytes]) .stream() .pipeThrough(new DecompressionStream("gzip")); return { url: result.url, body: await new Response(stream).text() }; } export async function getBody(url, limit = 360000, timeout = 9500, env) { if (env?.REGIONAL_FETCH_KEY && regionalHosts.has(new URL(url).hostname)) { try { return await regionalBody(url, limit, env); } catch (error) { console.warn( "Regional reader:", new URL(url).hostname, String(error.message).slice(0, 75), ); } } const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeout); try { const r = await fetch(url, { redirect: "follow", signal: controller.signal, headers: { "User-Agent": "AlexanderMorningDigest/1.0 (+https://digest.kovrizhkin.ru)", Accept: "*/*", }, }); if (!r.ok) { await r.body?.cancel(); throw Error("HTTP " + r.status); } const reader = r.body.getReader(), decoder = new TextDecoder(); let size = 0, body = ""; try { while (size < limit) { const c = await reader.read(); if (c.done) break; const take = c.value.subarray(0, limit - size); size += take.length; body += decoder.decode(take, { stream: true }); } body += decoder.decode(); } finally { await reader.cancel().catch(() => {}); } if (body.trim().length < 20) throw Error("Пустой ответ"); return { body, url: r.url || url }; } finally { clearTimeout(timer); } } export function parseFeed(body) { const items = []; // Parse complete entries individually so multi-megabyte feeds never exhaust a Worker step. for (const m of body.matchAll( /<(item|entry)(?=[\s>])[^>]*>[\s\S]*?<\/\1\s*>/gi, )) { const item = xml.parse(m[0])[m[1]]; if (!item) continue; const links = [item.link].flat().filter(Boolean); let href = links.find((v) => v["@rel"] === "alternate")?.["@href"] || links.find((v) => v["@href"])?.["@href"] || str(links[0]) || str(item.guid); const url = urlKey(href); const date = dateOf( item.published || item.pubDate || item["dc:date"] || item.updated, ); if (!url || !date) continue; items.push({ url, date, title: clean(str(item.title)), excerpt: clean( str( item["content:encoded"] || item.content || item.description || item.summary, ), ).slice(0, 4500), }); if (items.length >= 35) break; } if ( !items.length && (!/<(?:rss|feed)\b/i.test(body) || /<(?:entry|item)\b/i.test(body)) ) throw Error("Не удалось разобрать записи ленты"); return items; } function issueWithin(date, since, until) { return date && date > since && date <= until; } function skipItem(x) { return ( /(?:canary|nightly|alpha[-.]?\d|dependency updates|self-host weekly)/i.test( x.title, ) || !x.title || !x.url ); } export function releases(data) { if (!Array.isArray(data)) { if (data?.html_url && data?.published_at) data = [data]; else throw Error("Неверный ответ GitHub"); } return data .filter((r) => !r.prerelease && !r.draft) .map((r) => ({ title: r.name || r.tag_name, date: r.published_at, url: r.html_url, excerpt: clean(r.body || "").slice(0, 4800), primary: true, entity: r.tag_name, })); } export function eventsFromPage(source, body) { const links = new Map(), pattern = new RegExp(source.pattern, "i"), host = new URL(source.url).host; for (const m of body.matchAll( /]*?href\s*=\s*["']([^"']+)["'][^>]*>([\s\S]*?)<\/a>/gi, )) { let url; try { url = new URL(m[1].replace(/&/g, "&"), source.url); } catch { continue; } if (url.host !== host || !pattern.test(url.pathname + url.search)) continue; const title = clean(m[2]); const key = urlKey(url); const prev = links.get(key); if (!prev || prev.title.length < title.length) links.set(key, { url: key, title: title.slice(0, 220), excerpt: "", event: true, }); } if (!links.size) throw Error("В афише не найдены страницы событий"); return [...links.values()].slice(0, 9); } export async function discover(source, since, until, env) { const byteLimit = source.kind === "vdnh" ? 4200000 : source.kind === "crocus" ? 2200000 : source.kind === "releases" ? 1250000 : 600000; let body, githubFallback = false; try { ({ body } = await getBody(source.url, byteLimit, 13000, env)); } catch (e) { if (source.kind !== "releases") throw e; const repo = new URL(source.url).pathname .replace(/^\/repos\//, "") .replace(/\/releases.*$/, ""); ({ body } = await getBody( "https://github.com/" + repo + "/releases.atom", 700000, 10000, )); githubFallback = true; } if (source.kind === "crocus") return crocusCalendar(body, moscowDate(until), clean); if (source.kind === "vdnh") return vdnhCalendar(body, moscowDate(until)); if (source.kind === "events") return eventsFromPage(source, body); let items; if (githubFallback) items = parseFeed(body) .filter( (item) => !/(?:alpha|beta|canary|nightly|preview|[-.]rc\d)/i.test(item.title), ) .map((item) => ({ ...item, primary: true })); else if (source.kind === "releases") items = releases(JSON.parse(body)); else if (source.kind === "feed") items = parseFeed(body); else throw Error("Неизвестный тип источника: " + source.kind); return items .filter((x) => !skipItem(x) && issueWithin(x.date, since, until)) .sort((a, b) => b.date.localeCompare(a.date)) .slice(0, 6); } function metadata(body, key) { const meta = body.match(/]*>/gi) || []; for (const tag of meta) { const id = tag .match(/\b(?:name|property)\s*=\s*["']([^"']+)["']/i)?.[1] ?.toLowerCase(); if (id === key) return clean(tag.match(/\bcontent\s*=\s*["']([^"']*)["']/i)?.[1] || ""); } return ""; } export async function verifyCandidate(item, source, env) { if (item.primary && item.excerpt.length > 45) return { ...item, source: source.name, origin: source.id }; const { body, url } = await getBody( item.url, source.kind === "vdnh" ? 2200000 : item.calendar ? 900000 : 460000, 12000, env, ); if ( /\s*(?:just a moment|access denied|security verification)/i.test( body, ) ) throw Error("Защитная страница источника"); const title = metadata(body, "og:title") || item.title || clean(body.match(/<h1\b[^>]*>([\s\S]*?)<\/h1>/i)?.[1] || ""); if (item.calendar) { const summary = metadata(body, "description") || metadata(body, "og:description"); const years = summary.match(/\b20\d{2}\b/g) || []; const description = source.kind === "crocus" && years.length && !years.includes(item.ends.slice(0, 4)) ? "" : summary; const practical = source.kind === "vdnh" ? vdnhDetails(body, url, clean) : organizerDetails(body, clean); const text = [item.excerpt, description, practical] .filter(Boolean) .join(" "); if (!title || text.length < 65) throw Error("Недостаточно сведений на странице события"); return { ...item, url: urlKey(url), title: item.title, excerpt: text.slice(0, 6500), source: source.name, origin: source.id, }; } const article = body.match(/<(article|main)\b[^>]*>([\s\S]*?)(?:<\/\1>|$)/i)?.[2] || body; const paragraphs = [ ...article.matchAll(/<(p|h[1-4]|li)\b[^>]*>([\s\S]*?)<\/\1>/gi), ] .map((m) => clean(m[2])) .filter((s) => s.length > 45 && s.length < 4000); const published = metadata(body, "article:published_time") || body.match(/"datePublished"\s*:\s*"([^"]+)"/i)?.[1]; const eventDates = [ ...body.matchAll(/"(?:startDate|endDate)"\s*:\s*"([^"]+)"/gi), ] .map((m) => m[0]) .join(" "); const text = [ metadata(body, "description") || metadata(body, "og:description"), eventDates, item.excerpt, ...paragraphs, ] .filter(Boolean) .join(" ") .slice(0, 6800); if (!title || text.length < 80) throw Error("Страница без проверяемого текста"); return { ...item, url: urlKey(url), title: title.slice(0, 220), date: dateOf(published) || item.date, excerpt: text, origin: source.id, source: source.name, }; } export async function gather(source, since, until, env) { const found = await discover(source, since, until, env); const errors = [], items = []; // Sequential reads bound concurrent outbound connections; each source gets its own durable step. for (const item of found) { try { const v = await verifyCandidate(item, source, env); if (v.event || issueWithin(v.date, since, until)) items.push(v); } catch (e) { const problem = String(e.message); // Official full-text feeds remain primary sources when their reading page rate-limits the data center. if ( source.kind === "feed" && !item.event && item.excerpt?.length >= 120 && /HTTP (403|429)|abort/i.test(problem) && (source.id === "openai" || source.id === "raspberrypi" || source.id === "hse" || source.id === "huggingface" || source.id === "benchmarks") ) { items.push({ ...item, source: source.name, origin: source.id }); } else errors.push(item.url + " " + problem.slice(0, 100)); } } return { id: source.id, section: source.section, items, errors, status: errors.length ? (items.length ? "partial" : "error") : "ok", checkedAt: until, }; }