const Parser = require('rss-parser'); const sanitizeHtml = require('sanitize-html'); const db = require('./db'); const parser = new Parser({ timeout: 15000, headers: { 'User-Agent': 'Dispatch RSS Reader/1.0' }, customFields: { item: [ ['media:content', 'mediaContent'], ['media:thumbnail', 'mediaThumbnail'], ['content:encoded', 'contentEncoded'], ], }, }); function cleanHtml(html) { if (!html) return null; return sanitizeHtml(html, { allowedTags: sanitizeHtml.defaults.allowedTags.concat(['img', 'figure', 'figcaption', 'iframe', 'h1', 'h2']), allowedAttributes: { ...sanitizeHtml.defaults.allowedAttributes, img: ['src', 'alt', 'title', 'width', 'height', 'loading'], iframe: ['src', 'allowfullscreen', 'frameborder'], a: ['href', 'name', 'target', 'rel'], }, allowedIframeHostnames: ['www.youtube.com', 'player.vimeo.com'], transformTags: { a: sanitizeHtml.simpleTransform('a', { rel: 'noopener noreferrer', target: '_blank' }), }, }); } function extractImage(item) { if (item.mediaContent && item.mediaContent.$ && item.mediaContent.$.url) return item.mediaContent.$.url; if (item.mediaThumbnail && item.mediaThumbnail.$ && item.mediaThumbnail.$.url) return item.mediaThumbnail.$.url; if (item.enclosure && item.enclosure.url && /image/.test(item.enclosure.type || '')) return item.enclosure.url; const html = item.contentEncoded || item.content || ''; const match = html.match(/]+src=["']([^"']+)["']/i); return match ? match[1] : null; } function stripToSummary(html, len = 220) { if (!html) return ''; const text = html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim(); return text.length > len ? text.slice(0, len).trim() + '…' : text; } async function fetchFeedMeta(url) { const feed = await parser.parseURL(url); return { title: feed.title || url, siteUrl: feed.link || null, description: feed.description || null, }; } async function fetchAndStoreFeed(feedRow) { const insertArticle = db.prepare(` INSERT INTO articles (feed_id, guid, title, link, author, content, summary, image_url, published_at) VALUES (@feed_id, @guid, @title, @link, @author, @content, @summary, @image_url, @published_at) ON CONFLICT(feed_id, guid) DO UPDATE SET title=excluded.title, link=excluded.link, content=excluded.content, summary=excluded.summary, image_url=excluded.image_url WHERE articles.title != excluded.title OR articles.content != excluded.content `); try { const feed = await parser.parseURL(feedRow.url); const tx = db.transaction((items) => { for (const item of items) { const guid = item.guid || item.id || item.link || item.title; if (!guid) continue; const rawContent = item.contentEncoded || item.content || item.summary || ''; const content = cleanHtml(rawContent); insertArticle.run({ feed_id: feedRow.id, guid, title: item.title || '(untitled)', link: item.link || null, author: item.creator || item.author || null, content, summary: stripToSummary(rawContent), image_url: extractImage(item), published_at: item.isoDate || item.pubDate || new Date().toISOString(), }); } }); tx(feed.items || []); db.prepare(`UPDATE feeds SET last_fetched_at = datetime('now'), last_error = NULL, title = CASE WHEN title IS NULL OR title = '' THEN @title ELSE title END, site_url = COALESCE(site_url, @siteUrl) WHERE id = @id`).run({ id: feedRow.id, title: feed.title || feedRow.title, siteUrl: feed.link || null, }); return { ok: true, count: (feed.items || []).length }; } catch (err) { db.prepare(`UPDATE feeds SET last_fetched_at = datetime('now'), last_error = ? WHERE id = ?`) .run(err.message || String(err), feedRow.id); return { ok: false, error: err.message }; } } async function fetchAllFeeds() { const feeds = db.prepare('SELECT * FROM feeds').all(); const results = []; for (const feed of feeds) { const res = await fetchAndStoreFeed(feed); results.push({ feedId: feed.id, ...res }); } return results; } module.exports = { fetchAndStoreFeed, fetchAllFeeds, fetchFeedMeta };