backend first draft

backend/feedFetcher.js

@@ -0,0 +1,121 @@
+const Parser = require('rss-parser');
+const sanitizeHtml = require('sanitize-html');
+const db = require('./db');
+
+const parser = new Parser({
+  timeout: 15000,
+  headers: { 'User-Agent': 'Dispatch RSS Reader/1.0' },
+  customFields: {
+    item: [
+      ['media:content', 'mediaContent'],
+      ['media:thumbnail', 'mediaThumbnail'],
+      ['content:encoded', 'contentEncoded'],
+    ],
+  },
+});
+
+function cleanHtml(html) {
+  if (!html) return null;
+  return sanitizeHtml(html, {
+    allowedTags: sanitizeHtml.defaults.allowedTags.concat(['img', 'figure', 'figcaption', 'iframe', 'h1', 'h2']),
+    allowedAttributes: {
+      ...sanitizeHtml.defaults.allowedAttributes,
+      img: ['src', 'alt', 'title', 'width', 'height', 'loading'],
+      iframe: ['src', 'allowfullscreen', 'frameborder'],
+      a: ['href', 'name', 'target', 'rel'],
+    },
+    allowedIframeHostnames: ['www.youtube.com', 'player.vimeo.com'],
+    transformTags: {
+      a: sanitizeHtml.simpleTransform('a', { rel: 'noopener noreferrer', target: '_blank' }),
+    },
+  });
+}
+
+function extractImage(item) {
+  if (item.mediaContent && item.mediaContent.$ && item.mediaContent.$.url) return item.mediaContent.$.url;
+  if (item.mediaThumbnail && item.mediaThumbnail.$ && item.mediaThumbnail.$.url) return item.mediaThumbnail.$.url;
+  if (item.enclosure && item.enclosure.url && /image/.test(item.enclosure.type || '')) return item.enclosure.url;
+  const html = item.contentEncoded || item.content || '';
+  const match = html.match(/<img[^>]+src=["']([^"']+)["']/i);
+  return match ? match[1] : null;
+}
+
+function stripToSummary(html, len = 220) {
+  if (!html) return '';
+  const text = html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
+  return text.length > len ? text.slice(0, len).trim() + '…' : text;
+}
+
+async function fetchFeedMeta(url) {
+  const feed = await parser.parseURL(url);
+  return {
+    title: feed.title || url,
+    siteUrl: feed.link || null,
+    description: feed.description || null,
+  };
+}
+
+async function fetchAndStoreFeed(feedRow) {
+  const insertArticle = db.prepare(`
+    INSERT INTO articles (feed_id, guid, title, link, author, content, summary, image_url, published_at)
+    VALUES (@feed_id, @guid, @title, @link, @author, @content, @summary, @image_url, @published_at)
+    ON CONFLICT(feed_id, guid) DO UPDATE SET
+      title=excluded.title,
+      link=excluded.link,
+      content=excluded.content,
+      summary=excluded.summary,
+      image_url=excluded.image_url
+      WHERE articles.title != excluded.title OR articles.content != excluded.content
+  `);
+
+  try {
+    const feed = await parser.parseURL(feedRow.url);
+    const tx = db.transaction((items) => {
+      for (const item of items) {
+        const guid = item.guid || item.id || item.link || item.title;
+        if (!guid) continue;
+        const rawContent = item.contentEncoded || item.content || item.summary || '';
+        const content = cleanHtml(rawContent);
+        insertArticle.run({
+          feed_id: feedRow.id,
+          guid,
+          title: item.title || '(untitled)',
+          link: item.link || null,
+          author: item.creator || item.author || null,
+          content,
+          summary: stripToSummary(rawContent),
+          image_url: extractImage(item),
+          published_at: item.isoDate || item.pubDate || new Date().toISOString(),
+        });
+      }
+    });
+    tx(feed.items || []);
+
+    db.prepare(`UPDATE feeds SET last_fetched_at = datetime('now'), last_error = NULL,
+      title = CASE WHEN title IS NULL OR title = '' THEN @title ELSE title END,
+      site_url = COALESCE(site_url, @siteUrl)
+      WHERE id = @id`).run({
+      id: feedRow.id,
+      title: feed.title || feedRow.title,
+      siteUrl: feed.link || null,
+    });
+
+    return { ok: true, count: (feed.items || []).length };
+  } catch (err) {
+    db.prepare(`UPDATE feeds SET last_fetched_at = datetime('now'), last_error = ? WHERE id = ?`)
+      .run(err.message || String(err), feedRow.id);
+    return { ok: false, error: err.message };
+  }
+}
+
+async function fetchAllFeeds() {
+  const feeds = db.prepare('SELECT * FROM feeds').all();
+  const results = [];
+  for (const feed of feeds) {
+    const res = await fetchAndStoreFeed(feed);
+    results.push({ feedId: feed.id, ...res });
+  }
+  return results;
+}
+
+module.exports = { fetchAndStoreFeed, fetchAllFeeds, fetchFeedMeta };