dispatch/backend/feedFetcher.js

const Parser = require('rss-parser');
const sanitizeHtml = require('sanitize-html');
const db = require('./db');

const parser = new Parser({
  timeout: 15000,
  headers: { 'User-Agent': 'Dispatch RSS Reader/1.0' },
  customFields: {
    item: [
      ['media:content', 'mediaContent'],
      ['media:thumbnail', 'mediaThumbnail'],
      ['content:encoded', 'contentEncoded'],
    ],
  },
});

function cleanHtml(html) {
  if (!html) return null;
  return sanitizeHtml(html, {
    allowedTags: sanitizeHtml.defaults.allowedTags.concat(['img', 'figure', 'figcaption', 'iframe', 'h1', 'h2']),
    allowedAttributes: {
      ...sanitizeHtml.defaults.allowedAttributes,
      img: ['src', 'alt', 'title', 'width', 'height', 'loading'],
      iframe: ['src', 'allowfullscreen', 'frameborder'],
      a: ['href', 'name', 'target', 'rel'],
    },
    allowedIframeHostnames: ['www.youtube.com', 'player.vimeo.com'],
    transformTags: {
      a: sanitizeHtml.simpleTransform('a', { rel: 'noopener noreferrer', target: '_blank' }),
    },
  });
}

function extractImage(item) {
  if (item.mediaContent && item.mediaContent.$ && item.mediaContent.$.url) return item.mediaContent.$.url;
  if (item.mediaThumbnail && item.mediaThumbnail.$ && item.mediaThumbnail.$.url) return item.mediaThumbnail.$.url;
  if (item.enclosure && item.enclosure.url && /image/.test(item.enclosure.type || '')) return item.enclosure.url;
  const html = item.contentEncoded || item.content || '';
  const match = html.match(/<img[^>]+src=["']([^"']+)["']/i);
  return match ? match[1] : null;
}

function stripToSummary(html, len = 220) {
  if (!html) return '';
  const text = html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
  return text.length > len ? text.slice(0, len).trim() + '…' : text;
}

async function fetchFeedMeta(url) {
  const feed = await parser.parseURL(url);
  return {
    title: feed.title || url,
    siteUrl: feed.link || null,
    description: feed.description || null,
  };
}

async function fetchAndStoreFeed(feedRow) {
  const insertArticle = db.prepare(`
    INSERT INTO articles (feed_id, guid, title, link, author, content, summary, image_url, published_at)
    VALUES (@feed_id, @guid, @title, @link, @author, @content, @summary, @image_url, @published_at)
    ON CONFLICT(feed_id, guid) DO UPDATE SET
      title=excluded.title,
      link=excluded.link,
      content=excluded.content,
      summary=excluded.summary,
      image_url=excluded.image_url
      WHERE articles.title != excluded.title OR articles.content != excluded.content
  `);

  try {
    const feed = await parser.parseURL(feedRow.url);
    const tx = db.transaction((items) => {
      for (const item of items) {
        const guid = item.guid || item.id || item.link || item.title;
        if (!guid) continue;
        const rawContent = item.contentEncoded || item.content || item.summary || '';
        const content = cleanHtml(rawContent);
        insertArticle.run({
          feed_id: feedRow.id,
          guid,
          title: item.title || '(untitled)',
          link: item.link || null,
          author: item.creator || item.author || null,
          content,
          summary: stripToSummary(rawContent),
          image_url: extractImage(item),
          published_at: item.isoDate || item.pubDate || new Date().toISOString(),
        });
      }
    });
    tx(feed.items || []);

    db.prepare(`UPDATE feeds SET last_fetched_at = datetime('now'), last_error = NULL,
      title = CASE WHEN title IS NULL OR title = '' THEN @title ELSE title END,
      site_url = COALESCE(site_url, @siteUrl)
      WHERE id = @id`).run({
      id: feedRow.id,
      title: feed.title || feedRow.title,
      siteUrl: feed.link || null,
    });

    return { ok: true, count: (feed.items || []).length };
  } catch (err) {
    db.prepare(`UPDATE feeds SET last_fetched_at = datetime('now'), last_error = ? WHERE id = ?`)
      .run(err.message || String(err), feedRow.id);
    return { ok: false, error: err.message };
  }
}

async function fetchAllFeeds() {
  const feeds = db.prepare('SELECT * FROM feeds').all();
  const results = [];
  for (const feed of feeds) {
    const res = await fetchAndStoreFeed(feed);
    results.push({ feedId: feed.id, ...res });
  }
  return results;
}

module.exports = { fetchAndStoreFeed, fetchAllFeeds, fetchFeedMeta };