backend first draft

2026-07-10 15:48:21 +02:00
parent 614dfd0229
commit 9cf0eda983
7 changed files with 482 additions and 0 deletions
@@ -0,0 +1,65 @@
+const Database = require('better-sqlite3');
+const path = require('path');
+const fs = require('fs');
+
+const DATA_DIR = process.env.DATA_DIR || path.join(__dirname, '..', 'data');
+if (!fs.existsSync(DATA_DIR)) {
+  fs.mkdirSync(DATA_DIR, { recursive: true });
+}
+
+const db = new Database(path.join(DATA_DIR, 'dispatch.db'));
+db.pragma('journal_mode = WAL');
+db.pragma('foreign_keys = ON');
+
+db.exec(`
+CREATE TABLE IF NOT EXISTS folders (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  name TEXT NOT NULL,
+  sort_order INTEGER DEFAULT 0,
+  created_at TEXT DEFAULT (datetime('now'))
+);
+
+CREATE TABLE IF NOT EXISTS feeds (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  folder_id INTEGER REFERENCES folders(id) ON DELETE SET NULL,
+  title TEXT NOT NULL,
+  url TEXT NOT NULL UNIQUE,
+  site_url TEXT,
+  favicon TEXT,
+  description TEXT,
+  last_fetched_at TEXT,
+  last_error TEXT,
+  fetch_interval_minutes INTEGER DEFAULT 30,
+  sort_order INTEGER DEFAULT 0,
+  created_at TEXT DEFAULT (datetime('now'))
+);
+
+CREATE TABLE IF NOT EXISTS articles (
+  id INTEGER PRIMARY KEY AUTOINCREMENT,
+  feed_id INTEGER NOT NULL REFERENCES feeds(id) ON DELETE CASCADE,
+  guid TEXT NOT NULL,
+  title TEXT,
+  link TEXT,
+  author TEXT,
+  content TEXT,
+  summary TEXT,
+  image_url TEXT,
+  published_at TEXT,
+  fetched_at TEXT DEFAULT (datetime('now')),
+  is_read INTEGER DEFAULT 0,
+  is_starred INTEGER DEFAULT 0,
+  UNIQUE(feed_id, guid)
+);
+
+CREATE INDEX IF NOT EXISTS idx_articles_feed ON articles(feed_id);
+CREATE INDEX IF NOT EXISTS idx_articles_published ON articles(published_at);
+CREATE INDEX IF NOT EXISTS idx_articles_read ON articles(is_read);
+CREATE INDEX IF NOT EXISTS idx_articles_starred ON articles(is_starred);
+
+CREATE TABLE IF NOT EXISTS settings (
+  key TEXT PRIMARY KEY,
+  value TEXT
+);
+`);
+
+module.exports = db;
@@ -0,0 +1,121 @@
+const Parser = require('rss-parser');
+const sanitizeHtml = require('sanitize-html');
+const db = require('./db');
+
+const parser = new Parser({
+  timeout: 15000,
+  headers: { 'User-Agent': 'Dispatch RSS Reader/1.0' },
+  customFields: {
+    item: [
+      ['media:content', 'mediaContent'],
+      ['media:thumbnail', 'mediaThumbnail'],
+      ['content:encoded', 'contentEncoded'],
+    ],
+  },
+});
+
+function cleanHtml(html) {
+  if (!html) return null;
+  return sanitizeHtml(html, {
+    allowedTags: sanitizeHtml.defaults.allowedTags.concat(['img', 'figure', 'figcaption', 'iframe', 'h1', 'h2']),
+    allowedAttributes: {
+      ...sanitizeHtml.defaults.allowedAttributes,
+      img: ['src', 'alt', 'title', 'width', 'height', 'loading'],
+      iframe: ['src', 'allowfullscreen', 'frameborder'],
+      a: ['href', 'name', 'target', 'rel'],
+    },
+    allowedIframeHostnames: ['www.youtube.com', 'player.vimeo.com'],
+    transformTags: {
+      a: sanitizeHtml.simpleTransform('a', { rel: 'noopener noreferrer', target: '_blank' }),
+    },
+  });
+}
+
+function extractImage(item) {
+  if (item.mediaContent && item.mediaContent.$ && item.mediaContent.$.url) return item.mediaContent.$.url;
+  if (item.mediaThumbnail && item.mediaThumbnail.$ && item.mediaThumbnail.$.url) return item.mediaThumbnail.$.url;
+  if (item.enclosure && item.enclosure.url && /image/.test(item.enclosure.type || '')) return item.enclosure.url;
+  const html = item.contentEncoded || item.content || '';
+  const match = html.match(/<img[^>]+src=["']([^"']+)["']/i);
+  return match ? match[1] : null;
+}
+
+function stripToSummary(html, len = 220) {
+  if (!html) return '';
+  const text = html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim();
+  return text.length > len ? text.slice(0, len).trim() + '…' : text;
+}
+
+async function fetchFeedMeta(url) {
+  const feed = await parser.parseURL(url);
+  return {
+    title: feed.title || url,
+    siteUrl: feed.link || null,
+    description: feed.description || null,
+  };
+}
+
+async function fetchAndStoreFeed(feedRow) {
+  const insertArticle = db.prepare(`
+    INSERT INTO articles (feed_id, guid, title, link, author, content, summary, image_url, published_at)
+    VALUES (@feed_id, @guid, @title, @link, @author, @content, @summary, @image_url, @published_at)
+    ON CONFLICT(feed_id, guid) DO UPDATE SET
+      title=excluded.title,
+      link=excluded.link,
+      content=excluded.content,
+      summary=excluded.summary,
+      image_url=excluded.image_url
+      WHERE articles.title != excluded.title OR articles.content != excluded.content
+  `);
+
+  try {
+    const feed = await parser.parseURL(feedRow.url);
+    const tx = db.transaction((items) => {
+      for (const item of items) {
+        const guid = item.guid || item.id || item.link || item.title;
+        if (!guid) continue;
+        const rawContent = item.contentEncoded || item.content || item.summary || '';
+        const content = cleanHtml(rawContent);
+        insertArticle.run({
+          feed_id: feedRow.id,
+          guid,
+          title: item.title || '(untitled)',
+          link: item.link || null,
+          author: item.creator || item.author || null,
+          content,
+          summary: stripToSummary(rawContent),
+          image_url: extractImage(item),
+          published_at: item.isoDate || item.pubDate || new Date().toISOString(),
+        });
+      }
+    });
+    tx(feed.items || []);
+
+    db.prepare(`UPDATE feeds SET last_fetched_at = datetime('now'), last_error = NULL,
+      title = CASE WHEN title IS NULL OR title = '' THEN @title ELSE title END,
+      site_url = COALESCE(site_url, @siteUrl)
+      WHERE id = @id`).run({
+      id: feedRow.id,
+      title: feed.title || feedRow.title,
+      siteUrl: feed.link || null,
+    });
+
+    return { ok: true, count: (feed.items || []).length };
+  } catch (err) {
+    db.prepare(`UPDATE feeds SET last_fetched_at = datetime('now'), last_error = ? WHERE id = ?`)
+      .run(err.message || String(err), feedRow.id);
+    return { ok: false, error: err.message };
+  }
+}
+
+async function fetchAllFeeds() {
+  const feeds = db.prepare('SELECT * FROM feeds').all();
+  const results = [];
+  for (const feed of feeds) {
+    const res = await fetchAndStoreFeed(feed);
+    results.push({ feedId: feed.id, ...res });
+  }
+  return results;
+}
+
+module.exports = { fetchAndStoreFeed, fetchAllFeeds, fetchFeedMeta };
@@ -0,0 +1,17 @@
+{
+  "name": "dispatch-rss-backend",
+  "version": "1.0.0",
+  "main": "server.js",
+  "type": "commonjs",
+  "scripts": {
+    "start": "node server.js"
+  },
+  "dependencies": {
+    "better-sqlite3": "^11.10.0",
+    "cors": "^2.8.5",
+    "express": "^4.19.2",
+    "node-cron": "^3.0.3",
+    "rss-parser": "^3.13.0",
+    "sanitize-html": "^2.13.0"
+  }
+}
@@ -0,0 +1,73 @@
+const express = require('express');
+const router = express.Router();
+const db = require('../db');
+
+// GET /api/articles?feed_id=&folder_id=&status=unread|starred|all&search=&page=&limit=
+router.get('/', (req, res) => {
+  const { feed_id, folder_id, status = 'all', search, page = 1, limit = 30 } = req.query;
+  const clauses = [];
+  const params = {};
+
+  if (feed_id) { clauses.push('a.feed_id = @feed_id'); params.feed_id = feed_id; }
+  if (folder_id) { clauses.push('f.folder_id = @folder_id'); params.folder_id = folder_id; }
+  if (status === 'unread') clauses.push('a.is_read = 0');
+  if (status === 'starred') clauses.push('a.is_starred = 1');
+  if (search) { clauses.push('(a.title LIKE @search OR a.summary LIKE @search)'); params.search = `%${search}%`; }
+
+  const where = clauses.length ? `WHERE ${clauses.join(' AND ')}` : '';
+  const offset = (Number(page) - 1) * Number(limit);
+
+  const articles = db.prepare(`
+    SELECT a.id, a.feed_id, a.title, a.link, a.author, a.summary, a.image_url,
+           a.published_at, a.is_read, a.is_starred,
+           f.title AS feed_title, f.favicon AS feed_favicon
+    FROM articles a
+    JOIN feeds f ON f.id = a.feed_id
+    ${where}
+    ORDER BY a.published_at DESC
+    LIMIT @limit OFFSET @offset
+  `).all({ ...params, limit: Number(limit), offset });
+
+  const total = db.prepare(`
+    SELECT COUNT(*) AS count FROM articles a JOIN feeds f ON f.id = a.feed_id ${where}
+  `).get(params).count;
+
+  res.json({ articles, total, page: Number(page), limit: Number(limit) });
+});
+
+router.get('/:id', (req, res) => {
+  const article = db.prepare(`
+    SELECT a.*, f.title AS feed_title, f.site_url AS feed_site_url, f.favicon AS feed_favicon
+    FROM articles a JOIN feeds f ON f.id = a.feed_id WHERE a.id = ?
+  `).get(req.params.id);
+  if (!article) return res.status(404).json({ error: 'Article not found' });
+  res.json(article);
+});
+
+router.patch('/:id', (req, res) => {
+  const { is_read, is_starred } = req.body;
+  const sets = [];
+  const params = { id: req.params.id };
+  if (is_read !== undefined) { sets.push('is_read = @is_read'); params.is_read = is_read ? 1 : 0; }
+  if (is_starred !== undefined) { sets.push('is_starred = @is_starred'); params.is_starred = is_starred ? 1 : 0; }
+  if (!sets.length) return res.status(400).json({ error: 'Nothing to update' });
+  db.prepare(`UPDATE articles SET ${sets.join(', ')} WHERE id = @id`).run(params);
+  res.json(db.prepare('SELECT * FROM articles WHERE id = ?').get(req.params.id));
+});
+
+// Bulk mark-as-read for a feed/folder/all
+router.post('/mark-read', (req, res) => {
+  const { feed_id, folder_id, all } = req.body;
+  if (all) {
+    db.prepare('UPDATE articles SET is_read = 1 WHERE is_read = 0').run();
+  } else if (feed_id) {
+    db.prepare('UPDATE articles SET is_read = 1 WHERE feed_id = ? AND is_read = 0').run(feed_id);
+  } else if (folder_id) {
+    db.prepare(`UPDATE articles SET is_read = 1 WHERE is_read = 0 AND feed_id IN (SELECT id FROM feeds WHERE folder_id = ?)`).run(folder_id);
+  } else {
+    return res.status(400).json({ error: 'Specify feed_id, folder_id, or all' });
+  }
+  res.json({ ok: true });
+});
+
+module.exports = router;
@@ -0,0 +1,129 @@
+const express = require('express');
+const router = express.Router();
+const db = require('../db');
+const { fetchFeedMeta, fetchAndStoreFeed } = require('../feedFetcher');
+
+router.get('/', (req, res) => {
+  const feeds = db.prepare(`
+    SELECT f.*,
+      (SELECT COUNT(*) FROM articles a WHERE a.feed_id = f.id AND a.is_read = 0) AS unread_count,
+      (SELECT COUNT(*) FROM articles a WHERE a.feed_id = f.id) AS total_count
+    FROM feeds f ORDER BY f.sort_order, f.title COLLATE NOCASE
+  `).all();
+  res.json(feeds);
+});
+
+router.post('/', async (req, res) => {
+  const { url, folder_id } = req.body;
+  if (!url || !url.trim()) return res.status(400).json({ error: 'Feed URL is required' });
+  const cleanUrl = url.trim();
+
+  const existing = db.prepare('SELECT * FROM feeds WHERE url = ?').get(cleanUrl);
+  if (existing) return res.status(409).json({ error: 'This feed is already in your list', feed: existing });
+
+  try {
+    const meta = await fetchFeedMeta(cleanUrl);
+    const info = db.prepare(`
+      INSERT INTO feeds (url, title, site_url, description, folder_id)
+      VALUES (?, ?, ?, ?, ?)
+    `).run(cleanUrl, meta.title, meta.siteUrl, meta.description, folder_id || null);
+
+    const feed = db.prepare('SELECT * FROM feeds WHERE id = ?').get(info.lastInsertRowid);
+    fetchAndStoreFeed(feed).catch(() => {});
+    res.status(201).json(feed);
+  } catch (err) {
+    res.status(422).json({ error: `Couldn't read that feed. Check the URL and try again. (${err.message})` });
+  }
+});
+
+router.put('/:id', (req, res) => {
+  const { title, folder_id, fetch_interval_minutes } = req.body;
+  const feed = db.prepare('SELECT * FROM feeds WHERE id = ?').get(req.params.id);
+  if (!feed) return res.status(404).json({ error: 'Feed not found' });
+
+  db.prepare(`UPDATE feeds SET
+    title = COALESCE(?, title),
+    folder_id = ?,
+    fetch_interval_minutes = COALESCE(?, fetch_interval_minutes)
+    WHERE id = ?`).run(
+    title, folder_id === undefined ? feed.folder_id : folder_id, fetch_interval_minutes, req.params.id
+  );
+  res.json(db.prepare('SELECT * FROM feeds WHERE id = ?').get(req.params.id));
+});
+
+router.delete('/:id', (req, res) => {
+  db.prepare('DELETE FROM feeds WHERE id = ?').run(req.params.id);
+  res.status(204).end();
+});
+
+router.post('/:id/refresh', async (req, res) => {
+  const feed = db.prepare('SELECT * FROM feeds WHERE id = ?').get(req.params.id);
+  if (!feed) return res.status(404).json({ error: 'Feed not found' });
+  const result = await fetchAndStoreFeed(feed);
+  res.json(result);
+});
+
+router.post('/refresh-all', async (req, res) => {
+  const feeds = db.prepare('SELECT * FROM feeds').all();
+  const results = [];
+  for (const feed of feeds) {
+    results.push({ feedId: feed.id, ...(await fetchAndStoreFeed(feed)) });
+  }
+  res.json(results);
+});
+
+// OPML export
+router.get('/export/opml', (req, res) => {
+  const feeds = db.prepare('SELECT f.*, fo.name AS folder_name FROM feeds f LEFT JOIN folders fo ON f.folder_id = fo.id').all();
+  const byFolder = {};
+  const noFolder = [];
+  for (const f of feeds) {
+    if (f.folder_name) {
+      byFolder[f.folder_name] = byFolder[f.folder_name] || [];
+      byFolder[f.folder_name].push(f);
+    } else {
+      noFolder.push(f);
+    }
+  }
+  const esc = (s) => String(s || '').replace(/&/g, '&amp;').replace(/"/g, '&quot;').replace(/</g, '&lt;');
+  let body = '';
+  for (const [folderName, list] of Object.entries(byFolder)) {
+    body += `<outline text="${esc(folderName)}" title="${esc(folderName)}">\n`;
+    for (const f of list) {
+      body += `  <outline type="rss" text="${esc(f.title)}" title="${esc(f.title)}" xmlUrl="${esc(f.url)}" htmlUrl="${esc(f.site_url)}"/>\n`;
+    }
+    body += `</outline>\n`;
+  }
+  for (const f of noFolder) {
+    body += `<outline type="rss" text="${esc(f.title)}" title="${esc(f.title)}" xmlUrl="${esc(f.url)}" htmlUrl="${esc(f.site_url)}"/>\n`;
+  }
+  const opml = `<?xml version="1.0" encoding="UTF-8"?>\n<opml version="2.0">\n<head><title>Dispatch Feed Export</title></head>\n<body>\n${body}</body>\n</opml>`;
+  res.set('Content-Type', 'text/x-opml');
+  res.set('Content-Disposition', 'attachment; filename="dispatch-feeds.opml"');
+  res.send(opml);
+});
+
+// OPML import
+router.post('/import/opml', express.text({ type: '*/*', limit: '5mb' }), async (req, res) => {
+  const xml = req.body;
+  const urlMatches = [...xml.matchAll(/xmlUrl="([^"]+)"/g)].map(m => m[1].replace(/&amp;/g, '&'));
+  const titleMatches = [...xml.matchAll(/xmlUrl="[^"]+"\s+(?:htmlUrl="[^"]*"\s*)?/g)];
+  const results = [];
+  for (const url of urlMatches) {
+    const existing = db.prepare('SELECT id FROM feeds WHERE url = ?').get(url);
+    if (existing) { results.push({ url, status: 'skipped' }); continue; }
+    try {
+      const meta = await fetchFeedMeta(url);
+      const info = db.prepare('INSERT INTO feeds (url, title, site_url, description) VALUES (?, ?, ?, ?)')
+        .run(url, meta.title, meta.siteUrl, meta.description);
+      const feed = db.prepare('SELECT * FROM feeds WHERE id = ?').get(info.lastInsertRowid);
+      fetchAndStoreFeed(feed).catch(() => {});
+      results.push({ url, status: 'added' });
+    } catch (err) {
+      results.push({ url, status: 'failed', error: err.message });
+    }
+  }
+  res.json({ imported: results.length, results });
+});
+
+module.exports = router;
@@ -0,0 +1,29 @@
+const express = require('express');
+const router = express.Router();
+const db = require('../db');
+
+router.get('/', (req, res) => {
+  const folders = db.prepare('SELECT * FROM folders ORDER BY sort_order, name').all();
+  res.json(folders);
+});
+
+router.post('/', (req, res) => {
+  const { name } = req.body;
+  if (!name || !name.trim()) return res.status(400).json({ error: 'Name is required' });
+  const info = db.prepare('INSERT INTO folders (name) VALUES (?)').run(name.trim());
+  const folder = db.prepare('SELECT * FROM folders WHERE id = ?').get(info.lastInsertRowid);
+  res.status(201).json(folder);
+});
+
+router.put('/:id', (req, res) => {
+  const { name } = req.body;
+  db.prepare('UPDATE folders SET name = ? WHERE id = ?').run(name, req.params.id);
+  res.json(db.prepare('SELECT * FROM folders WHERE id = ?').get(req.params.id));
+});
+
+router.delete('/:id', (req, res) => {
+  db.prepare('DELETE FROM folders WHERE id = ?').run(req.params.id);
+  res.status(204).end();
+});
+
+module.exports = router;
@@ -0,0 +1,48 @@
+const express = require('express');
+const cors = require('cors');
+const path = require('path');
+const cron = require('node-cron');
+
+const db = require('./db');
+const { fetchAllFeeds } = require('./feedFetcher');
+const foldersRouter = require('./routes/folders');
+const feedsRouter = require('./routes/feeds');
+const articlesRouter = require('./routes/articles');
+
+const app = express();
+const PORT = process.env.PORT || 8080;
+const FETCH_INTERVAL_MIN = Number(process.env.FETCH_INTERVAL_MINUTES || 15);
+
+app.use(cors());
+app.use(express.json({ limit: '2mb' }));
+
+app.use('/api/folders', foldersRouter);
+app.use('/api/feeds', feedsRouter);
+app.use('/api/articles', articlesRouter);
+
+app.get('/api/stats', (req, res) => {
+  const feedCount = db.prepare('SELECT COUNT(*) c FROM feeds').get().c;
+  const unread = db.prepare('SELECT COUNT(*) c FROM articles WHERE is_read = 0').get().c;
+  const starred = db.prepare('SELECT COUNT(*) c FROM articles WHERE is_starred = 1').get().c;
+  res.json({ feedCount, unread, starred });
+});
+
+app.get('/api/health', (req, res) => res.json({ ok: true }));
+
+// Serve built frontend
+const FRONTEND_DIST = path.join(__dirname, 'public');
+app.use(express.static(FRONTEND_DIST));
+app.get(/^(?!\/api).*/, (req, res) => {
+  res.sendFile(path.join(FRONTEND_DIST, 'index.html'));
+});
+
+app.listen(PORT, () => {
+  console.log(`Dispatch RSS reader listening on port ${PORT}`);
+  // Fetch on boot, then on a schedule
+  fetchAllFeeds().then((r) => console.log(`Initial fetch complete: ${r.length} feeds`)).catch((e) => console.error(e));
+});
+
+cron.schedule(`*/${FETCH_INTERVAL_MIN} * * * *`, () => {
+  console.log('Running scheduled feed fetch...');
+  fetchAllFeeds().then((r) => console.log(`Scheduled fetch complete: ${r.length} feeds`)).catch((e) => console.error(e));
+});