From 9cf0eda983eb79b57de13e86decb7fb46c65c6c5 Mon Sep 17 00:00:00 2001 From: Quinta0 <0pietroquintavalle0@gmail.com> Date: Fri, 10 Jul 2026 15:48:21 +0200 Subject: [PATCH] backend first draft --- backend/db/index.js | 65 +++++++++++++++++++ backend/feedFetcher.js | 121 ++++++++++++++++++++++++++++++++++ backend/package.json | 17 +++++ backend/routes/articles.js | 73 +++++++++++++++++++++ backend/routes/feeds.js | 129 +++++++++++++++++++++++++++++++++++++ backend/routes/folders.js | 29 +++++++++ backend/server.js | 48 ++++++++++++++ 7 files changed, 482 insertions(+) create mode 100644 backend/db/index.js create mode 100644 backend/feedFetcher.js create mode 100644 backend/package.json create mode 100644 backend/routes/articles.js create mode 100644 backend/routes/feeds.js create mode 100644 backend/routes/folders.js create mode 100644 backend/server.js diff --git a/backend/db/index.js b/backend/db/index.js new file mode 100644 index 0000000..179647e --- /dev/null +++ b/backend/db/index.js @@ -0,0 +1,65 @@ +const Database = require('better-sqlite3'); +const path = require('path'); +const fs = require('fs'); + +const DATA_DIR = process.env.DATA_DIR || path.join(__dirname, '..', 'data'); +if (!fs.existsSync(DATA_DIR)) { + fs.mkdirSync(DATA_DIR, { recursive: true }); +} + +const db = new Database(path.join(DATA_DIR, 'dispatch.db')); +db.pragma('journal_mode = WAL'); +db.pragma('foreign_keys = ON'); + +db.exec(` +CREATE TABLE IF NOT EXISTS folders ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL, + sort_order INTEGER DEFAULT 0, + created_at TEXT DEFAULT (datetime('now')) +); + +CREATE TABLE IF NOT EXISTS feeds ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + folder_id INTEGER REFERENCES folders(id) ON DELETE SET NULL, + title TEXT NOT NULL, + url TEXT NOT NULL UNIQUE, + site_url TEXT, + favicon TEXT, + description TEXT, + last_fetched_at TEXT, + last_error TEXT, + fetch_interval_minutes INTEGER DEFAULT 30, + sort_order INTEGER DEFAULT 0, + created_at TEXT DEFAULT (datetime('now')) +); + +CREATE TABLE IF NOT EXISTS articles ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + feed_id INTEGER NOT NULL REFERENCES feeds(id) ON DELETE CASCADE, + guid TEXT NOT NULL, + title TEXT, + link TEXT, + author TEXT, + content TEXT, + summary TEXT, + image_url TEXT, + published_at TEXT, + fetched_at TEXT DEFAULT (datetime('now')), + is_read INTEGER DEFAULT 0, + is_starred INTEGER DEFAULT 0, + UNIQUE(feed_id, guid) +); + +CREATE INDEX IF NOT EXISTS idx_articles_feed ON articles(feed_id); +CREATE INDEX IF NOT EXISTS idx_articles_published ON articles(published_at); +CREATE INDEX IF NOT EXISTS idx_articles_read ON articles(is_read); +CREATE INDEX IF NOT EXISTS idx_articles_starred ON articles(is_starred); + +CREATE TABLE IF NOT EXISTS settings ( + key TEXT PRIMARY KEY, + value TEXT +); +`); + +module.exports = db; diff --git a/backend/feedFetcher.js b/backend/feedFetcher.js new file mode 100644 index 0000000..4961033 --- /dev/null +++ b/backend/feedFetcher.js @@ -0,0 +1,121 @@ +const Parser = require('rss-parser'); +const sanitizeHtml = require('sanitize-html'); +const db = require('./db'); + +const parser = new Parser({ + timeout: 15000, + headers: { 'User-Agent': 'Dispatch RSS Reader/1.0' }, + customFields: { + item: [ + ['media:content', 'mediaContent'], + ['media:thumbnail', 'mediaThumbnail'], + ['content:encoded', 'contentEncoded'], + ], + }, +}); + +function cleanHtml(html) { + if (!html) return null; + return sanitizeHtml(html, { + allowedTags: sanitizeHtml.defaults.allowedTags.concat(['img', 'figure', 'figcaption', 'iframe', 'h1', 'h2']), + allowedAttributes: { + ...sanitizeHtml.defaults.allowedAttributes, + img: ['src', 'alt', 'title', 'width', 'height', 'loading'], + iframe: ['src', 'allowfullscreen', 'frameborder'], + a: ['href', 'name', 'target', 'rel'], + }, + allowedIframeHostnames: ['www.youtube.com', 'player.vimeo.com'], + transformTags: { + a: sanitizeHtml.simpleTransform('a', { rel: 'noopener noreferrer', target: '_blank' }), + }, + }); +} + +function extractImage(item) { + if (item.mediaContent && item.mediaContent.$ && item.mediaContent.$.url) return item.mediaContent.$.url; + if (item.mediaThumbnail && item.mediaThumbnail.$ && item.mediaThumbnail.$.url) return item.mediaThumbnail.$.url; + if (item.enclosure && item.enclosure.url && /image/.test(item.enclosure.type || '')) return item.enclosure.url; + const html = item.contentEncoded || item.content || ''; + const match = html.match(/]+src=["']([^"']+)["']/i); + return match ? match[1] : null; +} + +function stripToSummary(html, len = 220) { + if (!html) return ''; + const text = html.replace(/<[^>]+>/g, ' ').replace(/\s+/g, ' ').trim(); + return text.length > len ? text.slice(0, len).trim() + '…' : text; +} + +async function fetchFeedMeta(url) { + const feed = await parser.parseURL(url); + return { + title: feed.title || url, + siteUrl: feed.link || null, + description: feed.description || null, + }; +} + +async function fetchAndStoreFeed(feedRow) { + const insertArticle = db.prepare(` + INSERT INTO articles (feed_id, guid, title, link, author, content, summary, image_url, published_at) + VALUES (@feed_id, @guid, @title, @link, @author, @content, @summary, @image_url, @published_at) + ON CONFLICT(feed_id, guid) DO UPDATE SET + title=excluded.title, + link=excluded.link, + content=excluded.content, + summary=excluded.summary, + image_url=excluded.image_url + WHERE articles.title != excluded.title OR articles.content != excluded.content + `); + + try { + const feed = await parser.parseURL(feedRow.url); + const tx = db.transaction((items) => { + for (const item of items) { + const guid = item.guid || item.id || item.link || item.title; + if (!guid) continue; + const rawContent = item.contentEncoded || item.content || item.summary || ''; + const content = cleanHtml(rawContent); + insertArticle.run({ + feed_id: feedRow.id, + guid, + title: item.title || '(untitled)', + link: item.link || null, + author: item.creator || item.author || null, + content, + summary: stripToSummary(rawContent), + image_url: extractImage(item), + published_at: item.isoDate || item.pubDate || new Date().toISOString(), + }); + } + }); + tx(feed.items || []); + + db.prepare(`UPDATE feeds SET last_fetched_at = datetime('now'), last_error = NULL, + title = CASE WHEN title IS NULL OR title = '' THEN @title ELSE title END, + site_url = COALESCE(site_url, @siteUrl) + WHERE id = @id`).run({ + id: feedRow.id, + title: feed.title || feedRow.title, + siteUrl: feed.link || null, + }); + + return { ok: true, count: (feed.items || []).length }; + } catch (err) { + db.prepare(`UPDATE feeds SET last_fetched_at = datetime('now'), last_error = ? WHERE id = ?`) + .run(err.message || String(err), feedRow.id); + return { ok: false, error: err.message }; + } +} + +async function fetchAllFeeds() { + const feeds = db.prepare('SELECT * FROM feeds').all(); + const results = []; + for (const feed of feeds) { + const res = await fetchAndStoreFeed(feed); + results.push({ feedId: feed.id, ...res }); + } + return results; +} + +module.exports = { fetchAndStoreFeed, fetchAllFeeds, fetchFeedMeta }; diff --git a/backend/package.json b/backend/package.json new file mode 100644 index 0000000..0fd991f --- /dev/null +++ b/backend/package.json @@ -0,0 +1,17 @@ +{ + "name": "dispatch-rss-backend", + "version": "1.0.0", + "main": "server.js", + "type": "commonjs", + "scripts": { + "start": "node server.js" + }, + "dependencies": { + "better-sqlite3": "^11.10.0", + "cors": "^2.8.5", + "express": "^4.19.2", + "node-cron": "^3.0.3", + "rss-parser": "^3.13.0", + "sanitize-html": "^2.13.0" + } +} diff --git a/backend/routes/articles.js b/backend/routes/articles.js new file mode 100644 index 0000000..a68fe19 --- /dev/null +++ b/backend/routes/articles.js @@ -0,0 +1,73 @@ +const express = require('express'); +const router = express.Router(); +const db = require('../db'); + +// GET /api/articles?feed_id=&folder_id=&status=unread|starred|all&search=&page=&limit= +router.get('/', (req, res) => { + const { feed_id, folder_id, status = 'all', search, page = 1, limit = 30 } = req.query; + const clauses = []; + const params = {}; + + if (feed_id) { clauses.push('a.feed_id = @feed_id'); params.feed_id = feed_id; } + if (folder_id) { clauses.push('f.folder_id = @folder_id'); params.folder_id = folder_id; } + if (status === 'unread') clauses.push('a.is_read = 0'); + if (status === 'starred') clauses.push('a.is_starred = 1'); + if (search) { clauses.push('(a.title LIKE @search OR a.summary LIKE @search)'); params.search = `%${search}%`; } + + const where = clauses.length ? `WHERE ${clauses.join(' AND ')}` : ''; + const offset = (Number(page) - 1) * Number(limit); + + const articles = db.prepare(` + SELECT a.id, a.feed_id, a.title, a.link, a.author, a.summary, a.image_url, + a.published_at, a.is_read, a.is_starred, + f.title AS feed_title, f.favicon AS feed_favicon + FROM articles a + JOIN feeds f ON f.id = a.feed_id + ${where} + ORDER BY a.published_at DESC + LIMIT @limit OFFSET @offset + `).all({ ...params, limit: Number(limit), offset }); + + const total = db.prepare(` + SELECT COUNT(*) AS count FROM articles a JOIN feeds f ON f.id = a.feed_id ${where} + `).get(params).count; + + res.json({ articles, total, page: Number(page), limit: Number(limit) }); +}); + +router.get('/:id', (req, res) => { + const article = db.prepare(` + SELECT a.*, f.title AS feed_title, f.site_url AS feed_site_url, f.favicon AS feed_favicon + FROM articles a JOIN feeds f ON f.id = a.feed_id WHERE a.id = ? + `).get(req.params.id); + if (!article) return res.status(404).json({ error: 'Article not found' }); + res.json(article); +}); + +router.patch('/:id', (req, res) => { + const { is_read, is_starred } = req.body; + const sets = []; + const params = { id: req.params.id }; + if (is_read !== undefined) { sets.push('is_read = @is_read'); params.is_read = is_read ? 1 : 0; } + if (is_starred !== undefined) { sets.push('is_starred = @is_starred'); params.is_starred = is_starred ? 1 : 0; } + if (!sets.length) return res.status(400).json({ error: 'Nothing to update' }); + db.prepare(`UPDATE articles SET ${sets.join(', ')} WHERE id = @id`).run(params); + res.json(db.prepare('SELECT * FROM articles WHERE id = ?').get(req.params.id)); +}); + +// Bulk mark-as-read for a feed/folder/all +router.post('/mark-read', (req, res) => { + const { feed_id, folder_id, all } = req.body; + if (all) { + db.prepare('UPDATE articles SET is_read = 1 WHERE is_read = 0').run(); + } else if (feed_id) { + db.prepare('UPDATE articles SET is_read = 1 WHERE feed_id = ? AND is_read = 0').run(feed_id); + } else if (folder_id) { + db.prepare(`UPDATE articles SET is_read = 1 WHERE is_read = 0 AND feed_id IN (SELECT id FROM feeds WHERE folder_id = ?)`).run(folder_id); + } else { + return res.status(400).json({ error: 'Specify feed_id, folder_id, or all' }); + } + res.json({ ok: true }); +}); + +module.exports = router; diff --git a/backend/routes/feeds.js b/backend/routes/feeds.js new file mode 100644 index 0000000..663cb5d --- /dev/null +++ b/backend/routes/feeds.js @@ -0,0 +1,129 @@ +const express = require('express'); +const router = express.Router(); +const db = require('../db'); +const { fetchFeedMeta, fetchAndStoreFeed } = require('../feedFetcher'); + +router.get('/', (req, res) => { + const feeds = db.prepare(` + SELECT f.*, + (SELECT COUNT(*) FROM articles a WHERE a.feed_id = f.id AND a.is_read = 0) AS unread_count, + (SELECT COUNT(*) FROM articles a WHERE a.feed_id = f.id) AS total_count + FROM feeds f ORDER BY f.sort_order, f.title COLLATE NOCASE + `).all(); + res.json(feeds); +}); + +router.post('/', async (req, res) => { + const { url, folder_id } = req.body; + if (!url || !url.trim()) return res.status(400).json({ error: 'Feed URL is required' }); + const cleanUrl = url.trim(); + + const existing = db.prepare('SELECT * FROM feeds WHERE url = ?').get(cleanUrl); + if (existing) return res.status(409).json({ error: 'This feed is already in your list', feed: existing }); + + try { + const meta = await fetchFeedMeta(cleanUrl); + const info = db.prepare(` + INSERT INTO feeds (url, title, site_url, description, folder_id) + VALUES (?, ?, ?, ?, ?) + `).run(cleanUrl, meta.title, meta.siteUrl, meta.description, folder_id || null); + + const feed = db.prepare('SELECT * FROM feeds WHERE id = ?').get(info.lastInsertRowid); + fetchAndStoreFeed(feed).catch(() => {}); + res.status(201).json(feed); + } catch (err) { + res.status(422).json({ error: `Couldn't read that feed. Check the URL and try again. (${err.message})` }); + } +}); + +router.put('/:id', (req, res) => { + const { title, folder_id, fetch_interval_minutes } = req.body; + const feed = db.prepare('SELECT * FROM feeds WHERE id = ?').get(req.params.id); + if (!feed) return res.status(404).json({ error: 'Feed not found' }); + + db.prepare(`UPDATE feeds SET + title = COALESCE(?, title), + folder_id = ?, + fetch_interval_minutes = COALESCE(?, fetch_interval_minutes) + WHERE id = ?`).run( + title, folder_id === undefined ? feed.folder_id : folder_id, fetch_interval_minutes, req.params.id + ); + res.json(db.prepare('SELECT * FROM feeds WHERE id = ?').get(req.params.id)); +}); + +router.delete('/:id', (req, res) => { + db.prepare('DELETE FROM feeds WHERE id = ?').run(req.params.id); + res.status(204).end(); +}); + +router.post('/:id/refresh', async (req, res) => { + const feed = db.prepare('SELECT * FROM feeds WHERE id = ?').get(req.params.id); + if (!feed) return res.status(404).json({ error: 'Feed not found' }); + const result = await fetchAndStoreFeed(feed); + res.json(result); +}); + +router.post('/refresh-all', async (req, res) => { + const feeds = db.prepare('SELECT * FROM feeds').all(); + const results = []; + for (const feed of feeds) { + results.push({ feedId: feed.id, ...(await fetchAndStoreFeed(feed)) }); + } + res.json(results); +}); + +// OPML export +router.get('/export/opml', (req, res) => { + const feeds = db.prepare('SELECT f.*, fo.name AS folder_name FROM feeds f LEFT JOIN folders fo ON f.folder_id = fo.id').all(); + const byFolder = {}; + const noFolder = []; + for (const f of feeds) { + if (f.folder_name) { + byFolder[f.folder_name] = byFolder[f.folder_name] || []; + byFolder[f.folder_name].push(f); + } else { + noFolder.push(f); + } + } + const esc = (s) => String(s || '').replace(/&/g, '&').replace(/"/g, '"').replace(/\n`; + for (const f of list) { + body += ` \n`; + } + body += `\n`; + } + for (const f of noFolder) { + body += `\n`; + } + const opml = `\n\nDispatch Feed Export\n\n${body}\n`; + res.set('Content-Type', 'text/x-opml'); + res.set('Content-Disposition', 'attachment; filename="dispatch-feeds.opml"'); + res.send(opml); +}); + +// OPML import +router.post('/import/opml', express.text({ type: '*/*', limit: '5mb' }), async (req, res) => { + const xml = req.body; + const urlMatches = [...xml.matchAll(/xmlUrl="([^"]+)"/g)].map(m => m[1].replace(/&/g, '&')); + const titleMatches = [...xml.matchAll(/xmlUrl="[^"]+"\s+(?:htmlUrl="[^"]*"\s*)?/g)]; + const results = []; + for (const url of urlMatches) { + const existing = db.prepare('SELECT id FROM feeds WHERE url = ?').get(url); + if (existing) { results.push({ url, status: 'skipped' }); continue; } + try { + const meta = await fetchFeedMeta(url); + const info = db.prepare('INSERT INTO feeds (url, title, site_url, description) VALUES (?, ?, ?, ?)') + .run(url, meta.title, meta.siteUrl, meta.description); + const feed = db.prepare('SELECT * FROM feeds WHERE id = ?').get(info.lastInsertRowid); + fetchAndStoreFeed(feed).catch(() => {}); + results.push({ url, status: 'added' }); + } catch (err) { + results.push({ url, status: 'failed', error: err.message }); + } + } + res.json({ imported: results.length, results }); +}); + +module.exports = router; diff --git a/backend/routes/folders.js b/backend/routes/folders.js new file mode 100644 index 0000000..377e7c2 --- /dev/null +++ b/backend/routes/folders.js @@ -0,0 +1,29 @@ +const express = require('express'); +const router = express.Router(); +const db = require('../db'); + +router.get('/', (req, res) => { + const folders = db.prepare('SELECT * FROM folders ORDER BY sort_order, name').all(); + res.json(folders); +}); + +router.post('/', (req, res) => { + const { name } = req.body; + if (!name || !name.trim()) return res.status(400).json({ error: 'Name is required' }); + const info = db.prepare('INSERT INTO folders (name) VALUES (?)').run(name.trim()); + const folder = db.prepare('SELECT * FROM folders WHERE id = ?').get(info.lastInsertRowid); + res.status(201).json(folder); +}); + +router.put('/:id', (req, res) => { + const { name } = req.body; + db.prepare('UPDATE folders SET name = ? WHERE id = ?').run(name, req.params.id); + res.json(db.prepare('SELECT * FROM folders WHERE id = ?').get(req.params.id)); +}); + +router.delete('/:id', (req, res) => { + db.prepare('DELETE FROM folders WHERE id = ?').run(req.params.id); + res.status(204).end(); +}); + +module.exports = router; diff --git a/backend/server.js b/backend/server.js new file mode 100644 index 0000000..0d8a7eb --- /dev/null +++ b/backend/server.js @@ -0,0 +1,48 @@ +const express = require('express'); +const cors = require('cors'); +const path = require('path'); +const cron = require('node-cron'); + +const db = require('./db'); +const { fetchAllFeeds } = require('./feedFetcher'); +const foldersRouter = require('./routes/folders'); +const feedsRouter = require('./routes/feeds'); +const articlesRouter = require('./routes/articles'); + +const app = express(); +const PORT = process.env.PORT || 8080; +const FETCH_INTERVAL_MIN = Number(process.env.FETCH_INTERVAL_MINUTES || 15); + +app.use(cors()); +app.use(express.json({ limit: '2mb' })); + +app.use('/api/folders', foldersRouter); +app.use('/api/feeds', feedsRouter); +app.use('/api/articles', articlesRouter); + +app.get('/api/stats', (req, res) => { + const feedCount = db.prepare('SELECT COUNT(*) c FROM feeds').get().c; + const unread = db.prepare('SELECT COUNT(*) c FROM articles WHERE is_read = 0').get().c; + const starred = db.prepare('SELECT COUNT(*) c FROM articles WHERE is_starred = 1').get().c; + res.json({ feedCount, unread, starred }); +}); + +app.get('/api/health', (req, res) => res.json({ ok: true })); + +// Serve built frontend +const FRONTEND_DIST = path.join(__dirname, 'public'); +app.use(express.static(FRONTEND_DIST)); +app.get(/^(?!\/api).*/, (req, res) => { + res.sendFile(path.join(FRONTEND_DIST, 'index.html')); +}); + +app.listen(PORT, () => { + console.log(`Dispatch RSS reader listening on port ${PORT}`); + // Fetch on boot, then on a schedule + fetchAllFeeds().then((r) => console.log(`Initial fetch complete: ${r.length} feeds`)).catch((e) => console.error(e)); +}); + +cron.schedule(`*/${FETCH_INTERVAL_MIN} * * * *`, () => { + console.log('Running scheduled feed fetch...'); + fetchAllFeeds().then((r) => console.log(`Scheduled fetch complete: ${r.length} feeds`)).catch((e) => console.error(e)); +});