From 643f1e0889eeb8165d6124951a57d7a48db40188 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 23 Sep 2020 23:45:02 +1200 Subject: [PATCH] Update feeds in background --- api/subscriptions.js | 4 +- api/utils/constants.js | 6 ++- api/utils/getuser.js | 21 ++++---- api/utils/youtube.js | 10 ---- background/feed-update.js | 92 ++++++++++++++++++++++++++++++++ package-lock.json | 5 ++ package.json | 1 + pug/includes/video-list-item.pug | 2 +- pug/video.pug | 2 +- 9 files changed, 118 insertions(+), 25 deletions(-) diff --git a/api/subscriptions.js b/api/subscriptions.js index c755f89..7c01566 100644 --- a/api/subscriptions.js +++ b/api/subscriptions.js @@ -20,8 +20,8 @@ module.exports = [ }) if (subscriptions.length) { hasSubscriptions = true - const all = await Promise.all(subscriptions.map(id => fetchChannelLatest(id))) - videos = all.flat(1).sort((a, b) => b.published - a.published).slice(0, 60) + const template = Array(subscriptions.length).fill("?").join(", ") + videos = db.prepare(`SELECT * FROM Videos WHERE authorId IN (${template}) ORDER BY published DESC LIMIT 60`).all(subscriptions) } } return render(200, "pug/subscriptions.pug", {hasSubscriptions, videos, channels}) diff --git a/api/utils/constants.js b/api/utils/constants.js index c274a44..88bdc32 100644 --- a/api/utils/constants.js +++ b/api/utils/constants.js @@ -2,7 +2,7 @@ const constants = { user_settings: { instance: { type: "string", - default: "https://invidious.snopyta.org" + default: "https://second.cadence.moe" }, save_history: { type: "boolean", @@ -11,7 +11,9 @@ const constants = { }, caching: { - csrf_time: 4*60*60*1000 + csrf_time: 4*60*60*1000, + seen_token_subscriptions_eligible: 40*60*60*1000, + subscriptions_refresh_loop_min: 5*60*1000, }, regex: { diff --git a/api/utils/getuser.js b/api/utils/getuser.js index c73a18b..33778b1 100644 --- a/api/utils/getuser.js +++ b/api/utils/getuser.js @@ -6,16 +6,19 @@ const db = require("./db") function getToken(req, responseHeaders) { if (!req.headers.cookie) req.headers.cookie = "" const cookie = parseCookie(req.headers.cookie) - const token = cookie.token - if (token) return token - if (responseHeaders) { // we should create a token - const setCookie = responseHeaders["set-cookie"] || [] - const token = crypto.randomBytes(18).toString("base64").replace(/\W/g, "_") - setCookie.push(`token=${token}; Path=/; Max-Age=2147483648; HttpOnly; SameSite=Lax`) - responseHeaders["set-cookie"] = setCookie - return token + let token = cookie.token + if (!token) { + if (responseHeaders) { // we should create a token + const setCookie = responseHeaders["set-cookie"] || [] + token = crypto.randomBytes(18).toString("base64").replace(/\W/g, "_") + setCookie.push(`token=${token}; Path=/; Max-Age=2147483648; HttpOnly; SameSite=Lax`) + responseHeaders["set-cookie"] = setCookie + } else { + return null + } } - return null + db.prepare("REPLACE INTO SeenTokens (token, seen) VALUES (?, ?)").run([token, Date.now()]) + return token } class User { diff --git a/api/utils/youtube.js b/api/utils/youtube.js index dcee701..1987394 100644 --- a/api/utils/youtube.js +++ b/api/utils/youtube.js @@ -13,14 +13,4 @@ async function fetchChannel(ucid, instance) { return channel } -function fetchChannelLatest(ucid) { - return fetch(`http://localhost:3000/api/v1/channels/${ucid}/latest`).then(res => res.json()).then(root => { - root.forEach(video => { - video.descriptionHtml = video.descriptionHtml.replace(/ ? ORDER BY SeenTokens.seen DESC" + ).pluck().all(afterTime) + this.addLast(channels) + this.lastLoadTime = Date.now() + } + + addNext(items) { + for (const i of items) { + this.queue.unshift(i) + this.set.add(i) + } + } + + addLast(items) { + for (const i of items) { + this.queue.push(i) + this.set.add(i) + } + } + + next() { + const item = this.queue.shift() + this.set.delete(item) + return item + } +} + +const refreshQueue = new RefreshQueue() + +function refreshChannel(ucid) { + return fetch(`http://localhost:3000/api/v1/channels/${ucid}/latest`).then(res => res.json()).then(root => { + if (Array.isArray(root)) { + root.forEach(video => { + // organise + video.descriptionHtml = video.descriptionHtml.replace(/