Update feeds in background

api/subscriptions.js

@@ -20,8 +20,8 @@ module.exports = [
 				})
 				if (subscriptions.length) {
 					hasSubscriptions = true
-					const all = await Promise.all(subscriptions.map(id => fetchChannelLatest(id)))
+					const template = Array(subscriptions.length).fill("?").join(", ")
-					videos = all.flat(1).sort((a, b) => b.published - a.published).slice(0, 60)
+					videos = db.prepare(`SELECT * FROM Videos WHERE authorId IN (${template}) ORDER BY published DESC LIMIT 60`).all(subscriptions)
 				}
 			}
 			return render(200, "pug/subscriptions.pug", {hasSubscriptions, videos, channels})

api/utils/constants.js

@@ -2,7 +2,7 @@ const constants = {
 	user_settings: {
 		instance: {
 			type: "string",
-			default: "https://invidious.snopyta.org"
+			default: "https://second.cadence.moe"
 		},
 		save_history: {
 			type: "boolean",
@@ -11,7 +11,9 @@ const constants = {
 	},
 
 	caching: {
-		csrf_time: 4*60*60*1000
+		csrf_time: 4*60*60*1000,
+		seen_token_subscriptions_eligible: 40*60*60*1000,
+		subscriptions_refresh_loop_min: 5*60*1000,
 	},
 
 	regex: {

api/utils/getuser.js

@@ -6,16 +6,19 @@ const db = require("./db")
 function getToken(req, responseHeaders) {
 	if (!req.headers.cookie) req.headers.cookie = ""
 	const cookie = parseCookie(req.headers.cookie)
-	const token = cookie.token
+	let token = cookie.token
-	if (token) return token
+	if (!token) {
-	if (responseHeaders) { // we should create a token
+		if (responseHeaders) { // we should create a token
-		const setCookie = responseHeaders["set-cookie"] || []
+			const setCookie = responseHeaders["set-cookie"] || []
-		const token = crypto.randomBytes(18).toString("base64").replace(/\W/g, "_")
+			token = crypto.randomBytes(18).toString("base64").replace(/\W/g, "_")
-		setCookie.push(`token=${token}; Path=/; Max-Age=2147483648; HttpOnly; SameSite=Lax`)
+			setCookie.push(`token=${token}; Path=/; Max-Age=2147483648; HttpOnly; SameSite=Lax`)
-		responseHeaders["set-cookie"] = setCookie
+			responseHeaders["set-cookie"] = setCookie
-		return token
+		} else {
+			return null
+		}
 	}
-	return null
+	db.prepare("REPLACE INTO SeenTokens (token, seen) VALUES (?, ?)").run([token, Date.now()])
+	return token
 }
 
 class User {

api/utils/youtube.js

@@ -13,14 +13,4 @@ async function fetchChannel(ucid, instance) {
 	return channel
 }
 
-function fetchChannelLatest(ucid) {
-	return fetch(`http://localhost:3000/api/v1/channels/${ucid}/latest`).then(res => res.json()).then(root => {
-		root.forEach(video => {
-			video.descriptionHtml = video.descriptionHtml.replace(/<a /g, '<a tabindex="-1" ')
-		})
-		return root
-	})
-}
-
 module.exports.fetchChannel = fetchChannel
-module.exports.fetchChannelLatest = fetchChannelLatest

background/feed-update.js

@@ -1,2 +1,94 @@
+const Denque = require("denque")
+const fetch = require("node-fetch")
+const constants = require("../api/utils/constants")
 const db = require("../api/utils/db")
 
+const prepared = {
+	video_insert: db.prepare(
+		"INSERT OR IGNORE INTO Videos"
+			+ " ( videoId,  title,  author,  authorId,  published,  publishedText,  viewCountText,  descriptionHtml)"
+			+ " VALUES"
+			+ " (@videoId, @title, @author, @authorId, @published, @publishedText, @viewCountText, @descriptionHtml)"
+	)
+}
+
+class RefreshQueue {
+	constructor() {
+		this.set = new Set()
+		this.queue = new Denque()
+		this.lastLoadTime = 0
+	}
+
+	isEmpty() {
+		return this.queue.isEmpty()
+	}
+
+	load() {
+		// get the next set of scheduled channels to refresh
+		const afterTime = Date.now() - constants.caching.seen_token_subscriptions_eligible
+		const channels = db.prepare(
+			"SELECT DISTINCT Subscriptions.ucid FROM SeenTokens INNER JOIN Subscriptions ON SeenTokens.token = Subscriptions.token AND SeenTokens.seen > ? ORDER BY SeenTokens.seen DESC"
+		).pluck().all(afterTime)
+		this.addLast(channels)
+		this.lastLoadTime = Date.now()
+	}
+
+	addNext(items) {
+		for (const i of items) {
+			this.queue.unshift(i)
+			this.set.add(i)
+		}
+	}
+
+	addLast(items) {
+		for (const i of items) {
+			this.queue.push(i)
+			this.set.add(i)
+		}
+	}
+
+	next() {
+		const item = this.queue.shift()
+		this.set.delete(item)
+		return item
+	}
+}
+
+const refreshQueue = new RefreshQueue()
+
+function refreshChannel(ucid) {
+	return fetch(`http://localhost:3000/api/v1/channels/${ucid}/latest`).then(res => res.json()).then(root => {
+		if (Array.isArray(root)) {
+			root.forEach(video => {
+				// organise
+				video.descriptionHtml = video.descriptionHtml.replace(/<a /g, '<a tabindex="-1" ') // should be safe
+				video.viewCountText = null //TODO?
+				// store
+				prepared.video_insert.run(video)
+			})
+			console.log(`updated ${root.length} videos for channel ${ucid}`)
+		} else if (root.identifier === "PUBLISHED_DATES_NOT_PROVIDED") {
+			return [] // nothing we can do. skip this iteration.
+		} else {
+			throw new Error(root.error)
+		}
+	})
+}
+
+function refreshNext() {
+	if (refreshQueue.isEmpty()) {
+		const timeSinceLastLoop = Date.now() - refreshQueue.lastLoadTime
+		if (timeSinceLastLoop < constants.caching.subscriptions_refresh_loop_min) {
+			const timeToWait = constants.caching.subscriptions_refresh_loop_min - timeSinceLastLoop
+			console.log(`waiting ${timeToWait} before next loop`)
+			return setTimeout(refreshNext, timeToWait)
+		} else {
+			refreshQueue.load()
+		}
+	}
+
+	const ucid = refreshQueue.next()
+	refreshChannel(ucid).then(refreshNext)
+}
+
+refreshNext()

package-lock.json

@@ -217,6 +217,11 @@
       "resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
       "integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o="
     },
+    "denque": {
+      "version": "1.4.1",
+      "resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz",
+      "integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ=="
+    },
     "detect-libc": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz",

package.json

@@ -12,6 +12,7 @@
   "dependencies": {
     "better-sqlite3": "^7.1.0",
     "cookie": "^0.4.1",
+    "denque": "^1.4.1",
     "node-fetch": "^2.6.0",
     "pinski": "git+https://git.sr.ht/~cadence/nodejs-pinski#04cd72482574f689b78670b7f2b80209f5449a21"
   }

pug/includes/video-list-item.pug

@@ -2,7 +2,7 @@ mixin video_list_item(video)
   - let link = `/watch?v=${video.videoId}`
   a(href=link tabindex="-1").thumbnail
     img(src=`https://i.ytimg.com/vi/${video.videoId}/mqdefault.jpg` width=320 height=180 alt="").image
-    if video.second__lengthText !== undefined
+    if video.second__lengthText != undefined
       span.duration= video.second__lengthText
   .info
     div.title: a(href=link).title-link= video.title

pug/video.pug

@@ -70,4 +70,4 @@ block content
     main.video-error-page
       h2 Error
       != message
-      p: a(href=`https://www.youtube.com/watch?v=${video.videoId}`) Watch on YouTube →
+      p: a(href=`https://www.youtube.com/watch?v=${video.videoId}#cloudtube`) Watch on YouTube →