const Denque = require("denque") const fetch = require("node-fetch") const constants = require("../utils/constants") const db = require("../utils/db") const prepared = { video_insert: db.prepare( "INSERT OR IGNORE INTO Videos" + " ( videoId, title, author, authorId, published, viewCountText, descriptionHtml)" + " VALUES" + " (@videoId, @title, @author, @authorId, @published, @viewCountText, @descriptionHtml)" ), channel_refreshed_update: db.prepare( "UPDATE Channels SET refreshed = ? WHERE ucid = ?" ) } class RefreshQueue { constructor() { this.set = new Set() this.queue = new Denque() this.lastLoadTime = 0 } isEmpty() { return this.queue.isEmpty() } load() { // get the next set of scheduled channels to refresh const afterTime = Date.now() - constants.caching.seen_token_subscriptions_eligible const channels = db.prepare( "SELECT DISTINCT Subscriptions.ucid FROM SeenTokens INNER JOIN Subscriptions ON SeenTokens.token = Subscriptions.token AND SeenTokens.seen > ? ORDER BY SeenTokens.seen DESC" ).pluck().all(afterTime) this.addLast(channels) this.lastLoadTime = Date.now() } addNext(items) { for (const i of items) { this.queue.unshift(i) this.set.add(i) } } addLast(items) { for (const i of items) { this.queue.push(i) this.set.add(i) } } next() { if (this.isEmpty()) { throw new Error("Cannot get next of empty refresh queue") } const item = this.queue.shift() this.set.delete(item) return item } } const refreshQueue = new RefreshQueue() function refreshChannel(ucid) { return fetch(`http://localhost:3000/api/v1/channels/${ucid}/latest`).then(res => res.json()).then(root => { if (Array.isArray(root)) { root.forEach(video => { // organise video.descriptionHtml = video.descriptionHtml.replace(/