1
0
mirror of https://git.sr.ht/~cadence/cloudtube synced 2024-11-09 18:17:30 +00:00

Update feeds in background

This commit is contained in:
Cadence Ember 2020-09-23 23:45:02 +12:00
parent 4a3c1e2ac3
commit 643f1e0889
No known key found for this signature in database
GPG Key ID: BC1C2C61CF521B17
9 changed files with 118 additions and 25 deletions

View File

@ -20,8 +20,8 @@ module.exports = [
})
if (subscriptions.length) {
hasSubscriptions = true
const all = await Promise.all(subscriptions.map(id => fetchChannelLatest(id)))
videos = all.flat(1).sort((a, b) => b.published - a.published).slice(0, 60)
const template = Array(subscriptions.length).fill("?").join(", ")
videos = db.prepare(`SELECT * FROM Videos WHERE authorId IN (${template}) ORDER BY published DESC LIMIT 60`).all(subscriptions)
}
}
return render(200, "pug/subscriptions.pug", {hasSubscriptions, videos, channels})

View File

@ -2,7 +2,7 @@ const constants = {
user_settings: {
instance: {
type: "string",
default: "https://invidious.snopyta.org"
default: "https://second.cadence.moe"
},
save_history: {
type: "boolean",
@ -11,7 +11,9 @@ const constants = {
},
caching: {
csrf_time: 4*60*60*1000
csrf_time: 4*60*60*1000,
seen_token_subscriptions_eligible: 40*60*60*1000,
subscriptions_refresh_loop_min: 5*60*1000,
},
regex: {

View File

@ -6,16 +6,19 @@ const db = require("./db")
function getToken(req, responseHeaders) {
if (!req.headers.cookie) req.headers.cookie = ""
const cookie = parseCookie(req.headers.cookie)
const token = cookie.token
if (token) return token
if (responseHeaders) { // we should create a token
const setCookie = responseHeaders["set-cookie"] || []
const token = crypto.randomBytes(18).toString("base64").replace(/\W/g, "_")
setCookie.push(`token=${token}; Path=/; Max-Age=2147483648; HttpOnly; SameSite=Lax`)
responseHeaders["set-cookie"] = setCookie
return token
let token = cookie.token
if (!token) {
if (responseHeaders) { // we should create a token
const setCookie = responseHeaders["set-cookie"] || []
token = crypto.randomBytes(18).toString("base64").replace(/\W/g, "_")
setCookie.push(`token=${token}; Path=/; Max-Age=2147483648; HttpOnly; SameSite=Lax`)
responseHeaders["set-cookie"] = setCookie
} else {
return null
}
}
return null
db.prepare("REPLACE INTO SeenTokens (token, seen) VALUES (?, ?)").run([token, Date.now()])
return token
}
class User {

View File

@ -13,14 +13,4 @@ async function fetchChannel(ucid, instance) {
return channel
}
function fetchChannelLatest(ucid) {
return fetch(`http://localhost:3000/api/v1/channels/${ucid}/latest`).then(res => res.json()).then(root => {
root.forEach(video => {
video.descriptionHtml = video.descriptionHtml.replace(/<a /g, '<a tabindex="-1" ')
})
return root
})
}
module.exports.fetchChannel = fetchChannel
module.exports.fetchChannelLatest = fetchChannelLatest

View File

@ -1,2 +1,94 @@
const Denque = require("denque")
const fetch = require("node-fetch")
const constants = require("../api/utils/constants")
const db = require("../api/utils/db")
const prepared = {
video_insert: db.prepare(
"INSERT OR IGNORE INTO Videos"
+ " ( videoId, title, author, authorId, published, publishedText, viewCountText, descriptionHtml)"
+ " VALUES"
+ " (@videoId, @title, @author, @authorId, @published, @publishedText, @viewCountText, @descriptionHtml)"
)
}
class RefreshQueue {
constructor() {
this.set = new Set()
this.queue = new Denque()
this.lastLoadTime = 0
}
isEmpty() {
return this.queue.isEmpty()
}
load() {
// get the next set of scheduled channels to refresh
const afterTime = Date.now() - constants.caching.seen_token_subscriptions_eligible
const channels = db.prepare(
"SELECT DISTINCT Subscriptions.ucid FROM SeenTokens INNER JOIN Subscriptions ON SeenTokens.token = Subscriptions.token AND SeenTokens.seen > ? ORDER BY SeenTokens.seen DESC"
).pluck().all(afterTime)
this.addLast(channels)
this.lastLoadTime = Date.now()
}
addNext(items) {
for (const i of items) {
this.queue.unshift(i)
this.set.add(i)
}
}
addLast(items) {
for (const i of items) {
this.queue.push(i)
this.set.add(i)
}
}
next() {
const item = this.queue.shift()
this.set.delete(item)
return item
}
}
const refreshQueue = new RefreshQueue()
function refreshChannel(ucid) {
return fetch(`http://localhost:3000/api/v1/channels/${ucid}/latest`).then(res => res.json()).then(root => {
if (Array.isArray(root)) {
root.forEach(video => {
// organise
video.descriptionHtml = video.descriptionHtml.replace(/<a /g, '<a tabindex="-1" ') // should be safe
video.viewCountText = null //TODO?
// store
prepared.video_insert.run(video)
})
console.log(`updated ${root.length} videos for channel ${ucid}`)
} else if (root.identifier === "PUBLISHED_DATES_NOT_PROVIDED") {
return [] // nothing we can do. skip this iteration.
} else {
throw new Error(root.error)
}
})
}
function refreshNext() {
if (refreshQueue.isEmpty()) {
const timeSinceLastLoop = Date.now() - refreshQueue.lastLoadTime
if (timeSinceLastLoop < constants.caching.subscriptions_refresh_loop_min) {
const timeToWait = constants.caching.subscriptions_refresh_loop_min - timeSinceLastLoop
console.log(`waiting ${timeToWait} before next loop`)
return setTimeout(refreshNext, timeToWait)
} else {
refreshQueue.load()
}
}
const ucid = refreshQueue.next()
refreshChannel(ucid).then(refreshNext)
}
refreshNext()

5
package-lock.json generated
View File

@ -217,6 +217,11 @@
"resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
"integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o="
},
"denque": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz",
"integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ=="
},
"detect-libc": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz",

View File

@ -12,6 +12,7 @@
"dependencies": {
"better-sqlite3": "^7.1.0",
"cookie": "^0.4.1",
"denque": "^1.4.1",
"node-fetch": "^2.6.0",
"pinski": "git+https://git.sr.ht/~cadence/nodejs-pinski#04cd72482574f689b78670b7f2b80209f5449a21"
}

View File

@ -2,7 +2,7 @@ mixin video_list_item(video)
- let link = `/watch?v=${video.videoId}`
a(href=link tabindex="-1").thumbnail
img(src=`https://i.ytimg.com/vi/${video.videoId}/mqdefault.jpg` width=320 height=180 alt="").image
if video.second__lengthText !== undefined
if video.second__lengthText != undefined
span.duration= video.second__lengthText
.info
div.title: a(href=link).title-link= video.title

View File

@ -70,4 +70,4 @@ block content
main.video-error-page
h2 Error
!= message
p: a(href=`https://www.youtube.com/watch?v=${video.videoId}`) Watch on YouTube →
p: a(href=`https://www.youtube.com/watch?v=${video.videoId}#cloudtube`) Watch on YouTube →