mirror of
https://git.sr.ht/~cadence/cloudtube
synced 2024-11-22 07:37:29 +00:00
Update feeds in background
This commit is contained in:
parent
4a3c1e2ac3
commit
643f1e0889
@ -20,8 +20,8 @@ module.exports = [
|
||||
})
|
||||
if (subscriptions.length) {
|
||||
hasSubscriptions = true
|
||||
const all = await Promise.all(subscriptions.map(id => fetchChannelLatest(id)))
|
||||
videos = all.flat(1).sort((a, b) => b.published - a.published).slice(0, 60)
|
||||
const template = Array(subscriptions.length).fill("?").join(", ")
|
||||
videos = db.prepare(`SELECT * FROM Videos WHERE authorId IN (${template}) ORDER BY published DESC LIMIT 60`).all(subscriptions)
|
||||
}
|
||||
}
|
||||
return render(200, "pug/subscriptions.pug", {hasSubscriptions, videos, channels})
|
||||
|
@ -2,7 +2,7 @@ const constants = {
|
||||
user_settings: {
|
||||
instance: {
|
||||
type: "string",
|
||||
default: "https://invidious.snopyta.org"
|
||||
default: "https://second.cadence.moe"
|
||||
},
|
||||
save_history: {
|
||||
type: "boolean",
|
||||
@ -11,7 +11,9 @@ const constants = {
|
||||
},
|
||||
|
||||
caching: {
|
||||
csrf_time: 4*60*60*1000
|
||||
csrf_time: 4*60*60*1000,
|
||||
seen_token_subscriptions_eligible: 40*60*60*1000,
|
||||
subscriptions_refresh_loop_min: 5*60*1000,
|
||||
},
|
||||
|
||||
regex: {
|
||||
|
@ -6,16 +6,19 @@ const db = require("./db")
|
||||
function getToken(req, responseHeaders) {
|
||||
if (!req.headers.cookie) req.headers.cookie = ""
|
||||
const cookie = parseCookie(req.headers.cookie)
|
||||
const token = cookie.token
|
||||
if (token) return token
|
||||
if (responseHeaders) { // we should create a token
|
||||
const setCookie = responseHeaders["set-cookie"] || []
|
||||
const token = crypto.randomBytes(18).toString("base64").replace(/\W/g, "_")
|
||||
setCookie.push(`token=${token}; Path=/; Max-Age=2147483648; HttpOnly; SameSite=Lax`)
|
||||
responseHeaders["set-cookie"] = setCookie
|
||||
return token
|
||||
let token = cookie.token
|
||||
if (!token) {
|
||||
if (responseHeaders) { // we should create a token
|
||||
const setCookie = responseHeaders["set-cookie"] || []
|
||||
token = crypto.randomBytes(18).toString("base64").replace(/\W/g, "_")
|
||||
setCookie.push(`token=${token}; Path=/; Max-Age=2147483648; HttpOnly; SameSite=Lax`)
|
||||
responseHeaders["set-cookie"] = setCookie
|
||||
} else {
|
||||
return null
|
||||
}
|
||||
}
|
||||
return null
|
||||
db.prepare("REPLACE INTO SeenTokens (token, seen) VALUES (?, ?)").run([token, Date.now()])
|
||||
return token
|
||||
}
|
||||
|
||||
class User {
|
||||
|
@ -13,14 +13,4 @@ async function fetchChannel(ucid, instance) {
|
||||
return channel
|
||||
}
|
||||
|
||||
function fetchChannelLatest(ucid) {
|
||||
return fetch(`http://localhost:3000/api/v1/channels/${ucid}/latest`).then(res => res.json()).then(root => {
|
||||
root.forEach(video => {
|
||||
video.descriptionHtml = video.descriptionHtml.replace(/<a /g, '<a tabindex="-1" ')
|
||||
})
|
||||
return root
|
||||
})
|
||||
}
|
||||
|
||||
module.exports.fetchChannel = fetchChannel
|
||||
module.exports.fetchChannelLatest = fetchChannelLatest
|
||||
|
@ -1,2 +1,94 @@
|
||||
const Denque = require("denque")
|
||||
const fetch = require("node-fetch")
|
||||
const constants = require("../api/utils/constants")
|
||||
const db = require("../api/utils/db")
|
||||
|
||||
const prepared = {
|
||||
video_insert: db.prepare(
|
||||
"INSERT OR IGNORE INTO Videos"
|
||||
+ " ( videoId, title, author, authorId, published, publishedText, viewCountText, descriptionHtml)"
|
||||
+ " VALUES"
|
||||
+ " (@videoId, @title, @author, @authorId, @published, @publishedText, @viewCountText, @descriptionHtml)"
|
||||
)
|
||||
}
|
||||
|
||||
class RefreshQueue {
|
||||
constructor() {
|
||||
this.set = new Set()
|
||||
this.queue = new Denque()
|
||||
this.lastLoadTime = 0
|
||||
}
|
||||
|
||||
isEmpty() {
|
||||
return this.queue.isEmpty()
|
||||
}
|
||||
|
||||
load() {
|
||||
// get the next set of scheduled channels to refresh
|
||||
const afterTime = Date.now() - constants.caching.seen_token_subscriptions_eligible
|
||||
const channels = db.prepare(
|
||||
"SELECT DISTINCT Subscriptions.ucid FROM SeenTokens INNER JOIN Subscriptions ON SeenTokens.token = Subscriptions.token AND SeenTokens.seen > ? ORDER BY SeenTokens.seen DESC"
|
||||
).pluck().all(afterTime)
|
||||
this.addLast(channels)
|
||||
this.lastLoadTime = Date.now()
|
||||
}
|
||||
|
||||
addNext(items) {
|
||||
for (const i of items) {
|
||||
this.queue.unshift(i)
|
||||
this.set.add(i)
|
||||
}
|
||||
}
|
||||
|
||||
addLast(items) {
|
||||
for (const i of items) {
|
||||
this.queue.push(i)
|
||||
this.set.add(i)
|
||||
}
|
||||
}
|
||||
|
||||
next() {
|
||||
const item = this.queue.shift()
|
||||
this.set.delete(item)
|
||||
return item
|
||||
}
|
||||
}
|
||||
|
||||
const refreshQueue = new RefreshQueue()
|
||||
|
||||
function refreshChannel(ucid) {
|
||||
return fetch(`http://localhost:3000/api/v1/channels/${ucid}/latest`).then(res => res.json()).then(root => {
|
||||
if (Array.isArray(root)) {
|
||||
root.forEach(video => {
|
||||
// organise
|
||||
video.descriptionHtml = video.descriptionHtml.replace(/<a /g, '<a tabindex="-1" ') // should be safe
|
||||
video.viewCountText = null //TODO?
|
||||
// store
|
||||
prepared.video_insert.run(video)
|
||||
})
|
||||
console.log(`updated ${root.length} videos for channel ${ucid}`)
|
||||
} else if (root.identifier === "PUBLISHED_DATES_NOT_PROVIDED") {
|
||||
return [] // nothing we can do. skip this iteration.
|
||||
} else {
|
||||
throw new Error(root.error)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
function refreshNext() {
|
||||
if (refreshQueue.isEmpty()) {
|
||||
const timeSinceLastLoop = Date.now() - refreshQueue.lastLoadTime
|
||||
if (timeSinceLastLoop < constants.caching.subscriptions_refresh_loop_min) {
|
||||
const timeToWait = constants.caching.subscriptions_refresh_loop_min - timeSinceLastLoop
|
||||
console.log(`waiting ${timeToWait} before next loop`)
|
||||
return setTimeout(refreshNext, timeToWait)
|
||||
} else {
|
||||
refreshQueue.load()
|
||||
}
|
||||
}
|
||||
|
||||
const ucid = refreshQueue.next()
|
||||
refreshChannel(ucid).then(refreshNext)
|
||||
}
|
||||
|
||||
refreshNext()
|
||||
|
5
package-lock.json
generated
5
package-lock.json
generated
@ -217,6 +217,11 @@
|
||||
"resolved": "https://registry.npmjs.org/delegates/-/delegates-1.0.0.tgz",
|
||||
"integrity": "sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o="
|
||||
},
|
||||
"denque": {
|
||||
"version": "1.4.1",
|
||||
"resolved": "https://registry.npmjs.org/denque/-/denque-1.4.1.tgz",
|
||||
"integrity": "sha512-OfzPuSZKGcgr96rf1oODnfjqBFmr1DVoc/TrItj3Ohe0Ah1C5WX5Baquw/9U9KovnQ88EqmJbD66rKYUQYN1tQ=="
|
||||
},
|
||||
"detect-libc": {
|
||||
"version": "1.0.3",
|
||||
"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-1.0.3.tgz",
|
||||
|
@ -12,6 +12,7 @@
|
||||
"dependencies": {
|
||||
"better-sqlite3": "^7.1.0",
|
||||
"cookie": "^0.4.1",
|
||||
"denque": "^1.4.1",
|
||||
"node-fetch": "^2.6.0",
|
||||
"pinski": "git+https://git.sr.ht/~cadence/nodejs-pinski#04cd72482574f689b78670b7f2b80209f5449a21"
|
||||
}
|
||||
|
@ -2,7 +2,7 @@ mixin video_list_item(video)
|
||||
- let link = `/watch?v=${video.videoId}`
|
||||
a(href=link tabindex="-1").thumbnail
|
||||
img(src=`https://i.ytimg.com/vi/${video.videoId}/mqdefault.jpg` width=320 height=180 alt="").image
|
||||
if video.second__lengthText !== undefined
|
||||
if video.second__lengthText != undefined
|
||||
span.duration= video.second__lengthText
|
||||
.info
|
||||
div.title: a(href=link).title-link= video.title
|
||||
|
@ -70,4 +70,4 @@ block content
|
||||
main.video-error-page
|
||||
h2 Error
|
||||
!= message
|
||||
p: a(href=`https://www.youtube.com/watch?v=${video.videoId}`) Watch on YouTube →
|
||||
p: a(href=`https://www.youtube.com/watch?v=${video.videoId}#cloudtube`) Watch on YouTube →
|
||||
|
Loading…
Reference in New Issue
Block a user