From 109dcd22de2f044bfc9728ca255706a3a5d27988 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 10 Jan 2022 14:18:45 +1300 Subject: [PATCH] Rework subscribing to deleted channels --- api/channels.js | 12 +++++- api/formapi.js | 2 - api/subscriptions.js | 4 +- background/feed-update.js | 30 ++++++++++---- pug/channel-error.pug | 27 +++++++++++++ pug/subscriptions.pug | 16 +++++++- sass/includes/_channel-page.sass | 13 ++++++ sass/includes/_subscriptions-page.sass | 4 ++ utils/constants.js | 1 + utils/getuser.js | 4 +- utils/upgradedb.js | 20 +++++++++ utils/youtube.js | 56 +++++++++++++++++++++++--- 12 files changed, 167 insertions(+), 22 deletions(-) create mode 100644 pug/channel-error.pug diff --git a/api/channels.js b/api/channels.js index de9bc2c..56320a3 100644 --- a/api/channels.js +++ b/api/channels.js @@ -13,6 +13,16 @@ module.exports = [ const data = await fetchChannel(id, settings.instance) const subscribed = user.isSubscribed(id) const instanceOrigin = settings.instance + + // problem with the channel? fetchChannel has collected the necessary information for us. + // we can render a skeleton page, display the message, and provide the option to unsubscribe. + if (data.error) { + const statusCode = data.missing ? 410 : 500 + return render(statusCode, "pug/channel-error.pug", {settings, data, subscribed, instanceOrigin}) + } + + // everything is fine + // normalise info, apply watched status if (!data.second__subCountText && data.subCount) { data.second__subCountText = converters.subscriberCountToText(data.subCount) @@ -24,7 +34,7 @@ module.exports = [ video.watched = watchedVideos.includes(video.videoId) }) } - return render(200, "pug/channel.pug", {settings, url, data, subscribed, instanceOrigin}) + return render(200, "pug/channel.pug", {settings, data, subscribed, instanceOrigin}) } } ] diff --git a/api/formapi.js b/api/formapi.js index 14f10d0..0f994d3 100644 --- a/api/formapi.js +++ b/api/formapi.js @@ -26,7 +26,6 @@ module.exports = [ await fetchChannel(ucid, settings.instance) db.prepare( "INSERT INTO Subscriptions (token, ucid) VALUES (?, ?)" - + " ON CONFLICT (token, ucid) DO UPDATE SET channel_missing = 0" ).run(token, ucid) } else { db.prepare("DELETE FROM Subscriptions WHERE token = ? AND ucid = ?").run(token, ucid) @@ -41,7 +40,6 @@ module.exports = [ }), content: "Success, redirecting..." } - return redirect(params.get("referrer"), 303) } else { return { statusCode: 200, diff --git a/api/subscriptions.js b/api/subscriptions.js index 6bb439b..bddce0c 100644 --- a/api/subscriptions.js +++ b/api/subscriptions.js @@ -11,12 +11,14 @@ module.exports = [ let hasSubscriptions = false let videos = [] let channels = [] + let missingChannelCount = 0 let refreshed = null if (user.token) { // trigger a background refresh, needed if they came back from being inactive refresher.skipWaiting() // get channels channels = db.prepare(`SELECT Channels.* FROM Channels INNER JOIN Subscriptions ON Channels.ucid = Subscriptions.ucid WHERE token = ? ORDER BY name`).all(user.token) + missingChannelCount = channels.reduce((a, c) => a + c.missing, 0) // get refreshed status refreshed = db.prepare(`SELECT min(refreshed) as min, max(refreshed) as max, count(refreshed) as count FROM Channels INNER JOIN Subscriptions ON Channels.ucid = Subscriptions.ucid WHERE token = ?`).get(user.token) // get watched videos @@ -37,7 +39,7 @@ module.exports = [ } const settings = user.getSettingsOrDefaults() const instanceOrigin = settings.instance - return render(200, "pug/subscriptions.pug", {url, settings, hasSubscriptions, videos, channels, refreshed, timeToPastText, instanceOrigin}) + return render(200, "pug/subscriptions.pug", {url, settings, hasSubscriptions, videos, channels, missingChannelCount, refreshed, timeToPastText, instanceOrigin}) } } ] diff --git a/background/feed-update.js b/background/feed-update.js index 069ee0d..6e46cfc 100644 --- a/background/feed-update.js +++ b/background/feed-update.js @@ -15,8 +15,8 @@ const prepared = { channel_refreshed_update: db.prepare( "UPDATE Channels SET refreshed = ? WHERE ucid = ?" ), - unsubscribe_all_from_channel: db.prepare( - "UPDATE Subscriptions SET channel_missing = 1 WHERE ucid = ?" + channel_mark_as_missing: db.prepare( + "UPDATE Channels SET missing = 1, missing_reason = ? WHERE ucid = ?" ) } @@ -35,7 +35,7 @@ class RefreshQueue { // get the next set of scheduled channels to refresh const afterTime = Date.now() - constants.caching.seen_token_subscriptions_eligible const channels = db.prepare( - "SELECT DISTINCT Subscriptions.ucid FROM SeenTokens INNER JOIN Subscriptions ON SeenTokens.token = Subscriptions.token AND SeenTokens.seen > ? WHERE Subscriptions.channel_missing = 0 ORDER BY SeenTokens.seen DESC" + "SELECT DISTINCT Subscriptions.ucid FROM SeenTokens INNER JOIN Subscriptions ON SeenTokens.token = Subscriptions.token INNER JOIN Channels ON Channels.ucid = Subscriptions.ucid WHERE Channels.missing = 0 AND SeenTokens.seen > ? ORDER BY SeenTokens.seen DESC" ).pluck().all(afterTime) this.addLast(channels) this.lastLoadTime = Date.now() @@ -72,11 +72,12 @@ class Refresher { this.refreshQueue = new RefreshQueue() this.state = this.sym.ACTIVE this.waitingTimeout = null + this.lastFakeNotFoundTime = 0 this.next() } refreshChannel(ucid) { - return fetch(`${constants.server_setup.local_instance_origin}/api/v1/channels/${ucid}/latest`).then(res => res.json()).then(root => { + return fetch(`${constants.server_setup.local_instance_origin}/api/v1/channels/${ucid}/latest`).then(res => res.json()).then(/** @param {any} root */ root => { if (Array.isArray(root)) { root.forEach(video => { // organise @@ -89,11 +90,24 @@ class Refresher { prepared.channel_refreshed_update.run(Date.now(), ucid) // console.log(`updated ${root.length} videos for channel ${ucid}`) } else if (root.identifier === "PUBLISHED_DATES_NOT_PROVIDED") { - return [] // nothing we can do. skip this iteration. + // nothing we can do. skip this iteration. } else if (root.identifier === "NOT_FOUND") { - // the channel does not exist. we should unsubscribe all users so we don't try again. - // console.log(`channel ${ucid} does not exist, unsubscribing all users`) - prepared.unsubscribe_all_from_channel.run(ucid) + // YouTube sometimes returns not found for absolutely no reason. + // There is no way to distinguish between a fake missing channel and a real missing channel without requesting the real endpoint. + // These fake missing channels often happen in bursts, which is why there is a cooldown. + const timeSinceLastFakeNotFound = Date.now() - this.lastFakeNotFoundTime + if (timeSinceLastFakeNotFound >= constants.caching.subscriptions_refesh_fake_not_found_cooldown) { + // We'll request the real endpoint to verify. + fetch(`${constants.server_setup.local_instance_origin}/api/v1/channels/${ucid}`).then(res => res.json()).then(/** @param {any} root */ root => { + if (root.error && (root.identifier === "NOT_FOUND" || root.identifier === "ACCOUNT_TERMINATED")) { + // The channel is really gone, and we should mark it as missing for everyone. + prepared.channel_mark_as_missing.run(root.error, ucid) + } else { + // The channel is not actually gone and YouTube is trolling us. + this.lastFakeNotFoundTime = Date.now() + } + }) + } // else youtube is currently trolling us, skip this until later. } else { throw new Error(root.error) } diff --git a/pug/channel-error.pug b/pug/channel-error.pug new file mode 100644 index 0000000..db4d226 --- /dev/null +++ b/pug/channel-error.pug @@ -0,0 +1,27 @@ +extends includes/layout + +include includes/video-list-item +include includes/subscribe-button + +block head + title= `${data.row ? data.row.name : "Deleted channel"} - CloudTube` + script(type="module" src=getStaticURL("html", "/static/js/channel.js")) + +block content + main.channel-page + if data.row + .channel-data + .info + - const iconURL = data.row.icon_url + if iconURL + .logo + img(src=iconURL alt="").thumbnail-image + .about + h1.name= data.row.name + +subscribe_button(data.ucid, subscribed, `/channel/${data.ucid}`).subscribe-button.base-border-look + + .channel-error + div= data.message + + if data.missing && subscribed + .you-should-unsubscribe To remove this channel from your subscriptions list, click Unsubscribe. diff --git a/pug/subscriptions.pug b/pug/subscriptions.pug index 2846b01..7710f64 100644 --- a/pug/subscriptions.pug +++ b/pug/subscriptions.pug @@ -11,12 +11,24 @@ block content if hasSubscriptions section details.channels-details - summary #{channels.length} subscriptions + summary + | #{channels.length} subscriptions + if missingChannelCount === 1 + = ` - ${missingChannelCount} channel is gone` + else if missingChannelCount > 1 + = ` - ${missingChannelCount} channels are gone` .channels-list for channel in channels a(href=`/channel/${channel.ucid}`).channel-item img(src=channel.icon_url width=512 height=512 alt="").thumbnail - span.name= channel.name + div + div.name= channel.name + if channel.missing + div.missing-reason + if channel.missing_reason + = channel.missing_reason + else + | This channel appears to be deleted or terminated. Click to check it. if refreshed section diff --git a/sass/includes/_channel-page.sass b/sass/includes/_channel-page.sass index 7f91f61..f1a8952 100644 --- a/sass/includes/_channel-page.sass +++ b/sass/includes/_channel-page.sass @@ -74,6 +74,19 @@ $_theme: () !default .channel-video @include channel-video +.channel-error + background-color: map.get($_theme, "bg-1") + padding: 24px + margin: 12px 0px 24px + border-radius: 8px + border: 1px solid map.get($_theme, "edge-grey") + font-size: 20px + color: map.get($_theme, "fg-warning") + +.you-should-unsubscribe + margin-top: 20px + color: map.get($_theme, "fg-main") + .about-description // class provided by youtube pre font-size: inherit diff --git a/sass/includes/_subscriptions-page.sass b/sass/includes/_subscriptions-page.sass index bd70f37..86c41b9 100644 --- a/sass/includes/_subscriptions-page.sass +++ b/sass/includes/_subscriptions-page.sass @@ -37,6 +37,10 @@ $_theme: () !default font-size: 22px color: map.get($_theme, "fg-main") + .missing-reason + font-size: 16px + color: map.get($_theme, "fg-warning") + @include forms.checkbox-hider("watched-videos-display") #watched-videos-display:checked ~ .video-list-item--watched diff --git a/utils/constants.js b/utils/constants.js index 585ee9f..55e916c 100644 --- a/utils/constants.js +++ b/utils/constants.js @@ -50,6 +50,7 @@ let constants = { csrf_time: 4*60*60*1000, seen_token_subscriptions_eligible: 40*60*60*1000, subscriptions_refresh_loop_min: 5*60*1000, + subscriptions_refesh_fake_not_found_cooldown: 10*60*1000, }, // Pattern matching. diff --git a/utils/getuser.js b/utils/getuser.js index d23fa0b..efc25ce 100644 --- a/utils/getuser.js +++ b/utils/getuser.js @@ -54,7 +54,7 @@ class User { getSubscriptions() { if (this.token) { - return db.prepare("SELECT ucid FROM Subscriptions WHERE token = ? AND channel_missing = 0").pluck().all(this.token) + return db.prepare("SELECT ucid FROM Subscriptions WHERE token = ?").pluck().all(this.token) } else { return [] } @@ -62,7 +62,7 @@ class User { isSubscribed(ucid) { if (this.token) { - return !!db.prepare("SELECT * FROM Subscriptions WHERE token = ? AND ucid = ? AND channel_missing = 0").get([this.token, ucid]) + return !!db.prepare("SELECT * FROM Subscriptions WHERE token = ? AND ucid = ?").get([this.token, ucid]) } else { return false } diff --git a/utils/upgradedb.js b/utils/upgradedb.js index 27607c5..545ed61 100644 --- a/utils/upgradedb.js +++ b/utils/upgradedb.js @@ -75,6 +75,26 @@ const deltas = [ function() { db.prepare("ALTER TABLE Settings ADD COLUMN theme INTEGER DEFAULT 0") .run() + }, + // 12: Channels +missing +missing_reason, Subscriptions - + // Better management for missing channels + // We totally discard the existing Subscriptions.channel_missing since it is unreliable. + function() { + db.prepare("ALTER TABLE Channels ADD COLUMN missing INTEGER NOT NULL DEFAULT 0") + .run() + db.prepare("ALTER TABLE Channels ADD COLUMN missing_reason TEXT") + .run() + // https://www.sqlite.org/lang_altertable.html#making_other_kinds_of_table_schema_changes + db.transaction(() => { + db.prepare("CREATE TABLE NEW_Subscriptions (token TEXT NOT NULL, ucid TEXT NOT NULL, PRIMARY KEY (token, ucid))") + .run() + db.prepare("INSERT INTO NEW_Subscriptions (token, ucid) SELECT token, ucid FROM Subscriptions") + .run() + db.prepare("DROP TABLE Subscriptions") + .run() + db.prepare("ALTER TABLE NEW_Subscriptions RENAME TO Subscriptions") + .run() + })() } ] diff --git a/utils/youtube.js b/utils/youtube.js index cc7e15a..991b884 100644 --- a/utils/youtube.js +++ b/utils/youtube.js @@ -2,14 +2,58 @@ const {request} = require("./request") const db = require("./db") async function fetchChannel(ucid, instance) { + function updateGoodData(channel) { + const bestIcon = channel.authorThumbnails.slice(-1)[0] + const iconURL = bestIcon ? bestIcon.url : null + db.prepare("REPLACE INTO Channels (ucid, name, icon_url, missing, missing_reason) VALUES (?, ?, ?, 0, NULL)").run(channel.authorId, channel.author, iconURL) + } + + function updateBadData(channel) { + if (channel.identifier === "NOT_FOUND" || channel.identifier === "ACCOUNT_TERMINATED") { + db.prepare("UPDATE Channels SET missing = 1, missing_reason = ? WHERE ucid = ?").run(channel.error, channel.authorId) + return { + missing: true, + message: channel.error + } + } else { + return { + missing: false, + message: channel.error + } + } + } + if (!instance) throw new Error("No instance parameter provided") - // fetch + + const row = db.prepare("SELECT * FROM Channels WHERE ucid = ?").get(ucid) + + // handle the case where the channel has a known error + if (row && row.missing_reason) { + return { + error: true, + ucid, + row, + missing: true, + message: row.missing_reason + } + } + + /** @type {any} */ const channel = await request(`${instance}/api/v1/channels/${ucid}`).then(res => res.json()) - // update database - const bestIcon = channel.authorThumbnails.slice(-1)[0] - const iconURL = bestIcon ? bestIcon.url : null - db.prepare("REPLACE INTO Channels (ucid, name, icon_url) VALUES (?, ?, ?)").run([channel.authorId, channel.author, iconURL]) - // return + + // handle the case where the channel has a newly discovered error + if (channel.error) { + const missingData = updateBadData(channel) + return { + error: true, + ucid, + row, + ...missingData + } + } + + // handle the case where the channel returns good data (this is the only remaining scenario) + updateGoodData(channel) return channel }