Rework subscribing to deleted channels

api/channels.js

@@ -13,6 +13,16 @@ module.exports = [
 			const data = await fetchChannel(id, settings.instance)
 			const subscribed = user.isSubscribed(id)
 			const instanceOrigin = settings.instance
+
+			// problem with the channel? fetchChannel has collected the necessary information for us.
+			// we can render a skeleton page, display the message, and provide the option to unsubscribe.
+			if (data.error) {
+				const statusCode = data.missing ? 410 : 500
+				return render(statusCode, "pug/channel-error.pug", {settings, data, subscribed, instanceOrigin})
+			}
+
+			// everything is fine
+
 			// normalise info, apply watched status
 			if (!data.second__subCountText && data.subCount) {
 				data.second__subCountText = converters.subscriberCountToText(data.subCount)
@@ -24,7 +34,7 @@ module.exports = [
 					video.watched = watchedVideos.includes(video.videoId)
 				})
 			}
-			return render(200, "pug/channel.pug", {settings, url, data, subscribed, instanceOrigin})
+			return render(200, "pug/channel.pug", {settings, data, subscribed, instanceOrigin})
 		}
 	}
 ]

api/formapi.js

@@ -26,7 +26,6 @@ module.exports = [
 						await fetchChannel(ucid, settings.instance)
 						db.prepare(
 							"INSERT INTO Subscriptions (token, ucid) VALUES (?, ?)"
-								+ " ON CONFLICT (token, ucid) DO UPDATE SET channel_missing = 0"
 						).run(token, ucid)
 					} else {
 						db.prepare("DELETE FROM Subscriptions WHERE token = ? AND ucid = ?").run(token, ucid)
@@ -41,7 +40,6 @@ module.exports = [
 							}),
 							content: "Success, redirecting..."
 						}
-						return redirect(params.get("referrer"), 303)
 					} else {
 						return {
 							statusCode: 200,

api/subscriptions.js

@@ -11,12 +11,14 @@ module.exports = [
 			let hasSubscriptions = false
 			let videos = []
 			let channels = []
+			let missingChannelCount = 0
 			let refreshed = null
 			if (user.token) {
 				// trigger a background refresh, needed if they came back from being inactive
 				refresher.skipWaiting()
 				// get channels
 				channels = db.prepare(`SELECT Channels.* FROM Channels INNER JOIN Subscriptions ON Channels.ucid = Subscriptions.ucid WHERE token = ? ORDER BY name`).all(user.token)
+				missingChannelCount = channels.reduce((a, c) => a + c.missing, 0)
 				// get refreshed status
 				refreshed = db.prepare(`SELECT min(refreshed) as min, max(refreshed) as max, count(refreshed) as count FROM Channels INNER JOIN Subscriptions ON Channels.ucid = Subscriptions.ucid WHERE token = ?`).get(user.token)
 				// get watched videos
@@ -37,7 +39,7 @@ module.exports = [
 			}
 			const settings = user.getSettingsOrDefaults()
 			const instanceOrigin = settings.instance
-			return render(200, "pug/subscriptions.pug", {url, settings, hasSubscriptions, videos, channels, refreshed, timeToPastText, instanceOrigin})
+			return render(200, "pug/subscriptions.pug", {url, settings, hasSubscriptions, videos, channels, missingChannelCount, refreshed, timeToPastText, instanceOrigin})
 		}
 	}
 ]

background/feed-update.js

@@ -15,8 +15,8 @@ const prepared = {
 	channel_refreshed_update: db.prepare(
 		"UPDATE Channels SET refreshed = ? WHERE ucid = ?"
 	),
-	unsubscribe_all_from_channel: db.prepare(
-		"UPDATE Subscriptions SET channel_missing = 1 WHERE ucid = ?"
+	channel_mark_as_missing: db.prepare(
+		"UPDATE Channels SET missing = 1, missing_reason = ? WHERE ucid = ?"
 	)
 }
 
@@ -35,7 +35,7 @@ class RefreshQueue {
 		// get the next set of scheduled channels to refresh
 		const afterTime = Date.now() - constants.caching.seen_token_subscriptions_eligible
 		const channels = db.prepare(
-			"SELECT DISTINCT Subscriptions.ucid FROM SeenTokens INNER JOIN Subscriptions ON SeenTokens.token = Subscriptions.token AND SeenTokens.seen > ? WHERE Subscriptions.channel_missing = 0 ORDER BY SeenTokens.seen DESC"
+			"SELECT DISTINCT Subscriptions.ucid FROM SeenTokens INNER JOIN Subscriptions ON SeenTokens.token = Subscriptions.token INNER JOIN Channels ON Channels.ucid = Subscriptions.ucid WHERE Channels.missing = 0 AND SeenTokens.seen > ? ORDER BY SeenTokens.seen DESC"
 		).pluck().all(afterTime)
 		this.addLast(channels)
 		this.lastLoadTime = Date.now()
@@ -72,11 +72,12 @@ class Refresher {
 		this.refreshQueue = new RefreshQueue()
 		this.state = this.sym.ACTIVE
 		this.waitingTimeout = null
+		this.lastFakeNotFoundTime = 0
 		this.next()
 	}
 
 	refreshChannel(ucid) {
-		return fetch(`${constants.server_setup.local_instance_origin}/api/v1/channels/${ucid}/latest`).then(res => res.json()).then(root => {
+		return fetch(`${constants.server_setup.local_instance_origin}/api/v1/channels/${ucid}/latest`).then(res => res.json()).then(/** @param {any} root */ root => {
 			if (Array.isArray(root)) {
 				root.forEach(video => {
 					// organise
@@ -89,11 +90,24 @@ class Refresher {
 				prepared.channel_refreshed_update.run(Date.now(), ucid)
 				// console.log(`updated ${root.length} videos for channel ${ucid}`)
 			} else if (root.identifier === "PUBLISHED_DATES_NOT_PROVIDED") {
-				return [] // nothing we can do. skip this iteration.
+				// nothing we can do. skip this iteration.
 			} else if (root.identifier === "NOT_FOUND") {
-				// the channel does not exist. we should unsubscribe all users so we don't try again.
-				// console.log(`channel ${ucid} does not exist, unsubscribing all users`)
-				prepared.unsubscribe_all_from_channel.run(ucid)
+				// YouTube sometimes returns not found for absolutely no reason.
+				// There is no way to distinguish between a fake missing channel and a real missing channel without requesting the real endpoint.
+				// These fake missing channels often happen in bursts, which is why there is a cooldown.
+				const timeSinceLastFakeNotFound = Date.now() - this.lastFakeNotFoundTime
+				if (timeSinceLastFakeNotFound >= constants.caching.subscriptions_refesh_fake_not_found_cooldown) {
+					// We'll request the real endpoint to verify.
+					fetch(`${constants.server_setup.local_instance_origin}/api/v1/channels/${ucid}`).then(res => res.json()).then(/** @param {any} root */ root => {
+						if (root.error && (root.identifier === "NOT_FOUND" || root.identifier === "ACCOUNT_TERMINATED")) {
+							// The channel is really gone, and we should mark it as missing for everyone.
+							prepared.channel_mark_as_missing.run(root.error, ucid)
+						} else {
+							// The channel is not actually gone and YouTube is trolling us.
+							this.lastFakeNotFoundTime = Date.now()
+						}
+					})
+				} // else youtube is currently trolling us, skip this until later.
 			} else {
 				throw new Error(root.error)
 			}

pug/channel-error.pug

@@ -0,0 +1,27 @@
+extends includes/layout
+
+include includes/video-list-item
+include includes/subscribe-button
+
+block head
+  title= `${data.row ? data.row.name : "Deleted channel"} - CloudTube`
+  script(type="module" src=getStaticURL("html", "/static/js/channel.js"))
+
+block content
+  main.channel-page
+    if data.row
+      .channel-data
+        .info
+          - const iconURL = data.row.icon_url
+          if iconURL
+            .logo
+              img(src=iconURL alt="").thumbnail-image
+          .about
+            h1.name= data.row.name
+          +subscribe_button(data.ucid, subscribed, `/channel/${data.ucid}`).subscribe-button.base-border-look
+
+    .channel-error
+      div= data.message
+
+      if data.missing && subscribed
+        .you-should-unsubscribe To remove this channel from your subscriptions list, click Unsubscribe.

pug/subscriptions.pug

@@ -11,12 +11,24 @@ block content
     if hasSubscriptions
       section
         details.channels-details
-          summary #{channels.length} subscriptions
+          summary
+            | #{channels.length} subscriptions
+            if missingChannelCount === 1
+              = ` - ${missingChannelCount} channel is gone`
+            else if missingChannelCount > 1
+              = ` - ${missingChannelCount} channels are gone`
           .channels-list
             for channel in channels
               a(href=`/channel/${channel.ucid}`).channel-item
                 img(src=channel.icon_url width=512 height=512 alt="").thumbnail
-                span.name= channel.name
+                div
+                  div.name= channel.name
+                  if channel.missing
+                    div.missing-reason
+                      if channel.missing_reason
+                        = channel.missing_reason
+                      else
+                        | This channel appears to be deleted or terminated. Click to check it.
 
       if refreshed
         section

sass/includes/_channel-page.sass

@@ -74,6 +74,19 @@ $_theme: () !default
 .channel-video
   @include channel-video
 
+.channel-error
+  background-color: map.get($_theme, "bg-1")
+  padding: 24px
+  margin: 12px 0px 24px
+  border-radius: 8px
+  border: 1px solid map.get($_theme, "edge-grey")
+  font-size: 20px
+  color: map.get($_theme, "fg-warning")
+
+.you-should-unsubscribe
+  margin-top: 20px
+  color: map.get($_theme, "fg-main")
+
 .about-description // class provided by youtube
   pre
     font-size: inherit

sass/includes/_subscriptions-page.sass

@@ -37,6 +37,10 @@ $_theme: () !default
     font-size: 22px
     color: map.get($_theme, "fg-main")
 
+  .missing-reason
+    font-size: 16px
+    color: map.get($_theme, "fg-warning")
+
 @include forms.checkbox-hider("watched-videos-display")
 
 #watched-videos-display:checked ~ .video-list-item--watched

utils/constants.js

@@ -50,6 +50,7 @@ let constants = {
 		csrf_time: 4*60*60*1000,
 		seen_token_subscriptions_eligible: 40*60*60*1000,
 		subscriptions_refresh_loop_min: 5*60*1000,
+		subscriptions_refesh_fake_not_found_cooldown: 10*60*1000,
 	},
 
 	// Pattern matching.

utils/getuser.js

@@ -54,7 +54,7 @@ class User {
 
 	getSubscriptions() {
 		if (this.token) {
-			return db.prepare("SELECT ucid FROM Subscriptions WHERE token = ? AND channel_missing = 0").pluck().all(this.token)
+			return db.prepare("SELECT ucid FROM Subscriptions WHERE token = ?").pluck().all(this.token)
 		} else {
 			return []
 		}
@@ -62,7 +62,7 @@ class User {
 
 	isSubscribed(ucid) {
 		if (this.token) {
-			return !!db.prepare("SELECT * FROM Subscriptions WHERE token = ? AND ucid = ? AND channel_missing = 0").get([this.token, ucid])
+			return !!db.prepare("SELECT * FROM Subscriptions WHERE token = ? AND ucid = ?").get([this.token, ucid])
 		} else {
 			return false
 		}

utils/upgradedb.js

@@ -75,6 +75,26 @@ const deltas = [
 	function() {
 		db.prepare("ALTER TABLE Settings ADD COLUMN theme INTEGER DEFAULT 0")
 			.run()
+	},
+	// 12: Channels +missing +missing_reason, Subscriptions -
+	// Better management for missing channels
+	// We totally discard the existing Subscriptions.channel_missing since it is unreliable.
+	function() {
+		db.prepare("ALTER TABLE Channels ADD COLUMN missing INTEGER NOT NULL DEFAULT 0")
+			.run()
+		db.prepare("ALTER TABLE Channels ADD COLUMN missing_reason TEXT")
+			.run()
+		// https://www.sqlite.org/lang_altertable.html#making_other_kinds_of_table_schema_changes
+		db.transaction(() => {
+			db.prepare("CREATE TABLE NEW_Subscriptions (token TEXT NOT NULL, ucid TEXT NOT NULL, PRIMARY KEY (token, ucid))")
+				.run()
+			db.prepare("INSERT INTO NEW_Subscriptions (token, ucid) SELECT token, ucid FROM Subscriptions")
+				.run()
+			db.prepare("DROP TABLE Subscriptions")
+				.run()
+			db.prepare("ALTER TABLE NEW_Subscriptions RENAME TO Subscriptions")
+				.run()
+		})()
 	}
 ]
 

utils/youtube.js

@@ -2,14 +2,58 @@ const {request} = require("./request")
 const db = require("./db")
 
 async function fetchChannel(ucid, instance) {
+	function updateGoodData(channel) {
+		const bestIcon = channel.authorThumbnails.slice(-1)[0]
+		const iconURL = bestIcon ? bestIcon.url : null
+		db.prepare("REPLACE INTO Channels (ucid, name, icon_url, missing, missing_reason) VALUES (?, ?, ?, 0, NULL)").run(channel.authorId, channel.author, iconURL)
+	}
+
+	function updateBadData(channel) {
+		if (channel.identifier === "NOT_FOUND" || channel.identifier === "ACCOUNT_TERMINATED") {
+			db.prepare("UPDATE Channels SET missing = 1, missing_reason = ? WHERE ucid = ?").run(channel.error, channel.authorId)
+			return {
+				missing: true,
+				message: channel.error
+			}
+		} else {
+			return {
+				missing: false,
+				message: channel.error
+			}
+		}
+	}
+
 	if (!instance) throw new Error("No instance parameter provided")
-	// fetch
+
+	const row = db.prepare("SELECT * FROM Channels WHERE ucid = ?").get(ucid)
+
+	// handle the case where the channel has a known error
+	if (row && row.missing_reason) {
+		return {
+			error: true,
+			ucid,
+			row,
+			missing: true,
+			message: row.missing_reason
+		}
+	}
+
+	/** @type {any} */
 	const channel = await request(`${instance}/api/v1/channels/${ucid}`).then(res => res.json())
-	// update database
-	const bestIcon = channel.authorThumbnails.slice(-1)[0]
-	const iconURL = bestIcon ? bestIcon.url : null
-	db.prepare("REPLACE INTO Channels (ucid, name, icon_url) VALUES (?, ?, ?)").run([channel.authorId, channel.author, iconURL])
-	// return
+
+	// handle the case where the channel has a newly discovered error
+	if (channel.error) {
+		const missingData = updateBadData(channel)
+		return {
+			error: true,
+			ucid,
+			row,
+			...missingData
+		}
+	}
+
+	// handle the case where the channel returns good data (this is the only remaining scenario)
+	updateGoodData(channel)
 	return channel
 }