Tor check bug fixes; recognise december blocking

src/lib/collectors.js

@@ -280,6 +280,7 @@ function fetchTimelinePage(userID, after) {
 	}))
 	return requestCache.getOrFetchPromise(`page/${userID}/${after}`, () => {
 		return switcher.request("timeline_graphql", `https://www.instagram.com/graphql/query/?${p.toString()}`, async res => {
+			if (res.status === 302) throw constants.symbols.INSTAGRAM_BLOCK_TYPE_DECEMBER
 			if (res.status === 429) throw constants.symbols.RATE_LIMITED
 		}).then(g => g.json()).then(root => {
 			if (root.data.user === null) {
@@ -317,6 +318,7 @@ function fetchIGTVPage(userID, after) {
 	return requestCache.getOrFetchPromise(`igtv/${userID}/${after}`, () => {
 		// assuming this uses the same bucket as timeline, which may not be the case
 		return switcher.request("timeline_graphql", `https://www.instagram.com/graphql/query/?${p.toString()}`, async res => {
+			if (res.status === 302) throw constants.symbols.INSTAGRAM_BLOCK_TYPE_DECEMBER
 			if (res.status === 429) throw constants.symbols.RATE_LIMITED
 		}).then(g => g.json()).then(root => {
 			/** @type {import("./types").PagedEdges<import("./types").TimelineEntryN2>} */
@@ -347,6 +349,7 @@ function verifyUserPair(userID, username) {
 	}))
 	return requestCache.getOrFetchPromise("userID/"+userID, () => {
 		return switcher.request("reel_graphql", `https://www.instagram.com/graphql/query/?${p.toString()}`, async res => {
+			if (res.status === 302) throw constants.symbols.INSTAGRAM_BLOCK_TYPE_DECEMBER
 			if (res.status === 429) throw constants.symbols.RATE_LIMITED
 			return res
 		}).then(res => res.json()).then(root => {
@@ -402,6 +405,7 @@ function fetchShortcodeData(shortcode) {
 	p.set("variables", JSON.stringify({shortcode}))
 	return requestCache.getOrFetchPromise("shortcode/"+shortcode, () => {
 		return switcher.request("post_graphql", `https://www.instagram.com/graphql/query/?${p.toString()}`, async res => {
+			if (res.status === 302) throw constants.symbols.INSTAGRAM_BLOCK_TYPE_DECEMBER
 			if (res.status === 429) throw constants.symbols.RATE_LIMITED
 		}).then(res => res.json()).then(root => {
 			/** @type {import("./types").TimelineEntryN3} */

src/lib/constants.js

@@ -29,10 +29,10 @@ let constants = {
 		enabled: true, // If false, everything else in this block has no effect.
 		password: null, // If `null`, Bibliogram will run its own Tor process instead.
 		for: {
-			user_html: true, // User HTML page seems to have less forgiving rates, and Tor always fails, so it's disabled by default.
+			user_html: true,
-			timeline_graphql: true,
+			timeline_graphql: false,
-			post_graphql: true,
+			post_graphql: false,
-			reel_graphql: true
+			reel_graphql: false
 		}
 	},
 	request_backend: "node-fetch", // one of: "node-fetch", "got"
@@ -262,6 +262,7 @@ let constants = {
 		NOT_FOUND: Symbol("NOT_FOUND"),
 		INSTAGRAM_DEMANDS_LOGIN: Symbol("INSTAGRAM_DEMANDS_LOGIN"),
 		RATE_LIMITED: Symbol("RATE_LIMITED"),
+		INSTAGRAM_BLOCK_TYPE_DECEMBER: Symbol("INSTAGRAM_BLOCK_TYPE_DECEMBER"),
 		ENDPOINT_OVERRIDDEN: Symbol("ENDPOINT_OVERRIDDEN"),
 		NO_ASSISTANTS_AVAILABLE: Symbol("NO_ASSISTANTS_AVAILABLE"),
 		QUOTA_REACHED: Symbol("QUOTA_REACHED"),

src/lib/utils/tor.js

@@ -17,18 +17,18 @@ class TorManager {
 	}
 
 	async request(url, test) {
-		let result = null
 		let done = false
+		let g
 		while (!done) {
-			const req = await request(url, {agent: this.agent}, {log: true, statusLine: "TOR"})
+			g = await request(url, {agent: this.agent}, {log: true, statusLine: "TOR"})
 			try {
-				result = await test(req)
+				await g.check(test)
-				done = true
+				break
 			} catch (e) {
 				await this.newCircuit()
 			}
 		}
-		return result
+		return g
 	}
 
 	newCircuit() {

src/site/api/routes.js

@@ -154,6 +154,8 @@ module.exports = [
 							lang
 						})
 					}
+				} else if (error === constants.symbols.INSTAGRAM_BLOCK_TYPE_DECEMBER) {
+					return render(503, "pug/blocked_december.pug")
 				} else if (error === constants.symbols.RATE_LIMITED) {
 					return render(503, "pug/blocked_graphql.pug")
 				} else if (error === constants.symbols.extractor_results.AGE_RESTRICTED) {
@@ -219,6 +221,8 @@ module.exports = [
 						message: "This user doesn't exist.",
 						withInstancesLink: false
 					})
+				} else if (error === constants.symbols.INSTAGRAM_BLOCK_TYPE_DECEMBER) {
+					return render(502, "pug/fragments/timeline_loading_blocked_december.pug")
 				} else if (error === constants.symbols.INSTAGRAM_DEMANDS_LOGIN || error === constants.symbols.RATE_LIMITED) {
 					return render(503, "pug/fragments/timeline_loading_blocked.pug")
 				} else if (error === constants.symbols.QUOTA_REACHED) {

src/site/pug/blocked_december.pug

@@ -0,0 +1,15 @@
+include includes/error.pug
+
+doctype html
+html
+	head
+		title= `Blocked | Bibliogram`
+		include includes/head
+	body.error-page
+		+error(502, "Blocked by Instagram", true)
+			| Instagram is refusing to provide data to this server.
+			| This server will only be able to load the first page of timelines.
+			| This error is permanent.
+			| #[a(href="https://lists.sr.ht/~cadence/bibliogram-announce/%3C20201218014302.855fa8a816be2f19da2f56e3%40disroot.org%3E") See here for more information.]
+			|
+			|

src/site/pug/fragments/timeline_loading_blocked_december.pug

@@ -0,0 +1,8 @@
+.error-fragment
+	.message Timeline loading blocked
+	.explanation.
+		Instagram is refusing to provide data to this server.
+		This server will only be able to load the first page of timelines.
+		This error is permanent.
+		#[a(href="https://lists.sr.ht/~cadence/bibliogram-announce/%3C20201218014302.855fa8a816be2f19da2f56e3%40disroot.org%3E") See here for more information.]
+		Or, you could try #[a(href="https://git.sr.ht/~cadence/bibliogram-docs/tree/master/docs/Instances.md") browsing Bibliogram on another instance.]