Update blocked page with command line to unblock

src/lib/collectors.js

@@ -287,6 +287,66 @@ function fetchTimelinePage(userID, after) {
 	})
 }
 
+/**
+ * @param {string} userID
+ * @param {string} after
+ * @returns {Promise<import("./types").PagedEdges<import("./types").TimelineEntryN2>>}
+ */
+function fetchIGTVPage(userID, after) {
+	const p = new URLSearchParams()
+	p.set("query_hash", constants.external.igtv_query_hash)
+	p.set("variables", JSON.stringify({
+		id: userID,
+		first: constants.external.igtv_fetch_first,
+		after: after
+	}))
+	return requestCache.getOrFetchPromise(`igtv/${userID}/${after}`, () => {
+		// assuming this uses the same bucket as timeline, which may not be the case
+		return switcher.request("timeline_graphql", `https://www.instagram.com/graphql/query/?${p.toString()}`, async res => {
+			if (res.status === 429) throw constants.symbols.RATE_LIMITED
+		}).then(g => g.json()).then(root => {
+			/** @type {import("./types").PagedEdges<import("./types").TimelineEntryN2>} */
+			const timeline = root.data.user.edge_owner_to_timeline_media
+			history.report("timeline", true)
+			return timeline
+		}).catch(error => {
+			if (error === constants.symbols.RATE_LIMITED) {
+				history.report("timeline", false)
+			}
+			throw error
+		})
+	})
+}
+
+/**
+ * @param {string} userID
+ * @param {string} username
+ * @returns {Promise<boolean>}
+ */
+function verifyUserPair(userID, username) {
+	// Fetch basic user information
+	const p = new URLSearchParams()
+	p.set("query_hash", constants.external.reel_query_hash)
+	p.set("variables", JSON.stringify({
+		user_id: userID,
+		include_reel: true
+	}))
+	return requestCache.getOrFetchPromise("userID/"+userID, () => {
+		return switcher.request("reel_graphql", `https://www.instagram.com/graphql/query/?${p.toString()}`, async res => {
+			if (res.status === 429) throw constants.symbols.RATE_LIMITED
+			return res
+		}).then(res => res.json()).then(root => {
+			let user = root.data.user
+			if (!user) throw constants.symbols.NOT_FOUND
+			user = user.reel.user
+			history.report("reel", true)
+			return user.id === userID && user.username === username
+		}).catch(error => {
+			throw error
+		})
+	})
+}
+
 /**
  * @param {string} shortcode
  * @returns {import("./structures/TimelineEntry")}
@@ -371,3 +431,4 @@ module.exports.updateProfilePictureFromReel = updateProfilePictureFromReel
 module.exports.history = history
 module.exports.fetchUserFromSaved = fetchUserFromSaved
 module.exports.assistantSwitcher = assistantSwitcher
+module.exports.verifyUserPair = verifyUserPair

src/site/api/feed.js

@@ -80,6 +80,8 @@ module.exports = [
 								"Retry-After": userRequestCache.getTtl("user/"+fill[0], 1000)
 							},
 							content: pugCache.get("pug/blocked.pug").web({
+								website_origin: constants.website_origin,
+								username: fill[0],
 								expiresMinutes: userRequestCache.getTtl("user/"+fill[0], 1000*60),
 								getStaticURL
 							})

src/site/api/routes.js

@@ -102,6 +102,8 @@ module.exports = [
 							"Retry-After": userRequestCache.getTtl("user/"+fill[0], 1000)
 						},
 						content: pugCache.get("pug/blocked.pug").web({
+							website_origin: constants.website_origin,
+							username: fill[0],
 							expiresMinutes: userRequestCache.getTtl("user/"+fill[0], 1000*60),
 							getStaticURL
 						})

src/site/api/suggest.js

@@ -0,0 +1,204 @@
+const {Readable} = require("stream")
+
+const db = require("../../lib/db")
+const collectors = require("../../lib/collectors")
+const constants = require("../../lib/constants")
+
+/** @type {Set<Waiter>} */
+const waiters = new Set()
+
+setInterval((new (function() {
+	const payload = `:keepalive ${Date.now()}\n\n`
+	for (const waiter of waiters.values()) {
+		waiter.stream.push(payload)
+	}
+})).constructor, 50000)
+
+class Waiter {
+	constructor(username) {
+		const _this = this
+		this.username = username
+		this.stream = new Readable({
+			autoDestroy: true,
+			// @ts-ignore
+			emitClose: true,
+			read: function() {},
+			destroy: function() {
+				waiters.delete(_this)
+			}
+		})
+		this.stream.push(":connected\n\n")
+		this.stream.on("end", () => {
+			waiters.delete(this)
+		})
+		waiters.add(this)
+	}
+
+	complete() {
+		this.stream.push("event: profile_available\ndata: profile_available\n\n")
+		this.stream.push(null)
+		waiters.delete(this)
+	}
+}
+
+module.exports = [
+	{
+		route: "/api/suggest_user/v1", methods: ["POST"], upload: true, code: async ({url, body}) => {
+			body = body.toString()
+			const respondAsPlaintext = url.searchParams.has("plaintext")
+			const params = new URLSearchParams(body)
+			const missingParams = []
+			if (!params.has("username")) missingParams.push("username")
+			if (!params.has("user_id")) missingParams.push("user_id")
+			if (missingParams.length) {
+				return {
+					statusCode: 400,
+					contentType: "application/json",
+					content: {
+						status: "fail",
+						version: "1.0",
+						generatedAt: Date.now(),
+						message: "These required POST body parameters were missing: " + missingParams.join(", "),
+						fields: missingParams.map(p => `bp:${p}`),
+						identifier: "MISSING_REQUIRED_PARAMETERS"
+					}
+				}
+			}
+			const username = params.get("username")
+			const userID = (params.get("user_id").match(/\d+/) || [])[0]
+			if (!userID) {
+				return {
+					statusCode: 400,
+					contentType: "application/json",
+					content: {
+						status: "fail",
+						version: "1.0",
+						generatedAt: Date.now(),
+						message: "The user_id parameter must be a number.",
+						fields: ["bp:user_id"],
+						identifier: "MALFORMED_USER_ID"
+					}
+				}
+			}
+			const existing = db.prepare("SELECT * FROM Users WHERE user_id = ?").get(userID)
+			if (existing) {
+				if (respondAsPlaintext) {
+					return {
+						statusCode: 403,
+						contentType: "text/plain",
+						content: "The user is already known. Nothing has changed.\n"
+					}
+				} else {
+					return {
+						statusCode: 403,
+						contentType: "application/json",
+						content: {
+							status: "fail",
+							version: "1.0",
+							generatedAt: Date.now(),
+							message: "The user is already known. Nothing has changed.",
+							identifier: "USER_ALREADY_KNOWN"
+						}
+					}
+				}
+			}
+			return collectors.verifyUserPair(userID, username).then(valid => {
+				if (!valid) {
+					return {
+						statusCode: 400,
+						contentType: "application/json",
+						content: {
+							status: "fail",
+							version: "1.0",
+							generatedAt: Date.now(),
+							message: "The user_id and username do not refer to the same user.",
+							identifier: "IDENTIFIERS_DO_NOT_MATCH"
+						}
+					}
+				}
+				db.prepare(
+					"INSERT INTO Users (user_id, username, created, updated, updated_version, post_count, following_count, followed_by_count, is_private, is_verified, profile_pic_url)"
+					+" VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
+				).run(
+					userID, username, Date.now(), Date.now(), 1, 0, 0, 0, 0, 0, ""
+				)
+				collectors.userRequestCache.cache.delete("user/"+username)
+				for (const waiter of waiters) {
+					if (waiter.username === username) waiter.complete()
+				}
+				if (respondAsPlaintext) {
+					return {
+						statusCode: 201,
+						contentType: "text/plain",
+						content: "User added! Go back to your web browser.\n"
+					}
+				} else {
+					return {
+						statusCode: 201,
+						contentType: "application/json",
+						content: {
+							status: "ok",
+							version: "1.0",
+							generatedAt: Date.now()
+						}
+					}
+				}
+			}).catch(error => {
+				if (error === constants.symbols.NOT_FOUND) {
+					return {
+						statusCode: 400,
+						contentType: "application/json",
+						content: {
+							status: "fail",
+							version: "1.0",
+							generatedAt: Date.now(),
+							message: "That user does not exist.",
+							identifier: "USER_DOES_NOT_EXIST"
+						}
+					}
+				} else {
+					return {
+						statusCode: 500,
+						contentType: "application/json",
+						content: {
+							status: "fail",
+							version: "1.0",
+							generatedAt: Date.now(),
+							message: "An unknown server error occurred.",
+							identifier: "UNKNOWN_ERROR"
+						}
+					}
+				}
+			})
+		}
+	},
+	{
+		route: `/api/user_available_stream/v1/(${constants.external.username_regex})`, methods: ["GET"], code: async ({fill}) => {
+			const username = fill[0]
+			const waiter = new Waiter(username)
+			return {
+				statusCode: 200,
+				contentType: "text/event-stream",
+				headers: {
+					"X-Accel-Buffering": "no"
+				},
+				stream: waiter.stream
+			}
+		}
+	},
+	{
+		route: `/u/(${constants.external.username_regex})/unblock.sh`, methods: ["GET"], code: async ({fill}) => {
+			const username = fill[0]
+			return {
+				statusCode: 200,
+				contentType: "text/plain",
+				content:
+					`# Good on you for looking at shell scripts before blindly running them.`
+					+`\n# This script contacts Instagram to get the profile's user ID, then sends the ID to Bibliogram. Bibliogram can take over from there.`
+					+`\ncurl 'https://www.instagram.com/${username}/' -Ss | grep -oE '"id":"[0-9]+"'`
+					+` | head -n 1 | grep -oE '[0-9]+' | curl --data-urlencode 'username=${username}' --data-urlencode 'user_id@-'`
+					+` '${constants.website_origin}/api/suggest_user/v1?plaintext=1'`
+			}
+		}
+	}
+]

src/site/html/static/js/user_available_waiter.js

@@ -0,0 +1,9 @@
+const data = document.getElementById("data")
+const username = data.getAttribute("data-username")
+const source = new EventSource(`/api/user_available_stream/v1/${username}`)
+source.addEventListener("open", () => {
+	console.log("Connected to profile waiter stream")
+})
+source.addEventListener("profile_available", () => {
+	window.location.reload()
+})

src/site/pug/blocked.pug

@@ -1,4 +1,4 @@
-//- Needs expiresMinutes, instancesURL
+//- Needs website_origin, instancesURL, username, expiresMinutes?
 
 include includes/error.pug
 
@@ -7,11 +7,21 @@ html
 	head
 		title= `Blocked | Bibliogram`
 		include includes/head
-	body.error-page
-		+error(503, "Blocked by Instagram", true)
-			| Instagram is refusing to provide data to this server. Try again later to see if the block has been lifted.
-			| This error has been cached. The internal cache will expire in #{expiresMinutes} minutes.
-			|
-			a(href="https://github.com/cloudrac3r/bibliogram/wiki/Rate-limits") Read more about blocking.
-			|
-			|
+		script(src=getStaticURL("html", "/static/js/user_available_waiter.js") type="module")
+	body
+		div(data-username=username)#data
+		.error-page
+			h1.code 503
+			p.message Blocked by Instagram
+			p.explanation
+				| Instagram is temporarily refusing to provide data to this server.
+
+			.width-block
+				p If you use Mac or Linux, you can unblock this profile now! Run this in your favourite terminal:
+				pre curl -Ss #{website_origin}/u/#{username}/unblock.sh | $SHELL
+				ul
+					li To learn more, #[a(href="https://github.com/cloudrac3r/bibliogram/wiki/Rate-limits") read about blocking.]
+					li You may be able to avoid this by #[a(href="https://github.com/cloudrac3r/bibliogram/wiki/Instances") browsing on another instance.]
+					li It's good to read scripts to see what they do. #[a(href=`${website_origin}/u/${username}/unblock.sh`) Read ./unblock.sh]
+
+			a(href="/").back ↵ Return home

src/site/sass/includes/_main.sass

@@ -424,6 +424,18 @@ body
 	a, a:visited
 		color: map-get($theme, "link-error-page")
 
+	p
+		white-space: pre-line
+
+	code, pre
+		font-size: 0.8em
+		padding: 3px 5px
+		background-color: rgba(255, 255, 255, 0.15)
+		border-radius: 3px
+
+	pre
+		white-space: pre-line
+
 	.code, .message, .explanation, .back-link
 		line-height: 1.2
 		margin: 0px
@@ -442,12 +454,18 @@ body
 		margin-top: 10px
 		font-size: 20px
 		color: map-get($theme, "foreground-error-explanation")
-		white-space: pre-line
 
 	.back
-		margin-top: 15vh
+		margin-top: 40px
 		font-size: 25px
 
+	.width-block
+		max-width: 600px
+		text-align: left
+		color: map-get($theme, "foreground-error-explanation")
+		margin-top: 10px
+		font-size: 20px
+
 .homepage
 	display: flex
 	flex-direction: column