Auto-refresh expired profile pictures

src/lib/collectors.js

@@ -8,6 +8,7 @@ const db = require("./db")
 require("./testimports")(constants, request, extractSharedData, UserRequestCache, RequestHistory, db)
 
 const requestCache = new RequestCache(constants.caching.resource_cache_time)
+/** @type {import("./cache").UserRequestCache<import("./structures/User")|import("./structures/ReelUser")>} */
 const userRequestCache = new UserRequestCache(constants.caching.resource_cache_time)
 /** @type {import("./cache").TtlCache<import("./structures/TimelineEntry")>} */
 const timelineEntryCache = new TtlCache(constants.caching.resource_cache_time)
@@ -116,10 +117,43 @@ function fetchUserFromHTML(username) {
 	})
 }
 
+/**
+ * @param {string} userID
+ */
+function updateProfilePictureFromReel(userID) {
+	const p = new URLSearchParams()
+	p.set("query_hash", constants.external.reel_query_hash)
+	p.set("variables", JSON.stringify({
+		user_id: userID,
+		include_reel: true
+	}))
+	return switcher.request("reel_graphql", `https://www.instagram.com/graphql/query/?${p.toString()}`, async res => {
+		if (res.status === 429) throw constants.symbols.RATE_LIMITED
+		return res
+	}).then(res => res.json()).then(root => {
+		const result = root.data.user
+		if (!result) throw constants.symbols.NOT_FOUND
+		const profilePicURL = result.reel.user.profile_pic_url
+		if (!profilePicURL) throw constants.symbols.NOT_FOUND
+		db.prepare("UPDATE Users SET profile_pic_url = ? WHERE user_id = ?").run(profilePicURL, userID)
+		for (const user of userRequestCache.cache.values()) {
+			// yes, data.data is correct.
+			if (user.data.data.id === userID) {
+				user.data.data.profile_pic_url = profilePicURL
+				user.data.computeProxyProfilePic()
+				break // stop checking entries from the cache since we won't find any more
+			}
+		}
+		return profilePicURL
+	}).catch(error => {
+		throw error
+	})
+}
+
 /**
  * @param {string} userID
  * @param {string} username
- * @returns {Promise<import("./structures/ReelUser")>}
+ * @returns {Promise<import("./structures/ReelUser")|import("./structures/User")>}
  */
 function fetchUserFromCombined(userID, username) {
 	// Fetch basic user information
@@ -296,4 +330,5 @@ module.exports.fetchShortcodeData = fetchShortcodeData
 module.exports.userRequestCache = userRequestCache
 module.exports.timelineEntryCache = timelineEntryCache
 module.exports.getOrFetchShortcode = getOrFetchShortcode
+module.exports.updateProfilePictureFromReel = updateProfilePictureFromReel
 module.exports.history = history

src/lib/structures/ReelUser.js

@@ -1,5 +1,5 @@
 const constants = require("../constants")
-const {proxyImage} = require("../utils/proxyurl")
+const {proxyProfilePic} = require("../utils/proxyurl")
 const {structure} = require("../utils/structuretext")
 const Timeline = require("./Timeline")
 require("../testimports")(constants, Timeline)
@@ -15,7 +15,11 @@ class ReelUser {
 		/** @type {import("./Timeline")} */
 		this.timeline = new Timeline(this)
 		this.cachedAt = Date.now()
-		this.proxyProfilePicture = proxyImage(this.data.profile_pic_url)
+		this.computeProxyProfilePic()
+	}
+
+	computeProxyProfilePic() {
+		this.proxyProfilePicture = proxyProfilePic(this.data.profile_pic_url, this.data.id)
 	}
 
 	getStructuredBio() {

src/lib/structures/User.js

@@ -1,5 +1,5 @@
 const constants = require("../constants")
-const {proxyImage} = require("../utils/proxyurl")
+const {proxyProfilePic} = require("../utils/proxyurl")
 const {structure} = require("../utils/structuretext")
 const Timeline = require("./Timeline")
 require("../testimports")(constants, Timeline)
@@ -15,7 +15,11 @@ class User {
 		this.posts = data.edge_owner_to_timeline_media.count
 		this.timeline = new Timeline(this)
 		this.cachedAt = Date.now()
-		this.proxyProfilePicture = proxyImage(this.data.profile_pic_url)
+		this.computeProxyProfilePic()
+	}
+
+	computeProxyProfilePic() {
+		this.proxyProfilePicture = proxyProfilePic(this.data.profile_pic_url, this.data.id)
 	}
 
 	getStructuredBio() {

src/lib/utils/proxyurl.js

@@ -5,6 +5,13 @@ function proxyImage(url, width) {
 	return "/imageproxy?"+params.toString()
 }
 
+function proxyProfilePic(url, userID) {
+	const params = new URLSearchParams()
+	params.set("userID", userID)
+	params.set("url", url)
+	return "/imageproxy?"+params.toString()
+}
+
 function proxyVideo(url) {
 	const params = new URLSearchParams()
 	params.set("url", url)
@@ -21,5 +28,6 @@ function proxyExtendedOwner(owner) {
 }
 
 module.exports.proxyImage = proxyImage
+module.exports.proxyProfilePic = proxyProfilePic
 module.exports.proxyVideo = proxyVideo
 module.exports.proxyExtendedOwner = proxyExtendedOwner

src/lib/utils/requestbackends/got.js

@@ -25,7 +25,8 @@ class Got {
 	 */
 	response() {
 		return this.send().instance.then(res => ({
-			status: res.statusCode
+			status: res.statusCode,
+			headers: new Map(Object.entries(res.headers))
 		}))
 	}
 

src/lib/utils/requestbackends/reference.js

@@ -1,6 +1,7 @@
 /**
  * @typedef GrabResponse
  * @property {number} status
+ * @property {Map<string, string|string[]} headers
  */
 
 // @ts-nocheck
@@ -14,7 +15,7 @@ class GrabReference {
 		throw new Error("This is the reference class, do not instantiate it.")
 	}
 
-	// Please help me type this
+	// Please help me write typings for stream()
 	/**
 	 * @returns {Promise<any>}
 	 */

src/site/api/proxy.js

@@ -1,8 +1,11 @@
-const constants = require("../../lib/constants")
-const {request} = require("../../lib/utils/request")
-const {proxy} = require("pinski/plugins")
 const sharp = require("sharp")
 
+const constants = require("../../lib/constants")
+const collectors = require("../../lib/collectors")
+const {request} = require("../../lib/utils/request")
+const db = require("../../lib/db")
+require("../../lib/testimports")(constants, request, db)
+
 /**
  * Check that a resource is on Instagram.
  * @param {URL} completeURL
@@ -21,6 +24,47 @@ function verifyURL(completeURL) {
 	if (!["fbcdn.net", "cdninstagram.com"].some(host => url.host.endsWith(host))) return {status: "fail", value: [400, "URL host is not allowed"]}
 	return {status: "ok", url}
 }
+
+function statusCodeIsAcceptable(status) {
+	return (status >= 200 && status < 300) || status === 304
+}
+
+/**
+ * @param {string} url
+ */
+async function proxyResource(url, suggestedHeaders = {}, refreshCallback = null) {
+	// console.log(`Asked to proxy ${url}\n`, suggestedHeaders)
+	const headersToSend = {}
+	for (const key of ["accept", "accept-encoding", "accept-language", "range"]) {
+		if (suggestedHeaders[key]) headersToSend[key] = suggestedHeaders[key]
+	}
+	const sent = request(url, {headers: headersToSend})
+	const stream = await sent.stream()
+	const response = await sent.response()
+	// console.log(response.status, response.headers)
+	if (statusCodeIsAcceptable(response.status)) {
+		const headersToReturn = {}
+		for (const key of ["content-type", "date", "last-modified", "expires", "cache-control", "accept-ranges", "origin", "etag", "content-length", "transfer-encoding"]) {
+			headersToReturn[key] = response.headers.get(key)
+		}
+		return {
+			statusCode: response.status,
+			headers: headersToReturn,
+			stream: stream
+		}
+	} else if (refreshCallback && response.status === 410) { // 410 GONE, profile picture has since changed
+		return refreshCallback()
+	} else {
+		return {
+			statusCode: 502,
+			headers: {
+				"Content-Type": "text/plain; charset=UTF-8"
+			},
+			content: `Instagram returned HTTP status ${response.status}, which is not a success code.`
+		}
+	}
+}
+
 module.exports = [
 	{
 		route: "/imageproxy", methods: ["GET"], code: async (input) => {
@@ -59,9 +103,41 @@ module.exports = [
 				})
 			} else {
 				// No specific size was requested, so just stream proxy the file directly.
-				return proxy(verifyResult.url, {
+				if (params.has("userID")) {
-					"Cache-Control": constants.caching.image_cache_control
+					// Users get special handling, because we need to update their profile picture if an expired version is cached.
-				})
+					return proxyResource(verifyResult.url.toString(), input.req.headers, () => {
+						// If we get here, we got HTTP 410 GONE.
+						const userID = params.get("userID")
+						const storedProfilePicURL = db.prepare("SELECT profile_pic_url FROM Users WHERE user_id = ?").pluck().get(userID)
+						if (storedProfilePicURL === verifyResult.url.toString()) {
+							// Everything looks fine, find out what the new URL for the provided user ID is and store it.
+							return collectors.updateProfilePictureFromReel(userID).then(url => {
+								// Updated. Return the new picture (without recursing)
+								return proxyResource(url, input.req.headers)
+							}).catch(error => {
+								console.error(error)
+								return {
+									statusCode: 500,
+									headers: {
+										"Content-Type": "text/plain; charset=UTF-8"
+									},
+									content: String(error)
+								}
+							})
+						} else {
+							// The request is a lie!
+							return {
+								statusCode: 400,
+								headers: {
+									"Content-Type": "text/plain; charset=UTF-8"
+								},
+								content: "Profile picture must be refreshed, but provided userID parameter does not match the stored profile_pic_url."
+							}
+						}
+					})
+				} else {
+					return proxyResource(verifyResult.url.toString(), input.req.headers)
+				}
 			}
 		}
 	},
@@ -71,9 +147,7 @@ module.exports = [
 			if (verifyResult.status !== "ok") return verifyResult.value
 			const url = verifyResult.url
 			if (!["mp4"].some(ext => url.pathname.endsWith(ext))) return [400, "URL extension is not allowed"]
-			return proxy(url, {
+			return proxyResource(url.toString(), input.req.headers)
-				"Cache-Control": constants.caching.image_cache_control
-			})
 		}
 	}
 ]