Show if blocked in /api/stats

This is experimental and is not listed in /api/stats:features yet.

src/lib/collectors.js

@@ -2,23 +2,29 @@ const constants = require("./constants")
 const {request} = require("./utils/request")
 const {extractSharedData} = require("./utils/body")
 const {TtlCache, RequestCache} = require("./cache")
-require("./testimports")(constants, request, extractSharedData, RequestCache)
+const RequestHistory = require("./structures/RequestHistory")
+require("./testimports")(constants, request, extractSharedData, RequestCache, RequestHistory)
 
 const requestCache = new RequestCache(constants.caching.resource_cache_time)
 /** @type {import("./cache").TtlCache<import("./structures/TimelineEntry")>} */
 const timelineEntryCache = new TtlCache(constants.caching.resource_cache_time)
+const history = new RequestHistory(["user", "timeline", "post"])
 
 function fetchUser(username) {
 	return requestCache.getOrFetch("user/"+username, () => {
 		return request(`https://www.instagram.com/${username}/`).then(res => {
-			if (res.status === 302) throw constants.symbols.INSTAGRAM_DEMANDS_LOGIN
-			else if (res.status === 404) throw constants.symbols.NOT_FOUND
-			else return res.text().then(text => {
+			if (res.status === 302) {
+				history.report("user", false)
+				throw constants.symbols.INSTAGRAM_DEMANDS_LOGIN
+			} else if (res.status === 404) {
+				throw constants.symbols.NOT_FOUND
+			} else return res.text().then(text => {
 				// require down here or have to deal with require loop. require cache will take care of it anyway.
 				// User -> Timeline -> TimelineImage -> collectors -/> User
 				const User = require("./structures/User")
 				const sharedData = extractSharedData(text)
 				const user = new User(sharedData.entry_data.ProfilePage[0].graphql.user)
+				history.report("user", true)
 				return user
 			})
 		})
@@ -40,10 +46,16 @@ function fetchTimelinePage(userID, after) {
 	}))
 	return requestCache.getOrFetchPromise("page/"+after, () => {
 		return request(`https://www.instagram.com/graphql/query/?${p.toString()}`).then(res => res.json()).then(root => {
-			if (!root.data) console.error("missing data:", root) //todo: please make this better.
-			/** @type {import("./types").PagedEdges<import("./types").TimelineEntryN2>} */
-			const timeline = root.data.user.edge_owner_to_timeline_media
-			return timeline
+			if (!root.data) {
+				history.report("timeline", false)
+				console.error("missing data from timeline request, 429?", root) //todo: please make this better.
+				throw new Error("missing data from timeline request, 429?")
+			} else {
+				/** @type {import("./types").PagedEdges<import("./types").TimelineEntryN2>} */
+				const timeline = root.data.user.edge_owner_to_timeline_media
+				history.report("timeline", true)
+				return timeline
+			}
 		})
 	})
 }
@@ -89,13 +101,20 @@ function fetchShortcodeData(shortcode) {
 	p.set("variables", JSON.stringify({shortcode}))
 	return requestCache.getOrFetchPromise("shortcode/"+shortcode, () => {
 		return request(`https://www.instagram.com/graphql/query/?${p.toString()}`).then(res => res.json()).then(root => {
+			if (!root.data) {
+				history.report("post", false)
+				console.error("missing data from post request, 429?", root) //todo: please make this better.
+				throw new Error("missing data from post request, 429?")
 			/** @type {import("./types").TimelineEntryN3} */
-			const data = root.data.shortcode_media
-			if (data == null) {
-				// the thing doesn't exist
-				throw constants.symbols.NOT_FOUND
 			} else {
-				return data
+				const data = root.data.shortcode_media
+				if (data == null) {
+					// the thing doesn't exist
+					throw constants.symbols.NOT_FOUND
+				} else {
+					history.report("post", true)
+					return data
+				}
 			}
 		})
 	})
@@ -108,3 +127,4 @@ module.exports.fetchShortcodeData = fetchShortcodeData
 module.exports.requestCache = requestCache
 module.exports.timelineEntryCache = timelineEntryCache
 module.exports.getOrFetchShortcode = getOrFetchShortcode
+module.exports.history = history

src/lib/constants.js

@@ -17,6 +17,7 @@ let constants = {
 
 	// Instagram uses this stuff. This shouldn't be changed, except to fix a bug that hasn't yet been fixed upstream.
 	external: {
+		user_query_hash: "c9100bf9110dd6361671f113dd02e7d6",
 		timeline_query_hash: "e769aa130647d2354c40ea6a439bfc08",
 		shortcode_query_hash: "2b0673e0dc4580674a88d426fe00ea90",
 		timeline_fetch_first: 12,

src/lib/structures/RequestHistory.js

@@ -0,0 +1,37 @@
+class RequestHistory {
+	/**
+	 * @param {string[]} tracked list of things that can be tracked
+	 */
+	constructor(tracked) {
+		this.tracked = new Set(tracked)
+		/** @type {Map<string, {lastRequestAt: number | null, lastRequestSuccessful: boolean | null}>} */
+		this.store = new Map()
+		for (const key of tracked) {
+			this.store.set(key, {
+				lastRequestAt: null,
+				lastRequestSuccessful: null
+			})
+		}
+	}
+
+	/**
+	 * @param {string} key
+	 * @param {boolean} success
+	 */
+	report(key, success) {
+		if (!this.tracked.has(key)) throw new Error(`Trying to report key ${key}, but is not tracked`)
+		const entry = this.store.get(key)
+		entry.lastRequestAt = Date.now()
+		entry.lastRequestSuccessful = success
+	}
+
+	export() {
+		const result = {}
+		for (const key of this.store.keys()) {
+			result[key] = this.store.get(key)
+		}
+		return result
+	}
+}
+
+module.exports = RequestHistory

src/site/api/api.js

@@ -1,6 +1,7 @@
 const constants = require("../../lib/constants")
 const child_process = require("child_process")
-const {fetchUser} = require("../../lib/collectors")
+const {history} = require("../../lib/collectors")
+const {redirect} = require("pinski/plugins")
 
 function reply(statusCode, content) {
 	return {
@@ -21,13 +22,6 @@ let commit = ""
 }
 
 module.exports = [
-	{
-		route: `/api/user/(${constants.external.username_regex})`, methods: ["GET"], code: async ({fill}) => {
-			const user = await fetchUser(fill[0])
-			const data = user.export()
-			return reply(200, data)
-		}
-	},
 	{
 		route: "/.well-known/nodeinfo", methods: ["GET"], code: async ({fill}) => {
 			return reply(200, {
@@ -41,7 +35,39 @@ module.exports = [
 		}
 	},
 	{
-		route: "/api/stats/2.0", methods: ["GET"], code: async ({fill}) => {
+		route: "/api/stats", methods: ["GET"], code: async () => {
+			return redirect("/api/stats/2.0", 302)
+		}
+	},
+	{
+		route: "/api/stats/2.0", methods: ["GET"], code: async ({url}) => {
+			const versions = ["1.0", "1.1"]
+			const features = [
+				"PAGE_PROFILE",
+				"PAGE_POST",
+				"API_STATS",
+				"PAGE_HOME",
+				"API_INSTANCES"
+			]
+			const inner = (
+				new Map([
+					["1.0", {
+						version: "1.0",
+						features
+					}],
+					["1.1", {
+						version: "1.1",
+						availableVersions: versions,
+						features,
+						history: history.export()
+					}]
+				])
+			).get(url.searchParams.get("bv") || versions[0])
+			if (!inner) return reply(400, {
+				status: "fail",
+				fields: ["q:bv"],
+				message: "query parameter `bv` selects version, must be either missing or any of " + versions.map(v => "`"+v+"`").join(", ") + "."
+			})
 			return reply(200, {
 				version: "2.0",
 				software: {
@@ -64,16 +90,7 @@ module.exports = [
 					}
 				},
 				metadata: {
-					bibliogram: {
-						version: "1.0",
-						features: [
-							"PAGE_PROFILE",
-							"PAGE_POST",
-							"API_STATS",
-							"PAGE_HOME",
-							"API_INSTANCES"
-						]
-					}
+					bibliogram: inner
 				}
 			})
 		}