From 94ed25adadf036c3d036a0046fb0c0eb31d9c049 Mon Sep 17 00:00:00 2001 From: Cadence Fish Date: Fri, 31 Jan 2020 01:51:59 +1300 Subject: [PATCH] Show if blocked in /api/stats This is experimental and is not listed in /api/stats:features yet. --- src/lib/collectors.js | 46 ++++++++++++++++------- src/lib/constants.js | 1 + src/lib/structures/RequestHistory.js | 37 +++++++++++++++++++ src/site/api/api.js | 55 ++++++++++++++++++---------- 4 files changed, 107 insertions(+), 32 deletions(-) create mode 100644 src/lib/structures/RequestHistory.js diff --git a/src/lib/collectors.js b/src/lib/collectors.js index 05d36e7..b51ddca 100644 --- a/src/lib/collectors.js +++ b/src/lib/collectors.js @@ -2,23 +2,29 @@ const constants = require("./constants") const {request} = require("./utils/request") const {extractSharedData} = require("./utils/body") const {TtlCache, RequestCache} = require("./cache") -require("./testimports")(constants, request, extractSharedData, RequestCache) +const RequestHistory = require("./structures/RequestHistory") +require("./testimports")(constants, request, extractSharedData, RequestCache, RequestHistory) const requestCache = new RequestCache(constants.caching.resource_cache_time) /** @type {import("./cache").TtlCache} */ const timelineEntryCache = new TtlCache(constants.caching.resource_cache_time) +const history = new RequestHistory(["user", "timeline", "post"]) function fetchUser(username) { return requestCache.getOrFetch("user/"+username, () => { return request(`https://www.instagram.com/${username}/`).then(res => { - if (res.status === 302) throw constants.symbols.INSTAGRAM_DEMANDS_LOGIN - else if (res.status === 404) throw constants.symbols.NOT_FOUND - else return res.text().then(text => { + if (res.status === 302) { + history.report("user", false) + throw constants.symbols.INSTAGRAM_DEMANDS_LOGIN + } else if (res.status === 404) { + throw constants.symbols.NOT_FOUND + } else return res.text().then(text => { // require down here or have to deal with require loop. require cache will take care of it anyway. // User -> Timeline -> TimelineImage -> collectors -/> User const User = require("./structures/User") const sharedData = extractSharedData(text) const user = new User(sharedData.entry_data.ProfilePage[0].graphql.user) + history.report("user", true) return user }) }) @@ -40,10 +46,16 @@ function fetchTimelinePage(userID, after) { })) return requestCache.getOrFetchPromise("page/"+after, () => { return request(`https://www.instagram.com/graphql/query/?${p.toString()}`).then(res => res.json()).then(root => { - if (!root.data) console.error("missing data:", root) //todo: please make this better. - /** @type {import("./types").PagedEdges} */ - const timeline = root.data.user.edge_owner_to_timeline_media - return timeline + if (!root.data) { + history.report("timeline", false) + console.error("missing data from timeline request, 429?", root) //todo: please make this better. + throw new Error("missing data from timeline request, 429?") + } else { + /** @type {import("./types").PagedEdges} */ + const timeline = root.data.user.edge_owner_to_timeline_media + history.report("timeline", true) + return timeline + } }) }) } @@ -89,13 +101,20 @@ function fetchShortcodeData(shortcode) { p.set("variables", JSON.stringify({shortcode})) return requestCache.getOrFetchPromise("shortcode/"+shortcode, () => { return request(`https://www.instagram.com/graphql/query/?${p.toString()}`).then(res => res.json()).then(root => { + if (!root.data) { + history.report("post", false) + console.error("missing data from post request, 429?", root) //todo: please make this better. + throw new Error("missing data from post request, 429?") /** @type {import("./types").TimelineEntryN3} */ - const data = root.data.shortcode_media - if (data == null) { - // the thing doesn't exist - throw constants.symbols.NOT_FOUND } else { - return data + const data = root.data.shortcode_media + if (data == null) { + // the thing doesn't exist + throw constants.symbols.NOT_FOUND + } else { + history.report("post", true) + return data + } } }) }) @@ -108,3 +127,4 @@ module.exports.fetchShortcodeData = fetchShortcodeData module.exports.requestCache = requestCache module.exports.timelineEntryCache = timelineEntryCache module.exports.getOrFetchShortcode = getOrFetchShortcode +module.exports.history = history diff --git a/src/lib/constants.js b/src/lib/constants.js index b525a8a..4b6459e 100644 --- a/src/lib/constants.js +++ b/src/lib/constants.js @@ -17,6 +17,7 @@ let constants = { // Instagram uses this stuff. This shouldn't be changed, except to fix a bug that hasn't yet been fixed upstream. external: { + user_query_hash: "c9100bf9110dd6361671f113dd02e7d6", timeline_query_hash: "e769aa130647d2354c40ea6a439bfc08", shortcode_query_hash: "2b0673e0dc4580674a88d426fe00ea90", timeline_fetch_first: 12, diff --git a/src/lib/structures/RequestHistory.js b/src/lib/structures/RequestHistory.js new file mode 100644 index 0000000..8063ff1 --- /dev/null +++ b/src/lib/structures/RequestHistory.js @@ -0,0 +1,37 @@ +class RequestHistory { + /** + * @param {string[]} tracked list of things that can be tracked + */ + constructor(tracked) { + this.tracked = new Set(tracked) + /** @type {Map} */ + this.store = new Map() + for (const key of tracked) { + this.store.set(key, { + lastRequestAt: null, + lastRequestSuccessful: null + }) + } + } + + /** + * @param {string} key + * @param {boolean} success + */ + report(key, success) { + if (!this.tracked.has(key)) throw new Error(`Trying to report key ${key}, but is not tracked`) + const entry = this.store.get(key) + entry.lastRequestAt = Date.now() + entry.lastRequestSuccessful = success + } + + export() { + const result = {} + for (const key of this.store.keys()) { + result[key] = this.store.get(key) + } + return result + } +} + +module.exports = RequestHistory diff --git a/src/site/api/api.js b/src/site/api/api.js index 01e3090..5de3a61 100644 --- a/src/site/api/api.js +++ b/src/site/api/api.js @@ -1,6 +1,7 @@ const constants = require("../../lib/constants") const child_process = require("child_process") -const {fetchUser} = require("../../lib/collectors") +const {history} = require("../../lib/collectors") +const {redirect} = require("pinski/plugins") function reply(statusCode, content) { return { @@ -21,13 +22,6 @@ let commit = "" } module.exports = [ - { - route: `/api/user/(${constants.external.username_regex})`, methods: ["GET"], code: async ({fill}) => { - const user = await fetchUser(fill[0]) - const data = user.export() - return reply(200, data) - } - }, { route: "/.well-known/nodeinfo", methods: ["GET"], code: async ({fill}) => { return reply(200, { @@ -41,7 +35,39 @@ module.exports = [ } }, { - route: "/api/stats/2.0", methods: ["GET"], code: async ({fill}) => { + route: "/api/stats", methods: ["GET"], code: async () => { + return redirect("/api/stats/2.0", 302) + } + }, + { + route: "/api/stats/2.0", methods: ["GET"], code: async ({url}) => { + const versions = ["1.0", "1.1"] + const features = [ + "PAGE_PROFILE", + "PAGE_POST", + "API_STATS", + "PAGE_HOME", + "API_INSTANCES" + ] + const inner = ( + new Map([ + ["1.0", { + version: "1.0", + features + }], + ["1.1", { + version: "1.1", + availableVersions: versions, + features, + history: history.export() + }] + ]) + ).get(url.searchParams.get("bv") || versions[0]) + if (!inner) return reply(400, { + status: "fail", + fields: ["q:bv"], + message: "query parameter `bv` selects version, must be either missing or any of " + versions.map(v => "`"+v+"`").join(", ") + "." + }) return reply(200, { version: "2.0", software: { @@ -64,16 +90,7 @@ module.exports = [ } }, metadata: { - bibliogram: { - version: "1.0", - features: [ - "PAGE_PROFILE", - "PAGE_POST", - "API_STATS", - "PAGE_HOME", - "API_INSTANCES" - ] - } + bibliogram: inner } }) }