From 934e6baf068f5f0cc7fa45f4fc9b62e8384e03c5 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 19 Jan 2021 01:45:22 +1300 Subject: [PATCH] Tor check bug fixes; recognise december blocking --- src/lib/collectors.js | 4 ++++ src/lib/constants.js | 9 +++++---- src/lib/utils/tor.js | 10 +++++----- src/site/api/routes.js | 4 ++++ src/site/pug/blocked_december.pug | 15 +++++++++++++++ .../timeline_loading_blocked_december.pug | 8 ++++++++ 6 files changed, 41 insertions(+), 9 deletions(-) create mode 100644 src/site/pug/blocked_december.pug create mode 100644 src/site/pug/fragments/timeline_loading_blocked_december.pug diff --git a/src/lib/collectors.js b/src/lib/collectors.js index 0c493a1..5de17b2 100644 --- a/src/lib/collectors.js +++ b/src/lib/collectors.js @@ -280,6 +280,7 @@ function fetchTimelinePage(userID, after) { })) return requestCache.getOrFetchPromise(`page/${userID}/${after}`, () => { return switcher.request("timeline_graphql", `https://www.instagram.com/graphql/query/?${p.toString()}`, async res => { + if (res.status === 302) throw constants.symbols.INSTAGRAM_BLOCK_TYPE_DECEMBER if (res.status === 429) throw constants.symbols.RATE_LIMITED }).then(g => g.json()).then(root => { if (root.data.user === null) { @@ -317,6 +318,7 @@ function fetchIGTVPage(userID, after) { return requestCache.getOrFetchPromise(`igtv/${userID}/${after}`, () => { // assuming this uses the same bucket as timeline, which may not be the case return switcher.request("timeline_graphql", `https://www.instagram.com/graphql/query/?${p.toString()}`, async res => { + if (res.status === 302) throw constants.symbols.INSTAGRAM_BLOCK_TYPE_DECEMBER if (res.status === 429) throw constants.symbols.RATE_LIMITED }).then(g => g.json()).then(root => { /** @type {import("./types").PagedEdges} */ @@ -347,6 +349,7 @@ function verifyUserPair(userID, username) { })) return requestCache.getOrFetchPromise("userID/"+userID, () => { return switcher.request("reel_graphql", `https://www.instagram.com/graphql/query/?${p.toString()}`, async res => { + if (res.status === 302) throw constants.symbols.INSTAGRAM_BLOCK_TYPE_DECEMBER if (res.status === 429) throw constants.symbols.RATE_LIMITED return res }).then(res => res.json()).then(root => { @@ -402,6 +405,7 @@ function fetchShortcodeData(shortcode) { p.set("variables", JSON.stringify({shortcode})) return requestCache.getOrFetchPromise("shortcode/"+shortcode, () => { return switcher.request("post_graphql", `https://www.instagram.com/graphql/query/?${p.toString()}`, async res => { + if (res.status === 302) throw constants.symbols.INSTAGRAM_BLOCK_TYPE_DECEMBER if (res.status === 429) throw constants.symbols.RATE_LIMITED }).then(res => res.json()).then(root => { /** @type {import("./types").TimelineEntryN3} */ diff --git a/src/lib/constants.js b/src/lib/constants.js index 8005b25..2e3f9c3 100644 --- a/src/lib/constants.js +++ b/src/lib/constants.js @@ -29,10 +29,10 @@ let constants = { enabled: true, // If false, everything else in this block has no effect. password: null, // If `null`, Bibliogram will run its own Tor process instead. for: { - user_html: true, // User HTML page seems to have less forgiving rates, and Tor always fails, so it's disabled by default. - timeline_graphql: true, - post_graphql: true, - reel_graphql: true + user_html: true, + timeline_graphql: false, + post_graphql: false, + reel_graphql: false } }, request_backend: "node-fetch", // one of: "node-fetch", "got" @@ -262,6 +262,7 @@ let constants = { NOT_FOUND: Symbol("NOT_FOUND"), INSTAGRAM_DEMANDS_LOGIN: Symbol("INSTAGRAM_DEMANDS_LOGIN"), RATE_LIMITED: Symbol("RATE_LIMITED"), + INSTAGRAM_BLOCK_TYPE_DECEMBER: Symbol("INSTAGRAM_BLOCK_TYPE_DECEMBER"), ENDPOINT_OVERRIDDEN: Symbol("ENDPOINT_OVERRIDDEN"), NO_ASSISTANTS_AVAILABLE: Symbol("NO_ASSISTANTS_AVAILABLE"), QUOTA_REACHED: Symbol("QUOTA_REACHED"), diff --git a/src/lib/utils/tor.js b/src/lib/utils/tor.js index e56bd95..456af26 100644 --- a/src/lib/utils/tor.js +++ b/src/lib/utils/tor.js @@ -17,18 +17,18 @@ class TorManager { } async request(url, test) { - let result = null let done = false + let g while (!done) { - const req = await request(url, {agent: this.agent}, {log: true, statusLine: "TOR"}) + g = await request(url, {agent: this.agent}, {log: true, statusLine: "TOR"}) try { - result = await test(req) - done = true + await g.check(test) + break } catch (e) { await this.newCircuit() } } - return result + return g } newCircuit() { diff --git a/src/site/api/routes.js b/src/site/api/routes.js index 78f6607..10ada19 100644 --- a/src/site/api/routes.js +++ b/src/site/api/routes.js @@ -154,6 +154,8 @@ module.exports = [ lang }) } + } else if (error === constants.symbols.INSTAGRAM_BLOCK_TYPE_DECEMBER) { + return render(503, "pug/blocked_december.pug") } else if (error === constants.symbols.RATE_LIMITED) { return render(503, "pug/blocked_graphql.pug") } else if (error === constants.symbols.extractor_results.AGE_RESTRICTED) { @@ -219,6 +221,8 @@ module.exports = [ message: "This user doesn't exist.", withInstancesLink: false }) + } else if (error === constants.symbols.INSTAGRAM_BLOCK_TYPE_DECEMBER) { + return render(502, "pug/fragments/timeline_loading_blocked_december.pug") } else if (error === constants.symbols.INSTAGRAM_DEMANDS_LOGIN || error === constants.symbols.RATE_LIMITED) { return render(503, "pug/fragments/timeline_loading_blocked.pug") } else if (error === constants.symbols.QUOTA_REACHED) { diff --git a/src/site/pug/blocked_december.pug b/src/site/pug/blocked_december.pug new file mode 100644 index 0000000..aaee973 --- /dev/null +++ b/src/site/pug/blocked_december.pug @@ -0,0 +1,15 @@ +include includes/error.pug + +doctype html +html + head + title= `Blocked | Bibliogram` + include includes/head + body.error-page + +error(502, "Blocked by Instagram", true) + | Instagram is refusing to provide data to this server. + | This server will only be able to load the first page of timelines. + | This error is permanent. + | #[a(href="https://lists.sr.ht/~cadence/bibliogram-announce/%3C20201218014302.855fa8a816be2f19da2f56e3%40disroot.org%3E") See here for more information.] + | + | diff --git a/src/site/pug/fragments/timeline_loading_blocked_december.pug b/src/site/pug/fragments/timeline_loading_blocked_december.pug new file mode 100644 index 0000000..f74c451 --- /dev/null +++ b/src/site/pug/fragments/timeline_loading_blocked_december.pug @@ -0,0 +1,8 @@ +.error-fragment + .message Timeline loading blocked + .explanation. + Instagram is refusing to provide data to this server. + This server will only be able to load the first page of timelines. + This error is permanent. + #[a(href="https://lists.sr.ht/~cadence/bibliogram-announce/%3C20201218014302.855fa8a816be2f19da2f56e3%40disroot.org%3E") See here for more information.] + Or, you could try #[a(href="https://git.sr.ht/~cadence/bibliogram-docs/tree/master/docs/Instances.md") browsing Bibliogram on another instance.]