diff --git a/src/lib/collectors.js b/src/lib/collectors.js index 0e8ace0..f2a481b 100644 --- a/src/lib/collectors.js +++ b/src/lib/collectors.js @@ -287,6 +287,66 @@ function fetchTimelinePage(userID, after) { }) } +/** + * @param {string} userID + * @param {string} after + * @returns {Promise>} + */ +function fetchIGTVPage(userID, after) { + const p = new URLSearchParams() + p.set("query_hash", constants.external.igtv_query_hash) + p.set("variables", JSON.stringify({ + id: userID, + first: constants.external.igtv_fetch_first, + after: after + })) + return requestCache.getOrFetchPromise(`igtv/${userID}/${after}`, () => { + // assuming this uses the same bucket as timeline, which may not be the case + return switcher.request("timeline_graphql", `https://www.instagram.com/graphql/query/?${p.toString()}`, async res => { + if (res.status === 429) throw constants.symbols.RATE_LIMITED + }).then(g => g.json()).then(root => { + /** @type {import("./types").PagedEdges} */ + const timeline = root.data.user.edge_owner_to_timeline_media + history.report("timeline", true) + return timeline + }).catch(error => { + if (error === constants.symbols.RATE_LIMITED) { + history.report("timeline", false) + } + throw error + }) + }) +} + +/** + * @param {string} userID + * @param {string} username + * @returns {Promise} + */ +function verifyUserPair(userID, username) { + // Fetch basic user information + const p = new URLSearchParams() + p.set("query_hash", constants.external.reel_query_hash) + p.set("variables", JSON.stringify({ + user_id: userID, + include_reel: true + })) + return requestCache.getOrFetchPromise("userID/"+userID, () => { + return switcher.request("reel_graphql", `https://www.instagram.com/graphql/query/?${p.toString()}`, async res => { + if (res.status === 429) throw constants.symbols.RATE_LIMITED + return res + }).then(res => res.json()).then(root => { + let user = root.data.user + if (!user) throw constants.symbols.NOT_FOUND + user = user.reel.user + history.report("reel", true) + return user.id === userID && user.username === username + }).catch(error => { + throw error + }) + }) +} + /** * @param {string} shortcode * @returns {import("./structures/TimelineEntry")} @@ -371,3 +431,4 @@ module.exports.updateProfilePictureFromReel = updateProfilePictureFromReel module.exports.history = history module.exports.fetchUserFromSaved = fetchUserFromSaved module.exports.assistantSwitcher = assistantSwitcher +module.exports.verifyUserPair = verifyUserPair diff --git a/src/site/api/feed.js b/src/site/api/feed.js index fd78857..9569a07 100644 --- a/src/site/api/feed.js +++ b/src/site/api/feed.js @@ -80,6 +80,8 @@ module.exports = [ "Retry-After": userRequestCache.getTtl("user/"+fill[0], 1000) }, content: pugCache.get("pug/blocked.pug").web({ + website_origin: constants.website_origin, + username: fill[0], expiresMinutes: userRequestCache.getTtl("user/"+fill[0], 1000*60), getStaticURL }) diff --git a/src/site/api/routes.js b/src/site/api/routes.js index 083a94f..8362317 100644 --- a/src/site/api/routes.js +++ b/src/site/api/routes.js @@ -102,6 +102,8 @@ module.exports = [ "Retry-After": userRequestCache.getTtl("user/"+fill[0], 1000) }, content: pugCache.get("pug/blocked.pug").web({ + website_origin: constants.website_origin, + username: fill[0], expiresMinutes: userRequestCache.getTtl("user/"+fill[0], 1000*60), getStaticURL }) diff --git a/src/site/api/suggest.js b/src/site/api/suggest.js new file mode 100644 index 0000000..9efb12c --- /dev/null +++ b/src/site/api/suggest.js @@ -0,0 +1,204 @@ +const {Readable} = require("stream") + +const db = require("../../lib/db") +const collectors = require("../../lib/collectors") +const constants = require("../../lib/constants") + +/** @type {Set} */ +const waiters = new Set() + +setInterval((new (function() { + const payload = `:keepalive ${Date.now()}\n\n` + for (const waiter of waiters.values()) { + waiter.stream.push(payload) + } +})).constructor, 50000) + +class Waiter { + constructor(username) { + const _this = this + this.username = username + this.stream = new Readable({ + autoDestroy: true, + // @ts-ignore + emitClose: true, + read: function() {}, + destroy: function() { + waiters.delete(_this) + } + }) + this.stream.push(":connected\n\n") + this.stream.on("end", () => { + waiters.delete(this) + }) + waiters.add(this) + } + + complete() { + this.stream.push("event: profile_available\ndata: profile_available\n\n") + this.stream.push(null) + waiters.delete(this) + } +} + +module.exports = [ + { + route: "/api/suggest_user/v1", methods: ["POST"], upload: true, code: async ({url, body}) => { + body = body.toString() + const respondAsPlaintext = url.searchParams.has("plaintext") + const params = new URLSearchParams(body) + const missingParams = [] + if (!params.has("username")) missingParams.push("username") + if (!params.has("user_id")) missingParams.push("user_id") + if (missingParams.length) { + return { + statusCode: 400, + contentType: "application/json", + content: { + status: "fail", + version: "1.0", + generatedAt: Date.now(), + message: "These required POST body parameters were missing: " + missingParams.join(", "), + fields: missingParams.map(p => `bp:${p}`), + identifier: "MISSING_REQUIRED_PARAMETERS" + } + } + } + const username = params.get("username") + const userID = (params.get("user_id").match(/\d+/) || [])[0] + if (!userID) { + return { + statusCode: 400, + contentType: "application/json", + content: { + status: "fail", + version: "1.0", + generatedAt: Date.now(), + message: "The user_id parameter must be a number.", + fields: ["bp:user_id"], + identifier: "MALFORMED_USER_ID" + } + } + } + const existing = db.prepare("SELECT * FROM Users WHERE user_id = ?").get(userID) + if (existing) { + if (respondAsPlaintext) { + return { + statusCode: 403, + contentType: "text/plain", + content: "The user is already known. Nothing has changed.\n" + } + } else { + return { + statusCode: 403, + contentType: "application/json", + content: { + status: "fail", + version: "1.0", + generatedAt: Date.now(), + message: "The user is already known. Nothing has changed.", + identifier: "USER_ALREADY_KNOWN" + } + } + } + } + return collectors.verifyUserPair(userID, username).then(valid => { + if (!valid) { + return { + statusCode: 400, + contentType: "application/json", + content: { + status: "fail", + version: "1.0", + generatedAt: Date.now(), + message: "The user_id and username do not refer to the same user.", + identifier: "IDENTIFIERS_DO_NOT_MATCH" + } + } + } + db.prepare( + "INSERT INTO Users (user_id, username, created, updated, updated_version, post_count, following_count, followed_by_count, is_private, is_verified, profile_pic_url)" + +" VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" + ).run( + userID, username, Date.now(), Date.now(), 1, 0, 0, 0, 0, 0, "" + ) + collectors.userRequestCache.cache.delete("user/"+username) + for (const waiter of waiters) { + if (waiter.username === username) waiter.complete() + } + if (respondAsPlaintext) { + return { + statusCode: 201, + contentType: "text/plain", + content: "User added! Go back to your web browser.\n" + } + } else { + return { + statusCode: 201, + contentType: "application/json", + content: { + status: "ok", + version: "1.0", + generatedAt: Date.now() + } + } + } + }).catch(error => { + if (error === constants.symbols.NOT_FOUND) { + return { + statusCode: 400, + contentType: "application/json", + content: { + status: "fail", + version: "1.0", + generatedAt: Date.now(), + message: "That user does not exist.", + identifier: "USER_DOES_NOT_EXIST" + } + } + } else { + return { + statusCode: 500, + contentType: "application/json", + content: { + status: "fail", + version: "1.0", + generatedAt: Date.now(), + message: "An unknown server error occurred.", + identifier: "UNKNOWN_ERROR" + } + } + } + }) + } + }, + { + route: `/api/user_available_stream/v1/(${constants.external.username_regex})`, methods: ["GET"], code: async ({fill}) => { + const username = fill[0] + const waiter = new Waiter(username) + return { + statusCode: 200, + contentType: "text/event-stream", + headers: { + "X-Accel-Buffering": "no" + }, + stream: waiter.stream + } + } + }, + { + route: `/u/(${constants.external.username_regex})/unblock.sh`, methods: ["GET"], code: async ({fill}) => { + const username = fill[0] + return { + statusCode: 200, + contentType: "text/plain", + content: + `# Good on you for looking at shell scripts before blindly running them.` + +`\n# This script contacts Instagram to get the profile's user ID, then sends the ID to Bibliogram. Bibliogram can take over from there.` + +`\ncurl 'https://www.instagram.com/${username}/' -Ss | grep -oE '"id":"[0-9]+"'` + +` | head -n 1 | grep -oE '[0-9]+' | curl --data-urlencode 'username=${username}' --data-urlencode 'user_id@-'` + +` '${constants.website_origin}/api/suggest_user/v1?plaintext=1'` + } + } + } +] diff --git a/src/site/html/static/js/user_available_waiter.js b/src/site/html/static/js/user_available_waiter.js new file mode 100644 index 0000000..8a1a2c0 --- /dev/null +++ b/src/site/html/static/js/user_available_waiter.js @@ -0,0 +1,9 @@ +const data = document.getElementById("data") +const username = data.getAttribute("data-username") +const source = new EventSource(`/api/user_available_stream/v1/${username}`) +source.addEventListener("open", () => { + console.log("Connected to profile waiter stream") +}) +source.addEventListener("profile_available", () => { + window.location.reload() +}) diff --git a/src/site/pug/blocked.pug b/src/site/pug/blocked.pug index ef57029..822aebc 100644 --- a/src/site/pug/blocked.pug +++ b/src/site/pug/blocked.pug @@ -1,4 +1,4 @@ -//- Needs expiresMinutes, instancesURL +//- Needs website_origin, instancesURL, username, expiresMinutes? include includes/error.pug @@ -7,11 +7,21 @@ html head title= `Blocked | Bibliogram` include includes/head - body.error-page - +error(503, "Blocked by Instagram", true) - | Instagram is refusing to provide data to this server. Try again later to see if the block has been lifted. - | This error has been cached. The internal cache will expire in #{expiresMinutes} minutes. - | - a(href="https://github.com/cloudrac3r/bibliogram/wiki/Rate-limits") Read more about blocking. - | - | + script(src=getStaticURL("html", "/static/js/user_available_waiter.js") type="module") + body + div(data-username=username)#data + .error-page + h1.code 503 + p.message Blocked by Instagram + p.explanation + | Instagram is temporarily refusing to provide data to this server. + + .width-block + p If you use Mac or Linux, you can unblock this profile now! Run this in your favourite terminal: + pre curl -Ss #{website_origin}/u/#{username}/unblock.sh | $SHELL + ul + li To learn more, #[a(href="https://github.com/cloudrac3r/bibliogram/wiki/Rate-limits") read about blocking.] + li You may be able to avoid this by #[a(href="https://github.com/cloudrac3r/bibliogram/wiki/Instances") browsing on another instance.] + li It's good to read scripts to see what they do. #[a(href=`${website_origin}/u/${username}/unblock.sh`) Read ./unblock.sh] + + a(href="/").back ↵ Return home diff --git a/src/site/sass/includes/_main.sass b/src/site/sass/includes/_main.sass index 440220f..69bbebe 100644 --- a/src/site/sass/includes/_main.sass +++ b/src/site/sass/includes/_main.sass @@ -424,6 +424,18 @@ body a, a:visited color: map-get($theme, "link-error-page") + p + white-space: pre-line + + code, pre + font-size: 0.8em + padding: 3px 5px + background-color: rgba(255, 255, 255, 0.15) + border-radius: 3px + + pre + white-space: pre-line + .code, .message, .explanation, .back-link line-height: 1.2 margin: 0px @@ -442,12 +454,18 @@ body margin-top: 10px font-size: 20px color: map-get($theme, "foreground-error-explanation") - white-space: pre-line .back - margin-top: 15vh + margin-top: 40px font-size: 25px + .width-block + max-width: 600px + text-align: left + color: map-get($theme, "foreground-error-explanation") + margin-top: 10px + font-size: 20px + .homepage display: flex flex-direction: column