diff --git a/src/lib/collectors.js b/src/lib/collectors.js index 914c6a0..24a1717 100644 --- a/src/lib/collectors.js +++ b/src/lib/collectors.js @@ -8,6 +8,7 @@ const db = require("./db") require("./testimports")(constants, request, extractSharedData, UserRequestCache, RequestHistory, db) const requestCache = new RequestCache(constants.caching.resource_cache_time) +/** @type {import("./cache").UserRequestCache} */ const userRequestCache = new UserRequestCache(constants.caching.resource_cache_time) /** @type {import("./cache").TtlCache} */ const timelineEntryCache = new TtlCache(constants.caching.resource_cache_time) @@ -116,10 +117,43 @@ function fetchUserFromHTML(username) { }) } +/** + * @param {string} userID + */ +function updateProfilePictureFromReel(userID) { + const p = new URLSearchParams() + p.set("query_hash", constants.external.reel_query_hash) + p.set("variables", JSON.stringify({ + user_id: userID, + include_reel: true + })) + return switcher.request("reel_graphql", `https://www.instagram.com/graphql/query/?${p.toString()}`, async res => { + if (res.status === 429) throw constants.symbols.RATE_LIMITED + return res + }).then(res => res.json()).then(root => { + const result = root.data.user + if (!result) throw constants.symbols.NOT_FOUND + const profilePicURL = result.reel.user.profile_pic_url + if (!profilePicURL) throw constants.symbols.NOT_FOUND + db.prepare("UPDATE Users SET profile_pic_url = ? WHERE user_id = ?").run(profilePicURL, userID) + for (const user of userRequestCache.cache.values()) { + // yes, data.data is correct. + if (user.data.data.id === userID) { + user.data.data.profile_pic_url = profilePicURL + user.data.computeProxyProfilePic() + break // stop checking entries from the cache since we won't find any more + } + } + return profilePicURL + }).catch(error => { + throw error + }) +} + /** * @param {string} userID * @param {string} username - * @returns {Promise} + * @returns {Promise} */ function fetchUserFromCombined(userID, username) { // Fetch basic user information @@ -296,4 +330,5 @@ module.exports.fetchShortcodeData = fetchShortcodeData module.exports.userRequestCache = userRequestCache module.exports.timelineEntryCache = timelineEntryCache module.exports.getOrFetchShortcode = getOrFetchShortcode +module.exports.updateProfilePictureFromReel = updateProfilePictureFromReel module.exports.history = history diff --git a/src/lib/structures/ReelUser.js b/src/lib/structures/ReelUser.js index 3b2ab6f..97eab91 100644 --- a/src/lib/structures/ReelUser.js +++ b/src/lib/structures/ReelUser.js @@ -1,5 +1,5 @@ const constants = require("../constants") -const {proxyImage} = require("../utils/proxyurl") +const {proxyProfilePic} = require("../utils/proxyurl") const {structure} = require("../utils/structuretext") const Timeline = require("./Timeline") require("../testimports")(constants, Timeline) @@ -15,7 +15,11 @@ class ReelUser { /** @type {import("./Timeline")} */ this.timeline = new Timeline(this) this.cachedAt = Date.now() - this.proxyProfilePicture = proxyImage(this.data.profile_pic_url) + this.computeProxyProfilePic() + } + + computeProxyProfilePic() { + this.proxyProfilePicture = proxyProfilePic(this.data.profile_pic_url, this.data.id) } getStructuredBio() { diff --git a/src/lib/structures/User.js b/src/lib/structures/User.js index 71d31a2..4fe3e57 100644 --- a/src/lib/structures/User.js +++ b/src/lib/structures/User.js @@ -1,5 +1,5 @@ const constants = require("../constants") -const {proxyImage} = require("../utils/proxyurl") +const {proxyProfilePic} = require("../utils/proxyurl") const {structure} = require("../utils/structuretext") const Timeline = require("./Timeline") require("../testimports")(constants, Timeline) @@ -15,7 +15,11 @@ class User { this.posts = data.edge_owner_to_timeline_media.count this.timeline = new Timeline(this) this.cachedAt = Date.now() - this.proxyProfilePicture = proxyImage(this.data.profile_pic_url) + this.computeProxyProfilePic() + } + + computeProxyProfilePic() { + this.proxyProfilePicture = proxyProfilePic(this.data.profile_pic_url, this.data.id) } getStructuredBio() { diff --git a/src/lib/utils/proxyurl.js b/src/lib/utils/proxyurl.js index 17ce50c..dbac697 100644 --- a/src/lib/utils/proxyurl.js +++ b/src/lib/utils/proxyurl.js @@ -5,6 +5,13 @@ function proxyImage(url, width) { return "/imageproxy?"+params.toString() } +function proxyProfilePic(url, userID) { + const params = new URLSearchParams() + params.set("userID", userID) + params.set("url", url) + return "/imageproxy?"+params.toString() +} + function proxyVideo(url) { const params = new URLSearchParams() params.set("url", url) @@ -21,5 +28,6 @@ function proxyExtendedOwner(owner) { } module.exports.proxyImage = proxyImage +module.exports.proxyProfilePic = proxyProfilePic module.exports.proxyVideo = proxyVideo module.exports.proxyExtendedOwner = proxyExtendedOwner diff --git a/src/lib/utils/requestbackends/got.js b/src/lib/utils/requestbackends/got.js index a6dc342..28f1909 100644 --- a/src/lib/utils/requestbackends/got.js +++ b/src/lib/utils/requestbackends/got.js @@ -25,7 +25,8 @@ class Got { */ response() { return this.send().instance.then(res => ({ - status: res.statusCode + status: res.statusCode, + headers: new Map(Object.entries(res.headers)) })) } diff --git a/src/lib/utils/requestbackends/reference.js b/src/lib/utils/requestbackends/reference.js index aaefdea..b4fa900 100644 --- a/src/lib/utils/requestbackends/reference.js +++ b/src/lib/utils/requestbackends/reference.js @@ -1,6 +1,7 @@ /** * @typedef GrabResponse * @property {number} status + * @property {Map} */ diff --git a/src/site/api/proxy.js b/src/site/api/proxy.js index 27a5fdd..fac218f 100644 --- a/src/site/api/proxy.js +++ b/src/site/api/proxy.js @@ -1,8 +1,11 @@ -const constants = require("../../lib/constants") -const {request} = require("../../lib/utils/request") -const {proxy} = require("pinski/plugins") const sharp = require("sharp") +const constants = require("../../lib/constants") +const collectors = require("../../lib/collectors") +const {request} = require("../../lib/utils/request") +const db = require("../../lib/db") +require("../../lib/testimports")(constants, request, db) + /** * Check that a resource is on Instagram. * @param {URL} completeURL @@ -21,6 +24,47 @@ function verifyURL(completeURL) { if (!["fbcdn.net", "cdninstagram.com"].some(host => url.host.endsWith(host))) return {status: "fail", value: [400, "URL host is not allowed"]} return {status: "ok", url} } + +function statusCodeIsAcceptable(status) { + return (status >= 200 && status < 300) || status === 304 +} + +/** + * @param {string} url + */ +async function proxyResource(url, suggestedHeaders = {}, refreshCallback = null) { + // console.log(`Asked to proxy ${url}\n`, suggestedHeaders) + const headersToSend = {} + for (const key of ["accept", "accept-encoding", "accept-language", "range"]) { + if (suggestedHeaders[key]) headersToSend[key] = suggestedHeaders[key] + } + const sent = request(url, {headers: headersToSend}) + const stream = await sent.stream() + const response = await sent.response() + // console.log(response.status, response.headers) + if (statusCodeIsAcceptable(response.status)) { + const headersToReturn = {} + for (const key of ["content-type", "date", "last-modified", "expires", "cache-control", "accept-ranges", "origin", "etag", "content-length", "transfer-encoding"]) { + headersToReturn[key] = response.headers.get(key) + } + return { + statusCode: response.status, + headers: headersToReturn, + stream: stream + } + } else if (refreshCallback && response.status === 410) { // 410 GONE, profile picture has since changed + return refreshCallback() + } else { + return { + statusCode: 502, + headers: { + "Content-Type": "text/plain; charset=UTF-8" + }, + content: `Instagram returned HTTP status ${response.status}, which is not a success code.` + } + } +} + module.exports = [ { route: "/imageproxy", methods: ["GET"], code: async (input) => { @@ -59,9 +103,41 @@ module.exports = [ }) } else { // No specific size was requested, so just stream proxy the file directly. - return proxy(verifyResult.url, { - "Cache-Control": constants.caching.image_cache_control - }) + if (params.has("userID")) { + // Users get special handling, because we need to update their profile picture if an expired version is cached. + return proxyResource(verifyResult.url.toString(), input.req.headers, () => { + // If we get here, we got HTTP 410 GONE. + const userID = params.get("userID") + const storedProfilePicURL = db.prepare("SELECT profile_pic_url FROM Users WHERE user_id = ?").pluck().get(userID) + if (storedProfilePicURL === verifyResult.url.toString()) { + // Everything looks fine, find out what the new URL for the provided user ID is and store it. + return collectors.updateProfilePictureFromReel(userID).then(url => { + // Updated. Return the new picture (without recursing) + return proxyResource(url, input.req.headers) + }).catch(error => { + console.error(error) + return { + statusCode: 500, + headers: { + "Content-Type": "text/plain; charset=UTF-8" + }, + content: String(error) + } + }) + } else { + // The request is a lie! + return { + statusCode: 400, + headers: { + "Content-Type": "text/plain; charset=UTF-8" + }, + content: "Profile picture must be refreshed, but provided userID parameter does not match the stored profile_pic_url." + } + } + }) + } else { + return proxyResource(verifyResult.url.toString(), input.req.headers) + } } } }, @@ -71,9 +147,7 @@ module.exports = [ if (verifyResult.status !== "ok") return verifyResult.value const url = verifyResult.url if (!["mp4"].some(ext => url.pathname.endsWith(ext))) return [400, "URL extension is not allowed"] - return proxy(url, { - "Cache-Control": constants.caching.image_cache_control - }) + return proxyResource(url.toString(), input.req.headers) } } ]