Allow got as request backend

package-lock.json

@@ -204,6 +204,21 @@
         "ip-address": "^5.8.9"
       }
     },
+    "@sindresorhus/is": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/@sindresorhus/is/-/is-2.1.0.tgz",
+      "integrity": "sha512-lXKXfypKo644k4Da4yXkPCrwcvn6SlUW2X2zFbuflKHNjf0w9htru01bo26uMhleMXsDmnZ12eJLdrAZa9MANg==",
+      "optional": true
+    },
+    "@szmarczak/http-timer": {
+      "version": "4.0.5",
+      "resolved": "https://registry.npmjs.org/@szmarczak/http-timer/-/http-timer-4.0.5.tgz",
+      "integrity": "sha512-PyRA9sm1Yayuj5OIoJ1hGt2YISX45w9WcFbh6ddT0Z/0yaFxOtGLInr4jUfU1EAFVs0Yfyfev4RNwBlUaHdlDQ==",
+      "optional": true,
+      "requires": {
+        "defer-to-connect": "^2.0.0"
+      }
+    },
     "@types/babel-types": {
       "version": "7.0.7",
       "resolved": "https://registry.npmjs.org/@types/babel-types/-/babel-types-7.0.7.tgz",
@@ -217,12 +232,48 @@
         "@types/babel-types": "*"
       }
     },
+    "@types/cacheable-request": {
+      "version": "6.0.1",
+      "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.1.tgz",
+      "integrity": "sha512-ykFq2zmBGOCbpIXtoVbz4SKY5QriWPh3AjyU4G74RYbtt5yOc5OfaY75ftjg7mikMOla1CTGpX3lLbuJh8DTrQ==",
+      "optional": true,
+      "requires": {
+        "@types/http-cache-semantics": "*",
+        "@types/keyv": "*",
+        "@types/node": "*",
+        "@types/responselike": "*"
+      }
+    },
+    "@types/http-cache-semantics": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/@types/http-cache-semantics/-/http-cache-semantics-4.0.0.tgz",
+      "integrity": "sha512-c3Xy026kOF7QOTn00hbIllV1dLR9hG9NkSrLQgCVs8NF6sBU+VGWjD3wLPhmh1TYAc7ugCFsvHYMN4VcBN1U1A==",
+      "optional": true
+    },
+    "@types/keyv": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/@types/keyv/-/keyv-3.1.1.tgz",
+      "integrity": "sha512-MPtoySlAZQ37VoLaPcTHCu1RWJ4llDkULYZIzOYxlhxBqYPB0RsRlmMU0R6tahtFe27mIdkHV+551ZWV4PLmVw==",
+      "optional": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "@types/node": {
       "version": "13.7.0",
       "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.0.tgz",
       "integrity": "sha512-GnZbirvmqZUzMgkFn70c74OQpTTUcCzlhQliTzYjQMqg+hVKcDnxdL19Ne3UdYzdMA/+W3eb646FWn/ZaT1NfQ==",
       "optional": true
     },
+    "@types/responselike": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/@types/responselike/-/responselike-1.0.0.tgz",
+      "integrity": "sha512-85Y2BjiufFzaMIlvJDvTTB8Fxl2xfLo4HgmHzVBz08w4wDePCTjYw66PdrolO0kzli3yam/YCgRufyo1DdQVTA==",
+      "optional": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "abbrev": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
@@ -584,6 +635,30 @@
       "integrity": "sha512-MQcXEUbCKtEo7bhqEs6560Hyd4XaovZlO/k9V3hjVUF/zwW7KBVdSK4gIt/bzwS9MbR5qob+F5jusZsb0YQK2A==",
       "dev": true
     },
+    "cacheable-lookup": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/cacheable-lookup/-/cacheable-lookup-2.0.0.tgz",
+      "integrity": "sha512-s2piO6LvA7xnL1AR03wuEdSx3BZT3tIJpZ56/lcJwzO/6DTJZlTs7X3lrvPxk6d1PlDe6PrVe2TjlUIZNFglAQ==",
+      "optional": true,
+      "requires": {
+        "keyv": "^4.0.0"
+      }
+    },
+    "cacheable-request": {
+      "version": "7.0.1",
+      "resolved": "https://registry.npmjs.org/cacheable-request/-/cacheable-request-7.0.1.tgz",
+      "integrity": "sha512-lt0mJ6YAnsrBErpTMWeu5kl/tg9xMAWjavYTN6VQXM1A/teBITuNcccXsCxF0tDQQJf9DfAaX5O4e0zp0KlfZw==",
+      "optional": true,
+      "requires": {
+        "clone-response": "^1.0.2",
+        "get-stream": "^5.1.0",
+        "http-cache-semantics": "^4.0.0",
+        "keyv": "^4.0.0",
+        "lowercase-keys": "^2.0.0",
+        "normalize-url": "^4.1.0",
+        "responselike": "^2.0.0"
+      }
+    },
     "caching-transform": {
       "version": "3.0.2",
       "resolved": "https://registry.npmjs.org/caching-transform/-/caching-transform-3.0.2.tgz",
@@ -717,6 +792,23 @@
         "wordwrap": "0.0.2"
       }
     },
+    "clone-response": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/clone-response/-/clone-response-1.0.2.tgz",
+      "integrity": "sha1-0dyXOSAxTfZ/vrlCI7TuNQI56Ws=",
+      "optional": true,
+      "requires": {
+        "mimic-response": "^1.0.0"
+      },
+      "dependencies": {
+        "mimic-response": {
+          "version": "1.0.1",
+          "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-1.0.1.tgz",
+          "integrity": "sha512-j5EctnkH7amfV/q5Hgmoal1g2QHFJRraOtmx0JpIqkxhBhI/lJSl1nMpQ45hVarwNETOoWEimndZ4QK0RHxuxQ==",
+          "optional": true
+        }
+      }
+    },
     "code-point-at": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/code-point-at/-/code-point-at-1.1.0.tgz",
@@ -933,6 +1025,12 @@
         }
       }
     },
+    "defer-to-connect": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.0.tgz",
+      "integrity": "sha512-bYL2d05vOSf1JEZNx5vSAtPuBMkX8K9EUutg7zlKvTqKXHt7RhWJFbmd7qakVuf13i+IkGmp6FwSsONOf6VYIg==",
+      "optional": true
+    },
     "delayed-stream": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
@@ -1000,6 +1098,12 @@
         "domelementtype": "1"
       }
     },
+    "duplexer3": {
+      "version": "0.1.4",
+      "resolved": "https://registry.npmjs.org/duplexer3/-/duplexer3-0.1.4.tgz",
+      "integrity": "sha1-7gHdHKwO08vH/b6jfcCo8c4ALOI=",
+      "optional": true
+    },
     "ecc-jsbn": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
@@ -1298,6 +1402,15 @@
       "resolved": "https://registry.npmjs.org/get-stdin/-/get-stdin-4.0.1.tgz",
       "integrity": "sha1-uWjGsKBDhDJJAui/Gl3zJXmkUP4="
     },
+    "get-stream": {
+      "version": "5.1.0",
+      "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz",
+      "integrity": "sha512-EXr1FOzrzTfGeL0gQdeFEvOMm2mzMOglyiOXSTpPC+iAjAKftbr3jpCMWynogwYnM+eSj9sHGc6wjIcDvYiygw==",
+      "optional": true,
+      "requires": {
+        "pump": "^3.0.0"
+      }
+    },
     "getpass": {
       "version": "0.1.7",
       "resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
@@ -1349,6 +1462,46 @@
         "minimatch": "~3.0.2"
       }
     },
+    "got": {
+      "version": "10.6.0",
+      "resolved": "https://registry.npmjs.org/got/-/got-10.6.0.tgz",
+      "integrity": "sha512-3LIdJNTdCFbbJc+h/EH0V5lpNpbJ6Bfwykk21lcQvQsEcrzdi/ltCyQehFHLzJ/ka0UMH4Slg0hkYvAZN9qUDg==",
+      "optional": true,
+      "requires": {
+        "@sindresorhus/is": "^2.0.0",
+        "@szmarczak/http-timer": "^4.0.0",
+        "@types/cacheable-request": "^6.0.1",
+        "cacheable-lookup": "^2.0.0",
+        "cacheable-request": "^7.0.1",
+        "decompress-response": "^5.0.0",
+        "duplexer3": "^0.1.4",
+        "get-stream": "^5.0.0",
+        "lowercase-keys": "^2.0.0",
+        "mimic-response": "^2.1.0",
+        "p-cancelable": "^2.0.0",
+        "p-event": "^4.0.0",
+        "responselike": "^2.0.0",
+        "to-readable-stream": "^2.0.0",
+        "type-fest": "^0.10.0"
+      },
+      "dependencies": {
+        "decompress-response": {
+          "version": "5.0.0",
+          "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-5.0.0.tgz",
+          "integrity": "sha512-TLZWWybuxWgoW7Lykv+gq9xvzOsUjQ9tF09Tj6NSTYGMTCHNXzrPnD6Hi+TgZq19PyTAGH4Ll/NIM/eTGglnMw==",
+          "optional": true,
+          "requires": {
+            "mimic-response": "^2.0.0"
+          }
+        },
+        "mimic-response": {
+          "version": "2.1.0",
+          "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-2.1.0.tgz",
+          "integrity": "sha512-wXqjST+SLt7R009ySCglWBCFpjUygmCIfD790/kVbiGmUgfYGuB14PiTd5DwVxSV4NcYHjzMkoj5LjQZwTQLEA==",
+          "optional": true
+        }
+      }
+    },
     "graceful-fs": {
       "version": "4.2.3",
       "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.3.tgz",
@@ -1442,6 +1595,12 @@
         }
       }
     },
+    "http-cache-semantics": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/http-cache-semantics/-/http-cache-semantics-4.1.0.tgz",
+      "integrity": "sha512-carPklcUh7ROWRK7Cv27RPtdhYhUsela/ue5/jKzjegVvXDqM2ILE9Q2BGn9JZJh1g87cp56su/FgQSzcWS8cQ==",
+      "optional": true
+    },
     "http-signature": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
@@ -1841,6 +2000,12 @@
       "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==",
       "dev": true
     },
+    "json-buffer": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/json-buffer/-/json-buffer-3.0.1.tgz",
+      "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==",
+      "optional": true
+    },
     "json-parse-better-errors": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz",
@@ -1882,6 +2047,15 @@
         "promise": "^7.0.1"
       }
     },
+    "keyv": {
+      "version": "4.0.0",
+      "resolved": "https://registry.npmjs.org/keyv/-/keyv-4.0.0.tgz",
+      "integrity": "sha512-U7ioE8AimvRVLfw4LffyOIRhL2xVgmE8T22L6i0BucSnBUyv4w+I7VN/zVZwRKHOI6ZRUcdMdWHQ8KSUvGpEog==",
+      "optional": true,
+      "requires": {
+        "json-buffer": "3.0.1"
+      }
+    },
     "kind-of": {
       "version": "3.2.2",
       "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
@@ -1999,6 +2173,12 @@
         "signal-exit": "^3.0.0"
       }
     },
+    "lowercase-keys": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/lowercase-keys/-/lowercase-keys-2.0.0.tgz",
+      "integrity": "sha512-tqNXrS78oMOE73NMxK4EMLQsQowWf8jKooH9g7xPavRT706R6bkQJ6DY2Te7QukaZsulxa30wQ7bk0pm4XiHmA==",
+      "optional": true
+    },
     "lru-cache": {
       "version": "4.1.5",
       "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-4.1.5.tgz",
@@ -2351,6 +2531,12 @@
       "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==",
       "dev": true
     },
+    "normalize-url": {
+      "version": "4.5.0",
+      "resolved": "https://registry.npmjs.org/normalize-url/-/normalize-url-4.5.0.tgz",
+      "integrity": "sha512-2s47yzUxdexf1OhyRi4Em83iQk0aPvwTddtFz4hnSSw9dCEsLEGf6SwIO8ss/19S9iBb5sJaOuTvTGDeZI00BQ==",
+      "optional": true
+    },
     "npmlog": {
       "version": "4.1.2",
       "resolved": "https://registry.npmjs.org/npmlog/-/npmlog-4.1.2.tgz",
@@ -2607,6 +2793,27 @@
         "own-or": "^1.0.0"
       }
     },
+    "p-cancelable": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-2.0.0.tgz",
+      "integrity": "sha512-wvPXDmbMmu2ksjkB4Z3nZWTSkJEb9lqVdMaCKpZUGJG9TMiNp9XcbG3fn9fPKjem04fJMJnXoyFPk2FmgiaiNg==",
+      "optional": true
+    },
+    "p-event": {
+      "version": "4.1.0",
+      "resolved": "https://registry.npmjs.org/p-event/-/p-event-4.1.0.tgz",
+      "integrity": "sha512-4vAd06GCsgflX4wHN1JqrMzBh/8QZ4j+rzp0cd2scXRwuBEv+QR3wrVA5aLhWDLw4y2WgDKvzWF3CCLmVM1UgA==",
+      "optional": true,
+      "requires": {
+        "p-timeout": "^2.0.1"
+      }
+    },
+    "p-finally": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz",
+      "integrity": "sha1-P7z7FbiZpEEjs0ttzBi3JDNqLK4=",
+      "optional": true
+    },
     "p-limit": {
       "version": "2.2.2",
       "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.2.2.tgz",
@@ -2625,6 +2832,15 @@
         "p-limit": "^2.0.0"
       }
     },
+    "p-timeout": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-2.0.1.tgz",
+      "integrity": "sha512-88em58dDVB/KzPEx1X0N3LwFfYZPyDc4B6eF38M1rk9VTZMbxXXgjugz8mmwpS9Ox4BDZ+t6t3QP5+/gazweIA==",
+      "optional": true,
+      "requires": {
+        "p-finally": "^1.0.0"
+      }
+    },
     "p-try": {
       "version": "2.2.0",
       "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz",
@@ -3114,6 +3330,15 @@
       "integrity": "sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g==",
       "dev": true
     },
+    "responselike": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/responselike/-/responselike-2.0.0.tgz",
+      "integrity": "sha512-xH48u3FTB9VsZw7R+vvgaKeLKzT6jOogbQhEe/jewwnZgzPcnyWui2Av6JpoYZF/91uueC+lqhWqeURw5/qhCw==",
+      "optional": true,
+      "requires": {
+        "lowercase-keys": "^2.0.0"
+      }
+    },
     "right-align": {
       "version": "0.1.3",
       "resolved": "https://registry.npmjs.org/right-align/-/right-align-0.1.3.tgz",
@@ -4761,6 +4986,12 @@
       "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-1.0.3.tgz",
       "integrity": "sha1-uDVx+k2MJbguIxsG46MFXeTKGkc="
     },
+    "to-readable-stream": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/to-readable-stream/-/to-readable-stream-2.1.0.tgz",
+      "integrity": "sha512-o3Qa6DGg1CEXshSdvWNX2sN4QHqg03SPq7U6jPXRahlQdl5dK8oXjkU/2/sGrnOZKeGV1zLSO8qPwyKklPPE7w==",
+      "optional": true
+    },
     "to-regex-range": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
@@ -4836,6 +5067,12 @@
       "resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
       "integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q="
     },
+    "type-fest": {
+      "version": "0.10.0",
+      "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.10.0.tgz",
+      "integrity": "sha512-EUV9jo4sffrwlg8s0zDhP0T2WD3pru5Xi0+HTE3zTUmBaZNhfkite9PdSJwdXLwPVW0jnAHT56pZHIOYckPEiw==",
+      "optional": true
+    },
     "typedarray-to-buffer": {
       "version": "3.1.5",
       "resolved": "https://registry.npmjs.org/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz",

package.json

@@ -1,5 +1,6 @@
 {
   "name": "bibliogram",
+  "private": true,
   "version": "1.0.0",
   "description": "",
   "main": "index.js",
@@ -23,7 +24,8 @@
     "socks-proxy-agent": "github:cloudrac3r/node-socks-proxy-agent#6a26d274b12098dfef6cc2faafd25b0c051f2467"
   },
   "optionalDependencies": {
-    "@deadcanaries/granax": "^3.2.5"
+    "@deadcanaries/granax": "^3.2.5",
+    "got": "^10.6.0"
   },
   "devDependencies": {
     "tap": "^14.10.6"

src/lib/collectors.js

@@ -71,11 +71,12 @@ function fetchUserFromHTML(username) {
 			if (res.status === 302) throw constants.symbols.INSTAGRAM_DEMANDS_LOGIN
 			if (res.status === 429) throw constants.symbols.RATE_LIMITED
 			return res
-		}).then(res => {
+		}).then(async g => {
+			const res = await g.response()
 			if (res.status === 404) {
 				throw constants.symbols.NOT_FOUND
 			} else {
-				return res.text().then(text => {
+				const text = await g.text()
 				// require down here or have to deal with require loop. require cache will take care of it anyway.
 				// User -> Timeline -> TimelineEntry -> collectors -/> User
 				const User = require("./structures/User")
@@ -105,7 +106,6 @@ function fetchUserFromHTML(username) {
 					})
 				}
 				return user
-				})
 			}
 		}).catch(error => {
 			if (error === constants.symbols.INSTAGRAM_DEMANDS_LOGIN || error === constants.symbols.RATE_LIMITED) {
@@ -202,8 +202,7 @@ function fetchTimelinePage(userID, after) {
 	return requestCache.getOrFetchPromise(`page/${userID}/${after}`, () => {
 		return switcher.request("timeline_graphql", `https://www.instagram.com/graphql/query/?${p.toString()}`, async res => {
 			if (res.status === 429) throw constants.symbols.RATE_LIMITED
-			return res
+		}).then(g => g.json()).then(root => {
-		}).then(res => res.json()).then(root => {
 			/** @type {import("./types").PagedEdges<import("./types").TimelineEntryN2>} */
 			const timeline = root.data.user.edge_owner_to_timeline_media
 			history.report("timeline", true)
@@ -259,7 +258,6 @@ function fetchShortcodeData(shortcode) {
 	return requestCache.getOrFetchPromise("shortcode/"+shortcode, () => {
 		return switcher.request("post_graphql", `https://www.instagram.com/graphql/query/?${p.toString()}`, async res => {
 			if (res.status === 429) throw constants.symbols.RATE_LIMITED
-			return res
 		}).then(res => res.json()).then(root => {
 			/** @type {import("./types").TimelineEntryN3} */
 			const data = root.data.shortcode_media

src/lib/constants.js

@@ -20,6 +20,7 @@ let constants = {
 			reel_graphql: true
 		}
 	},
+	request_backend: "node-fetch", // one of: "node-fetch", "got"
 	// After setting your privacy policy, I suggest you read src/site/html/.well-known/dnt-policy.txt. If you comply with it,
 	// change this to `true` to serve it, which will make extensions like Privacy Badger automatically whitelist the domain.
 	does_not_track: false,

src/lib/utils/request.js

@@ -1,16 +1,39 @@
-const fetch = require("node-fetch").default
+const NodeFetch = require("./requestbackends/node-fetch")
+const Got = require("./requestbackends/got")
 
+const constants = require("../constants")
+
+const userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"
+
+const backendStatusLineMap = new Map([
+	["node-fetch", "NF "],
+	["got", "GOT"]
+])
+
+/**
+ * @returns {import("./requestbackends/reference")}
+ */
 function request(url, options = {}, settings = {}) {
 	if (settings.statusLine === undefined) settings.statusLine = "OUT"
 	if (settings.log === undefined) settings.log = true
-	if (settings.log) console.log(`          -> [${settings.statusLine}] ${url}`) // todo: make more like pinski?
+	if (settings.log) console.log(`      -> [${settings.statusLine}-${backendStatusLineMap.get(constants.request_backend)}] ${url}`) // todo: make more like pinski?
-	// @ts-ignore
+	if (constants.request_backend === "node-fetch") {
-	return fetch(url, Object.assign({
+		return new NodeFetch(url, Object.assign({
 			headers: {
-			"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"
+				"User-Agent": userAgent
 			},
 			redirect: "manual"
 		}, options))
+	} else if (constants.request_backend === "got") {
+		return new Got(url, Object.assign({
+			headers: {
+				"User-Agent": userAgent
+			},
+			followRedirect: false
+		}, options))
+	} else {
+		throw new Error("Invalid value for setting `request_backend`.")
+	}
 }
 
 module.exports.request = request

src/lib/utils/requestbackends/got.js

@@ -0,0 +1,46 @@
+try {
+	var got = require("got").default
+} catch (e) {}
+
+class Got {
+	constructor(url, options, stream) {
+		if (!got) throw new Error("`got` is not installed, either install it or set a different request backend.")
+		this.url = url
+		this.options = options
+	}
+
+	stream() {
+		return Promise.resolve(got.stream(this.url, this.options))
+	}
+
+	send() {
+		if (!this.instance) {
+			this.instance = got(this.url, this.options)
+		}
+		return this
+	}
+
+	/**
+	 * @returns {Promise<import("./reference").GrabResponse>}
+	 */
+	response() {
+		return this.send().instance.then(res => ({
+			status: res.statusCode
+		}))
+	}
+
+	async check(test) {
+		await this.send().response().then(res => test(res))
+		return this
+	}
+
+	json() {
+		return this.send().instance.json()
+	}
+
+	text() {
+		return this.send().instance.text()
+	}
+}
+
+module.exports = Got

src/lib/utils/requestbackends/node-fetch.js

@@ -0,0 +1,30 @@
+const fetch = require("node-fetch").default
+
+class NodeFetch {
+	constructor(url, options) {
+		this.instance = fetch(url, options)
+	}
+
+	stream() {
+		return this.instance.then(res => res.body)
+	}
+
+	response() {
+		return this.instance
+	}
+
+	json() {
+		return this.instance.then(res => res.json())
+	}
+
+	text() {
+		return this.instance.then(res => res.text())
+	}
+
+	async check(test) {
+		await this.response().then(res => test(res))
+		return this
+	}
+}
+
+module.exports = NodeFetch

src/lib/utils/requestbackends/reference.js

@@ -0,0 +1,45 @@
+/**
+ * @typedef GrabResponse
+ * @property {number} status
+ */
+
+// @ts-nocheck
+
+class GrabReference {
+	/**
+	 * @param {string} url
+	 * @param {any} options
+	 */
+	constructor(url, options) {
+		throw new Error("This is the reference class, do not instantiate it.")
+	}
+
+	// Please help me type this
+	/**
+	 * @returns {Promise<any>}
+	 */
+	stream() {}
+
+	/**
+	 * @returns {Promise<GrabResponse>}
+	 */
+	response() {}
+
+	/**
+	 * @returns {Promise<any>}
+	 */
+	json() {}
+
+	/**
+	 * @returns {Promise<string>}
+	 */
+	text() {}
+
+	/**
+	 * @param {(res: GrabResponse) => any}
+	 * @returns {Promise<Reference>}
+	 */
+	check(test) {}
+}
+
+module.exports = GrabReference

src/lib/utils/torswitcher.js

@@ -21,15 +21,14 @@ class TorSwitcher {
 	 * If the test function fails, its error will be rejected here.
 	 * Only include rate limit logic in the test function!
 	 * @param {string} url
-	 * @param {(res: import("node-fetch").Response) => Promise<T>} test
+	 * @param {(res: import("./requestbackends/reference").GrabResponse) => any} test
-	 * @returns {Promise<T>}
+	 * @returns {Promise<import("./requestbackends/reference")>}
-	 * @template T the return value of the test function
 	 */
 	request(type, url, test) {
 		if (this.torManager && constants.tor.for[type]) {
 			return this.torManager.request(url, test)
 		} else {
-			return request(url).then(res => test(res))
+			return request(url).check(test)
 		}
 	}
 }

src/site/api/proxy.js

@@ -36,18 +36,25 @@ module.exports = [
 					Some thumbnails aren't square and would otherwise be stretched on the page without this.
 					If I cropped the images client side, it would have to be done with CSS background-image, which means no <img srcset>.
 				*/
-				return request(verifyResult.url, {}, {log: false}).then(res => {
+				return request(verifyResult.url, {}, {log: false}).stream().then(body => {
 					const converter = sharp().resize(width, width, {position: "entropy"})
+					body.on("error", error => {
+						console.error("Response stream emitted an error:", error)
+					})
 					converter.on("error", error => {
 						console.error("Sharp instance emitted an error:", error)
 					})
+					const piped = body.pipe(converter)
+					piped.on("error", error => {
+						console.error("Piped stream emitted na error:", error)
+					})
 					return {
 						statusCode: 200,
 						contentType: "image/jpeg",
 						headers: {
 							"Cache-Control": constants.caching.image_cache_control
 						},
-						stream: res.body.pipe(converter)
+						stream: piped
 					}
 				})
 			} else {