Add request quota system

src/lib/cache.js

@@ -106,15 +106,15 @@ class RequestCache extends TtlCache {
 	/**
 	 * @param {string} key
 	 * @param {() => Promise<T>} callback
-	 * @returns {Promise<T>}
+	 * @returns {Promise<{result: T, fromCache: boolean}>}
 	 */
 	getOrFetch(key, callback) {
 		this.cleanKey(key)
-		if (this.cache.has(key)) return Promise.resolve(this.get(key))
+		if (this.cache.has(key)) return Promise.resolve({result: this.get(key), fromCache: true})
 		else {
 			const pending = callback().then(result => {
 				this.set(key, result)
-				return result
+				return {result, fromCache: false}
 			})
 			this.set(key, pending)
 			return pending
@@ -124,7 +124,7 @@ class RequestCache extends TtlCache {
 	/**
 	 * @param {string} key
 	 * @param {() => Promise<T>} callback
-	 * @returns {Promise<T>}
+	 * @returns {Promise<{result: T, fromCache: boolean}>}
 	 */
 	getOrFetchPromise(key, callback) {
 		return this.getOrFetch(key, callback).then(result => {

src/lib/collectors.js

@@ -83,7 +83,7 @@ async function fetchUser(username, context) {
 
 /**
  * @param {string} username
- * @returns {Promise<import("./structures/User")>}
+ * @returns {Promise<{user: import("./structures/User"), quotaUsed: number}>}
  */
 function fetchUserFromHTML(username) {
 	if (constants.caching.self_blocked_status.enabled) {
@@ -151,7 +151,7 @@ function fetchUserFromHTML(username) {
 			}
 			throw error
 		})
-	})
+	}).then(user => ({user, quotaUsed: 0}))
 }
 
 /**
@@ -190,7 +190,7 @@ function updateProfilePictureFromReel(userID) {
 /**
  * @param {string} userID
  * @param {string} username
- * @returns {Promise<import("./structures/ReelUser")|import("./structures/User")>}
+ * @returns {Promise<{user: import("./structures/ReelUser")|import("./structures/User"), quotaUsed: number}>}
  */
 function fetchUserFromCombined(userID, username) {
 	// Fetch basic user information
@@ -220,11 +220,13 @@ function fetchUserFromCombined(userID, username) {
 		})
 	}).then(async user => {
 		// Add first timeline page
+		let quotaUsed = 0
 		if (!user.timeline.pages[0]) {
-			const page = await fetchTimelinePage(userID, "")
-			user.timeline.addPage(page)
+			const fetched = await fetchTimelinePage(userID, "")
+			if (!fetched.fromCache) quotaUsed++
+			user.timeline.addPage(fetched.result)
 		}
-		return user
+		return {user, quotaUsed}
 	}).catch(error => {
 		if (error === constants.symbols.RATE_LIMITED) {
 			history.report("reel", false)
@@ -234,6 +236,7 @@ function fetchUserFromCombined(userID, username) {
 }
 
 function fetchUserFromSaved(saved) {
+	let quotaUsed = 0
 	return userRequestCache.getOrFetch("user/"+saved.username, false, true, async () => {
 		// require down here or have to deal with require loop. require cache will take care of it anyway.
 		// ReelUser -> Timeline -> TimelineEntry -> collectors -/> ReelUser
@@ -252,17 +255,20 @@ function fetchUserFromSaved(saved) {
 		})
 		// Add first timeline page
 		if (!user.timeline.pages[0]) {
-			const page = await fetchTimelinePage(user.data.id, "")
+			const {result: page, fromCache} = await fetchTimelinePage(user.data.id, "")
+			if (!fromCache) quotaUsed++
 			user.timeline.addPage(page)
 		}
 		return user
+	}).then(user => {
+		return {user, quotaUsed}
 	})
 }
 
 /**
  * @param {string} userID
  * @param {string} after
- * @returns {Promise<import("./types").PagedEdges<import("./types").TimelineEntryN2>>}
+ * @returns {Promise<{result: import("./types").PagedEdges<import("./types").TimelineEntryN2>, fromCache: boolean}>}
  */
 function fetchTimelinePage(userID, after) {
 	const p = new URLSearchParams()
@@ -298,7 +304,7 @@ function fetchTimelinePage(userID, after) {
 /**
  * @param {string} userID
  * @param {string} after
- * @returns {Promise<import("./types").PagedEdges<import("./types").TimelineEntryN2>>}
+ * @returns {Promise<{result: import("./types").PagedEdges<import("./types").TimelineEntryN2>, fromCache: boolean}>}
  */
 function fetchIGTVPage(userID, after) {
 	const p = new URLSearchParams()
@@ -329,7 +335,7 @@ function fetchIGTVPage(userID, after) {
 /**
  * @param {string} userID
  * @param {string} username
- * @returns {Promise<boolean>}
+ * @returns {Promise<{result: boolean, fromCache: boolean}>}
  */
 function verifyUserPair(userID, username) {
 	// Fetch basic user information
@@ -378,14 +384,14 @@ async function getOrFetchShortcode(shortcode) {
 	} else {
 		const data = await fetchShortcodeData(shortcode)
 		const entry = getOrCreateShortcode(shortcode)
-		entry.applyN3(data)
+		entry.applyN3(data.result)
 		return entry
 	}
 }
 
 /**
  * @param {string} shortcode
- * @returns {Promise<import("./types").TimelineEntryN3>}
+ * @returns {Promise<{result: import("./types").TimelineEntryN3, fromCache: boolean}>}
  */
 function fetchShortcodeData(shortcode) {
 	// example actual query from web:

src/lib/constants.js

@@ -112,6 +112,14 @@ let constants = {
 		rewrite_twitter: "nitter.net"
 	},
 
+	quota: {
+		enabled: false,
+		timeframe: 20*60*60*1000,
+		count: 50,
+		ip_mode: "header", // one of: "header", "address"
+		ip_header: "x-forwarded-for"
+	},
+
 	user_settings: [
 		{
 			name: "language",
@@ -255,6 +263,7 @@ let constants = {
 		RATE_LIMITED: Symbol("RATE_LIMITED"),
 		ENDPOINT_OVERRIDDEN: Symbol("ENDPOINT_OVERRIDDEN"),
 		NO_ASSISTANTS_AVAILABLE: Symbol("NO_ASSISTANTS_AVAILABLE"),
+		QUOTA_REACHED: Symbol("QUOTA_REACHED"),
 		extractor_results: {
 			SUCCESS: Symbol("SUCCESS"),
 			AGE_RESTRICTED: Symbol("AGE_RESTRICTED"),

src/lib/quota/LimitByFrame.js

@@ -0,0 +1,54 @@
+const constants = require("../constants")
+
+class Frame {
+	constructor(identifier) {
+		this.identifier = identifier
+		this.frameStartedAt = 0
+		this.count = 0
+	}
+
+	refresh() {
+		if (Date.now() > this.frameStartedAt + constants.quota.timeframe) {
+			this.count = 0
+		}
+	}
+
+	remaining() {
+		this.refresh()
+		return Math.max(constants.quota.count - this.count, 0)
+	}
+
+	add(count) {
+		this.refresh()
+		if (this.count === 0) this.frameStartedAt = Date.now()
+		this.count += count
+		return this.remaining()
+	}
+}
+
+class LimitByFrame {
+	constructor() {
+		/** @type {Map<string, Frame>} */
+		this.frames = new Map()
+	}
+
+	getOrCreateFrame(identifier) {
+		if (!this.frames.has(identifier)) {
+			const frame = new Frame(identifier)
+			this.frames.set(identifier, frame)
+			return frame
+		} else {
+			return this.frames.get(identifier)
+		}
+	}
+
+	remaining(identifier) {
+		return this.getOrCreateFrame(identifier).remaining()
+	}
+
+	add(identifier, count) {
+		return this.getOrCreateFrame(identifier).add(count)
+	}
+}
+
+module.exports = LimitByFrame

src/lib/quota/get_identifier.js

@@ -0,0 +1,25 @@
+const {request} = require("../utils/request")
+const {log} = require("pinski/util/common")
+
+let addresses = []
+
+request("https://check.torproject.org/torbulkexitlist").text().then(text => {
+	const lines = text.split("\n").filter(l => l)
+	addresses = addresses.concat(lines)
+	log(`Loaded Tor exit node list (${addresses.length} total)`, "spam")
+})
+
+/*
+	request("https://meta.bibliogram.art/ip_proxy_list.txt").text().then(text => {
+		const lines = text.split("\n").filter(l => l)
+		addresses = addresses.concat(lines)
+		log(`Loaded Bibliogram proxy list (${addresses.length} total)`, "spam")
+	})
+*/
+
+function getIdentifier(address) {
+	if (addresses.includes(address)) return "proxy"
+	else return address
+}
+
+module.exports.getIdentifier = getIdentifier

src/lib/quota/index.js

@@ -0,0 +1,33 @@
+const constants = require("../constants")
+const LimitByFrame = require("./LimitByFrame")
+const {getIdentifier} = require("./get_identifier")
+require("../testimports")(LimitByFrame, getIdentifier)
+
+const limiter = new LimitByFrame()
+
+function getIPFromReq(req) {
+	if (constants.quota.ip_mode === "header") {
+		return req.headers[constants.quota.ip_header]
+	} else { // constants.quota.ip_mode === "address"
+		return req.connection.remoteAddress
+	}
+}
+
+function remaining(req) {
+	if (!constants.quota.enabled) return Infinity // sure.
+
+	const ip = getIPFromReq(req)
+	const identifier = getIdentifier(ip)
+	return limiter.remaining(identifier)
+}
+
+function add(req, count) {
+	if (!constants.quota.enabled) return Infinity // why not.
+
+	const ip = getIPFromReq(req)
+	const identifier = getIdentifier(ip)
+	return limiter.add(identifier, count)
+}
+
+module.exports.remaining = remaining
+module.exports.add = add

src/lib/structures/Timeline.js

@@ -45,16 +45,22 @@ class Timeline {
 			: this.type === "igtv" ? collectors.fetchIGTVPage
 			: null
 		const after = this.page_info ? this.page_info.end_cursor : ""
-		return method(this.user.data.id, after).then(page => {
+		return method(this.user.data.id, after).then(({result: page, fromCache}) => {
+			const quotaUsed = fromCache ? 0 : 1
 			this.addPage(page)
-			return this.pages.slice(-1)[0]
+			return {page: this.pages.slice(-1)[0], quotaUsed}
 		})
 	}
 
 	async fetchUpToPage(index) {
+		let quotaUsed = 0
 		while (this.pages[index] === undefined && this.hasNextPage()) {
-			await this.fetchNextPage()
+			const result = await this.fetchNextPage()
+			if (typeof result !== "symbol") {
+				quotaUsed += result.quotaUsed
+			}
 		}
+		return quotaUsed
 	}
 
 	addPage(page) {

src/lib/structures/TimelineEntry.js

@@ -32,7 +32,7 @@ class TimelineEntry extends TimelineBaseMethods {
 
 	async update() {
 		return collectors.fetchShortcodeData(this.data.shortcode).then(data => {
-			this.applyN3(data)
+			this.applyN3(data.result)
 		}).catch(error => {
 			console.error("TimelineEntry could not self-update; trying to continue anyway...")
 			console.error("E:", error)