Add request quota system

src/lib/cache.js

@@ -106,15 +106,15 @@ class RequestCache extends TtlCache {
 	/**
 	 * @param {string} key
 	 * @param {() => Promise<T>} callback
-	 * @returns {Promise<T>}
+	 * @returns {Promise<{result: T, fromCache: boolean}>}
 	 */
 	getOrFetch(key, callback) {
 		this.cleanKey(key)
-		if (this.cache.has(key)) return Promise.resolve(this.get(key))
+		if (this.cache.has(key)) return Promise.resolve({result: this.get(key), fromCache: true})
 		else {
 			const pending = callback().then(result => {
 				this.set(key, result)
-				return result
+				return {result, fromCache: false}
 			})
 			this.set(key, pending)
 			return pending
@@ -124,7 +124,7 @@ class RequestCache extends TtlCache {
 	/**
 	 * @param {string} key
 	 * @param {() => Promise<T>} callback
-	 * @returns {Promise<T>}
+	 * @returns {Promise<{result: T, fromCache: boolean}>}
 	 */
 	getOrFetchPromise(key, callback) {
 		return this.getOrFetch(key, callback).then(result => {

src/lib/collectors.js

@@ -83,7 +83,7 @@ async function fetchUser(username, context) {
 
 /**
  * @param {string} username
- * @returns {Promise<import("./structures/User")>}
+ * @returns {Promise<{user: import("./structures/User"), quotaUsed: number}>}
  */
 function fetchUserFromHTML(username) {
 	if (constants.caching.self_blocked_status.enabled) {
@@ -151,7 +151,7 @@ function fetchUserFromHTML(username) {
 			}
 			throw error
 		})
-	})
+	}).then(user => ({user, quotaUsed: 0}))
 }
 
 /**
@@ -190,7 +190,7 @@ function updateProfilePictureFromReel(userID) {
 /**
  * @param {string} userID
  * @param {string} username
- * @returns {Promise<import("./structures/ReelUser")|import("./structures/User")>}
+ * @returns {Promise<{user: import("./structures/ReelUser")|import("./structures/User"), quotaUsed: number}>}
  */
 function fetchUserFromCombined(userID, username) {
 	// Fetch basic user information
@@ -220,11 +220,13 @@ function fetchUserFromCombined(userID, username) {
 		})
 	}).then(async user => {
 		// Add first timeline page
+		let quotaUsed = 0
 		if (!user.timeline.pages[0]) {
-			const page = await fetchTimelinePage(userID, "")
+			const fetched = await fetchTimelinePage(userID, "")
-			user.timeline.addPage(page)
+			if (!fetched.fromCache) quotaUsed++
+			user.timeline.addPage(fetched.result)
 		}
-		return user
+		return {user, quotaUsed}
 	}).catch(error => {
 		if (error === constants.symbols.RATE_LIMITED) {
 			history.report("reel", false)
@@ -234,6 +236,7 @@ function fetchUserFromCombined(userID, username) {
 }
 
 function fetchUserFromSaved(saved) {
+	let quotaUsed = 0
 	return userRequestCache.getOrFetch("user/"+saved.username, false, true, async () => {
 		// require down here or have to deal with require loop. require cache will take care of it anyway.
 		// ReelUser -> Timeline -> TimelineEntry -> collectors -/> ReelUser
@@ -252,17 +255,20 @@ function fetchUserFromSaved(saved) {
 		})
 		// Add first timeline page
 		if (!user.timeline.pages[0]) {
-			const page = await fetchTimelinePage(user.data.id, "")
+			const {result: page, fromCache} = await fetchTimelinePage(user.data.id, "")
+			if (!fromCache) quotaUsed++
 			user.timeline.addPage(page)
 		}
 		return user
+	}).then(user => {
+		return {user, quotaUsed}
 	})
 }
 
 /**
  * @param {string} userID
  * @param {string} after
- * @returns {Promise<import("./types").PagedEdges<import("./types").TimelineEntryN2>>}
+ * @returns {Promise<{result: import("./types").PagedEdges<import("./types").TimelineEntryN2>, fromCache: boolean}>}
  */
 function fetchTimelinePage(userID, after) {
 	const p = new URLSearchParams()
@@ -298,7 +304,7 @@ function fetchTimelinePage(userID, after) {
 /**
  * @param {string} userID
  * @param {string} after
- * @returns {Promise<import("./types").PagedEdges<import("./types").TimelineEntryN2>>}
+ * @returns {Promise<{result: import("./types").PagedEdges<import("./types").TimelineEntryN2>, fromCache: boolean}>}
  */
 function fetchIGTVPage(userID, after) {
 	const p = new URLSearchParams()
@@ -329,7 +335,7 @@ function fetchIGTVPage(userID, after) {
 /**
  * @param {string} userID
  * @param {string} username
- * @returns {Promise<boolean>}
+ * @returns {Promise<{result: boolean, fromCache: boolean}>}
  */
 function verifyUserPair(userID, username) {
 	// Fetch basic user information
@@ -378,14 +384,14 @@ async function getOrFetchShortcode(shortcode) {
 	} else {
 		const data = await fetchShortcodeData(shortcode)
 		const entry = getOrCreateShortcode(shortcode)
-		entry.applyN3(data)
+		entry.applyN3(data.result)
 		return entry
 	}
 }
 
 /**
  * @param {string} shortcode
- * @returns {Promise<import("./types").TimelineEntryN3>}
+ * @returns {Promise<{result: import("./types").TimelineEntryN3, fromCache: boolean}>}
  */
 function fetchShortcodeData(shortcode) {
 	// example actual query from web:

src/lib/constants.js

@@ -112,6 +112,14 @@ let constants = {
 		rewrite_twitter: "nitter.net"
 	},
 
+	quota: {
+		enabled: false,
+		timeframe: 20*60*60*1000,
+		count: 50,
+		ip_mode: "header", // one of: "header", "address"
+		ip_header: "x-forwarded-for"
+	},
+
 	user_settings: [
 		{
 			name: "language",
@@ -255,6 +263,7 @@ let constants = {
 		RATE_LIMITED: Symbol("RATE_LIMITED"),
 		ENDPOINT_OVERRIDDEN: Symbol("ENDPOINT_OVERRIDDEN"),
 		NO_ASSISTANTS_AVAILABLE: Symbol("NO_ASSISTANTS_AVAILABLE"),
+		QUOTA_REACHED: Symbol("QUOTA_REACHED"),
 		extractor_results: {
 			SUCCESS: Symbol("SUCCESS"),
 			AGE_RESTRICTED: Symbol("AGE_RESTRICTED"),

src/lib/quota/LimitByFrame.js

@@ -0,0 +1,54 @@
+const constants = require("../constants")
+
+class Frame {
+	constructor(identifier) {
+		this.identifier = identifier
+		this.frameStartedAt = 0
+		this.count = 0
+	}
+
+	refresh() {
+		if (Date.now() > this.frameStartedAt + constants.quota.timeframe) {
+			this.count = 0
+		}
+	}
+
+	remaining() {
+		this.refresh()
+		return Math.max(constants.quota.count - this.count, 0)
+	}
+
+	add(count) {
+		this.refresh()
+		if (this.count === 0) this.frameStartedAt = Date.now()
+		this.count += count
+		return this.remaining()
+	}
+}
+
+class LimitByFrame {
+	constructor() {
+		/** @type {Map<string, Frame>} */
+		this.frames = new Map()
+	}
+
+	getOrCreateFrame(identifier) {
+		if (!this.frames.has(identifier)) {
+			const frame = new Frame(identifier)
+			this.frames.set(identifier, frame)
+			return frame
+		} else {
+			return this.frames.get(identifier)
+		}
+	}
+
+	remaining(identifier) {
+		return this.getOrCreateFrame(identifier).remaining()
+	}
+
+	add(identifier, count) {
+		return this.getOrCreateFrame(identifier).add(count)
+	}
+}
+
+module.exports = LimitByFrame

src/lib/quota/get_identifier.js

@@ -0,0 +1,25 @@
+const {request} = require("../utils/request")
+const {log} = require("pinski/util/common")
+
+let addresses = []
+
+request("https://check.torproject.org/torbulkexitlist").text().then(text => {
+	const lines = text.split("\n").filter(l => l)
+	addresses = addresses.concat(lines)
+	log(`Loaded Tor exit node list (${addresses.length} total)`, "spam")
+})
+
+/*
+	request("https://meta.bibliogram.art/ip_proxy_list.txt").text().then(text => {
+		const lines = text.split("\n").filter(l => l)
+		addresses = addresses.concat(lines)
+		log(`Loaded Bibliogram proxy list (${addresses.length} total)`, "spam")
+	})
+*/
+
+function getIdentifier(address) {
+	if (addresses.includes(address)) return "proxy"
+	else return address
+}
+
+module.exports.getIdentifier = getIdentifier

src/lib/quota/index.js

@@ -0,0 +1,33 @@
+const constants = require("../constants")
+const LimitByFrame = require("./LimitByFrame")
+const {getIdentifier} = require("./get_identifier")
+require("../testimports")(LimitByFrame, getIdentifier)
+
+const limiter = new LimitByFrame()
+
+function getIPFromReq(req) {
+	if (constants.quota.ip_mode === "header") {
+		return req.headers[constants.quota.ip_header]
+	} else { // constants.quota.ip_mode === "address"
+		return req.connection.remoteAddress
+	}
+}
+
+function remaining(req) {
+	if (!constants.quota.enabled) return Infinity // sure.
+
+	const ip = getIPFromReq(req)
+	const identifier = getIdentifier(ip)
+	return limiter.remaining(identifier)
+}
+
+function add(req, count) {
+	if (!constants.quota.enabled) return Infinity // why not.
+
+	const ip = getIPFromReq(req)
+	const identifier = getIdentifier(ip)
+	return limiter.add(identifier, count)
+}
+
+module.exports.remaining = remaining
+module.exports.add = add

src/lib/structures/Timeline.js

@@ -45,16 +45,22 @@ class Timeline {
 			: this.type === "igtv" ? collectors.fetchIGTVPage
 			: null
 		const after = this.page_info ? this.page_info.end_cursor : ""
-		return method(this.user.data.id, after).then(page => {
+		return method(this.user.data.id, after).then(({result: page, fromCache}) => {
+			const quotaUsed = fromCache ? 0 : 1
 			this.addPage(page)
-			return this.pages.slice(-1)[0]
+			return {page: this.pages.slice(-1)[0], quotaUsed}
 		})
 	}
 
 	async fetchUpToPage(index) {
+		let quotaUsed = 0
 		while (this.pages[index] === undefined && this.hasNextPage()) {
-			await this.fetchNextPage()
+			const result = await this.fetchNextPage()
+			if (typeof result !== "symbol") {
+				quotaUsed += result.quotaUsed
+			}
 		}
+		return quotaUsed
 	}
 
 	addPage(page) {

src/lib/structures/TimelineEntry.js

@@ -32,7 +32,7 @@ class TimelineEntry extends TimelineBaseMethods {
 
 	async update() {
 		return collectors.fetchShortcodeData(this.data.shortcode).then(data => {
-			this.applyN3(data)
+			this.applyN3(data.result)
 		}).catch(error => {
 			console.error("TimelineEntry could not self-update; trying to continue anyway...")
 			console.error("E:", error)

src/site/api/feed.js

@@ -57,7 +57,7 @@ module.exports = [
 		route: `/u/(${constants.external.username_regex})/(rss|atom)\\.xml`, methods: ["GET"], code: ({fill}) => {
 			const kind = fill[1]
 			if (constants.feeds.enabled) {
-				return fetchUser(fill[0], constants.symbols.fetch_context.RSS).then(async user => {
+				return fetchUser(fill[0], constants.symbols.fetch_context.RSS).then(async ({user}) => {
 					const feed = await user.timeline.fetchFeed()
 					if (constants.feeds.feed_message.enabled) {
 						addAnnouncementFeedItem(feed)

src/site/api/routes.js

@@ -6,6 +6,7 @@ const {render, redirect, getStaticURL} = require("pinski/plugins")
 const {pugCache} = require("../passthrough")
 const {getSettings} = require("./utils/getsettings")
 const {getSettingsReferrer} = require("./utils/settingsreferrer")
+const quota = require("../../lib/quota")
 
 /** @param {import("../../lib/structures/TimelineEntry")} post */
 function getPageTitle(post) {
@@ -66,21 +67,38 @@ module.exports = [
 		}
 	},
 	{
-		route: `/u/(${constants.external.username_regex})(/channel)?`, methods: ["GET"], code: ({req, url, fill}) => {
+		route: `/u/(${constants.external.username_regex})(/channel)?`, methods: ["GET"], code: async ({req, url, fill}) => {
 			const username = fill[0]
 			const type = fill[1] ? "igtv" : "timeline"
 
 			if (username !== username.toLowerCase()) { // some capital letters
-				return Promise.resolve(redirect(`/u/${username.toLowerCase()}`, 301))
+				return redirect(`/u/${username.toLowerCase()}`, 301)
 			}
 
 			const settings = getSettings(req)
 			const params = url.searchParams
-			return fetchUser(username).then(async user => {
+
+			try {
+				if (quota.remaining(req) === 0) {
+					throw constants.symbols.QUOTA_REACHED
+				}
+
+				const {user, quotaUsed} = await fetchUser(username)
+				let remaining = quota.add(req, quotaUsed)
+
 				const selectedTimeline = user[type]
 				let pageNumber = +params.get("page")
 				if (isNaN(pageNumber) || pageNumber < 1) pageNumber = 1
-				await selectedTimeline.fetchUpToPage(pageNumber - 1)
+				const pageIndex = pageNumber - 1
+
+				const pagesNeeded = pageNumber - selectedTimeline.pages.length
+				if (pagesNeeded > remaining) {
+					throw constants.symbols.QUOTA_REACHED
+				}
+
+				const quotaUsed2 = await selectedTimeline.fetchUpToPage(pageIndex)
+				remaining = quota.add(req, quotaUsed2)
+
 				const followerCountsAvailable = !(user.constructor.name === "ReelUser" && user.following === 0 && user.followedBy === 0)
 				return render(200, "pug/user.pug", {
 					url,
@@ -90,9 +108,10 @@ module.exports = [
 					followerCountsAvailable,
 					constants,
 					settings,
-					settingsReferrer: getSettingsReferrer(req.url)
+					settingsReferrer: getSettingsReferrer(req.url),
+					remaining
 				})
-			}).catch(error => {
+			} catch (error) {
 				if (error === constants.symbols.NOT_FOUND || error === constants.symbols.ENDPOINT_OVERRIDDEN) {
 					return render(404, "pug/friendlyerror.pug", {
 						statusCode: 404,
@@ -119,10 +138,21 @@ module.exports = [
 					}
 				} else if (error === constants.symbols.extractor_results.AGE_RESTRICTED) {
 					return render(403, "pug/age_gated.pug", {settings})
+				} else if (error === constants.symbols.QUOTA_REACHED) {
+					return render(429, "pug/friendlyerror.pug", {
+						title: "Quota reached",
+						statusCode: 429,
+						message: "Quota reached",
+						explanation:
+							"Each person has a limited number of requests to Bibliogram."
+							+"\nYou have reached that limit."
+							+"\nWait a while to for your counter to reset.\n",
+						withInstancesLink: true
+					})
 				} else {
 					throw error
 				}
-			})
+			}
 		}
 	},
 	{
@@ -139,11 +169,27 @@ module.exports = [
 			let type = url.searchParams.get("type")
 			if (!["timeline", "igtv"].includes(type)) type = "timeline"
 
-			const settings = getSettings(req)
+			try {
-			return fetchUser(username).then(async user => {
+				if (quota.remaining(req) === 0) {
+					throw constants.symbols.QUOTA_REACHED
+				}
+
+				const settings = getSettings(req)
+
+				const {user, quotaUsed} = await fetchUser(username)
+				const remaining = quota.add(req, quotaUsed)
+
 				const pageIndex = pageNumber - 1
 				const selectedTimeline = user[type]
-				await selectedTimeline.fetchUpToPage(pageIndex)
+
+				const pagesNeeded = pageNumber - selectedTimeline.pages.length
+				if (pagesNeeded > remaining) {
+					throw constants.symbols.QUOTA_REACHED
+				}
+
+				const quotaUsed2 = await selectedTimeline.fetchUpToPage(pageIndex)
+				quota.add(req, quotaUsed2)
+
 				if (selectedTimeline.pages[pageIndex]) {
 					return render(200, "pug/fragments/timeline_page.pug", {page: selectedTimeline.pages[pageIndex], selectedTimeline, type, pageIndex, user, url, settings})
 				} else {
@@ -153,7 +199,7 @@ module.exports = [
 						content: "That page does not exist."
 					}
 				}
-			}).catch(error => {
+			} catch (error) {
 				if (error === constants.symbols.NOT_FOUND || error === constants.symbols.ENDPOINT_OVERRIDDEN) {
 					return render(404, "pug/friendlyerror.pug", {
 						statusCode: 404,
@@ -163,10 +209,12 @@ module.exports = [
 					})
 				} else if (error === constants.symbols.INSTAGRAM_DEMANDS_LOGIN || error === constants.symbols.RATE_LIMITED) {
 					return render(503, "pug/fragments/timeline_loading_blocked.pug")
+				} else if (error === constants.symbols.QUOTA_REACHED) {
+					return render(429, "pug/fragments/timeline_quota_reached.pug")
 				} else {
 					throw error
 				}
-			})
+			}
 		}
 	},
 	{

src/site/assistant_api/user.js

@@ -65,7 +65,7 @@ module.exports = [
 				delete saved.updated_version
 				return Promise.resolve(replyWithUserData(saved))
 			} else {
-				return collectors.fetchUser(username, constants.symbols.fetch_context.ASSISTANT).then(user => {
+				return collectors.fetchUser(username, constants.symbols.fetch_context.ASSISTANT).then(({user}) => {
 					return replyWithUserData({
 						username: user.data.username,
 						user_id: user.data.id,

src/site/html/static/js/pagination.js

@@ -1,4 +1,5 @@
 import {ElemJS, q} from "./elemjs/elemjs.js"
+import {quota} from "./quota.js"
 
 class FreezeWidth extends ElemJS {
 	freeze(text) {
@@ -79,6 +80,7 @@ class NextPage extends FreezeWidth {
 		const type = this.element.getAttribute("data-type")
 
 		return fetch(`/fragment/user/${this.element.getAttribute("data-username")}/${this.nextPageNumber}?type=${type}`).then(res => res.text()).then(text => {
+			quota.change(-1)
 			q("#next-page-container").remove()
 			this.observer.disconnect()
 			q("#timeline").insertAdjacentHTML("beforeend", text)

src/site/html/static/js/quota.js

@@ -0,0 +1,18 @@
+import {ElemJS, q} from "./elemjs/elemjs.js"
+
+class Quota extends ElemJS {
+	constructor() {
+		super(q("#quota"))
+		this.value = +this.element.textContent
+	}
+
+	change(difference) {
+		this.value += difference
+		this.value = Math.max(0, this.value)
+		this.text(this.value)
+	}
+}
+
+const quota = new Quota()
+
+export {quota}

src/site/pug/fragments/timeline_quota_reached.pug

@@ -0,0 +1,7 @@
+.error-fragment
+	.message Quota reached
+	.explanation.
+		Each person has a limited number of requests to Bibliogram.
+		You have reached that limit. You cannot load any more data on this instance.
+		Your quota will reset automatically after some time has passed.
+		Or, you could try #[a(href="https://git.sr.ht/~cadence/bibliogram-docs/tree/master/docs/Instances.md") browsing Bibliogram on another instance.]

src/site/pug/user.pug

@@ -95,6 +95,8 @@ html
 						.links
 							a(href="/")= ll.t_home
 							a(href=settingsReferrer)= ll.t_settings
+						if constants.quota.enabled
+							.quota Quota left: #[span#quota= remaining]
 
 			- const hasPosts = !user.data.is_private && selectedTimeline.pages.length && selectedTimeline.pages[0].length
 			.timeline-section

src/site/sass/includes/_main.sass

@@ -163,6 +163,9 @@ body
 			> *
 				margin: 5px 8px
 
+		.quota
+			margin: 15px 0px
+
 		.bibliogram-meta
 			margin: 20px 10px
 			border-top: map-get($theme, "edge-context-divider")

test/screenshot.js

@@ -5,10 +5,15 @@ const fs = require("fs").promises
 const Jimp = require("jimp")
 const commands = require("./screenshots/commands")
 const child_process = require("child_process")
+const constants = require("../src/lib/constants")
 
 const browser = "firefox"
 
-const origin = "http://localhost:10407"
+const port = 10407 + (+process.env.TAP_CHILD_ID)
+constants.port = port
+constants.request_backend = "saved" // predictable request results
+const origin = `http://localhost:${port}`
+constants.website_origin = origin
 
 const dimensions = new Map([
 	["firefox", {
@@ -25,9 +30,6 @@ const dimensions = new Map([
 
 const browserDimensions = dimensions.get(browser)
 
-const constants = require("../src/lib/constants")
-constants.request_backend = "saved" // predictable request results
-
 process.chdir("src/site")
 const server = require("../src/site/server")