From 112d9cc90eeaca836bc9def6bddab744e7bf5daa Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Thu, 23 Jul 2020 00:58:21 +1200 Subject: [PATCH] Add request quota system --- src/lib/cache.js | 8 +-- src/lib/collectors.js | 30 ++++---- src/lib/constants.js | 9 +++ src/lib/quota/LimitByFrame.js | 54 ++++++++++++++ src/lib/quota/get_identifier.js | 25 +++++++ src/lib/quota/index.js | 33 +++++++++ src/lib/structures/Timeline.js | 12 +++- src/lib/structures/TimelineEntry.js | 2 +- src/site/api/feed.js | 2 +- src/site/api/routes.js | 72 +++++++++++++++---- src/site/assistant_api/user.js | 2 +- src/site/html/static/js/pagination.js | 2 + src/site/html/static/js/quota.js | 18 +++++ .../pug/fragments/timeline_quota_reached.pug | 7 ++ src/site/pug/user.pug | 2 + src/site/sass/includes/_main.sass | 3 + test/screenshot.js | 10 +-- 17 files changed, 253 insertions(+), 38 deletions(-) create mode 100644 src/lib/quota/LimitByFrame.js create mode 100644 src/lib/quota/get_identifier.js create mode 100644 src/lib/quota/index.js create mode 100644 src/site/html/static/js/quota.js create mode 100644 src/site/pug/fragments/timeline_quota_reached.pug diff --git a/src/lib/cache.js b/src/lib/cache.js index a35ffcf..b683def 100644 --- a/src/lib/cache.js +++ b/src/lib/cache.js @@ -106,15 +106,15 @@ class RequestCache extends TtlCache { /** * @param {string} key * @param {() => Promise} callback - * @returns {Promise} + * @returns {Promise<{result: T, fromCache: boolean}>} */ getOrFetch(key, callback) { this.cleanKey(key) - if (this.cache.has(key)) return Promise.resolve(this.get(key)) + if (this.cache.has(key)) return Promise.resolve({result: this.get(key), fromCache: true}) else { const pending = callback().then(result => { this.set(key, result) - return result + return {result, fromCache: false} }) this.set(key, pending) return pending @@ -124,7 +124,7 @@ class RequestCache extends TtlCache { /** * @param {string} key * @param {() => Promise} callback - * @returns {Promise} + * @returns {Promise<{result: T, fromCache: boolean}>} */ getOrFetchPromise(key, callback) { return this.getOrFetch(key, callback).then(result => { diff --git a/src/lib/collectors.js b/src/lib/collectors.js index 93a2044..6e23fd1 100644 --- a/src/lib/collectors.js +++ b/src/lib/collectors.js @@ -83,7 +83,7 @@ async function fetchUser(username, context) { /** * @param {string} username - * @returns {Promise} + * @returns {Promise<{user: import("./structures/User"), quotaUsed: number}>} */ function fetchUserFromHTML(username) { if (constants.caching.self_blocked_status.enabled) { @@ -151,7 +151,7 @@ function fetchUserFromHTML(username) { } throw error }) - }) + }).then(user => ({user, quotaUsed: 0})) } /** @@ -190,7 +190,7 @@ function updateProfilePictureFromReel(userID) { /** * @param {string} userID * @param {string} username - * @returns {Promise} + * @returns {Promise<{user: import("./structures/ReelUser")|import("./structures/User"), quotaUsed: number}>} */ function fetchUserFromCombined(userID, username) { // Fetch basic user information @@ -220,11 +220,13 @@ function fetchUserFromCombined(userID, username) { }) }).then(async user => { // Add first timeline page + let quotaUsed = 0 if (!user.timeline.pages[0]) { - const page = await fetchTimelinePage(userID, "") - user.timeline.addPage(page) + const fetched = await fetchTimelinePage(userID, "") + if (!fetched.fromCache) quotaUsed++ + user.timeline.addPage(fetched.result) } - return user + return {user, quotaUsed} }).catch(error => { if (error === constants.symbols.RATE_LIMITED) { history.report("reel", false) @@ -234,6 +236,7 @@ function fetchUserFromCombined(userID, username) { } function fetchUserFromSaved(saved) { + let quotaUsed = 0 return userRequestCache.getOrFetch("user/"+saved.username, false, true, async () => { // require down here or have to deal with require loop. require cache will take care of it anyway. // ReelUser -> Timeline -> TimelineEntry -> collectors -/> ReelUser @@ -252,17 +255,20 @@ function fetchUserFromSaved(saved) { }) // Add first timeline page if (!user.timeline.pages[0]) { - const page = await fetchTimelinePage(user.data.id, "") + const {result: page, fromCache} = await fetchTimelinePage(user.data.id, "") + if (!fromCache) quotaUsed++ user.timeline.addPage(page) } return user + }).then(user => { + return {user, quotaUsed} }) } /** * @param {string} userID * @param {string} after - * @returns {Promise>} + * @returns {Promise<{result: import("./types").PagedEdges, fromCache: boolean}>} */ function fetchTimelinePage(userID, after) { const p = new URLSearchParams() @@ -298,7 +304,7 @@ function fetchTimelinePage(userID, after) { /** * @param {string} userID * @param {string} after - * @returns {Promise>} + * @returns {Promise<{result: import("./types").PagedEdges, fromCache: boolean}>} */ function fetchIGTVPage(userID, after) { const p = new URLSearchParams() @@ -329,7 +335,7 @@ function fetchIGTVPage(userID, after) { /** * @param {string} userID * @param {string} username - * @returns {Promise} + * @returns {Promise<{result: boolean, fromCache: boolean}>} */ function verifyUserPair(userID, username) { // Fetch basic user information @@ -378,14 +384,14 @@ async function getOrFetchShortcode(shortcode) { } else { const data = await fetchShortcodeData(shortcode) const entry = getOrCreateShortcode(shortcode) - entry.applyN3(data) + entry.applyN3(data.result) return entry } } /** * @param {string} shortcode - * @returns {Promise} + * @returns {Promise<{result: import("./types").TimelineEntryN3, fromCache: boolean}>} */ function fetchShortcodeData(shortcode) { // example actual query from web: diff --git a/src/lib/constants.js b/src/lib/constants.js index 92a06b8..2a1522d 100644 --- a/src/lib/constants.js +++ b/src/lib/constants.js @@ -112,6 +112,14 @@ let constants = { rewrite_twitter: "nitter.net" }, + quota: { + enabled: false, + timeframe: 20*60*60*1000, + count: 50, + ip_mode: "header", // one of: "header", "address" + ip_header: "x-forwarded-for" + }, + user_settings: [ { name: "language", @@ -255,6 +263,7 @@ let constants = { RATE_LIMITED: Symbol("RATE_LIMITED"), ENDPOINT_OVERRIDDEN: Symbol("ENDPOINT_OVERRIDDEN"), NO_ASSISTANTS_AVAILABLE: Symbol("NO_ASSISTANTS_AVAILABLE"), + QUOTA_REACHED: Symbol("QUOTA_REACHED"), extractor_results: { SUCCESS: Symbol("SUCCESS"), AGE_RESTRICTED: Symbol("AGE_RESTRICTED"), diff --git a/src/lib/quota/LimitByFrame.js b/src/lib/quota/LimitByFrame.js new file mode 100644 index 0000000..a787e11 --- /dev/null +++ b/src/lib/quota/LimitByFrame.js @@ -0,0 +1,54 @@ +const constants = require("../constants") + +class Frame { + constructor(identifier) { + this.identifier = identifier + this.frameStartedAt = 0 + this.count = 0 + } + + refresh() { + if (Date.now() > this.frameStartedAt + constants.quota.timeframe) { + this.count = 0 + } + } + + remaining() { + this.refresh() + return Math.max(constants.quota.count - this.count, 0) + } + + add(count) { + this.refresh() + if (this.count === 0) this.frameStartedAt = Date.now() + this.count += count + return this.remaining() + } +} + +class LimitByFrame { + constructor() { + /** @type {Map} */ + this.frames = new Map() + } + + getOrCreateFrame(identifier) { + if (!this.frames.has(identifier)) { + const frame = new Frame(identifier) + this.frames.set(identifier, frame) + return frame + } else { + return this.frames.get(identifier) + } + } + + remaining(identifier) { + return this.getOrCreateFrame(identifier).remaining() + } + + add(identifier, count) { + return this.getOrCreateFrame(identifier).add(count) + } +} + +module.exports = LimitByFrame diff --git a/src/lib/quota/get_identifier.js b/src/lib/quota/get_identifier.js new file mode 100644 index 0000000..52454c9 --- /dev/null +++ b/src/lib/quota/get_identifier.js @@ -0,0 +1,25 @@ +const {request} = require("../utils/request") +const {log} = require("pinski/util/common") + +let addresses = [] + +request("https://check.torproject.org/torbulkexitlist").text().then(text => { + const lines = text.split("\n").filter(l => l) + addresses = addresses.concat(lines) + log(`Loaded Tor exit node list (${addresses.length} total)`, "spam") +}) + +/* + request("https://meta.bibliogram.art/ip_proxy_list.txt").text().then(text => { + const lines = text.split("\n").filter(l => l) + addresses = addresses.concat(lines) + log(`Loaded Bibliogram proxy list (${addresses.length} total)`, "spam") + }) +*/ + +function getIdentifier(address) { + if (addresses.includes(address)) return "proxy" + else return address +} + +module.exports.getIdentifier = getIdentifier diff --git a/src/lib/quota/index.js b/src/lib/quota/index.js new file mode 100644 index 0000000..a424cb2 --- /dev/null +++ b/src/lib/quota/index.js @@ -0,0 +1,33 @@ +const constants = require("../constants") +const LimitByFrame = require("./LimitByFrame") +const {getIdentifier} = require("./get_identifier") +require("../testimports")(LimitByFrame, getIdentifier) + +const limiter = new LimitByFrame() + +function getIPFromReq(req) { + if (constants.quota.ip_mode === "header") { + return req.headers[constants.quota.ip_header] + } else { // constants.quota.ip_mode === "address" + return req.connection.remoteAddress + } +} + +function remaining(req) { + if (!constants.quota.enabled) return Infinity // sure. + + const ip = getIPFromReq(req) + const identifier = getIdentifier(ip) + return limiter.remaining(identifier) +} + +function add(req, count) { + if (!constants.quota.enabled) return Infinity // why not. + + const ip = getIPFromReq(req) + const identifier = getIdentifier(ip) + return limiter.add(identifier, count) +} + +module.exports.remaining = remaining +module.exports.add = add diff --git a/src/lib/structures/Timeline.js b/src/lib/structures/Timeline.js index 6616d3c..1e444f4 100644 --- a/src/lib/structures/Timeline.js +++ b/src/lib/structures/Timeline.js @@ -45,16 +45,22 @@ class Timeline { : this.type === "igtv" ? collectors.fetchIGTVPage : null const after = this.page_info ? this.page_info.end_cursor : "" - return method(this.user.data.id, after).then(page => { + return method(this.user.data.id, after).then(({result: page, fromCache}) => { + const quotaUsed = fromCache ? 0 : 1 this.addPage(page) - return this.pages.slice(-1)[0] + return {page: this.pages.slice(-1)[0], quotaUsed} }) } async fetchUpToPage(index) { + let quotaUsed = 0 while (this.pages[index] === undefined && this.hasNextPage()) { - await this.fetchNextPage() + const result = await this.fetchNextPage() + if (typeof result !== "symbol") { + quotaUsed += result.quotaUsed + } } + return quotaUsed } addPage(page) { diff --git a/src/lib/structures/TimelineEntry.js b/src/lib/structures/TimelineEntry.js index 1b0ce8d..a06baff 100644 --- a/src/lib/structures/TimelineEntry.js +++ b/src/lib/structures/TimelineEntry.js @@ -32,7 +32,7 @@ class TimelineEntry extends TimelineBaseMethods { async update() { return collectors.fetchShortcodeData(this.data.shortcode).then(data => { - this.applyN3(data) + this.applyN3(data.result) }).catch(error => { console.error("TimelineEntry could not self-update; trying to continue anyway...") console.error("E:", error) diff --git a/src/site/api/feed.js b/src/site/api/feed.js index 9569a07..89f1902 100644 --- a/src/site/api/feed.js +++ b/src/site/api/feed.js @@ -57,7 +57,7 @@ module.exports = [ route: `/u/(${constants.external.username_regex})/(rss|atom)\\.xml`, methods: ["GET"], code: ({fill}) => { const kind = fill[1] if (constants.feeds.enabled) { - return fetchUser(fill[0], constants.symbols.fetch_context.RSS).then(async user => { + return fetchUser(fill[0], constants.symbols.fetch_context.RSS).then(async ({user}) => { const feed = await user.timeline.fetchFeed() if (constants.feeds.feed_message.enabled) { addAnnouncementFeedItem(feed) diff --git a/src/site/api/routes.js b/src/site/api/routes.js index 5a945bb..e23d78b 100644 --- a/src/site/api/routes.js +++ b/src/site/api/routes.js @@ -6,6 +6,7 @@ const {render, redirect, getStaticURL} = require("pinski/plugins") const {pugCache} = require("../passthrough") const {getSettings} = require("./utils/getsettings") const {getSettingsReferrer} = require("./utils/settingsreferrer") +const quota = require("../../lib/quota") /** @param {import("../../lib/structures/TimelineEntry")} post */ function getPageTitle(post) { @@ -66,21 +67,38 @@ module.exports = [ } }, { - route: `/u/(${constants.external.username_regex})(/channel)?`, methods: ["GET"], code: ({req, url, fill}) => { + route: `/u/(${constants.external.username_regex})(/channel)?`, methods: ["GET"], code: async ({req, url, fill}) => { const username = fill[0] const type = fill[1] ? "igtv" : "timeline" if (username !== username.toLowerCase()) { // some capital letters - return Promise.resolve(redirect(`/u/${username.toLowerCase()}`, 301)) + return redirect(`/u/${username.toLowerCase()}`, 301) } const settings = getSettings(req) const params = url.searchParams - return fetchUser(username).then(async user => { + + try { + if (quota.remaining(req) === 0) { + throw constants.symbols.QUOTA_REACHED + } + + const {user, quotaUsed} = await fetchUser(username) + let remaining = quota.add(req, quotaUsed) + const selectedTimeline = user[type] let pageNumber = +params.get("page") if (isNaN(pageNumber) || pageNumber < 1) pageNumber = 1 - await selectedTimeline.fetchUpToPage(pageNumber - 1) + const pageIndex = pageNumber - 1 + + const pagesNeeded = pageNumber - selectedTimeline.pages.length + if (pagesNeeded > remaining) { + throw constants.symbols.QUOTA_REACHED + } + + const quotaUsed2 = await selectedTimeline.fetchUpToPage(pageIndex) + remaining = quota.add(req, quotaUsed2) + const followerCountsAvailable = !(user.constructor.name === "ReelUser" && user.following === 0 && user.followedBy === 0) return render(200, "pug/user.pug", { url, @@ -90,9 +108,10 @@ module.exports = [ followerCountsAvailable, constants, settings, - settingsReferrer: getSettingsReferrer(req.url) + settingsReferrer: getSettingsReferrer(req.url), + remaining }) - }).catch(error => { + } catch (error) { if (error === constants.symbols.NOT_FOUND || error === constants.symbols.ENDPOINT_OVERRIDDEN) { return render(404, "pug/friendlyerror.pug", { statusCode: 404, @@ -119,10 +138,21 @@ module.exports = [ } } else if (error === constants.symbols.extractor_results.AGE_RESTRICTED) { return render(403, "pug/age_gated.pug", {settings}) + } else if (error === constants.symbols.QUOTA_REACHED) { + return render(429, "pug/friendlyerror.pug", { + title: "Quota reached", + statusCode: 429, + message: "Quota reached", + explanation: + "Each person has a limited number of requests to Bibliogram." + +"\nYou have reached that limit." + +"\nWait a while to for your counter to reset.\n", + withInstancesLink: true + }) } else { throw error } - }) + } } }, { @@ -139,11 +169,27 @@ module.exports = [ let type = url.searchParams.get("type") if (!["timeline", "igtv"].includes(type)) type = "timeline" - const settings = getSettings(req) - return fetchUser(username).then(async user => { + try { + if (quota.remaining(req) === 0) { + throw constants.symbols.QUOTA_REACHED + } + + const settings = getSettings(req) + + const {user, quotaUsed} = await fetchUser(username) + const remaining = quota.add(req, quotaUsed) + const pageIndex = pageNumber - 1 const selectedTimeline = user[type] - await selectedTimeline.fetchUpToPage(pageIndex) + + const pagesNeeded = pageNumber - selectedTimeline.pages.length + if (pagesNeeded > remaining) { + throw constants.symbols.QUOTA_REACHED + } + + const quotaUsed2 = await selectedTimeline.fetchUpToPage(pageIndex) + quota.add(req, quotaUsed2) + if (selectedTimeline.pages[pageIndex]) { return render(200, "pug/fragments/timeline_page.pug", {page: selectedTimeline.pages[pageIndex], selectedTimeline, type, pageIndex, user, url, settings}) } else { @@ -153,7 +199,7 @@ module.exports = [ content: "That page does not exist." } } - }).catch(error => { + } catch (error) { if (error === constants.symbols.NOT_FOUND || error === constants.symbols.ENDPOINT_OVERRIDDEN) { return render(404, "pug/friendlyerror.pug", { statusCode: 404, @@ -163,10 +209,12 @@ module.exports = [ }) } else if (error === constants.symbols.INSTAGRAM_DEMANDS_LOGIN || error === constants.symbols.RATE_LIMITED) { return render(503, "pug/fragments/timeline_loading_blocked.pug") + } else if (error === constants.symbols.QUOTA_REACHED) { + return render(429, "pug/fragments/timeline_quota_reached.pug") } else { throw error } - }) + } } }, { diff --git a/src/site/assistant_api/user.js b/src/site/assistant_api/user.js index f6fb393..140d82a 100644 --- a/src/site/assistant_api/user.js +++ b/src/site/assistant_api/user.js @@ -65,7 +65,7 @@ module.exports = [ delete saved.updated_version return Promise.resolve(replyWithUserData(saved)) } else { - return collectors.fetchUser(username, constants.symbols.fetch_context.ASSISTANT).then(user => { + return collectors.fetchUser(username, constants.symbols.fetch_context.ASSISTANT).then(({user}) => { return replyWithUserData({ username: user.data.username, user_id: user.data.id, diff --git a/src/site/html/static/js/pagination.js b/src/site/html/static/js/pagination.js index 1f26052..dc76b48 100644 --- a/src/site/html/static/js/pagination.js +++ b/src/site/html/static/js/pagination.js @@ -1,4 +1,5 @@ import {ElemJS, q} from "./elemjs/elemjs.js" +import {quota} from "./quota.js" class FreezeWidth extends ElemJS { freeze(text) { @@ -79,6 +80,7 @@ class NextPage extends FreezeWidth { const type = this.element.getAttribute("data-type") return fetch(`/fragment/user/${this.element.getAttribute("data-username")}/${this.nextPageNumber}?type=${type}`).then(res => res.text()).then(text => { + quota.change(-1) q("#next-page-container").remove() this.observer.disconnect() q("#timeline").insertAdjacentHTML("beforeend", text) diff --git a/src/site/html/static/js/quota.js b/src/site/html/static/js/quota.js new file mode 100644 index 0000000..68edd1b --- /dev/null +++ b/src/site/html/static/js/quota.js @@ -0,0 +1,18 @@ +import {ElemJS, q} from "./elemjs/elemjs.js" + +class Quota extends ElemJS { + constructor() { + super(q("#quota")) + this.value = +this.element.textContent + } + + change(difference) { + this.value += difference + this.value = Math.max(0, this.value) + this.text(this.value) + } +} + +const quota = new Quota() + +export {quota} diff --git a/src/site/pug/fragments/timeline_quota_reached.pug b/src/site/pug/fragments/timeline_quota_reached.pug new file mode 100644 index 0000000..59bb6f5 --- /dev/null +++ b/src/site/pug/fragments/timeline_quota_reached.pug @@ -0,0 +1,7 @@ +.error-fragment + .message Quota reached + .explanation. + Each person has a limited number of requests to Bibliogram. + You have reached that limit. You cannot load any more data on this instance. + Your quota will reset automatically after some time has passed. + Or, you could try #[a(href="https://git.sr.ht/~cadence/bibliogram-docs/tree/master/docs/Instances.md") browsing Bibliogram on another instance.] diff --git a/src/site/pug/user.pug b/src/site/pug/user.pug index d5fba4e..6d711f2 100644 --- a/src/site/pug/user.pug +++ b/src/site/pug/user.pug @@ -95,6 +95,8 @@ html .links a(href="/")= ll.t_home a(href=settingsReferrer)= ll.t_settings + if constants.quota.enabled + .quota Quota left: #[span#quota= remaining] - const hasPosts = !user.data.is_private && selectedTimeline.pages.length && selectedTimeline.pages[0].length .timeline-section diff --git a/src/site/sass/includes/_main.sass b/src/site/sass/includes/_main.sass index e5d7ec6..8942530 100644 --- a/src/site/sass/includes/_main.sass +++ b/src/site/sass/includes/_main.sass @@ -163,6 +163,9 @@ body > * margin: 5px 8px + .quota + margin: 15px 0px + .bibliogram-meta margin: 20px 10px border-top: map-get($theme, "edge-context-divider") diff --git a/test/screenshot.js b/test/screenshot.js index 7f1535c..0336d9b 100644 --- a/test/screenshot.js +++ b/test/screenshot.js @@ -5,10 +5,15 @@ const fs = require("fs").promises const Jimp = require("jimp") const commands = require("./screenshots/commands") const child_process = require("child_process") +const constants = require("../src/lib/constants") const browser = "firefox" -const origin = "http://localhost:10407" +const port = 10407 + (+process.env.TAP_CHILD_ID) +constants.port = port +constants.request_backend = "saved" // predictable request results +const origin = `http://localhost:${port}` +constants.website_origin = origin const dimensions = new Map([ ["firefox", { @@ -25,9 +30,6 @@ const dimensions = new Map([ const browserDimensions = dimensions.get(browser) -const constants = require("../src/lib/constants") -constants.request_backend = "saved" // predictable request results - process.chdir("src/site") const server = require("../src/site/server")