From 3adb0d0ac6cd8334ff7dbd70734601819aae1fd7 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Fri, 17 Apr 2020 01:40:20 +1200 Subject: [PATCH] Use block status to inform future request policy --- src/lib/collectors.js | 10 +++++++++- src/lib/constants.js | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/lib/collectors.js b/src/lib/collectors.js index 6df6a47..e054251 100644 --- a/src/lib/collectors.js +++ b/src/lib/collectors.js @@ -87,6 +87,14 @@ async function fetchUser(username, context) { */ function fetchUserFromHTML(username) { return userRequestCache.getOrFetch("user/"+username, false, true, () => { + if (constants.caching.self_blocked_status.enabled) { + if (history.store.has("user")) { + const entry = history.store.get("user") + if (!entry.lastRequestSuccessful && Date.now() < entry.lastRequestAt + constants.caching.self_blocked_status.time) { + return Promise.reject(constants.symbols.RATE_LIMITED) + } + } + } return switcher.request("user_html", `https://www.instagram.com/${username}/`, async res => { if (res.status === 301) throw constants.symbols.ENDPOINT_OVERRIDDEN if (res.status === 302) throw constants.symbols.INSTAGRAM_DEMANDS_LOGIN @@ -110,7 +118,7 @@ function fetchUserFromHTML(username) { const existing = db.prepare("SELECT created, updated_version FROM Users WHERE username = ?").get(user.data.username) db.prepare( "REPLACE INTO Users (username, user_id, created, updated, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url) VALUES " - +"(@username, @user_id, @created, @updated, @updated_version, @biography, @post_count, @following_count, @followed_by_count, @external_url, @full_name, @is_private, @is_verified, @profile_pic_url)" + +"(@username, @user_id, @created, @updated, @updated_version, @biography, @post_count, @following_count, @followed_by_count, @external_url, @full_name, @is_private, @is_verified, @profile_pic_url)" ).run({ username: user.data.username, user_id: user.data.id, diff --git a/src/lib/constants.js b/src/lib/constants.js index 61a70b5..2d75600 100644 --- a/src/lib/constants.js +++ b/src/lib/constants.js @@ -63,6 +63,10 @@ let constants = { instance_list_cache_time: 3*60*1000, updater_cache_time: 2*60*1000, cache_sweep_interval: 3*60*1000, + self_blocked_status: { + enabled: false, + time: 2*60*60*1000, + }, db_user_id: true, db_post_n3: true, db_request_history: false