Lomanic/bibliogram
1
0
Fork 0
mirror of https://git.sr.ht/~cadence/bibliogram synced 2024-11-26 09:37:28 +00:00
Code Issues Releases Wiki Activity

Use block status to inform future request policy

Browse Source
This commit is contained in:
Cadence Ember 2020-04-17 01:40:20 +12:00
parent d66d12e7da
commit 3adb0d0ac6
No known key found for this signature in database
GPG Key ID: 128B99B1B74A6412
2 changed files with 13 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
src/lib/collectors.js
View File

@ -87,6 +87,14 @@ async function fetchUser(username, context) {
*/ */
function fetchUserFromHTML(username) { function fetchUserFromHTML(username) {
return userRequestCache.getOrFetch("user/"+username, false, true, () => { return userRequestCache.getOrFetch("user/"+username, false, true, () => {
if (constants.caching.self_blocked_status.enabled) {
if (history.store.has("user")) {
const entry = history.store.get("user")
if (!entry.lastRequestSuccessful && Date.now() < entry.lastRequestAt + constants.caching.self_blocked_status.time) {
return Promise.reject(constants.symbols.RATE_LIMITED)
}
}
}
return switcher.request("user_html", `https://www.instagram.com/${username}/`, async res => { return switcher.request("user_html", `https://www.instagram.com/${username}/`, async res => {
if (res.status === 301) throw constants.symbols.ENDPOINT_OVERRIDDEN if (res.status === 301) throw constants.symbols.ENDPOINT_OVERRIDDEN
if (res.status === 302) throw constants.symbols.INSTAGRAM_DEMANDS_LOGIN if (res.status === 302) throw constants.symbols.INSTAGRAM_DEMANDS_LOGIN
@ -110,7 +118,7 @@ function fetchUserFromHTML(username) {
const existing = db.prepare("SELECT created, updated_version FROM Users WHERE username = ?").get(user.data.username) const existing = db.prepare("SELECT created, updated_version FROM Users WHERE username = ?").get(user.data.username)
db.prepare( db.prepare(
"REPLACE INTO Users (username, user_id, created, updated, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url) VALUES " "REPLACE INTO Users (username, user_id, created, updated, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url) VALUES "
+"(@username, @user_id, @created, @updated, @updated_version, @biography, @post_count, @following_count, @followed_by_count, @external_url, @full_name, @is_private, @is_verified, @profile_pic_url)" +"(@username, @user_id, @created, @updated, @updated_version, @biography, @post_count, @following_count, @followed_by_count, @external_url, @full_name, @is_private, @is_verified, @profile_pic_url)"
).run({ ).run({
username: user.data.username, username: user.data.username,
user_id: user.data.id, user_id: user.data.id,

4
src/lib/constants.js
View File

@ -63,6 +63,10 @@ let constants = {
instance_list_cache_time: 3*60*1000, instance_list_cache_time: 3*60*1000,
updater_cache_time: 2*60*1000, updater_cache_time: 2*60*1000,
cache_sweep_interval: 3*60*1000, cache_sweep_interval: 3*60*1000,
self_blocked_status: {
enabled: false,
time: 2*60*60*1000,
},
db_user_id: true, db_user_id: true,
db_post_n3: true, db_post_n3: true,
db_request_history: false db_request_history: false