Lomanic
/
bibliogram
mirror of https://git.sr.ht/~cadence/bibliogram
1
0
Fork 0
Code Issues Releases Wiki Activity

Add preferRSS setting

This commit is contained in:
Cadence Fish 2020-02-03 03:53:37 +13:00
parent c5d8b5788b
commit 6e136dc77a
No known key found for this signature in database
GPG Key ID: 81015DF9AA8607E1
7 changed files with 84 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

57
src/lib/cache.js
View File

@ -85,6 +85,10 @@ class TtlCache {
}
}
/**
* @extends TtlCache<T>
* @template T
*/
class RequestCache extends TtlCache {
/**
* @param {number} ttl time to keep each resource in milliseconds
@ -97,7 +101,6 @@ class RequestCache extends TtlCache {
* @param {string} key
* @param {() => Promise<T>} callback
* @returns {Promise<T>}
* @template T
*/
getOrFetch(key, callback) {
this.cleanKey(key)
@ -116,7 +119,6 @@ class RequestCache extends TtlCache {
* @param {string} key
* @param {() => Promise<T>} callback
* @returns {Promise<T>}
* @template T
*/
getOrFetchPromise(key, callback) {
return this.getOrFetch(key, callback).then(result => {
@ -126,5 +128,56 @@ class RequestCache extends TtlCache {
}
}
/**
* @template T
*/
class UserRequestCache extends TtlCache {
constructor(ttl) {
super(ttl)
/** @type {Map<string, {data: T, isReel: boolean, isFailedPromise: boolean, htmlFailed: boolean, time: number}>} */
this.cache
}
/**
* @param {string} key
* @param {boolean} isReel
* @param {any} [data]
*/
set(key, isReel, data) {
const existing = this.cache.get(key)
// Preserve html failure status if now requesting as reel
const htmlFailed = isReel && existing && existing.htmlFailed
this.cache.set(key, {data, isReel, isFailedPromise: false, htmlFailed, time: Date.now()})
}
/**
* @param {string} key
* @param {boolean} isHtmlPreferred
* @param {boolean} willFetchReel
* @param {() => Promise<T>} callback
* @returns {Promise<T>}
*/
getOrFetch(key, willFetchReel, isHtmlPreferred, callback) {
this.cleanKey(key)
if (this.cache.has(key)) {
const existing = this.cache.get(key)
if ((!existing.isReel || !isHtmlPreferred || existing.htmlFailed) && !existing.isFailedPromise) return Promise.resolve(existing.data)
}
const pending = callback().then(result => {
if (this.getWithoutClean(key) === pending) { // if nothing has replaced the current cache in the meantime
this.set(key, willFetchReel, result)
}
return result
}).catch(error => {
this.cache.get(key).htmlFailed = true
this.cache.get(key).isFailedPromise = true
throw error
})
this.set(key, willFetchReel, pending)
return pending
}
}
module.exports.TtlCache = TtlCache
module.exports.RequestCache = RequestCache
module.exports.UserRequestCache = UserRequestCache

27
src/lib/collectors.js
View File

@ -2,20 +2,26 @@ const constants = require("./constants")
const {request} = require("./utils/request")
const switcher = require("./utils/torswitcher")
const {extractSharedData} = require("./utils/body")
const {TtlCache, RequestCache} = require("./cache")
const {TtlCache, RequestCache, UserRequestCache} = require("./cache")
const RequestHistory = require("./structures/RequestHistory")
const db = require("./db")
require("./testimports")(constants, request, extractSharedData, RequestCache, RequestHistory)
require("./testimports")(constants, request, extractSharedData, UserRequestCache, RequestHistory)
const requestCache = new RequestCache(constants.caching.resource_cache_time)
const userRequestCache = new UserRequestCache(constants.caching.resource_cache_time)
/** @type {import("./cache").TtlCache<import("./structures/TimelineEntry")>} */
const timelineEntryCache = new TtlCache(constants.caching.resource_cache_time)
const history = new RequestHistory(["user", "timeline", "post", "reel"])
async function fetchUser(username) {
if (constants.allow_user_from_reel === "never") {
async function fetchUser(username, isRSS) {
let mode = constants.allow_user_from_reel
if (mode === "preferForRSS") {
if (isRSS) mode = "prefer"
else mode = "fallback"
}
if (mode === "never") {
return fetchUserFromHTML(username)
} else if (constants.allow_user_from_reel === "prefer") {
} else if (mode === "prefer") {
const userID = db.prepare("SELECT user_id FROM Users WHERE username = ?").pluck().get(username)
if (userID) return fetchUserFromCombined(userID, username)
else return fetchUserFromHTML(username)
@ -24,7 +30,6 @@ async function fetchUser(username) {
if (error === constants.symbols.INSTAGRAM_DEMANDS_LOGIN || error === constants.symbols.RATE_LIMITED) {
const userID = db.prepare("SELECT user_id FROM Users WHERE username = ?").pluck().get(username)
if (userID) {
requestCache.cache.delete("user/"+username)
return fetchUserFromCombined(userID, username)
}
}
@ -34,7 +39,7 @@ async function fetchUser(username) {
}
function fetchUserFromHTML(username) {
return requestCache.getOrFetch("user/"+username, () => {
return userRequestCache.getOrFetch("user/"+username, false, true, () => {
return switcher.request("user_html", `https://www.instagram.com/${username}/`, async res => {
if (res.status === 302) throw constants.symbols.INSTAGRAM_DEMANDS_LOGIN
if (res.status === 429) throw constants.symbols.RATE_LIMITED
@ -74,7 +79,7 @@ function fetchUserFromCombined(userID, username) {
user_id: userID,
include_reel: true
}))
return requestCache.getOrFetch("user/"+username, () => {
return userRequestCache.getOrFetch("user/"+username, true, false, () => {
return switcher.request("reel_graphql", `https://www.instagram.com/graphql/query/?${p.toString()}`, async res => {
if (res.status === 429) throw constants.symbols.RATE_LIMITED
return res
@ -192,8 +197,8 @@ function fetchShortcodeData(shortcode) {
.run({shortcode: data.shortcode, id: data.id, id_as_numeric: data.id, username: data.owner.username, json: JSON.stringify(data)})
}
// if we have the owner but only a reelUser, update it. this code is gross.
if (requestCache.hasNotPromise("user/"+data.owner.username)) {
const user = requestCache.getWithoutClean("user/"+data.owner.username)
if (userRequestCache.hasNotPromise("user/"+data.owner.username)) {
const user = userRequestCache.getWithoutClean("user/"+data.owner.username)
if (user.fromReel) {
user.data.full_name = data.owner.full_name
user.data.is_verified = data.owner.is_verified
@ -214,7 +219,7 @@ module.exports.fetchUser = fetchUser
module.exports.fetchTimelinePage = fetchTimelinePage
module.exports.getOrCreateShortcode = getOrCreateShortcode
module.exports.fetchShortcodeData = fetchShortcodeData
module.exports.requestCache = requestCache
module.exports.userRequestCache = userRequestCache
module.exports.timelineEntryCache = timelineEntryCache
module.exports.getOrFetchShortcode = getOrFetchShortcode
module.exports.history = history

2
src/lib/constants.js
View File

@ -20,7 +20,7 @@ let constants = {
}
},
allow_user_from_reel: "fallback", // one of: "never", "fallback", "prefer".
allow_user_from_reel: "preferForRSS", // one of: "never", "fallback", "prefer", "preferForRSS"
settings: {
rss_enabled: true

4
src/lib/structures/TimelineEntry.js
View File

@ -180,9 +180,9 @@ class TimelineEntry extends TimelineBaseMethods {
}
// The owner may be in the user cache, so copy from that.
// This could be implemented better.
else if (collectors.requestCache.hasNotPromise("user/"+this.data.owner.username)) {
else if (collectors.userRequestCache.hasNotPromise("user/"+this.data.owner.username)) {
/** @type {import("./User")} */
const user = collectors.requestCache.getWithoutClean("user/"+this.data.owner.username)
const user = collectors.userRequestCache.getWithoutClean("user/"+this.data.owner.username)
if (user.data.full_name) {
this.data.owner = {
id: user.data.id,

8
src/site/api/feed.js
View File

@ -1,12 +1,12 @@
const constants = require("../../lib/constants")
const {fetchUser, requestCache} = require("../../lib/collectors")
const {fetchUser, userRequestCache} = require("../../lib/collectors")
const {render} = require("pinski/plugins")
const {pugCache} = require("../passthrough")
module.exports = [
{route: `/u/(${constants.external.username_regex})/rss.xml`, methods: ["GET"], code: ({fill}) => {
if (constants.settings.rss_enabled) {
return fetchUser(fill[0]).then(async user => {
return fetchUser(fill[0], true).then(async user => {
const content = await user.timeline.fetchFeed()
const xml = content.xml()
return {
@ -27,10 +27,10 @@ module.exports = [
statusCode: 503,
contentType: "text/html",
headers: {
"Retry-After": requestCache.getTtl("user/"+fill[0], 1000)
"Retry-After": userRequestCache.getTtl("user/"+fill[0], 1000)
},
content: pugCache.get("pug/blocked.pug").web({
expiresMinutes: requestCache.getTtl("user/"+fill[0], 1000*60)
expiresMinutes: userRequestCache.getTtl("user/"+fill[0], 1000*60)
})
}
} else {

10
src/site/api/routes.js
View File

@ -1,5 +1,5 @@
const constants = require("../../lib/constants")
const {fetchUser, getOrFetchShortcode, requestCache, history} = require("../../lib/collectors")
const {fetchUser, getOrFetchShortcode, userRequestCache, history} = require("../../lib/collectors")
const {render, redirect} = require("pinski/plugins")
const {pugCache} = require("../passthrough")
@ -34,7 +34,7 @@ module.exports = [
{
route: `/u/(${constants.external.username_regex})`, methods: ["GET"], code: ({url, fill}) => {
const params = url.searchParams
return fetchUser(fill[0]).then(async user => {
return fetchUser(fill[0], false).then(async user => {
const page = +params.get("page")
if (typeof page === "number" && !isNaN(page) && page >= 1) {
await user.timeline.fetchUpToPage(page - 1)
@ -53,10 +53,10 @@ module.exports = [
statusCode: 503,
contentType: "text/html",
headers: {
"Retry-After": requestCache.getTtl("user/"+fill[0], 1000)
"Retry-After": userRequestCache.getTtl("user/"+fill[0], 1000)
},
content: pugCache.get("pug/blocked.pug").web({
expiresMinutes: requestCache.getTtl("user/"+fill[0], 1000*60)
expiresMinutes: userRequestCache.getTtl("user/"+fill[0], 1000*60)
})
}
} else {
@ -67,7 +67,7 @@ module.exports = [
},
{
route: `/fragment/user/(${constants.external.username_regex})/(\\d+)`, methods: ["GET"], code: async ({url, fill}) => {
return fetchUser(fill[0]).then(async user => {
return fetchUser(fill[0], false).then(async user => {
const pageNumber = +fill[1]
const pageIndex = pageNumber - 1
await user.timeline.fetchUpToPage(pageIndex)

2
src/site/repl.js
View File

@ -1,5 +1,5 @@
const {instance, pugCache, wss} = require("./passthrough")
const {requestCache, timelineEntryCache, history} = require("../lib/collectors")
const {userRequestCache, timelineEntryCache, history} = require("../lib/collectors")
const constants = require("../lib/constants")
const util = require("util")
const repl = require("repl")