From b22028aaa416c3dd96f70260ea7bd6ed98674ea7 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Tue, 7 Apr 2020 18:30:00 +1200 Subject: [PATCH] Add experimental assistant feature --- package.json | 1 + src/lib/collectors.js | 10 ++++ src/lib/constants.js | 18 +++++- src/lib/structures/Assistant.js | 42 ++++++++++++++ src/lib/structures/AssistantSwitcher.js | 49 ++++++++++++++++ src/site/api/routes.js | 2 +- src/site/assistant.js | 40 +++++++++++++ src/site/assistant_api/user.js | 75 +++++++++++++++++++++++++ src/site/repl.js | 1 + src/site/server.js | 1 - 10 files changed, 236 insertions(+), 3 deletions(-) create mode 100644 src/lib/structures/Assistant.js create mode 100644 src/lib/structures/AssistantSwitcher.js create mode 100644 src/site/assistant.js create mode 100644 src/site/assistant_api/user.js diff --git a/package.json b/package.json index 773ed0f..00b28f8 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,7 @@ "main": "index.js", "scripts": { "start": "cd src/site && node server.js", + "assistant": "cd src/site && node assistant.js", "test": "tap" }, "keywords": [], diff --git a/src/lib/collectors.js b/src/lib/collectors.js index 24a1717..0f5631b 100644 --- a/src/lib/collectors.js +++ b/src/lib/collectors.js @@ -14,6 +14,9 @@ const userRequestCache = new UserRequestCache(constants.caching.resource_cache_t const timelineEntryCache = new TtlCache(constants.caching.resource_cache_time) const history = new RequestHistory(["user", "timeline", "post", "reel"]) +const AssistantSwitcher = require("./structures/AssistantSwitcher") +const assistantSwitcher = new AssistantSwitcher() + /** * @param {string} username * @param {boolean} isRSS @@ -53,6 +56,11 @@ async function fetchUser(username, isRSS) { return fetchUserFromCombined(saved.user_id, username) } else if (saved && saved.updated_version >= 2) { return fetchUserFromSaved(saved) + } else if (assistantSwitcher.enabled()) { + return assistantSwitcher.requestUser(username).catch(error => { + if (error === constants.symbols.NO_ASSISTANTS_AVAILABLE) throw constants.symbols.RATE_LIMITED + else throw error + }) } } throw error @@ -332,3 +340,5 @@ module.exports.timelineEntryCache = timelineEntryCache module.exports.getOrFetchShortcode = getOrFetchShortcode module.exports.updateProfilePictureFromReel = updateProfilePictureFromReel module.exports.history = history +module.exports.fetchUserFromSaved = fetchUserFromSaved +module.exports.assistantSwitcher = assistantSwitcher diff --git a/src/lib/constants.js b/src/lib/constants.js index 85242e0..2a4151b 100644 --- a/src/lib/constants.js +++ b/src/lib/constants.js @@ -40,6 +40,15 @@ let constants = { enable_updater_page: false }, + assistant: { + enabled: false, + // List of assistant origin URLs, if you have any. + origins: [ + ], + offline_request_cooldown: 20*60*1000, + blocked_request_cooldown: 2*60*60*1000, + }, + caching: { image_cache_control: `public, max-age=${7*24*60*60}`, resource_cache_time: 30*60*1000, @@ -78,7 +87,14 @@ let constants = { NO_SHARED_DATA: Symbol("NO_SHARED_DATA"), INSTAGRAM_DEMANDS_LOGIN: Symbol("INSTAGRAM_DEMANDS_LOGIN"), RATE_LIMITED: Symbol("RATE_LIMITED"), - ENDPOINT_OVERRIDDEN: Symbol("ENDPOINT_OVERRIDDEN") + ENDPOINT_OVERRIDDEN: Symbol("ENDPOINT_OVERRIDDEN"), + NO_ASSISTANTS_AVAILABLE: Symbol("NO_ASSISTANTS_AVAILABLE"), + assistant_statuses: { + OFFLINE: Symbol("OFFLINE"), + BLOCKED: Symbol("BLOCKED"), + OK: Symbol("OK"), + NONE: Symbol("NONE") + } }, database_version: 2 diff --git a/src/lib/structures/Assistant.js b/src/lib/structures/Assistant.js new file mode 100644 index 0000000..4f12591 --- /dev/null +++ b/src/lib/structures/Assistant.js @@ -0,0 +1,42 @@ +const {request} = require("../utils/request") +const constants = require("../constants") + +class Assistant { + constructor(origin) { + this.origin = origin + this.lastRequest = 0 + this.lastRequestStatus = constants.symbols.assistant_statuses.NONE + } + + available() { + if (this.lastRequestStatus === constants.symbols.assistant_statuses.OFFLINE) { + return Date.now() - this.lastRequest > constants.assistant.offline_request_cooldown + } else if (this.lastRequestStatus === constants.symbols.assistant_statuses.BLOCKED) { + return Date.now() - this.lastRequest > constants.assistant.blocked_request_cooldown + } else { + return true + } + } + + requestUser(username) { + this.lastRequest = Date.now() + return new Promise((resolve, reject) => { + request(`${this.origin}/api/user/v1/${username}`).json().then(root => { + console.log(root) + if (root.status === "ok") { + this.lastRequestStatus = constants.symbols.assistant_statuses.OK + resolve(root.data.user) + } else { + this.lastRequestStatus = constants.symbols.assistant_statuses.BLOCKED + reject(constants.symbols.assistant_statuses.BLOCKED) + } + }).catch(error => { + console.error(error) + this.lastRequestStatus = constants.symbols.assistant_statuses.OFFLINE + reject(constants.symbols.assistant_statuses.OFFLINE) + }) + }) + } +} + +module.exports = Assistant diff --git a/src/lib/structures/AssistantSwitcher.js b/src/lib/structures/AssistantSwitcher.js new file mode 100644 index 0000000..549eb81 --- /dev/null +++ b/src/lib/structures/AssistantSwitcher.js @@ -0,0 +1,49 @@ +const constants = require("../constants") +const collectors = require("../collectors") +const Assistant = require("./Assistant") +const db = require("../db") + +class AssistantSwitcher { + constructor() { + this.assistants = constants.assistant.origins.map(origin => new Assistant(origin)) + } + + enabled() { + return constants.assistant.enabled && this.assistants.length + } + + getAvailableAssistants() { + return this.assistants.filter(assistant => assistant.available()).sort((a, b) => (a.lastRequest - b.lastRequest)) + } + + requestUser(username) { + return new Promise(async (resolve, reject) => { + const assistants = this.getAvailableAssistants() + while (assistants.length) { + const assistant = assistants.shift() + try { + const user = await assistant.requestUser(username) + return resolve(user) + } catch (e) { + // that assistant broke. try the next one. + } + } + return reject(constants.symbols.NO_ASSISTANTS_AVAILABLE) + }).then(user => { + const bind = {...user} + bind.created = Date.now() + bind.updated = Date.now() + bind.updated_version = constants.database_version + bind.is_private = +user.is_private + bind.is_verified = +user.is_verified + db.prepare( + "REPLACE INTO Users (username, user_id, created, updated, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url) VALUES " + +"(@username, @user_id, @created, @updated, @updated_version, @biography, @post_count, @following_count, @followed_by_count, @external_url, @full_name, @is_private, @is_verified, @profile_pic_url)" + ).run(bind) + collectors.userRequestCache.cache.delete(`user/${username}`) + return collectors.fetchUserFromSaved(user) + }) + } +} + +module.exports = AssistantSwitcher diff --git a/src/site/api/routes.js b/src/site/api/routes.js index b0d1168..8ce9e21 100644 --- a/src/site/api/routes.js +++ b/src/site/api/routes.js @@ -78,7 +78,7 @@ module.exports = [ message: "This user doesn't exist.", withInstancesLink: false }) - } else if (error === constants.symbols.INSTAGRAM_DEMANDS_LOGIN) { + } else if (error === constants.symbols.INSTAGRAM_DEMANDS_LOGIN || error === constants.symbols.RATE_LIMITED) { return { statusCode: 503, contentType: "text/html", diff --git a/src/site/assistant.js b/src/site/assistant.js new file mode 100644 index 0000000..030ca50 --- /dev/null +++ b/src/site/assistant.js @@ -0,0 +1,40 @@ +const {Pinski} = require("pinski") +const {subdirs} = require("node-dir") +const constants = require("../lib/constants") + +const passthrough = require("./passthrough") + +const pinski = new Pinski({ + port: constants.port, + relativeRoot: __dirname +}) + +;(async (err, dirs) => { + if (err) throw err + + // need to check for and run db upgrades before anything starts using it + await require("../lib/utils/upgradedb")() + + if (constants.tor.enabled) { + await require("../lib/utils/tor") // make sure tor state is known before going further + } + + pinski.addAPIDir("assistant_api") + pinski.startServer() + pinski.enableWS() + + require("pinski/plugins").setInstance(pinski) + + Object.assign(passthrough, pinski.getExports()) + + console.log("Assistant started") + + if (constants.allow_user_from_reel !== "never") { + constants.allow_user_from_reel = "never" + console.log(`[!] You are running the assistant, so \`constants.allow_user_from_reel\` has been set to "never" for this session.`) + } + + if (process.stdin.isTTY || process.argv.includes("--enable-repl")) { + require("./repl") + } +})() diff --git a/src/site/assistant_api/user.js b/src/site/assistant_api/user.js new file mode 100644 index 0000000..fef22f2 --- /dev/null +++ b/src/site/assistant_api/user.js @@ -0,0 +1,75 @@ +const constants = require("../../lib/constants") +const collectors = require("../../lib/collectors") +const db = require("../../lib/db") + +function reply(statusCode, content) { + return { + statusCode: statusCode, + contentType: "application/json", + content: JSON.stringify(content) + } +} + +module.exports = [ + { + route: `/api/user`, methods: ["GET"], code: () => { + return Promise.resolve(reply(200, { + status: "ok", + generatedAt: Date.now(), + availableVersions: ["1"] + })) + } + }, + { + route: `/api/user/v1/(${constants.external.username_regex})`, methods: ["GET"], code: async ({fill, url}) => { + function replyWithUserData(userData, type) { + return reply(200, { + status: "ok", + version: "1.0", + generatedAt: Date.now(), + data: { + type, + allow_user_from_reel: constants.allow_user_from_reel, + user: userData + } + }) + } + + const username = fill[0] + const saved = db.prepare("SELECT username, user_id, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url FROM Users WHERE username = ?").get(username) + if (saved && saved.updated_version >= 2) { // suitable data is already saved + delete saved.updated_version + return Promise.resolve(replyWithUserData(saved, "user")) + } else { + return collectors.fetchUser(username, false).then(user => { + const type = user.constructor.name === "User" ? "user" : "reel" + return replyWithUserData({ + username: user.data.username, + user_id: user.data.id, + biography: user.data.biography, + post_count: user.posts, + following_count: user.following, + followed_by_count: user.followedBy, + external_url: user.data.external_url, + full_name: user.data.full_name, + is_private: user.data.is_private, + is_verified: user.data.is_verified, + profile_pic_url: user.data.profile_pic_url + }, type) + }).catch(error => { + if (error === constants.symbols.RATE_LIMITED || error === constants.symbols.INSTAGRAM_DEMANDS_LOGIN) { + return reply(503, { + status: "fail", + version: "1.0", + generatedAt: Date.now(), + message: "Rate limited by Instagram.", + identifier: "RATE_LIMITED" + }) + } else { + throw error + } + }) + } + } + } +] diff --git a/src/site/repl.js b/src/site/repl.js index 4e249fe..517d3e5 100644 --- a/src/site/repl.js +++ b/src/site/repl.js @@ -1,6 +1,7 @@ const {instance, pugCache, wss} = require("./passthrough") const {userRequestCache, timelineEntryCache, history} = require("../lib/collectors") const constants = require("../lib/constants") +const collectors = require("../lib/collectors") const util = require("util") const repl = require("repl") const vm = require("vm") diff --git a/src/site/server.js b/src/site/server.js index cf6d741..c04bce1 100644 --- a/src/site/server.js +++ b/src/site/server.js @@ -34,7 +34,6 @@ subdirs("pug", async (err, dirs) => { pinski.addAPIDir("api") pinski.startServer() - pinski.enableWS() require("pinski/plugins").setInstance(pinski)