From 41cbffa95a7e06e1454ec9776e781724cf049775 Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Mon, 13 Apr 2020 02:52:04 +1200 Subject: [PATCH] Hopefully the final assistants changes --- src/lib/collectors.js | 18 ++++++++++-- src/lib/constants.js | 31 +++++++++++++++++---- src/lib/structures/Assistant.js | 16 ++++++++--- src/lib/structures/AssistantSwitcher.js | 7 +++-- src/site/api/feed.js | 2 +- src/site/api/routes.js | 4 +-- src/site/assistant.js | 5 ---- src/site/assistant_api/user.js | 37 ++++++++++++++++++++----- src/site/server.js | 6 ++++ 9 files changed, 97 insertions(+), 29 deletions(-) diff --git a/src/lib/collectors.js b/src/lib/collectors.js index 0f5631b..34ce5a5 100644 --- a/src/lib/collectors.js +++ b/src/lib/collectors.js @@ -19,14 +19,26 @@ const assistantSwitcher = new AssistantSwitcher() /** * @param {string} username - * @param {boolean} isRSS + * @param {symbol} [context] */ -async function fetchUser(username, isRSS) { +async function fetchUser(username, context) { + if (constants.external.reserved_paths.includes(username)) { + throw constants.symbols.ENDPOINT_OVERRIDDEN + } + let mode = constants.allow_user_from_reel if (mode === "preferForRSS") { - if (isRSS) mode = "prefer" + if (context === constants.symbols.fetch_context.RSS) mode = "prefer" else mode = "onlyPreferSaved" } + if (context === constants.symbols.fetch_context.ASSISTANT) { + const saved = db.prepare("SELECT username, user_id, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url FROM Users WHERE username = ?").get(username) + if (saved && saved.updated_version >= 2) { + return fetchUserFromSaved(saved) + } else { + return fetchUserFromHTML(username) + } + } if (mode === "never") { return fetchUserFromHTML(username) } diff --git a/src/lib/constants.js b/src/lib/constants.js index 2a4151b..3907fd2 100644 --- a/src/lib/constants.js +++ b/src/lib/constants.js @@ -40,15 +40,23 @@ let constants = { enable_updater_page: false }, - assistant: { + use_assistant: { enabled: false, - // List of assistant origin URLs, if you have any. - origins: [ + // Read the docs. + assistants: [ ], offline_request_cooldown: 20*60*1000, blocked_request_cooldown: 2*60*60*1000, }, + as_assistant: { + enabled: false, // You can still start just the assistant with npm run assistant. + require_key: false, + // List of keys that are allowed access. You can use any string. + // Try `crypto.randomBytes(20).toString("hex")` to get some randomness. + keys: [] + }, + caching: { image_cache_control: `public, max-age=${7*24*60*60}`, resource_cache_time: 30*60*1000, @@ -68,7 +76,15 @@ let constants = { timeline_fetch_first: 12, username_regex: "[\\w.]*[\\w]", shortcode_regex: "[\\w-]+", - hashtag_regex: "[^ \\n`~!@#\\$%^&*()\\-=+[\\]{};:\"',<.>/?\\\\]+" + hashtag_regex: "[^ \\n`~!@#\\$%^&*()\\-=+[\\]{};:\"',<.>/?\\\\]+", + reserved_paths: [ // https://github.com/cloudrac3r/bibliogram/wiki/Reserved-URLs + // Redirects + "about", "explore", "support", "press", "api", "privacy", "safety", "admin", + // Content + "embed.js", + // Not found, but likely reserved + "graphql", "accounts", "p", "help", "terms", "contact", "blog", "igtv" + ] }, resources: { @@ -93,7 +109,12 @@ let constants = { OFFLINE: Symbol("OFFLINE"), BLOCKED: Symbol("BLOCKED"), OK: Symbol("OK"), - NONE: Symbol("NONE") + NONE: Symbol("NONE"), + NOT_AUTHENTICATED: Symbol("NOT_AUTHENTICATED") + }, + fetch_context: { + RSS: Symbol("RSS"), + ASSISTANT: Symbol("ASSISTANT") } }, diff --git a/src/lib/structures/Assistant.js b/src/lib/structures/Assistant.js index 91edb03..202c4c0 100644 --- a/src/lib/structures/Assistant.js +++ b/src/lib/structures/Assistant.js @@ -2,17 +2,20 @@ const {request} = require("../utils/request") const constants = require("../constants") class Assistant { - constructor(origin) { + constructor(origin, key) { this.origin = origin + this.key = key this.lastRequest = 0 this.lastRequestStatus = constants.symbols.assistant_statuses.NONE } available() { if (this.lastRequestStatus === constants.symbols.assistant_statuses.OFFLINE) { - return Date.now() - this.lastRequest > constants.assistant.offline_request_cooldown + return Date.now() - this.lastRequest > constants.use_assistant.offline_request_cooldown } else if (this.lastRequestStatus === constants.symbols.assistant_statuses.BLOCKED) { - return Date.now() - this.lastRequest > constants.assistant.blocked_request_cooldown + return Date.now() - this.lastRequest > constants.use_assistant.blocked_request_cooldown + } else if (this.lastRequestStatus === constants.symbols.assistant_statuses.NOT_AUTHENTICATED) { + return false } else { return true } @@ -21,7 +24,9 @@ class Assistant { requestUser(username) { this.lastRequest = Date.now() return new Promise((resolve, reject) => { - request(`${this.origin}/api/user/v1/${username}`).json().then(root => { + const url = new URL(`${this.origin}/api/user/v1/${username}`) + if (this.key !== null) url.searchParams.append("key", this.key) + request(url.toString()).json().then(root => { // console.log(root) if (root.status === "ok") { this.lastRequestStatus = constants.symbols.assistant_statuses.OK @@ -30,6 +35,9 @@ class Assistant { if (root.identifier === "NOT_FOUND") { this.lastRequestStatus = constants.symbols.assistant_statuses.OK reject(constants.symbols.NOT_FOUND) + } else if (root.identifier === "NOT_AUTHENTICATED") { + this.lastRequestStatus = constants.symbols.assistant_statuses.NOT_AUTHENTICATED + reject(constants.symbols.assistant_statuses.NOT_AUTHENTICATED) } else { // blocked this.lastRequestStatus = constants.symbols.assistant_statuses.BLOCKED reject(constants.symbols.assistant_statuses.BLOCKED) diff --git a/src/lib/structures/AssistantSwitcher.js b/src/lib/structures/AssistantSwitcher.js index 13f72fb..e69a41e 100644 --- a/src/lib/structures/AssistantSwitcher.js +++ b/src/lib/structures/AssistantSwitcher.js @@ -5,11 +5,11 @@ const db = require("../db") class AssistantSwitcher { constructor() { - this.assistants = constants.assistant.origins.map(origin => new Assistant(origin)) + this.assistants = constants.use_assistant.assistants.map(data => new Assistant(data.origin, data.key)) } enabled() { - return constants.assistant.enabled && this.assistants.length + return constants.use_assistant.enabled && this.assistants.length } getAvailableAssistants() { @@ -30,6 +30,9 @@ class AssistantSwitcher { rejection.catch(() => {}) // otherwise we get a warning that the rejection was handled asynchronously collectors.userRequestCache.set(`user/${username}`, false, rejection) return reject(e) + } else if (e === constants.symbols.assistant_statuses.NOT_AUTHENTICATED) { + // no further requests will be successful. the assistant has already marked itself as not available. + console.error(`Assistant ${assistant.origin} refused request, not authenticated`) } // that assistant broke. try the next one. } diff --git a/src/site/api/feed.js b/src/site/api/feed.js index 853d4f3..a5fdb28 100644 --- a/src/site/api/feed.js +++ b/src/site/api/feed.js @@ -7,7 +7,7 @@ module.exports = [ {route: `/u/(${constants.external.username_regex})/(rss|atom)\\.xml`, methods: ["GET"], code: ({fill}) => { if (constants.settings.rss_enabled) { const kind = fill[1] - return fetchUser(fill[0], true).then(async user => { + return fetchUser(fill[0], constants.symbols.fetch_context.RSS).then(async user => { const feed = await user.timeline.fetchFeed() if (kind === "rss") { var data = { diff --git a/src/site/api/routes.js b/src/site/api/routes.js index 8ce9e21..53dc596 100644 --- a/src/site/api/routes.js +++ b/src/site/api/routes.js @@ -63,7 +63,7 @@ module.exports = [ } const params = url.searchParams - return fetchUser(fill[0], false).then(async user => { + return fetchUser(fill[0]).then(async user => { const page = +params.get("page") if (typeof page === "number" && !isNaN(page) && page >= 1) { await user.timeline.fetchUpToPage(page - 1) @@ -97,7 +97,7 @@ module.exports = [ }, { route: `/fragment/user/(${constants.external.username_regex})/(\\d+)`, methods: ["GET"], code: async ({url, fill}) => { - return fetchUser(fill[0], false).then(async user => { + return fetchUser(fill[0]).then(async user => { const pageNumber = +fill[1] const pageIndex = pageNumber - 1 await user.timeline.fetchUpToPage(pageIndex) diff --git a/src/site/assistant.js b/src/site/assistant.js index 030ca50..294b1c6 100644 --- a/src/site/assistant.js +++ b/src/site/assistant.js @@ -29,11 +29,6 @@ const pinski = new Pinski({ console.log("Assistant started") - if (constants.allow_user_from_reel !== "never") { - constants.allow_user_from_reel = "never" - console.log(`[!] You are running the assistant, so \`constants.allow_user_from_reel\` has been set to "never" for this session.`) - } - if (process.stdin.isTTY || process.argv.includes("--enable-repl")) { require("./repl") } diff --git a/src/site/assistant_api/user.js b/src/site/assistant_api/user.js index 7ad771c..0268e21 100644 --- a/src/site/assistant_api/user.js +++ b/src/site/assistant_api/user.js @@ -1,3 +1,4 @@ +const crypto = require("crypto") const constants = require("../../lib/constants") const collectors = require("../../lib/collectors") const db = require("../../lib/db") @@ -22,27 +23,49 @@ module.exports = [ }, { route: `/api/user/v1/(${constants.external.username_regex})`, methods: ["GET"], code: async ({fill, url}) => { - function replyWithUserData(userData, type) { + function replyWithUserData(userData) { return reply(200, { status: "ok", version: "1.0", generatedAt: Date.now(), data: { - type, - allow_user_from_reel: constants.allow_user_from_reel, user: userData } }) } + if (constants.as_assistant.require_key) { + if (url.searchParams.has("key")) { + const inputKey = url.searchParams.get("key") + if (!constants.as_assistant.keys.some(key => inputKey === key)) { + return reply(401, { + status: "fail", + version: "1.0", + generatedAt: Date.now(), + message: "The authentication key provided is not in the list of allowed keys.", + fields: ["q:key"], + identifier: "NOT_AUTHENTICATED" + }) + } + } else { + return reply(401, { + status: "fail", + version: "1.0", + generatedAt: Date.now(), + message: "This endpoint requires authentication. If you have a key, specify it with the `key` query parameter.", + fields: ["q:key"], + identifier: "NOT_AUTHENTICATED" + }) + } + } + const username = fill[0] const saved = db.prepare("SELECT username, user_id, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url FROM Users WHERE username = ?").get(username) if (saved && saved.updated_version >= 2) { // suitable data is already saved delete saved.updated_version - return Promise.resolve(replyWithUserData(saved, "user")) + return Promise.resolve(replyWithUserData(saved)) } else { - return collectors.fetchUser(username, false).then(user => { - const type = user.constructor.name === "User" ? "user" : "reel" + return collectors.fetchUser(username, constants.symbols.fetch_context.ASSISTANT).then(user => { return replyWithUserData({ username: user.data.username, user_id: user.data.id, @@ -55,7 +78,7 @@ module.exports = [ is_private: user.data.is_private, is_verified: user.data.is_verified, profile_pic_url: user.data.profile_pic_url - }, type) + }) }).catch(error => { if (error === constants.symbols.RATE_LIMITED || error === constants.symbols.INSTAGRAM_DEMANDS_LOGIN) { return reply(503, { diff --git a/src/site/server.js b/src/site/server.js index c04bce1..741b5fc 100644 --- a/src/site/server.js +++ b/src/site/server.js @@ -33,6 +33,12 @@ subdirs("pug", async (err, dirs) => { } pinski.addAPIDir("api") + + if (constants.as_assistant.enabled) { + console.log("Assistant API enabled") + pinski.addAPIDir("assistant_api") + } + pinski.startServer() require("pinski/plugins").setInstance(pinski)