Add experimental assistant feature

package.json

@@ -6,6 +6,7 @@
   "main": "index.js",
   "scripts": {
     "start": "cd src/site && node server.js",
+    "assistant": "cd src/site && node assistant.js",
     "test": "tap"
   },
   "keywords": [],

src/lib/collectors.js

@@ -14,6 +14,9 @@ const userRequestCache = new UserRequestCache(constants.caching.resource_cache_t
 const timelineEntryCache = new TtlCache(constants.caching.resource_cache_time)
 const history = new RequestHistory(["user", "timeline", "post", "reel"])
 
+const AssistantSwitcher = require("./structures/AssistantSwitcher")
+const assistantSwitcher = new AssistantSwitcher()
+
 /**
  * @param {string} username
  * @param {boolean} isRSS
@@ -53,6 +56,11 @@ async function fetchUser(username, isRSS) {
 					return fetchUserFromCombined(saved.user_id, username)
 				} else if (saved && saved.updated_version >= 2) {
 					return fetchUserFromSaved(saved)
+				} else if (assistantSwitcher.enabled()) {
+					return assistantSwitcher.requestUser(username).catch(error => {
+						if (error === constants.symbols.NO_ASSISTANTS_AVAILABLE) throw constants.symbols.RATE_LIMITED
+						else throw error
+					})
 				}
 			}
 			throw error
@@ -332,3 +340,5 @@ module.exports.timelineEntryCache = timelineEntryCache
 module.exports.getOrFetchShortcode = getOrFetchShortcode
 module.exports.updateProfilePictureFromReel = updateProfilePictureFromReel
 module.exports.history = history
+module.exports.fetchUserFromSaved = fetchUserFromSaved
+module.exports.assistantSwitcher = assistantSwitcher

src/lib/constants.js

@@ -40,6 +40,15 @@ let constants = {
 		enable_updater_page: false
 	},
 
+	assistant: {
+		enabled: false,
+		// List of assistant origin URLs, if you have any.
+		origins: [
+		],
+		offline_request_cooldown: 20*60*1000,
+		blocked_request_cooldown: 2*60*60*1000,
+	},
+
 	caching: {
 		image_cache_control: `public, max-age=${7*24*60*60}`,
 		resource_cache_time: 30*60*1000,
@@ -78,7 +87,14 @@ let constants = {
 		NO_SHARED_DATA: Symbol("NO_SHARED_DATA"),
 		INSTAGRAM_DEMANDS_LOGIN: Symbol("INSTAGRAM_DEMANDS_LOGIN"),
 		RATE_LIMITED: Symbol("RATE_LIMITED"),
-		ENDPOINT_OVERRIDDEN: Symbol("ENDPOINT_OVERRIDDEN")
+		ENDPOINT_OVERRIDDEN: Symbol("ENDPOINT_OVERRIDDEN"),
+		NO_ASSISTANTS_AVAILABLE: Symbol("NO_ASSISTANTS_AVAILABLE"),
+		assistant_statuses: {
+			OFFLINE: Symbol("OFFLINE"),
+			BLOCKED: Symbol("BLOCKED"),
+			OK: Symbol("OK"),
+			NONE: Symbol("NONE")
+		}
 	},
 
 	database_version: 2

src/lib/structures/Assistant.js

@@ -0,0 +1,42 @@
+const {request} = require("../utils/request")
+const constants = require("../constants")
+
+class Assistant {
+	constructor(origin) {
+		this.origin = origin
+		this.lastRequest = 0
+		this.lastRequestStatus = constants.symbols.assistant_statuses.NONE
+	}
+
+	available() {
+		if (this.lastRequestStatus === constants.symbols.assistant_statuses.OFFLINE) {
+			return Date.now() - this.lastRequest > constants.assistant.offline_request_cooldown
+		} else if (this.lastRequestStatus === constants.symbols.assistant_statuses.BLOCKED) {
+			return Date.now() - this.lastRequest > constants.assistant.blocked_request_cooldown
+		} else {
+			return true
+		}
+	}
+
+	requestUser(username) {
+		this.lastRequest = Date.now()
+		return new Promise((resolve, reject) => {
+			request(`${this.origin}/api/user/v1/${username}`).json().then(root => {
+				console.log(root)
+				if (root.status === "ok") {
+					this.lastRequestStatus = constants.symbols.assistant_statuses.OK
+					resolve(root.data.user)
+				} else {
+					this.lastRequestStatus = constants.symbols.assistant_statuses.BLOCKED
+					reject(constants.symbols.assistant_statuses.BLOCKED)
+				}
+			}).catch(error => {
+				console.error(error)
+				this.lastRequestStatus = constants.symbols.assistant_statuses.OFFLINE
+				reject(constants.symbols.assistant_statuses.OFFLINE)
+			})
+		})
+	}
+}
+
+module.exports = Assistant

src/lib/structures/AssistantSwitcher.js

@@ -0,0 +1,49 @@
+const constants = require("../constants")
+const collectors = require("../collectors")
+const Assistant = require("./Assistant")
+const db = require("../db")
+
+class AssistantSwitcher {
+	constructor() {
+		this.assistants = constants.assistant.origins.map(origin => new Assistant(origin))
+	}
+
+	enabled() {
+		return constants.assistant.enabled && this.assistants.length
+	}
+
+	getAvailableAssistants() {
+		return this.assistants.filter(assistant => assistant.available()).sort((a, b) => (a.lastRequest - b.lastRequest))
+	}
+
+	requestUser(username) {
+		return new Promise(async (resolve, reject) => {
+			const assistants = this.getAvailableAssistants()
+			while (assistants.length) {
+				const assistant = assistants.shift()
+				try {
+					const user = await assistant.requestUser(username)
+					return resolve(user)
+				} catch (e) {
+					// that assistant broke. try the next one.
+				}
+			}
+			return reject(constants.symbols.NO_ASSISTANTS_AVAILABLE)
+		}).then(user => {
+			const bind = {...user}
+			bind.created = Date.now()
+			bind.updated = Date.now()
+			bind.updated_version = constants.database_version
+			bind.is_private = +user.is_private
+			bind.is_verified = +user.is_verified
+			db.prepare(
+				"REPLACE INTO Users (username,  user_id,  created,  updated,  updated_version,  biography,  post_count,  following_count,  followed_by_count,  external_url,  full_name,  is_private,  is_verified,  profile_pic_url) VALUES "
+									  +"(@username, @user_id, @created, @updated, @updated_version, @biography, @post_count, @following_count, @followed_by_count, @external_url, @full_name, @is_private, @is_verified, @profile_pic_url)"
+			).run(bind)
+			collectors.userRequestCache.cache.delete(`user/${username}`)
+			return collectors.fetchUserFromSaved(user)
+		})
+	}
+}
+
+module.exports = AssistantSwitcher

src/site/api/routes.js

@@ -78,7 +78,7 @@ module.exports = [
 						message: "This user doesn't exist.",
 						withInstancesLink: false
 					})
-				} else if (error === constants.symbols.INSTAGRAM_DEMANDS_LOGIN) {
+				} else if (error === constants.symbols.INSTAGRAM_DEMANDS_LOGIN || error === constants.symbols.RATE_LIMITED) {
 					return {
 						statusCode: 503,
 						contentType: "text/html",

src/site/assistant.js

@@ -0,0 +1,40 @@
+const {Pinski} = require("pinski")
+const {subdirs} = require("node-dir")
+const constants = require("../lib/constants")
+
+const passthrough = require("./passthrough")
+
+const pinski = new Pinski({
+	port: constants.port,
+	relativeRoot: __dirname
+})
+
+;(async (err, dirs) => {
+	if (err) throw err
+
+	// need to check for and run db upgrades before anything starts using it
+	await require("../lib/utils/upgradedb")()
+
+	if (constants.tor.enabled) {
+		await require("../lib/utils/tor") // make sure tor state is known before going further
+	}
+
+	pinski.addAPIDir("assistant_api")
+	pinski.startServer()
+	pinski.enableWS()
+
+	require("pinski/plugins").setInstance(pinski)
+
+	Object.assign(passthrough, pinski.getExports())
+
+	console.log("Assistant started")
+
+	if (constants.allow_user_from_reel !== "never") {
+		constants.allow_user_from_reel = "never"
+		console.log(`[!] You are running the assistant, so \`constants.allow_user_from_reel\` has been set to "never" for this session.`)
+	}
+
+	if (process.stdin.isTTY || process.argv.includes("--enable-repl")) {
+		require("./repl")
+	}
+})()

src/site/assistant_api/user.js

@@ -0,0 +1,75 @@
+const constants = require("../../lib/constants")
+const collectors = require("../../lib/collectors")
+const db = require("../../lib/db")
+
+function reply(statusCode, content) {
+	return {
+		statusCode: statusCode,
+		contentType: "application/json",
+		content: JSON.stringify(content)
+	}
+}
+
+module.exports = [
+	{
+		route: `/api/user`, methods: ["GET"], code: () => {
+			return Promise.resolve(reply(200, {
+				status: "ok",
+				generatedAt: Date.now(),
+				availableVersions: ["1"]
+			}))
+		}
+	},
+	{
+		route: `/api/user/v1/(${constants.external.username_regex})`, methods: ["GET"], code: async ({fill, url}) => {
+			function replyWithUserData(userData, type) {
+				return reply(200, {
+					status: "ok",
+					version: "1.0",
+					generatedAt: Date.now(),
+					data: {
+						type,
+						allow_user_from_reel: constants.allow_user_from_reel,
+						user: userData
+					}
+				})
+			}
+
+			const username = fill[0]
+			const saved = db.prepare("SELECT username, user_id, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url FROM Users WHERE username = ?").get(username)
+			if (saved && saved.updated_version >= 2) { // suitable data is already saved
+				delete saved.updated_version
+				return Promise.resolve(replyWithUserData(saved, "user"))
+			} else {
+				return collectors.fetchUser(username, false).then(user => {
+					const type = user.constructor.name === "User" ? "user" : "reel"
+					return replyWithUserData({
+						username: user.data.username,
+						user_id: user.data.id,
+						biography: user.data.biography,
+						post_count: user.posts,
+						following_count: user.following,
+						followed_by_count: user.followedBy,
+						external_url: user.data.external_url,
+						full_name: user.data.full_name,
+						is_private: user.data.is_private,
+						is_verified: user.data.is_verified,
+						profile_pic_url: user.data.profile_pic_url
+					}, type)
+				}).catch(error => {
+					if (error === constants.symbols.RATE_LIMITED || error === constants.symbols.INSTAGRAM_DEMANDS_LOGIN) {
+						return reply(503, {
+							status: "fail",
+							version: "1.0",
+							generatedAt: Date.now(),
+							message: "Rate limited by Instagram.",
+							identifier: "RATE_LIMITED"
+						})
+					} else {
+						throw error
+					}
+				})
+			}
+		}
+	}
+]

src/site/repl.js

@@ -1,6 +1,7 @@
 const {instance, pugCache, wss} = require("./passthrough")
 const {userRequestCache, timelineEntryCache, history} = require("../lib/collectors")
 const constants = require("../lib/constants")
+const collectors = require("../lib/collectors")
 const util = require("util")
 const repl = require("repl")
 const vm = require("vm")

src/site/server.js

@@ -34,7 +34,6 @@ subdirs("pug", async (err, dirs) => {
 
 	pinski.addAPIDir("api")
 	pinski.startServer()
-	pinski.enableWS()
 
 	require("pinski/plugins").setInstance(pinski)