Hopefully the final assistants changes

src/lib/collectors.js

@@ -19,14 +19,26 @@ const assistantSwitcher = new AssistantSwitcher()
 
 /**
  * @param {string} username
- * @param {boolean} isRSS
+ * @param {symbol} [context]
  */
-async function fetchUser(username, isRSS) {
+async function fetchUser(username, context) {
+	if (constants.external.reserved_paths.includes(username)) {
+		throw constants.symbols.ENDPOINT_OVERRIDDEN
+	}
+
 	let mode = constants.allow_user_from_reel
 	if (mode === "preferForRSS") {
-		if (isRSS) mode = "prefer"
+		if (context === constants.symbols.fetch_context.RSS) mode = "prefer"
 		else mode = "onlyPreferSaved"
 	}
+	if (context === constants.symbols.fetch_context.ASSISTANT) {
+		const saved = db.prepare("SELECT username, user_id, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url FROM Users WHERE username = ?").get(username)
+		if (saved && saved.updated_version >= 2) {
+			return fetchUserFromSaved(saved)
+		} else {
+			return fetchUserFromHTML(username)
+		}
+	}
 	if (mode === "never") {
 		return fetchUserFromHTML(username)
 	}

src/lib/constants.js

@@ -40,15 +40,23 @@ let constants = {
 		enable_updater_page: false
 	},
 
-	assistant: {
+	use_assistant: {
 		enabled: false,
-		// List of assistant origin URLs, if you have any.
+		// Read the docs.
-		origins: [
+		assistants: [
 		],
 		offline_request_cooldown: 20*60*1000,
 		blocked_request_cooldown: 2*60*60*1000,
 	},
 
+	as_assistant: {
+		enabled: false, // You can still start just the assistant with npm run assistant.
+		require_key: false,
+		// List of keys that are allowed access. You can use any string.
+		// Try `crypto.randomBytes(20).toString("hex")` to get some randomness.
+		keys: []
+	},
+
 	caching: {
 		image_cache_control: `public, max-age=${7*24*60*60}`,
 		resource_cache_time: 30*60*1000,
@@ -68,7 +76,15 @@ let constants = {
 		timeline_fetch_first: 12,
 		username_regex: "[\\w.]*[\\w]",
 		shortcode_regex: "[\\w-]+",
-		hashtag_regex: "[^ \\n`~!@#\\$%^&*()\\-=+[\\]{};:\"',<.>/?\\\\]+"
+		hashtag_regex: "[^ \\n`~!@#\\$%^&*()\\-=+[\\]{};:\"',<.>/?\\\\]+",
+		reserved_paths: [ // https://github.com/cloudrac3r/bibliogram/wiki/Reserved-URLs
+			// Redirects
+			"about", "explore", "support", "press", "api", "privacy", "safety", "admin",
+			// Content
+			"embed.js",
+			// Not found, but likely reserved
+			"graphql", "accounts", "p", "help", "terms", "contact", "blog", "igtv"
+		]
 	},
 
 	resources: {
@@ -93,7 +109,12 @@ let constants = {
 			OFFLINE: Symbol("OFFLINE"),
 			BLOCKED: Symbol("BLOCKED"),
 			OK: Symbol("OK"),
-			NONE: Symbol("NONE")
+			NONE: Symbol("NONE"),
+			NOT_AUTHENTICATED: Symbol("NOT_AUTHENTICATED")
+		},
+		fetch_context: {
+			RSS: Symbol("RSS"),
+			ASSISTANT: Symbol("ASSISTANT")
 		}
 	},
 

src/lib/structures/Assistant.js

@@ -2,17 +2,20 @@ const {request} = require("../utils/request")
 const constants = require("../constants")
 
 class Assistant {
-	constructor(origin) {
+	constructor(origin, key) {
 		this.origin = origin
+		this.key = key
 		this.lastRequest = 0
 		this.lastRequestStatus = constants.symbols.assistant_statuses.NONE
 	}
 
 	available() {
 		if (this.lastRequestStatus === constants.symbols.assistant_statuses.OFFLINE) {
-			return Date.now() - this.lastRequest > constants.assistant.offline_request_cooldown
+			return Date.now() - this.lastRequest > constants.use_assistant.offline_request_cooldown
 		} else if (this.lastRequestStatus === constants.symbols.assistant_statuses.BLOCKED) {
-			return Date.now() - this.lastRequest > constants.assistant.blocked_request_cooldown
+			return Date.now() - this.lastRequest > constants.use_assistant.blocked_request_cooldown
+		} else if (this.lastRequestStatus === constants.symbols.assistant_statuses.NOT_AUTHENTICATED) {
+			return false
 		} else {
 			return true
 		}
@@ -21,7 +24,9 @@ class Assistant {
 	requestUser(username) {
 		this.lastRequest = Date.now()
 		return new Promise((resolve, reject) => {
-			request(`${this.origin}/api/user/v1/${username}`).json().then(root => {
+			const url = new URL(`${this.origin}/api/user/v1/${username}`)
+			if (this.key !== null) url.searchParams.append("key", this.key)
+			request(url.toString()).json().then(root => {
 				// console.log(root)
 				if (root.status === "ok") {
 					this.lastRequestStatus = constants.symbols.assistant_statuses.OK
@@ -30,6 +35,9 @@ class Assistant {
 					if (root.identifier === "NOT_FOUND") {
 						this.lastRequestStatus = constants.symbols.assistant_statuses.OK
 						reject(constants.symbols.NOT_FOUND)
+					} else if (root.identifier === "NOT_AUTHENTICATED") {
+						this.lastRequestStatus = constants.symbols.assistant_statuses.NOT_AUTHENTICATED
+						reject(constants.symbols.assistant_statuses.NOT_AUTHENTICATED)
 					} else { // blocked
 						this.lastRequestStatus = constants.symbols.assistant_statuses.BLOCKED
 						reject(constants.symbols.assistant_statuses.BLOCKED)

src/lib/structures/AssistantSwitcher.js

@@ -5,11 +5,11 @@ const db = require("../db")
 
 class AssistantSwitcher {
 	constructor() {
-		this.assistants = constants.assistant.origins.map(origin => new Assistant(origin))
+		this.assistants = constants.use_assistant.assistants.map(data => new Assistant(data.origin, data.key))
 	}
 
 	enabled() {
-		return constants.assistant.enabled && this.assistants.length
+		return constants.use_assistant.enabled && this.assistants.length
 	}
 
 	getAvailableAssistants() {
@@ -30,6 +30,9 @@ class AssistantSwitcher {
 						rejection.catch(() => {}) // otherwise we get a warning that the rejection was handled asynchronously
 						collectors.userRequestCache.set(`user/${username}`, false, rejection)
 						return reject(e)
+					} else if (e === constants.symbols.assistant_statuses.NOT_AUTHENTICATED) {
+						// no further requests will be successful. the assistant has already marked itself as not available.
+						console.error(`Assistant ${assistant.origin} refused request, not authenticated`)
 					}
 					// that assistant broke. try the next one.
 				}

src/site/api/feed.js

@@ -7,7 +7,7 @@ module.exports = [
 	{route: `/u/(${constants.external.username_regex})/(rss|atom)\\.xml`, methods: ["GET"], code: ({fill}) => {
 		if (constants.settings.rss_enabled) {
 			const kind = fill[1]
-			return fetchUser(fill[0], true).then(async user => {
+			return fetchUser(fill[0], constants.symbols.fetch_context.RSS).then(async user => {
 				const feed = await user.timeline.fetchFeed()
 				if (kind === "rss") {
 					var data = {

src/site/api/routes.js

@@ -63,7 +63,7 @@ module.exports = [
 			}
 
 			const params = url.searchParams
-			return fetchUser(fill[0], false).then(async user => {
+			return fetchUser(fill[0]).then(async user => {
 				const page = +params.get("page")
 				if (typeof page === "number" && !isNaN(page) && page >= 1) {
 					await user.timeline.fetchUpToPage(page - 1)
@@ -97,7 +97,7 @@ module.exports = [
 	},
 	{
 		route: `/fragment/user/(${constants.external.username_regex})/(\\d+)`, methods: ["GET"], code: async ({url, fill}) => {
-			return fetchUser(fill[0], false).then(async user => {
+			return fetchUser(fill[0]).then(async user => {
 				const pageNumber = +fill[1]
 				const pageIndex = pageNumber - 1
 				await user.timeline.fetchUpToPage(pageIndex)

src/site/assistant.js

@@ -29,11 +29,6 @@ const pinski = new Pinski({
 
 	console.log("Assistant started")
 
-	if (constants.allow_user_from_reel !== "never") {
-		constants.allow_user_from_reel = "never"
-		console.log(`[!] You are running the assistant, so \`constants.allow_user_from_reel\` has been set to "never" for this session.`)
-	}
-
 	if (process.stdin.isTTY || process.argv.includes("--enable-repl")) {
 		require("./repl")
 	}

src/site/assistant_api/user.js

@@ -1,3 +1,4 @@
+const crypto = require("crypto")
 const constants = require("../../lib/constants")
 const collectors = require("../../lib/collectors")
 const db = require("../../lib/db")
@@ -22,27 +23,49 @@ module.exports = [
 	},
 	{
 		route: `/api/user/v1/(${constants.external.username_regex})`, methods: ["GET"], code: async ({fill, url}) => {
-			function replyWithUserData(userData, type) {
+			function replyWithUserData(userData) {
 				return reply(200, {
 					status: "ok",
 					version: "1.0",
 					generatedAt: Date.now(),
 					data: {
-						type,
-						allow_user_from_reel: constants.allow_user_from_reel,
 						user: userData
 					}
 				})
 			}
 
+			if (constants.as_assistant.require_key) {
+				if (url.searchParams.has("key")) {
+					const inputKey = url.searchParams.get("key")
+					if (!constants.as_assistant.keys.some(key => inputKey === key)) {
+						return reply(401, {
+							status: "fail",
+							version: "1.0",
+							generatedAt: Date.now(),
+							message: "The authentication key provided is not in the list of allowed keys.",
+							fields: ["q:key"],
+							identifier: "NOT_AUTHENTICATED"
+						})
+					}
+				} else {
+					return reply(401, {
+						status: "fail",
+						version: "1.0",
+						generatedAt: Date.now(),
+						message: "This endpoint requires authentication. If you have a key, specify it with the `key` query parameter.",
+						fields: ["q:key"],
+						identifier: "NOT_AUTHENTICATED"
+					})
+				}
+			}
+
 			const username = fill[0]
 			const saved = db.prepare("SELECT username, user_id, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url FROM Users WHERE username = ?").get(username)
 			if (saved && saved.updated_version >= 2) { // suitable data is already saved
 				delete saved.updated_version
-				return Promise.resolve(replyWithUserData(saved, "user"))
+				return Promise.resolve(replyWithUserData(saved))
 			} else {
-				return collectors.fetchUser(username, false).then(user => {
+				return collectors.fetchUser(username, constants.symbols.fetch_context.ASSISTANT).then(user => {
-					const type = user.constructor.name === "User" ? "user" : "reel"
 					return replyWithUserData({
 						username: user.data.username,
 						user_id: user.data.id,
@@ -55,7 +78,7 @@ module.exports = [
 						is_private: user.data.is_private,
 						is_verified: user.data.is_verified,
 						profile_pic_url: user.data.profile_pic_url
-					}, type)
+					})
 				}).catch(error => {
 					if (error === constants.symbols.RATE_LIMITED || error === constants.symbols.INSTAGRAM_DEMANDS_LOGIN) {
 						return reply(503, {

src/site/server.js

@@ -33,6 +33,12 @@ subdirs("pug", async (err, dirs) => {
 	}
 
 	pinski.addAPIDir("api")
+
+	if (constants.as_assistant.enabled) {
+		console.log("Assistant API enabled")
+		pinski.addAPIDir("assistant_api")
+	}
+
 	pinski.startServer()
 
 	require("pinski/plugins").setInstance(pinski)