src/lib/collectors.js

@@ -28,18 +28,65 @@ async function fetchUser(username, context) {
 	}
 
 	let mode = constants.allow_user_from_reel
-	if (mode === "iweb") {
+	if (mode === "preferForRSS") {
-		return fetchUserFromIWeb(username)
+		if (context === constants.symbols.fetch_context.RSS) mode = "prefer"
+		else mode = "onlyPreferSaved"
 	}
-
+	if (context === constants.symbols.fetch_context.ASSISTANT) {
-	throw new Error(`Your instance admin selected fetch mode ${mode}, which is now unsupported. Please use "iweb" instead (the default).`)
+		const saved = db.prepare("SELECT username, user_id, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url FROM Users WHERE username = ?").get(username)
+		if (saved && saved.updated_version >= 2) {
+			return fetchUserFromSaved(saved)
+		} else {
+			return fetchUserFromHTML(username)
+		}
+	}
+	if (mode === "never") {
+		return fetchUserFromHTML(username)
+	}
+	if (mode === "prefer") {
+		const saved = db.prepare("SELECT username, user_id, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url FROM Users WHERE username = ?").get(username)
+		if (saved && saved.updated_version >= 2) {
+			return fetchUserFromSaved(saved)
+		} else if (saved && saved.updated_version === 1) {
+			return fetchUserFromCombined(saved.user_id, saved.username)
+		} else {
+			return fetchUserFromHTML(username)
+		}
+	}
+	if (mode === "onlyPreferSaved") {
+		const saved = db.prepare("SELECT username, user_id, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url FROM Users WHERE username = ?").get(username)
+		if (saved && saved.updated_version >= 2) {
+			return fetchUserFromSaved(saved)
+		} else {
+			mode = "fallback"
+		}
+	}
+	if (mode === "fallback") {
+		return fetchUserFromHTML(username).catch(error => {
+			if (error === constants.symbols.INSTAGRAM_DEMANDS_LOGIN || error === constants.symbols.RATE_LIMITED) {
+				const saved = db.prepare("SELECT username, user_id, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url FROM Users WHERE username = ?").get(username)
+				if (saved && saved.updated_version === 1) {
+					return fetchUserFromCombined(saved.user_id, username)
+				} else if (saved && saved.updated_version >= 2) {
+					return fetchUserFromSaved(saved)
+				} else if (assistantSwitcher.enabled()) {
+					return assistantSwitcher.requestUser(username).catch(error => {
+						if (error === constants.symbols.NO_ASSISTANTS_AVAILABLE) throw constants.symbols.RATE_LIMITED
+						else throw error
+					})
+				}
+			}
+			throw error
+		})
+	}
+	throw new Error(`Selected fetch mode ${mode} was unmatched.`)
 }
 
 /**
  * @param {string} username
  * @returns {Promise<{user: import("./structures/User"), quotaUsed: number}>}
  */
-function fetchUserFromIWeb(username) {
+function fetchUserFromHTML(username) {
 	const blockedCacheConfig = constants.caching.self_blocked_status.user_html
 	if (blockedCacheConfig) {
 		if (history.store.has("user")) {
@@ -52,44 +99,56 @@ function fetchUserFromIWeb(username) {
 	let quotaUsed = 0
 	return userRequestCache.getOrFetch("user/"+username, false, true, () => {
 		quotaUsed++
-		const params = new URLSearchParams({username})
+		return switcher.request("user_html", `https://www.instagram.com/${username}/feed/`, async res => {
-		return switcher.request("user_html", `https://i.instagram.com/api/v1/users/web_profile_info/?${params}`, async res => {
 			if (res.status === 301) throw constants.symbols.ENDPOINT_OVERRIDDEN
 			if (res.status === 302) throw constants.symbols.INSTAGRAM_DEMANDS_LOGIN
 			if (res.status === 429) throw constants.symbols.RATE_LIMITED
 			return res
 		}).then(async g => {
 			const res = await g.response()
-			const json = await g.json()
+			if (res.status === 404) {
-			// require down here or have to deal with require loop. require cache will take care of it anyway.
+				throw constants.symbols.NOT_FOUND
-			// User -> Timeline -> TimelineEntry -> collectors -/> User
+			} else {
-			const User = require("./structures/User")
+				const text = await g.text()
-			const user = new User(json.data.user)
+				// require down here or have to deal with require loop. require cache will take care of it anyway.
-			history.report("user", true)
+				// User -> Timeline -> TimelineEntry -> collectors -/> User
-			// sure, cache the user info. why not.
+				const User = require("./structures/User")
-			if (constants.caching.db_user_id) {
+				const result = extractSharedData(text)
-				const existing = db.prepare("SELECT created, updated_version FROM Users WHERE username = ?").get(user.data.username)
+				if (result.status === constants.symbols.extractor_results.SUCCESS) {
-				db.prepare(
+					const sharedData = result.value
-					"REPLACE INTO Users (username,  user_id,  created,  updated,  updated_version,  biography,  post_count,  following_count,  followed_by_count,  external_url,  full_name,  is_private,  is_verified,  profile_pic_url) VALUES "
+					const user = new User(sharedData.entry_data.ProfilePage[0].graphql.user)
-						+"(@username, @user_id, @created, @updated, @updated_version, @biography, @post_count, @following_count, @followed_by_count, @external_url, @full_name, @is_private, @is_verified, @profile_pic_url)"
+					history.report("user", true)
-				).run({
+					if (constants.caching.db_user_id) {
-					username: user.data.username,
+						const existing = db.prepare("SELECT created, updated_version FROM Users WHERE username = ?").get(user.data.username)
-					user_id: user.data.id,
+						db.prepare(
-					created: existing && existing.updated_version === constants.database_version ? existing.created : Date.now(),
+							"REPLACE INTO Users (username,  user_id,  created,  updated,  updated_version,  biography,  post_count,  following_count,  followed_by_count,  external_url,  full_name,  is_private,  is_verified,  profile_pic_url) VALUES "
-					updated: Date.now(),
+							                 +"(@username, @user_id, @created, @updated, @updated_version, @biography, @post_count, @following_count, @followed_by_count, @external_url, @full_name, @is_private, @is_verified, @profile_pic_url)"
-					updated_version: constants.database_version,
+						).run({
-					biography: user.data.biography || null,
+							username: user.data.username,
-					post_count: user.posts || 0,
+							user_id: user.data.id,
-					following_count: user.following || 0,
+							created: existing && existing.updated_version === constants.database_version ? existing.created : Date.now(),
-					followed_by_count: user.followedBy || 0,
+							updated: Date.now(),
-					external_url: user.data.external_url || null,
+							updated_version: constants.database_version,
-					full_name: user.data.full_name || null,
+							biography: user.data.biography || null,
-					is_private: +user.data.is_private,
+							post_count: user.posts || 0,
-					is_verified: +user.data.is_verified,
+							following_count: user.following || 0,
-					profile_pic_url: user.data.profile_pic_url
+							followed_by_count: user.followedBy || 0,
-				})
+							external_url: user.data.external_url || null,
+							full_name: user.data.full_name || null,
+							is_private: +user.data.is_private,
+							is_verified: +user.data.is_verified,
+							profile_pic_url: user.data.profile_pic_url
+						})
+					}
+					return user
+				} else if (result.status === constants.symbols.extractor_results.AGE_RESTRICTED) {
+					// I don't like this code.
+					history.report("user", true)
+					throw constants.symbols.extractor_results.AGE_RESTRICTED
+				} else {
+					throw result.status
+				}
 			}
-			return user
 		}).catch(error => {
 			if (error === constants.symbols.INSTAGRAM_DEMANDS_LOGIN || error === constants.symbols.RATE_LIMITED) {
 				history.report("user", false, error)
@@ -180,6 +239,36 @@ function fetchUserFromCombined(userID, username) {
 	})
 }
 
+function fetchUserFromSaved(saved) {
+	let quotaUsed = 0
+	return userRequestCache.getOrFetch("user/"+saved.username, false, true, async () => {
+		// require down here or have to deal with require loop. require cache will take care of it anyway.
+		// ReelUser -> Timeline -> TimelineEntry -> collectors -/> ReelUser
+		const ReelUser = require("./structures/ReelUser")
+		const user = new ReelUser({
+			username: saved.username,
+			id: saved.user_id,
+			biography: saved.biography,
+			edge_follow: {count: saved.following_count},
+			edge_followed_by: {count: saved.followed_by_count},
+			external_url: saved.external_url,
+			full_name: saved.full_name,
+			is_private: !!saved.is_private,
+			is_verified: !!saved.is_verified,
+			profile_pic_url: saved.profile_pic_url
+		})
+		// Add first timeline page
+		if (!user.timeline.pages[0]) {
+			const {result: page, fromCache} = await fetchTimelinePage(user.data.id, "")
+			if (!fromCache) quotaUsed++
+			user.timeline.addPage(page)
+		}
+		return user
+	}).then(user => {
+		return {user, quotaUsed}
+	})
+}
+
 /**
  * @param {string} userID
  * @param {string} after
@@ -459,5 +548,6 @@ module.exports.timelineEntryCache = timelineEntryCache
 module.exports.getOrFetchShortcode = getOrFetchShortcode
 module.exports.updateProfilePictureFromReel = updateProfilePictureFromReel
 module.exports.history = history
+module.exports.fetchUserFromSaved = fetchUserFromSaved
 module.exports.assistantSwitcher = assistantSwitcher
 module.exports.verifyUserPair = verifyUserPair

src/lib/constants.js

@@ -26,11 +26,11 @@ let constants = {
 
 	// Things that server owners _could_ change if they want to.
 	tor: {
-		enabled: false, // If false, everything else in this block has no effect.
+		enabled: true, // If false, everything else in this block has no effect.
 		password: null, // If `null`, Bibliogram will run its own Tor process instead.
 		port: 9051, // If a password is provided, Bibliogram will connect to Tor on this port. (This is ignored when running its own Tor process.)
 		for: {
-			user_html: false,
+			user_html: true,
 			timeline_graphql: false,
 			post_graphql: false,
 			reel_graphql: false
@@ -41,7 +41,7 @@ let constants = {
 	// change this to `true` to serve it, which will make extensions like Privacy Badger automatically whitelist the domain.
 	does_not_track: false,
 
-	allow_user_from_reel: "iweb", // legacy. this must be "iweb" now.
+	allow_user_from_reel: "fallback", // one of: "never", "fallback", "prefer", "onlyPreferSaved", "preferForRSS"
 	proxy_media: { // Whether to proxy media (images, videos, thumbnails) through Bibliogram. This is strongly recommended to protect user privacy. If proxy is turned off, some browser content blockers may break all images since they are served from Facebook domains.
 		image: true,
 		video: true,
@@ -155,7 +155,7 @@ let constants = {
 			replaceEmptyWithDefault: false
 		},{
 			name: "infinite_scroll",
-			default: "off",
+			default: "normal",
 			boolean: false,
 			replaceEmptyWithDefault: true
 		},{
@@ -223,12 +223,12 @@ let constants = {
 		csrf_time: 60*60*1000,
 		self_blocked_status: {
 			user_html: {
-				enabled: true,
+				enabled: false,
-				time: 2*60*60*1000
+				time: 15*60*1000
 			},
 			timeline_graphql: {
-				enabled: true,
+				enabled: false,
-				time: 2*60*60*1000
+				time: 24*60*60*1000
 			}
 		},
 		db_user_id: true,
@@ -239,7 +239,7 @@ let constants = {
 	// Instagram uses this stuff. This shouldn't be changed, except to fix a bug that hasn't yet been fixed upstream.
 	external: {
 		reel_query_hash: "c9100bf9110dd6361671f113dd02e7d6",
-		timeline_query_hash: "69cba40317214236af40e7efa697781d",
+		timeline_query_hash: "e769aa130647d2354c40ea6a439bfc08",
 		timeline_query_hash_2: "42323d64886122307be10013ad2dcc44", // https://github.com/rarcega/instagram-scraper/blob/dc022081dbefc81500c5f70cce5c70cfd2816e3c/instagram_scraper/constants.py#L30
 		shortcode_query_hash: "2b0673e0dc4580674a88d426fe00ea90",
 		igtv_query_hash: "bc78b344a68ed16dd5d7f264681c4c76",

src/lib/utils/request.js

@@ -4,12 +4,7 @@ const SavedRequestManager = require("./saved_requests/manager")
 
 const constants = require("../constants")
 
-const userAgent = "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:102.0) Gecko/20100101 Firefox/102.0"
+const userAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"
-
-const headers = {
-	"User-Agent": userAgent,
-	"X-IG-App-ID": 936619743392459 // needed for profile iweb to work
-}
 
 const backendStatusLineMap = new Map([
 	["node-fetch", "NF "],
@@ -28,12 +23,16 @@ function request(url, options = {}, settings = {}) {
 
 	if (constants.request_backend === "node-fetch") {
 		return new NodeFetch(url, Object.assign({
-			headers,
+			headers: {
+				"User-Agent": userAgent
+			},
 			redirect: "manual"
 		}, options))
 	} else if (constants.request_backend === "got") {
 		return new Got(url, Object.assign({
-			headers,
+			headers: {
+				"User-Agent": userAgent
+			},
 			followRedirect: false,
 			throwHttpErrors: false
 		}, options))

src/site/html/static/js/pagination.js

@@ -47,14 +47,11 @@ class NextPage extends FreezeWidth {
 		this.controller = controller
 		this.clicked = false
 		this.nextPageNumber = +this.element.getAttribute("data-page")
-		this.auto = this.element.getAttribute("data-auto")
 		this.attribute("href", "javascript:void(0)")
 		this.event("click", event => this.onClick(event))
 
 		this.observer = new IntersectionObserver(entries => this.onIntersect(entries), {rootMargin: "0px", threshold: intersectionThreshold})
-		if (this.auto !== "off") {
+		this.observer.observe(this.element)
-			this.observer.observe(this.element)
-		}
 	}
 
 	onClick(event) {

src/site/pug/includes/next_page_button.pug

@@ -10,7 +10,6 @@ mixin next_page_button(user, selectedTimeline, url, type)
 				data-username=(user.data.username)
 				data-type=type
 				data-loading-text=ll.next_page_button_loading
-				data-auto=settings.infinite_scroll
 			)#next-page.next-page= ll.next_page_button
 	else
 		div

src/site/pug/settings.pug

@@ -102,8 +102,9 @@ html(dir=ll.meta_direction, lang=settings.language)
 
 					+checkbox("spa", ll.fast_navigation, ll.t_enabled, false)
 
-					+select("infinite_scroll", ll.infinite_scroll, false, [
+					+select("infinite_scroll", ll.infinite_scroll, true, [
 						{value: "normal", text: ll.t_normal},
+						{value: "eager", text: ll.t_eager},
 						{value: "off", text: ll.t_manual}
 					])