Feeble attempt to support the new multi-format

This doesn't work, but it's a start. Somebody else can continue the effort.

src/lib/collectors.js

@@ -1,12 +1,12 @@
 const constants = require("./constants")
 const {request} = require("./utils/request")
 const switcher = require("./utils/torswitcher")
-const {extractPreloader} = require("./utils/body")
+const {selectExtractor} = require("./utils/body")
 const {TtlCache, RequestCache, UserRequestCache} = require("./cache")
 const RequestHistory = require("./structures/RequestHistory")
 const fhp = require("fast-html-parser")
 const db = require("./db")
-require("./testimports")(constants, request, extractPreloader, UserRequestCache, RequestHistory, db)
+require("./testimports")(constants, request, selectExtractor, UserRequestCache, RequestHistory, db)
 
 const requestCache = new RequestCache(constants.caching.resource_cache_time)
 /** @type {import("./cache").UserRequestCache<import("./structures/User")|import("./structures/ReelUser")>} */
@@ -34,7 +34,7 @@ async function fetchUser(username, context) {
 		return fetchUserFromHTML(username)
 	}
 
-	throw new Error(`Your instance admin selected fetch mode ${mode}, which is now unsupported. Please use "iweb" instead (the default).`)
+	throw new Error(`Your instance admin selected fetch mode ${mode}, which is now unsupported. Please ask them to use the default fetch mode by omitting that setting.`)
 }
 
 /**
@@ -68,12 +68,8 @@ function fetchUserFromHTML(username) {
 				// require down here or have to deal with require loop. require cache will take care of it anyway.
 				// User -> Timeline -> TimelineEntry -> collectors -/> User
 				const User = require("./structures/User")
-				const preloader = extractPreloader(text)
+				const userData = selectExtractor(text)
-				const profileInfoResponse = preloader.find(x => x.request.url === "/api/v1/users/web_profile_info/")
+				const user = new User(userData)
-				if (!profileInfoResponse) {
-					throw new Error("No profile info in the preloader.")
-				}
-				const user = new User(JSON.parse(profileInfoResponse.result.response).data.user)
 				history.report("user", true)
 				if (constants.caching.db_user_id) {
 					const existing = db.prepare("SELECT created, updated_version FROM Users WHERE username = ?").get(user.data.username)

src/lib/constants.js

@@ -30,7 +30,7 @@ let constants = {
 		password: null, // If `null`, Bibliogram will run its own Tor process instead.
 		port: 9051, // If a password is provided, Bibliogram will connect to Tor on this port. (This is ignored when running its own Tor process.)
 		for: {
-			user_html: false,
+			user_html: true,
 			timeline_graphql: false,
 			post_graphql: false,
 			reel_graphql: false

src/lib/utils/body.js

@@ -1,6 +1,16 @@
 const constants = require("../constants")
 const {Parser} = require("./parser/parser")
 
+function selectExtractor(text) {
+	if (text.includes("window._sharedData = ")) {
+		return extractSharedData(text)
+	} else if (text.includes("PolarisQueryPreloaderCache")) {
+		return extractPreloader(text)
+	} else {
+		throw constants.symbols.extractor_results.NO_SHARED_DATA
+	}
+}
+
 /**
  * @param {string} text
  * @returns {{status: symbol, value: any}}
@@ -12,21 +22,22 @@ function extractSharedData(text) {
 		// Maybe the profile is age restricted?
 		const age = getRestrictedAge(text)
 		if (age !== null) { // Correct.
-			return {status: constants.symbols.extractor_results.AGE_RESTRICTED, value: age}
+			throw constants.symbols.extractor_results.AGE_RESTRICTED
 		}
-		return {status: constants.symbols.extractor_results.NO_SHARED_DATA, value: null}
+		throw constants.symbols.extractor_results.NO_SHARED_DATA
 	}
 	parser.store()
 	const end = parser.seek(";</script>")
 	parser.restore()
 	const sharedDataString = parser.slice(end - parser.cursor)
 	const sharedData = JSON.parse(sharedDataString)
+	console.log(sharedData)
 	// check for alternate form of age restrictions
 	if (sharedData.entry_data && sharedData.entry_data.HttpGatedContentPage) {
-		// lazy fix; ideally extracting the age should be done here, but for the web ui it doesn't matter
+		// ideally extracting the age should be done here, but for the web ui it doesn't matter
-		return {status: constants.symbols.extractor_results.AGE_RESTRICTED, value: null}
+		throw constants.symbols.extractor_results.AGE_RESTRICTED
 	}
-	return {status: constants.symbols.extractor_results.SUCCESS, value: sharedData}
+	return sharedData.entry_data.ProfilePage[0].graphql.user
 }
 
 /**
@@ -43,7 +54,12 @@ function extractPreloader(text) {
 			entries.push(data)
 		}
 	}
-	return entries
+	// entries now has the things
+	const profileInfoResponse = entries.find(x => x.request.url === "/api/v1/users/web_profile_info/")
+	if (!profileInfoResponse) {
+		throw new Error("No profile info in the preloader.")
+	}
+	return JSON.parse(profileInfoResponse.result.response).data.user
 }
 
 /**
@@ -61,5 +77,6 @@ function getRestrictedAge(text) {
 	return +match[1] // the age
 }
 
+module.exports.selectExtractor = selectExtractor
 module.exports.extractSharedData = extractSharedData
 module.exports.extractPreloader = extractPreloader

src/lib/utils/tor.js

@@ -4,6 +4,8 @@ const constants = require("../constants")
 const {request} = require("./request")
 const {RequestCache} = require("../cache")
 
+let circuitIndex = 0
+
 class TorManager {
 	/**
 	 * @param {import("@deadcanaries/granax/lib/controller")} tor
@@ -20,12 +22,16 @@ class TorManager {
 		let done = false
 		let g
 		while (!done) {
+			const circuitIndexUsed = circuitIndex
 			g = await request(url, {agent: this.agent}, {log: true, statusLine: "TOR"})
 			try {
 				await g.check(test)
 				break
 			} catch (e) {
-				await this.newCircuit()
+				if (circuitIndexUsed === circuitIndex) {
+					circuitIndex++
+					await this.newCircuit()
+				}
 			}
 		}
 		return g

src/site/sass/includes/_main.sass

@@ -15,10 +15,7 @@ $theme: () !default
 		src: url(/static/fonts/cantarell-#{$weight}.otf) format("opentype")
 		font-display: swap // prefer a fallback font until the font file is loaded
 
-body
+body, body.use-boring-font
-	font-family: "Bariol", sans-serif
-
-body.use-boring-font
 	// this is useful so that cyrillic and other scripts don't look conspicuous compared to nearby latin letters.
 	// the use-boring-font class is activated based on the page and heuristics of its contents
 	font-family: "Cantarell", sans-serif