Store more in database for complete fallback

src/lib/collectors.js

@@ -5,7 +5,7 @@ const {extractSharedData} = require("./utils/body")
 const {TtlCache, RequestCache, UserRequestCache} = require("./cache")
 const RequestHistory = require("./structures/RequestHistory")
 const db = require("./db")
-require("./testimports")(constants, request, extractSharedData, UserRequestCache, RequestHistory)
+require("./testimports")(constants, request, extractSharedData, UserRequestCache, RequestHistory, db)
 
 const requestCache = new RequestCache(constants.caching.resource_cache_time)
 const userRequestCache = new UserRequestCache(constants.caching.resource_cache_time)
@@ -21,25 +21,43 @@ async function fetchUser(username, isRSS) {
 	let mode = constants.allow_user_from_reel
 	if (mode === "preferForRSS") {
 		if (isRSS) mode = "prefer"
-		else mode = "fallback"
+		else mode = "onlyPreferSaved"
 	}
 	if (mode === "never") {
 		return fetchUserFromHTML(username)
-	} else if (mode === "prefer") {
+	}
-		const userID = db.prepare("SELECT user_id FROM Users WHERE username = ?").pluck().get(username)
+	if (mode === "prefer") {
-		if (userID) return fetchUserFromCombined(userID, username)
+		const saved = db.prepare("SELECT username, user_id, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url FROM Users WHERE username = ?").get(username)
-		else return fetchUserFromHTML(username)
+		if (saved && saved.updated_version >= 2) {
-	} else { // === "fallback"
+			return fetchUserFromSaved(saved)
+		} else if (saved && saved.updated_version === 1) {
+			return fetchUserFromCombined(saved.user_id, saved.username)
+		} else {
+			return fetchUserFromHTML(username)
+		}
+	}
+	if (mode === "onlyPreferSaved") {
+		const saved = db.prepare("SELECT username, user_id, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url FROM Users WHERE username = ?").get(username)
+		if (saved && saved.updated_version >= 2) {
+			return fetchUserFromSaved(saved)
+		} else {
+			mode = "fallback"
+		}
+	}
+	if (mode === "fallback") {
 		return fetchUserFromHTML(username).catch(error => {
 			if (error === constants.symbols.INSTAGRAM_DEMANDS_LOGIN || error === constants.symbols.RATE_LIMITED) {
-				const userID = db.prepare("SELECT user_id FROM Users WHERE username = ?").pluck().get(username)
+				const saved = db.prepare("SELECT username, user_id, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url FROM Users WHERE username = ?").get(username)
-				if (userID) {
+				if (saved && saved.updated_version === 1) {
-					return fetchUserFromCombined(userID, username)
+					return fetchUserFromCombined(saved.user_id, username)
+				} else if (saved && saved.updated_version >= 2) {
+					return fetchUserFromSaved(saved)
 				}
 			}
 			throw error
 		})
 	}
+	throw new Error(`Selected fetch mode ${mode} was unmatched.`)
 }
 
 /**
@@ -65,8 +83,26 @@ function fetchUserFromHTML(username) {
 					const user = new User(sharedData.entry_data.ProfilePage[0].graphql.user)
 					history.report("user", true)
 					if (constants.caching.db_user_id) {
-						db.prepare("INSERT OR IGNORE INTO Users (username, user_id) VALUES (@username, @user_id)")
+						const existing = db.prepare("SELECT created, updated_version FROM Users WHERE username = ?").get(user.data.username)
-							.run({username: user.data.username, user_id: user.data.id})
+						db.prepare(
+							"REPLACE INTO Users (username,  user_id,  created,  updated,  updated_version,  biography,  post_count,  following_count,  followed_by_count,  external_url,  full_name,  is_private,  is_verified,  profile_pic_url) VALUES "
+							                 +"(@username, @user_id, @created, @updated, @updated_version, @biography, @post_count, @following_count, @followed_by_count, @external_url, @full_name, @is_private, @is_verified, @profile_pic_url)"
+						).run({
+							username: user.data.username,
+							user_id: user.data.id,
+							created: existing && existing.updated_version === constants.database_version ? existing.created : Date.now(),
+							updated: Date.now(),
+							updated_version: constants.database_version,
+							biography: user.data.biography || null,
+							post_count: user.posts || 0,
+							following_count: user.following || 0,
+							followed_by_count: user.followedBy || 0,
+							external_url: user.data.external_url || null,
+							full_name: user.data.full_name || null,
+							is_private: +user.data.is_private,
+							is_verified: +user.data.is_verified,
+							profile_pic_url: user.data.profile_pic_url
+						})
 					}
 					return user
 				})
@@ -104,6 +140,7 @@ function fetchUserFromCombined(userID, username) {
 			// ReelUser -> Timeline -> TimelineEntry -> collectors -/> User
 			const ReelUser = require("./structures/ReelUser")
 			const user = new ReelUser(result.reel.user)
+			history.report("reel", true)
 			return user
 		}).catch(error => {
 			throw error
@@ -114,7 +151,6 @@ function fetchUserFromCombined(userID, username) {
 			const page = await fetchTimelinePage(userID, "")
 			user.timeline.addPage(page)
 		}
-		history.report("reel", true)
 		return user
 	}).catch(error => {
 		if (error === constants.symbols.RATE_LIMITED) {
@@ -124,6 +160,32 @@ function fetchUserFromCombined(userID, username) {
 	})
 }
 
+function fetchUserFromSaved(saved) {
+	return userRequestCache.getOrFetch("user/"+saved.username, false, true, async () => {
+		// require down here or have to deal with require loop. require cache will take care of it anyway.
+		// ReelUser -> Timeline -> TimelineEntry -> collectors -/> ReelUser
+		const ReelUser = require("./structures/ReelUser")
+		const user = new ReelUser({
+			username: saved.username,
+			id: saved.user_id,
+			biography: saved.biography,
+			edge_follow: {count: saved.following_count},
+			edge_followed_by: {count: saved.followed_by_count},
+			external_url: saved.external_url,
+			full_name: saved.full_name,
+			is_private: !!saved.is_private,
+			is_verified: !!saved.is_verified,
+			profile_pic_url: saved.profile_pic_url
+		})
+		// Add first timeline page
+		if (!user.timeline.pages[0]) {
+			const page = await fetchTimelinePage(user.data.id, "")
+			user.timeline.addPage(page)
+		}
+		return user
+	})
+}
+
 /**
  * @param {string} userID
  * @param {string} after

src/lib/constants.js

@@ -21,7 +21,7 @@ let constants = {
 		}
 	},
 
-	allow_user_from_reel: "preferForRSS", // one of: "never", "fallback", "prefer", "preferForRSS"
+	allow_user_from_reel: "preferForRSS", // one of: "never", "fallback", "prefer", "onlyPreferSaved", "preferForRSS"
 
 	settings: {
 		rss_enabled: true
@@ -66,7 +66,7 @@ let constants = {
 		ENDPOINT_OVERRIDDEN: Symbol("ENDPOINT_OVERRIDDEN")
 	},
 
-	database_version: 1
+	database_version: 2
 }
 
 // Override values from config and export the result

src/lib/db.js

@@ -6,5 +6,3 @@ const dir = pj(__dirname, "../../db")
 fs.mkdirSync(pj(dir, "backups"), {recursive: true})
 const db = new sqlite(pj(dir, "bibliogram.db"))
 module.exports = db
-
-require("./utils/upgradedb")()

src/lib/structures/ReelUser.js

@@ -1,18 +1,17 @@
 const constants = require("../constants")
 const {proxyImage} = require("../utils/proxyurl")
+const {structure} = require("../utils/structuretext")
 const Timeline = require("./Timeline")
 require("../testimports")(constants, Timeline)
 
 class ReelUser {
-	/**
-	 * @param {import("../types").GraphUser} data
-	 */
 	constructor(data) {
+		/** @type {import("../types").GraphUser} */
 		this.data = data
 		this.fromReel = true
-		this.following = 0
-		this.followedBy = 0
 		this.posts = 0
+		this.following = data.edge_follow ? data.edge_follow.count : 0
+		this.followedBy = data.edge_followed_by ? data.edge_followed_by.count : 0
 		/** @type {import("./Timeline")} */
 		this.timeline = new Timeline(this)
 		this.cachedAt = Date.now()
@@ -20,7 +19,8 @@ class ReelUser {
 	}
 
 	getStructuredBio() {
-		return null
+		if (!this.data.biography) return null
+		return structure(this.data.biography)
 	}
 
 	getTtl(scale = 1) {

src/lib/structures/Timeline.js

@@ -51,6 +51,7 @@ class Timeline {
 	addPage(page) {
 		this.pages.push(transformEdges(page.edges))
 		this.page_info = page.page_info
+		this.user.posts = page.count
 	}
 
 	async fetchFeed() {

src/lib/utils/upgradedb.js

@@ -20,6 +20,24 @@ const deltas = new Map([
 		db.prepare("CREATE TABLE Posts (shortcode TEXT NOT NULL UNIQUE, id TEXT NOT NULL UNIQUE, id_as_numeric NUMERIC NOT NULL, username TEXT NOT NULL, json TEXT NOT NULL, PRIMARY KEY (shortcode))")
 			// for future investigation: may not be able to sort by id as a string, may not be able to fit entire id in numeric type
 			.run()
+	}],
+	// version 1 to version 2
+	[2, function() {
+		db.transaction(() => {
+			db.prepare(
+				"CREATE TABLE Users_New (username TEXT NOT NULL UNIQUE, user_id TEXT NOT NULL UNIQUE, created INTEGER NOT NULL, updated INTEGER NOT NULL"
+				+", updated_version INTEGER NOT NULL, biography TEXT, post_count INTEGER NOT NULL, following_count INTEGER NOT NULL, followed_by_count INTEGER NOT NULL, external_url TEXT"
+				+", full_name TEXT, is_private INTEGER NOT NULL, is_verified INTEGER NOT NULL, profile_pic_url TEXT NOT NULL"
+				+", PRIMARY KEY (username))"
+			)
+				.run()
+			db.prepare("INSERT INTO Users_New SELECT username, user_id, 0, 0, 1, NULL, 0, 0, 0, NULL, NULL, 0, 0, '' from Users")
+				.run()
+			db.prepare("DROP TABLE Users")
+				.run()
+			db.prepare("ALTER TABLE Users_New RENAME TO Users")
+				.run()
+		})()
 	}]
 ])
 

src/site/server.js

@@ -16,6 +16,9 @@ const pinski = new Pinski({
 subdirs("pug", async (err, dirs) => {
 	if (err) throw err
 
+	// need to check for and run db upgrades before anything starts using it
+	await require("../lib/utils/upgradedb")()
+
 	//pinski.addRoute("/", "pug/index.pug", "pug")
 	pinski.addRoute("/static/css/main.css", "sass/main.sass", "sass")
 	pinski.addPugDir("pug", dirs)