From d95f6950c4cf81511b15b808e43fb10d61311491 Mon Sep 17 00:00:00 2001 From: Cadence Fish Date: Tue, 18 Feb 2020 17:06:11 +1300 Subject: [PATCH] Store more in database for complete fallback --- src/lib/collectors.js | 88 +++++++++++++++++++++++++++++----- src/lib/constants.js | 4 +- src/lib/db.js | 2 - src/lib/structures/ReelUser.js | 12 ++--- src/lib/structures/Timeline.js | 1 + src/lib/utils/upgradedb.js | 18 +++++++ src/site/server.js | 3 ++ 7 files changed, 105 insertions(+), 23 deletions(-) diff --git a/src/lib/collectors.js b/src/lib/collectors.js index 95eb61a..aded837 100644 --- a/src/lib/collectors.js +++ b/src/lib/collectors.js @@ -5,7 +5,7 @@ const {extractSharedData} = require("./utils/body") const {TtlCache, RequestCache, UserRequestCache} = require("./cache") const RequestHistory = require("./structures/RequestHistory") const db = require("./db") -require("./testimports")(constants, request, extractSharedData, UserRequestCache, RequestHistory) +require("./testimports")(constants, request, extractSharedData, UserRequestCache, RequestHistory, db) const requestCache = new RequestCache(constants.caching.resource_cache_time) const userRequestCache = new UserRequestCache(constants.caching.resource_cache_time) @@ -21,25 +21,43 @@ async function fetchUser(username, isRSS) { let mode = constants.allow_user_from_reel if (mode === "preferForRSS") { if (isRSS) mode = "prefer" - else mode = "fallback" + else mode = "onlyPreferSaved" } if (mode === "never") { return fetchUserFromHTML(username) - } else if (mode === "prefer") { - const userID = db.prepare("SELECT user_id FROM Users WHERE username = ?").pluck().get(username) - if (userID) return fetchUserFromCombined(userID, username) - else return fetchUserFromHTML(username) - } else { // === "fallback" + } + if (mode === "prefer") { + const saved = db.prepare("SELECT username, user_id, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url FROM Users WHERE username = ?").get(username) + if (saved && saved.updated_version >= 2) { + return fetchUserFromSaved(saved) + } else if (saved && saved.updated_version === 1) { + return fetchUserFromCombined(saved.user_id, saved.username) + } else { + return fetchUserFromHTML(username) + } + } + if (mode === "onlyPreferSaved") { + const saved = db.prepare("SELECT username, user_id, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url FROM Users WHERE username = ?").get(username) + if (saved && saved.updated_version >= 2) { + return fetchUserFromSaved(saved) + } else { + mode = "fallback" + } + } + if (mode === "fallback") { return fetchUserFromHTML(username).catch(error => { if (error === constants.symbols.INSTAGRAM_DEMANDS_LOGIN || error === constants.symbols.RATE_LIMITED) { - const userID = db.prepare("SELECT user_id FROM Users WHERE username = ?").pluck().get(username) - if (userID) { - return fetchUserFromCombined(userID, username) + const saved = db.prepare("SELECT username, user_id, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url FROM Users WHERE username = ?").get(username) + if (saved && saved.updated_version === 1) { + return fetchUserFromCombined(saved.user_id, username) + } else if (saved && saved.updated_version >= 2) { + return fetchUserFromSaved(saved) } } throw error }) } + throw new Error(`Selected fetch mode ${mode} was unmatched.`) } /** @@ -65,8 +83,26 @@ function fetchUserFromHTML(username) { const user = new User(sharedData.entry_data.ProfilePage[0].graphql.user) history.report("user", true) if (constants.caching.db_user_id) { - db.prepare("INSERT OR IGNORE INTO Users (username, user_id) VALUES (@username, @user_id)") - .run({username: user.data.username, user_id: user.data.id}) + const existing = db.prepare("SELECT created, updated_version FROM Users WHERE username = ?").get(user.data.username) + db.prepare( + "REPLACE INTO Users (username, user_id, created, updated, updated_version, biography, post_count, following_count, followed_by_count, external_url, full_name, is_private, is_verified, profile_pic_url) VALUES " + +"(@username, @user_id, @created, @updated, @updated_version, @biography, @post_count, @following_count, @followed_by_count, @external_url, @full_name, @is_private, @is_verified, @profile_pic_url)" + ).run({ + username: user.data.username, + user_id: user.data.id, + created: existing && existing.updated_version === constants.database_version ? existing.created : Date.now(), + updated: Date.now(), + updated_version: constants.database_version, + biography: user.data.biography || null, + post_count: user.posts || 0, + following_count: user.following || 0, + followed_by_count: user.followedBy || 0, + external_url: user.data.external_url || null, + full_name: user.data.full_name || null, + is_private: +user.data.is_private, + is_verified: +user.data.is_verified, + profile_pic_url: user.data.profile_pic_url + }) } return user }) @@ -104,6 +140,7 @@ function fetchUserFromCombined(userID, username) { // ReelUser -> Timeline -> TimelineEntry -> collectors -/> User const ReelUser = require("./structures/ReelUser") const user = new ReelUser(result.reel.user) + history.report("reel", true) return user }).catch(error => { throw error @@ -114,7 +151,6 @@ function fetchUserFromCombined(userID, username) { const page = await fetchTimelinePage(userID, "") user.timeline.addPage(page) } - history.report("reel", true) return user }).catch(error => { if (error === constants.symbols.RATE_LIMITED) { @@ -124,6 +160,32 @@ function fetchUserFromCombined(userID, username) { }) } +function fetchUserFromSaved(saved) { + return userRequestCache.getOrFetch("user/"+saved.username, false, true, async () => { + // require down here or have to deal with require loop. require cache will take care of it anyway. + // ReelUser -> Timeline -> TimelineEntry -> collectors -/> ReelUser + const ReelUser = require("./structures/ReelUser") + const user = new ReelUser({ + username: saved.username, + id: saved.user_id, + biography: saved.biography, + edge_follow: {count: saved.following_count}, + edge_followed_by: {count: saved.followed_by_count}, + external_url: saved.external_url, + full_name: saved.full_name, + is_private: !!saved.is_private, + is_verified: !!saved.is_verified, + profile_pic_url: saved.profile_pic_url + }) + // Add first timeline page + if (!user.timeline.pages[0]) { + const page = await fetchTimelinePage(user.data.id, "") + user.timeline.addPage(page) + } + return user + }) +} + /** * @param {string} userID * @param {string} after diff --git a/src/lib/constants.js b/src/lib/constants.js index ae50ba5..ce7bf7a 100644 --- a/src/lib/constants.js +++ b/src/lib/constants.js @@ -21,7 +21,7 @@ let constants = { } }, - allow_user_from_reel: "preferForRSS", // one of: "never", "fallback", "prefer", "preferForRSS" + allow_user_from_reel: "preferForRSS", // one of: "never", "fallback", "prefer", "onlyPreferSaved", "preferForRSS" settings: { rss_enabled: true @@ -66,7 +66,7 @@ let constants = { ENDPOINT_OVERRIDDEN: Symbol("ENDPOINT_OVERRIDDEN") }, - database_version: 1 + database_version: 2 } // Override values from config and export the result diff --git a/src/lib/db.js b/src/lib/db.js index 02a1469..bbf470b 100644 --- a/src/lib/db.js +++ b/src/lib/db.js @@ -6,5 +6,3 @@ const dir = pj(__dirname, "../../db") fs.mkdirSync(pj(dir, "backups"), {recursive: true}) const db = new sqlite(pj(dir, "bibliogram.db")) module.exports = db - -require("./utils/upgradedb")() diff --git a/src/lib/structures/ReelUser.js b/src/lib/structures/ReelUser.js index d236cff..3b2ab6f 100644 --- a/src/lib/structures/ReelUser.js +++ b/src/lib/structures/ReelUser.js @@ -1,18 +1,17 @@ const constants = require("../constants") const {proxyImage} = require("../utils/proxyurl") +const {structure} = require("../utils/structuretext") const Timeline = require("./Timeline") require("../testimports")(constants, Timeline) class ReelUser { - /** - * @param {import("../types").GraphUser} data - */ constructor(data) { + /** @type {import("../types").GraphUser} */ this.data = data this.fromReel = true - this.following = 0 - this.followedBy = 0 this.posts = 0 + this.following = data.edge_follow ? data.edge_follow.count : 0 + this.followedBy = data.edge_followed_by ? data.edge_followed_by.count : 0 /** @type {import("./Timeline")} */ this.timeline = new Timeline(this) this.cachedAt = Date.now() @@ -20,7 +19,8 @@ class ReelUser { } getStructuredBio() { - return null + if (!this.data.biography) return null + return structure(this.data.biography) } getTtl(scale = 1) { diff --git a/src/lib/structures/Timeline.js b/src/lib/structures/Timeline.js index 3b353ee..f38f3a8 100644 --- a/src/lib/structures/Timeline.js +++ b/src/lib/structures/Timeline.js @@ -51,6 +51,7 @@ class Timeline { addPage(page) { this.pages.push(transformEdges(page.edges)) this.page_info = page.page_info + this.user.posts = page.count } async fetchFeed() { diff --git a/src/lib/utils/upgradedb.js b/src/lib/utils/upgradedb.js index ab702cc..2ef7445 100644 --- a/src/lib/utils/upgradedb.js +++ b/src/lib/utils/upgradedb.js @@ -20,6 +20,24 @@ const deltas = new Map([ db.prepare("CREATE TABLE Posts (shortcode TEXT NOT NULL UNIQUE, id TEXT NOT NULL UNIQUE, id_as_numeric NUMERIC NOT NULL, username TEXT NOT NULL, json TEXT NOT NULL, PRIMARY KEY (shortcode))") // for future investigation: may not be able to sort by id as a string, may not be able to fit entire id in numeric type .run() + }], + // version 1 to version 2 + [2, function() { + db.transaction(() => { + db.prepare( + "CREATE TABLE Users_New (username TEXT NOT NULL UNIQUE, user_id TEXT NOT NULL UNIQUE, created INTEGER NOT NULL, updated INTEGER NOT NULL" + +", updated_version INTEGER NOT NULL, biography TEXT, post_count INTEGER NOT NULL, following_count INTEGER NOT NULL, followed_by_count INTEGER NOT NULL, external_url TEXT" + +", full_name TEXT, is_private INTEGER NOT NULL, is_verified INTEGER NOT NULL, profile_pic_url TEXT NOT NULL" + +", PRIMARY KEY (username))" + ) + .run() + db.prepare("INSERT INTO Users_New SELECT username, user_id, 0, 0, 1, NULL, 0, 0, 0, NULL, NULL, 0, 0, '' from Users") + .run() + db.prepare("DROP TABLE Users") + .run() + db.prepare("ALTER TABLE Users_New RENAME TO Users") + .run() + })() }] ]) diff --git a/src/site/server.js b/src/site/server.js index 99f54f2..7b56d66 100644 --- a/src/site/server.js +++ b/src/site/server.js @@ -16,6 +16,9 @@ const pinski = new Pinski({ subdirs("pug", async (err, dirs) => { if (err) throw err + // need to check for and run db upgrades before anything starts using it + await require("../lib/utils/upgradedb")() + //pinski.addRoute("/", "pug/index.pug", "pug") pinski.addRoute("/static/css/main.css", "sass/main.sass", "sass") pinski.addPugDir("pug", dirs)