Add database

.gitignore

@@ -1,2 +1,3 @@
 node_modules
 .vscode
+db/**/*.db*

package-lock.json

@@ -173,6 +173,61 @@
         "tweetnacl": "^0.14.3"
       }
     },
+    "better-sqlite3": {
+      "version": "5.4.3",
+      "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-5.4.3.tgz",
+      "integrity": "sha512-fPp+8f363qQIhuhLyjI4bu657J/FfMtgiiHKfaTsj3RWDkHlWC1yT7c6kHZDnBxzQVoAINuzg553qKmZ4F1rEw==",
+      "requires": {
+        "integer": "^2.1.0",
+        "tar": "^4.4.10"
+      },
+      "dependencies": {
+        "fs-minipass": {
+          "version": "1.2.7",
+          "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz",
+          "integrity": "sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==",
+          "requires": {
+            "minipass": "^2.6.0"
+          }
+        },
+        "minipass": {
+          "version": "2.9.0",
+          "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.9.0.tgz",
+          "integrity": "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==",
+          "requires": {
+            "safe-buffer": "^5.1.2",
+            "yallist": "^3.0.0"
+          }
+        },
+        "minizlib": {
+          "version": "1.3.3",
+          "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.3.3.tgz",
+          "integrity": "sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==",
+          "requires": {
+            "minipass": "^2.9.0"
+          }
+        },
+        "tar": {
+          "version": "4.4.13",
+          "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.13.tgz",
+          "integrity": "sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA==",
+          "requires": {
+            "chownr": "^1.1.1",
+            "fs-minipass": "^1.2.5",
+            "minipass": "^2.8.6",
+            "minizlib": "^1.2.1",
+            "mkdirp": "^0.5.0",
+            "safe-buffer": "^5.1.2",
+            "yallist": "^3.0.3"
+          }
+        },
+        "yallist": {
+          "version": "3.1.1",
+          "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
+          "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
+        }
+      }
+    },
     "bl": {
       "version": "3.0.0",
       "resolved": "https://registry.npmjs.org/bl/-/bl-3.0.0.tgz",
@@ -691,6 +746,11 @@
       "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
       "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw=="
     },
+    "integer": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/integer/-/integer-2.1.0.tgz",
+      "integrity": "sha512-vBtiSgrEiNocWvvZX1RVfeOKa2mCHLZQ2p9nkQkQZ/BvEiY+6CcUz0eyjvIiewjJoeNidzg2I+tpPJvpyspL1w=="
+    },
     "invert-kv": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz",

package.json

@@ -10,6 +10,7 @@
   "author": "",
   "license": "AGPL-3.0-only",
   "dependencies": {
+    "better-sqlite3": "^5.4.3",
     "mixin-deep": "^2.0.1",
     "node-dir": "^0.1.17",
     "node-fetch": "^2.6.0",

src/lib/collectors.js

@@ -3,6 +3,7 @@ const {request} = require("./utils/request")
 const {extractSharedData} = require("./utils/body")
 const {TtlCache, RequestCache} = require("./cache")
 const RequestHistory = require("./structures/RequestHistory")
+const db = require("./db")
 require("./testimports")(constants, request, extractSharedData, RequestCache, RequestHistory)
 
 const requestCache = new RequestCache(constants.caching.resource_cache_time)
@@ -25,6 +26,10 @@ function fetchUser(username) {
 				const sharedData = extractSharedData(text)
 				const user = new User(sharedData.entry_data.ProfilePage[0].graphql.user)
 				history.report("user", true)
+				if (constants.caching.db_user_id) {
+					db.prepare("INSERT OR IGNORE INTO Users (username, user_id) VALUES (@username, @user_id)")
+						.run({username: user.data.username, user_id: user.data.id})
+				}
 				return user
 			})
 		})
@@ -105,14 +110,18 @@ function fetchShortcodeData(shortcode) {
 				history.report("post", false)
 				console.error("missing data from post request, 429?", root) //todo: please make this better.
 				throw new Error("missing data from post request, 429?")
-			/** @type {import("./types").TimelineEntryN3} */
 			} else {
+				/** @type {import("./types").TimelineEntryN3} */
 				const data = root.data.shortcode_media
 				if (data == null) {
 					// the thing doesn't exist
 					throw constants.symbols.NOT_FOUND
 				} else {
 					history.report("post", true)
+					if (constants.caching.db_post_n3) {
+						db.prepare("REPLACE INTO Posts (shortcode, id, id_as_numeric, username, json) VALUES (@shortcode, @id, @id_as_numeric, @username, @json)")
+							.run({shortcode: data.shortcode, id: data.id, id_as_numeric: data.id, username: data.owner.username, json: JSON.stringify(data)})
+					}
 					return data
 				}
 			}

src/lib/constants.js

@@ -16,7 +16,9 @@ let constants = {
 	caching: {
 		image_cache_control: `public, max-age=${7*24*60*60}`,
 		resource_cache_time: 30*60*1000,
-		instance_list_cache_time: 3*60*1000
+		instance_list_cache_time: 3*60*1000,
+		db_user_id: true,
+		db_post_n3: true
 	},
 
 	// Instagram uses this stuff. This shouldn't be changed, except to fix a bug that hasn't yet been fixed upstream.
@@ -44,7 +46,9 @@ let constants = {
 		NOT_FOUND: Symbol("NOT_FOUND"),
 		NO_SHARED_DATA: Symbol("NO_SHARED_DATA"),
 		INSTAGRAM_DEMANDS_LOGIN: Symbol("INSTAGRAM_DEMANDS_LOGIN")
-	}
+	},
+
+	database_version: 1
 }
 
 // Override values from config and export the result

src/lib/db.js

@@ -0,0 +1,10 @@
+const sqlite = require("better-sqlite3")
+const pj = require("path").join
+const fs = require("fs")
+
+const dir = pj(__dirname, "../../db")
+fs.mkdirSync(pj(dir, "backups"), {recursive: true})
+const db = new sqlite(pj(dir, "bibliogram.db"))
+module.exports = db
+
+require("./utils/upgradedb")()

src/lib/utils/upgradedb.js

@@ -0,0 +1,57 @@
+const constants = require("../constants")
+const pj = require("path").join
+const db = require("../db")
+require("../testimports")(db)
+
+const deltas = new Map([
+	// empty file to version 1
+	[1, function() {
+		db.prepare("DROP TABLE IF EXISTS Users")
+			.run()
+		db.prepare("DROP TABLE IF EXISTS DatabaseVersion")
+			.run()
+		db.prepare("DROP TABLE IF Exists Posts")
+			.run()
+
+		db.prepare("CREATE TABLE Users (username TEXT NOT NULL UNIQUE, user_id TEXT NOT NULL UNIQUE, PRIMARY KEY (username))")
+			.run()
+		db.prepare("CREATE TABLE DatabaseVersion (version INTEGER NOT NULL UNIQUE, PRIMARY KEY (version))")
+			.run()
+		db.prepare("CREATE TABLE Posts (shortcode TEXT NOT NULL UNIQUE, id TEXT NOT NULL UNIQUE, id_as_numeric NUMERIC NOT NULL, username TEXT NOT NULL, json TEXT NOT NULL, PRIMARY KEY (shortcode))")
+			// for future investigation: may not be able to sort by id as a string, may not be able to fit entire id in numeric type
+			.run()
+	}]
+])
+
+module.exports = async function() {
+	let currentVersion = 0
+	try {
+		currentVersion = db.prepare("SELECT version FROM DatabaseVersion").pluck().get() || 0
+	} catch (e) {}
+
+	const newVersion = constants.database_version
+
+	if (currentVersion !== newVersion) {
+		console.log(`Upgrading database from version ${currentVersion} to version ${newVersion}...`)
+		// go through the entire upgrade path
+		for (let entry = currentVersion+1; entry <= newVersion; entry++) {
+			// Back up current version
+			if (entry !== 1) {
+				const filename = `backups/bibliogram.db.bak-v${entry-1}`
+				process.stdout.write(`Backing up current to ${filename}... `)
+				await db.backup(pj(__dirname, "../../../db", filename))
+				process.stdout.write("done.\n")
+			}
+			// Run delta
+			process.stdout.write(`Using script ${entry}... `)
+			deltas.get(entry)()
+			db.prepare("DELETE FROM DatabaseVersion").run()
+			db.prepare("INSERT INTO DatabaseVersion (version) VALUES (?)").run(entry)
+			process.stdout.write("done.\n")
+		}
+		console.log(
+			   "Upgrade complete."
+			+"\n-----------------"
+		)
+	}
+}