From fbbb5052f10268b98ce22d68c3033d55fc2708bc Mon Sep 17 00:00:00 2001 From: Cadence Fish Date: Sat, 1 Feb 2020 17:44:40 +1300 Subject: [PATCH] Add database --- .gitignore | 1 + package-lock.json | 60 ++++++++++++++++++++++++++++++++++++++ package.json | 1 + src/lib/collectors.js | 11 ++++++- src/lib/constants.js | 8 +++-- src/lib/db.js | 10 +++++++ src/lib/utils/upgradedb.js | 57 ++++++++++++++++++++++++++++++++++++ 7 files changed, 145 insertions(+), 3 deletions(-) create mode 100644 src/lib/db.js create mode 100644 src/lib/utils/upgradedb.js diff --git a/.gitignore b/.gitignore index 76efb07..9fe2862 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ node_modules .vscode +db/**/*.db* diff --git a/package-lock.json b/package-lock.json index 9becd87..61b8f65 100644 --- a/package-lock.json +++ b/package-lock.json @@ -173,6 +173,61 @@ "tweetnacl": "^0.14.3" } }, + "better-sqlite3": { + "version": "5.4.3", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-5.4.3.tgz", + "integrity": "sha512-fPp+8f363qQIhuhLyjI4bu657J/FfMtgiiHKfaTsj3RWDkHlWC1yT7c6kHZDnBxzQVoAINuzg553qKmZ4F1rEw==", + "requires": { + "integer": "^2.1.0", + "tar": "^4.4.10" + }, + "dependencies": { + "fs-minipass": { + "version": "1.2.7", + "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz", + "integrity": "sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==", + "requires": { + "minipass": "^2.6.0" + } + }, + "minipass": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-2.9.0.tgz", + "integrity": "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==", + "requires": { + "safe-buffer": "^5.1.2", + "yallist": "^3.0.0" + } + }, + "minizlib": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.3.3.tgz", + "integrity": "sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==", + "requires": { + "minipass": "^2.9.0" + } + }, + "tar": { + "version": "4.4.13", + "resolved": "https://registry.npmjs.org/tar/-/tar-4.4.13.tgz", + "integrity": "sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA==", + "requires": { + "chownr": "^1.1.1", + "fs-minipass": "^1.2.5", + "minipass": "^2.8.6", + "minizlib": "^1.2.1", + "mkdirp": "^0.5.0", + "safe-buffer": "^5.1.2", + "yallist": "^3.0.3" + } + }, + "yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==" + } + } + }, "bl": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/bl/-/bl-3.0.0.tgz", @@ -691,6 +746,11 @@ "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" }, + "integer": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/integer/-/integer-2.1.0.tgz", + "integrity": "sha512-vBtiSgrEiNocWvvZX1RVfeOKa2mCHLZQ2p9nkQkQZ/BvEiY+6CcUz0eyjvIiewjJoeNidzg2I+tpPJvpyspL1w==" + }, "invert-kv": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz", diff --git a/package.json b/package.json index b8f9efb..1caafc0 100644 --- a/package.json +++ b/package.json @@ -10,6 +10,7 @@ "author": "", "license": "AGPL-3.0-only", "dependencies": { + "better-sqlite3": "^5.4.3", "mixin-deep": "^2.0.1", "node-dir": "^0.1.17", "node-fetch": "^2.6.0", diff --git a/src/lib/collectors.js b/src/lib/collectors.js index b51ddca..21c416f 100644 --- a/src/lib/collectors.js +++ b/src/lib/collectors.js @@ -3,6 +3,7 @@ const {request} = require("./utils/request") const {extractSharedData} = require("./utils/body") const {TtlCache, RequestCache} = require("./cache") const RequestHistory = require("./structures/RequestHistory") +const db = require("./db") require("./testimports")(constants, request, extractSharedData, RequestCache, RequestHistory) const requestCache = new RequestCache(constants.caching.resource_cache_time) @@ -25,6 +26,10 @@ function fetchUser(username) { const sharedData = extractSharedData(text) const user = new User(sharedData.entry_data.ProfilePage[0].graphql.user) history.report("user", true) + if (constants.caching.db_user_id) { + db.prepare("INSERT OR IGNORE INTO Users (username, user_id) VALUES (@username, @user_id)") + .run({username: user.data.username, user_id: user.data.id}) + } return user }) }) @@ -105,14 +110,18 @@ function fetchShortcodeData(shortcode) { history.report("post", false) console.error("missing data from post request, 429?", root) //todo: please make this better. throw new Error("missing data from post request, 429?") - /** @type {import("./types").TimelineEntryN3} */ } else { + /** @type {import("./types").TimelineEntryN3} */ const data = root.data.shortcode_media if (data == null) { // the thing doesn't exist throw constants.symbols.NOT_FOUND } else { history.report("post", true) + if (constants.caching.db_post_n3) { + db.prepare("REPLACE INTO Posts (shortcode, id, id_as_numeric, username, json) VALUES (@shortcode, @id, @id_as_numeric, @username, @json)") + .run({shortcode: data.shortcode, id: data.id, id_as_numeric: data.id, username: data.owner.username, json: JSON.stringify(data)}) + } return data } } diff --git a/src/lib/constants.js b/src/lib/constants.js index 7b236cf..b59a9c4 100644 --- a/src/lib/constants.js +++ b/src/lib/constants.js @@ -16,7 +16,9 @@ let constants = { caching: { image_cache_control: `public, max-age=${7*24*60*60}`, resource_cache_time: 30*60*1000, - instance_list_cache_time: 3*60*1000 + instance_list_cache_time: 3*60*1000, + db_user_id: true, + db_post_n3: true }, // Instagram uses this stuff. This shouldn't be changed, except to fix a bug that hasn't yet been fixed upstream. @@ -44,7 +46,9 @@ let constants = { NOT_FOUND: Symbol("NOT_FOUND"), NO_SHARED_DATA: Symbol("NO_SHARED_DATA"), INSTAGRAM_DEMANDS_LOGIN: Symbol("INSTAGRAM_DEMANDS_LOGIN") - } + }, + + database_version: 1 } // Override values from config and export the result diff --git a/src/lib/db.js b/src/lib/db.js new file mode 100644 index 0000000..02a1469 --- /dev/null +++ b/src/lib/db.js @@ -0,0 +1,10 @@ +const sqlite = require("better-sqlite3") +const pj = require("path").join +const fs = require("fs") + +const dir = pj(__dirname, "../../db") +fs.mkdirSync(pj(dir, "backups"), {recursive: true}) +const db = new sqlite(pj(dir, "bibliogram.db")) +module.exports = db + +require("./utils/upgradedb")() diff --git a/src/lib/utils/upgradedb.js b/src/lib/utils/upgradedb.js new file mode 100644 index 0000000..ab702cc --- /dev/null +++ b/src/lib/utils/upgradedb.js @@ -0,0 +1,57 @@ +const constants = require("../constants") +const pj = require("path").join +const db = require("../db") +require("../testimports")(db) + +const deltas = new Map([ + // empty file to version 1 + [1, function() { + db.prepare("DROP TABLE IF EXISTS Users") + .run() + db.prepare("DROP TABLE IF EXISTS DatabaseVersion") + .run() + db.prepare("DROP TABLE IF Exists Posts") + .run() + + db.prepare("CREATE TABLE Users (username TEXT NOT NULL UNIQUE, user_id TEXT NOT NULL UNIQUE, PRIMARY KEY (username))") + .run() + db.prepare("CREATE TABLE DatabaseVersion (version INTEGER NOT NULL UNIQUE, PRIMARY KEY (version))") + .run() + db.prepare("CREATE TABLE Posts (shortcode TEXT NOT NULL UNIQUE, id TEXT NOT NULL UNIQUE, id_as_numeric NUMERIC NOT NULL, username TEXT NOT NULL, json TEXT NOT NULL, PRIMARY KEY (shortcode))") + // for future investigation: may not be able to sort by id as a string, may not be able to fit entire id in numeric type + .run() + }] +]) + +module.exports = async function() { + let currentVersion = 0 + try { + currentVersion = db.prepare("SELECT version FROM DatabaseVersion").pluck().get() || 0 + } catch (e) {} + + const newVersion = constants.database_version + + if (currentVersion !== newVersion) { + console.log(`Upgrading database from version ${currentVersion} to version ${newVersion}...`) + // go through the entire upgrade path + for (let entry = currentVersion+1; entry <= newVersion; entry++) { + // Back up current version + if (entry !== 1) { + const filename = `backups/bibliogram.db.bak-v${entry-1}` + process.stdout.write(`Backing up current to ${filename}... `) + await db.backup(pj(__dirname, "../../../db", filename)) + process.stdout.write("done.\n") + } + // Run delta + process.stdout.write(`Using script ${entry}... `) + deltas.get(entry)() + db.prepare("DELETE FROM DatabaseVersion").run() + db.prepare("INSERT INTO DatabaseVersion (version) VALUES (?)").run(entry) + process.stdout.write("done.\n") + } + console.log( + "Upgrade complete." + +"\n-----------------" + ) + } +}