mirror of https://git.sr.ht/~cadence/bibliogram
Add database
This commit is contained in:
Cadence Fish
parent
341aade87c
commit
fbbb5052f1
|
|
@ -1,2 +1,3 @@
|
|||
node_modules
|
||||
.vscode
|
||||
db/**/*.db*
|
||||
|
|
|
|||
|
|
@ -173,6 +173,61 @@
|
|||
"tweetnacl": "^0.14.3"
|
||||
}
|
||||
},
|
||||
"better-sqlite3": {
|
||||
"version": "5.4.3",
|
||||
"resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-5.4.3.tgz",
|
||||
"integrity": "sha512-fPp+8f363qQIhuhLyjI4bu657J/FfMtgiiHKfaTsj3RWDkHlWC1yT7c6kHZDnBxzQVoAINuzg553qKmZ4F1rEw==",
|
||||
"requires": {
|
||||
"integer": "^2.1.0",
|
||||
"tar": "^4.4.10"
|
||||
},
|
||||
"dependencies": {
|
||||
"fs-minipass": {
|
||||
"version": "1.2.7",
|
||||
"resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz",
|
||||
"integrity": "sha512-GWSSJGFy4e9GUeCcbIkED+bgAoFyj7XF1mV8rma3QW4NIqX9Kyx79N/PF61H5udOV3aY1IaMLs6pGbH71nlCTA==",
|
||||
"requires": {
|
||||
"minipass": "^2.6.0"
|
||||
}
|
||||
},
|
||||
"minipass": {
|
||||
"version": "2.9.0",
|
||||
"resolved": "https://registry.npmjs.org/minipass/-/minipass-2.9.0.tgz",
|
||||
"integrity": "sha512-wxfUjg9WebH+CUDX/CdbRlh5SmfZiy/hpkxaRI16Y9W56Pa75sWgd/rvFilSgrauD9NyFymP/+JFV3KwzIsJeg==",
|
||||
"requires": {
|
||||
"safe-buffer": "^5.1.2",
|
||||
"yallist": "^3.0.0"
|
||||
}
|
||||
},
|
||||
"minizlib": {
|
||||
"version": "1.3.3",
|
||||
"resolved": "https://registry.npmjs.org/minizlib/-/minizlib-1.3.3.tgz",
|
||||
"integrity": "sha512-6ZYMOEnmVsdCeTJVE0W9ZD+pVnE8h9Hma/iOwwRDsdQoePpoX56/8B6z3P9VNwppJuBKNRuFDRNRqRWexT9G9Q==",
|
||||
"requires": {
|
||||
"minipass": "^2.9.0"
|
||||
}
|
||||
},
|
||||
"tar": {
|
||||
"version": "4.4.13",
|
||||
"resolved": "https://registry.npmjs.org/tar/-/tar-4.4.13.tgz",
|
||||
"integrity": "sha512-w2VwSrBoHa5BsSyH+KxEqeQBAllHhccyMFVHtGtdMpF4W7IRWfZjFiQceJPChOeTsSDVUpER2T8FA93pr0L+QA==",
|
||||
"requires": {
|
||||
"chownr": "^1.1.1",
|
||||
"fs-minipass": "^1.2.5",
|
||||
"minipass": "^2.8.6",
|
||||
"minizlib": "^1.2.1",
|
||||
"mkdirp": "^0.5.0",
|
||||
"safe-buffer": "^5.1.2",
|
||||
"yallist": "^3.0.3"
|
||||
}
|
||||
},
|
||||
"yallist": {
|
||||
"version": "3.1.1",
|
||||
"resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz",
|
||||
"integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g=="
|
||||
}
|
||||
}
|
||||
},
|
||||
"bl": {
|
||||
"version": "3.0.0",
|
||||
"resolved": "https://registry.npmjs.org/bl/-/bl-3.0.0.tgz",
|
||||
|
|
@ -691,6 +746,11 @@
|
|||
"resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz",
|
||||
"integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw=="
|
||||
},
|
||||
"integer": {
|
||||
"version": "2.1.0",
|
||||
"resolved": "https://registry.npmjs.org/integer/-/integer-2.1.0.tgz",
|
||||
"integrity": "sha512-vBtiSgrEiNocWvvZX1RVfeOKa2mCHLZQ2p9nkQkQZ/BvEiY+6CcUz0eyjvIiewjJoeNidzg2I+tpPJvpyspL1w=="
|
||||
},
|
||||
"invert-kv": {
|
||||
"version": "1.0.0",
|
||||
"resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz",
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@
|
|||
"author": "",
|
||||
"license": "AGPL-3.0-only",
|
||||
"dependencies": {
|
||||
"better-sqlite3": "^5.4.3",
|
||||
"mixin-deep": "^2.0.1",
|
||||
"node-dir": "^0.1.17",
|
||||
"node-fetch": "^2.6.0",
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ const {request} = require("./utils/request")
|
|||
const {extractSharedData} = require("./utils/body")
|
||||
const {TtlCache, RequestCache} = require("./cache")
|
||||
const RequestHistory = require("./structures/RequestHistory")
|
||||
const db = require("./db")
|
||||
require("./testimports")(constants, request, extractSharedData, RequestCache, RequestHistory)
|
||||
|
||||
const requestCache = new RequestCache(constants.caching.resource_cache_time)
|
||||
|
|
@ -25,6 +26,10 @@ function fetchUser(username) {
|
|||
const sharedData = extractSharedData(text)
|
||||
const user = new User(sharedData.entry_data.ProfilePage[0].graphql.user)
|
||||
history.report("user", true)
|
||||
if (constants.caching.db_user_id) {
|
||||
db.prepare("INSERT OR IGNORE INTO Users (username, user_id) VALUES (@username, @user_id)")
|
||||
.run({username: user.data.username, user_id: user.data.id})
|
||||
}
|
||||
return user
|
||||
})
|
||||
})
|
||||
|
|
@ -105,14 +110,18 @@ function fetchShortcodeData(shortcode) {
|
|||
history.report("post", false)
|
||||
console.error("missing data from post request, 429?", root) //todo: please make this better.
|
||||
throw new Error("missing data from post request, 429?")
|
||||
/** @type {import("./types").TimelineEntryN3} */
|
||||
} else {
|
||||
/** @type {import("./types").TimelineEntryN3} */
|
||||
const data = root.data.shortcode_media
|
||||
if (data == null) {
|
||||
// the thing doesn't exist
|
||||
throw constants.symbols.NOT_FOUND
|
||||
} else {
|
||||
history.report("post", true)
|
||||
if (constants.caching.db_post_n3) {
|
||||
db.prepare("REPLACE INTO Posts (shortcode, id, id_as_numeric, username, json) VALUES (@shortcode, @id, @id_as_numeric, @username, @json)")
|
||||
.run({shortcode: data.shortcode, id: data.id, id_as_numeric: data.id, username: data.owner.username, json: JSON.stringify(data)})
|
||||
}
|
||||
return data
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,7 +16,9 @@ let constants = {
|
|||
caching: {
|
||||
image_cache_control: `public, max-age=${7*24*60*60}`,
|
||||
resource_cache_time: 30*60*1000,
|
||||
instance_list_cache_time: 3*60*1000
|
||||
instance_list_cache_time: 3*60*1000,
|
||||
db_user_id: true,
|
||||
db_post_n3: true
|
||||
},
|
||||
|
||||
// Instagram uses this stuff. This shouldn't be changed, except to fix a bug that hasn't yet been fixed upstream.
|
||||
|
|
@ -44,7 +46,9 @@ let constants = {
|
|||
NOT_FOUND: Symbol("NOT_FOUND"),
|
||||
NO_SHARED_DATA: Symbol("NO_SHARED_DATA"),
|
||||
INSTAGRAM_DEMANDS_LOGIN: Symbol("INSTAGRAM_DEMANDS_LOGIN")
|
||||
}
|
||||
},
|
||||
|
||||
database_version: 1
|
||||
}
|
||||
|
||||
// Override values from config and export the result
|
||||
|
|
|
|||
|
|
@ -0,0 +1,10 @@
|
|||
const sqlite = require("better-sqlite3")
|
||||
const pj = require("path").join
|
||||
const fs = require("fs")
|
||||
|
||||
const dir = pj(__dirname, "../../db")
|
||||
fs.mkdirSync(pj(dir, "backups"), {recursive: true})
|
||||
const db = new sqlite(pj(dir, "bibliogram.db"))
|
||||
module.exports = db
|
||||
|
||||
require("./utils/upgradedb")()
|
||||
|
|
@ -0,0 +1,57 @@
|
|||
const constants = require("../constants")
|
||||
const pj = require("path").join
|
||||
const db = require("../db")
|
||||
require("../testimports")(db)
|
||||
|
||||
const deltas = new Map([
|
||||
// empty file to version 1
|
||||
[1, function() {
|
||||
db.prepare("DROP TABLE IF EXISTS Users")
|
||||
.run()
|
||||
db.prepare("DROP TABLE IF EXISTS DatabaseVersion")
|
||||
.run()
|
||||
db.prepare("DROP TABLE IF Exists Posts")
|
||||
.run()
|
||||
|
||||
db.prepare("CREATE TABLE Users (username TEXT NOT NULL UNIQUE, user_id TEXT NOT NULL UNIQUE, PRIMARY KEY (username))")
|
||||
.run()
|
||||
db.prepare("CREATE TABLE DatabaseVersion (version INTEGER NOT NULL UNIQUE, PRIMARY KEY (version))")
|
||||
.run()
|
||||
db.prepare("CREATE TABLE Posts (shortcode TEXT NOT NULL UNIQUE, id TEXT NOT NULL UNIQUE, id_as_numeric NUMERIC NOT NULL, username TEXT NOT NULL, json TEXT NOT NULL, PRIMARY KEY (shortcode))")
|
||||
// for future investigation: may not be able to sort by id as a string, may not be able to fit entire id in numeric type
|
||||
.run()
|
||||
}]
|
||||
])
|
||||
|
||||
module.exports = async function() {
|
||||
let currentVersion = 0
|
||||
try {
|
||||
currentVersion = db.prepare("SELECT version FROM DatabaseVersion").pluck().get() || 0
|
||||
} catch (e) {}
|
||||
|
||||
const newVersion = constants.database_version
|
||||
|
||||
if (currentVersion !== newVersion) {
|
||||
console.log(`Upgrading database from version ${currentVersion} to version ${newVersion}...`)
|
||||
// go through the entire upgrade path
|
||||
for (let entry = currentVersion+1; entry <= newVersion; entry++) {
|
||||
// Back up current version
|
||||
if (entry !== 1) {
|
||||
const filename = `backups/bibliogram.db.bak-v${entry-1}`
|
||||
process.stdout.write(`Backing up current to ${filename}... `)
|
||||
await db.backup(pj(__dirname, "../../../db", filename))
|
||||
process.stdout.write("done.\n")
|
||||
}
|
||||
// Run delta
|
||||
process.stdout.write(`Using script ${entry}... `)
|
||||
deltas.get(entry)()
|
||||
db.prepare("DELETE FROM DatabaseVersion").run()
|
||||
db.prepare("INSERT INTO DatabaseVersion (version) VALUES (?)").run(entry)
|
||||
process.stdout.write("done.\n")
|
||||
}
|
||||
console.log(
|
||||
"Upgrade complete."
|
||||
+"\n-----------------"
|
||||
)
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue