diff --git a/src/lib/utils/request_backends/saved.js b/src/lib/utils/request_backends/saved.js new file mode 100644 index 0000000..6dc35b9 --- /dev/null +++ b/src/lib/utils/request_backends/saved.js @@ -0,0 +1,40 @@ +const {Readable} = require("stream") +const fs = require("fs") +const fsp = fs.promises +const pj = require("path").join + +class Saved { + constructor(base) { + this.base = base + this.meta = fsp.readFile(`${this.base}.meta.json`, "utf8").then(data => JSON.parse(data)) + } + + stream() { + return Promise.resolve(fs.createReadStream(this.base)) + } + + response() { + return this.meta.then(res => ({ + status: res.status, + headers: new Map(Object.entries(res.headers).map(e => { + if (e[1].length === 1) e[1] = e[1][0] // collapse header arrays back to string if possible + return e + })) + })) + } + + json() { + return fsp.readFile(this.base, "utf8").then(data => JSON.parse(data)) + } + + text() { + return fsp.readFile(this.base, "utf8") + } + + async check(test) { + await this.response().then(res => test(res)) + return this + } +} + +module.exports = Saved diff --git a/src/lib/utils/saved_requests/manager.js b/src/lib/utils/saved_requests/manager.js new file mode 100644 index 0000000..a45d6aa --- /dev/null +++ b/src/lib/utils/saved_requests/manager.js @@ -0,0 +1,91 @@ +const fs = require("fs") +const fsp = fs.promises +const pj = require("path").join +const {promisify} = require("util") +const crypto = require("crypto") +const stream = require("stream") + +const db = require("../../db") +const Saved = require("../request_backends/saved") +const NodeFetch = require("../request_backends/node-fetch") +require("../../testimports")(db, Saved) + +const folder = __dirname + "/files" + +function generateName() { + return crypto.randomBytes(20).toString("hex") +} + +class DelayedBackend { + constructor(waiter) { + this.waiter = waiter + } + + stream() { + return this.waiter.then(instance => instance.stream()) + } + + response() { + return this.waiter.then(instance => instance.response()) + } + + json() { + return this.waiter.then(instance => instance.json()) + } + + text() { + return this.waiter.then(instance => instance.text()) + } + + check(test) { + return this.waiter.then(instance => instance.check(test)) + } +} + +class SavedRequestManager { + constructor(url, options) { + this.url = url.toString() + this.options = options + // console.log(this.url, this.options) + } + + clone() { + return new SavedRequestManager(this.url, this.options) + } + + request() { + const row = db.prepare("SELECT * FROM SavedRequests WHERE url = ?").get(this.url) + if (row) { + console.log("Found, using saved request for "+row.path) + const base = pj(folder, row.path) + return new Saved(base) + } else { + const name = generateName() + console.log("Not found, saving now as "+name) + + const internalRequest = new NodeFetch(this.url) + return new DelayedBackend( + Promise.all([ + internalRequest.instance.then(instance => { + return fsp.writeFile(pj(folder, name + ".meta.json"), JSON.stringify({ + status: instance.status, + headers: instance.headers.raw() + }, null, 4)) + }), + internalRequest.stream().then(readable => { + return promisify(stream.pipeline)( + readable, + fs.createWriteStream(pj(folder, name)) + ) + }) + ]).then(() => { + // console.log("Pipeline complete, writing database") + db.prepare("REPLACE INTO SavedRequests (url, path) VALUES (?, ?)").run(this.url, name) + return this.clone().request() + }) + ) + } + } +} + +module.exports = SavedRequestManager