Lomanic/bibliogram
1
0
Fork 0
mirror of https://git.sr.ht/~cadence/bibliogram synced 2024-11-22 08:07:30 +00:00
Code Issues Releases Wiki Activity

Use get-stream instead of reading by hand

Browse Source
This commit is contained in:
Cadence Ember 2020-07-13 03:32:04 +12:00
parent b4d21f5002
commit bfc8a1443f
No known key found for this signature in database
GPG Key ID: 128B99B1B74A6412
3 changed files with 3 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
package-lock.json generated
View File

@ -1803,7 +1803,6 @@
"version": "5.1.0", "version": "5.1.0",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.1.0.tgz",
"integrity": "sha512-EXr1FOzrzTfGeL0gQdeFEvOMm2mzMOglyiOXSTpPC+iAjAKftbr3jpCMWynogwYnM+eSj9sHGc6wjIcDvYiygw==", "integrity": "sha512-EXr1FOzrzTfGeL0gQdeFEvOMm2mzMOglyiOXSTpPC+iAjAKftbr3jpCMWynogwYnM+eSj9sHGc6wjIcDvYiygw==",
"optional": true,
"requires": { "requires": {
"pump": "^3.0.0" "pump": "^3.0.0"
} }

1
package.json
View File

@ -16,6 +16,7 @@
"better-sqlite3": "^6.0.1", "better-sqlite3": "^6.0.1",
"cookie": "^0.4.1", "cookie": "^0.4.1",
"feed": "github:cloudrac3r/feed#f42b4d7183fc2b6e566cb400ad083c4dd714c858", "feed": "github:cloudrac3r/feed#f42b4d7183fc2b6e566cb400ad083c4dd714c858",
"get-stream": "^5.1.0",
"mixin-deep": "^2.0.1", "mixin-deep": "^2.0.1",
"node-dir": "^0.1.17", "node-dir": "^0.1.17",
"node-fetch": "^2.6.0", "node-fetch": "^2.6.0",

7
scripts/import_users.js
View File

@ -1,7 +1,7 @@
const fs = require("fs") const fs = require("fs")
const {createGunzip} = require("zlib") const {createGunzip} = require("zlib")
const pj = require("path").join
const db = require("../src/lib/db") const db = require("../src/lib/db")
const getStream = require("get-stream")
const {request} = require("../src/lib/utils/request") const {request} = require("../src/lib/utils/request")
async function progress(message, callback) { async function progress(message, callback) {
@ -42,10 +42,7 @@ async function progress(message, callback) {
// Read out the stream into a buffer // Read out the stream into a buffer
/** @type {{username: string, user_id: string, created: number, updated: number, updated_version: number, biography: string, post_count: number, following_count: number, followed_by_count: number, external_url: string, full_name: string, is_private: number, is_verified: number, profile_pic_url: string}[]} */ /** @type {{username: string, user_id: string, created: number, updated: number, updated_version: number, biography: string, post_count: number, following_count: number, followed_by_count: number, external_url: string, full_name: string, is_private: number, is_verified: number, profile_pic_url: string}[]} */
const incomingUsers = await progress("Reading data... ", async () => { const incomingUsers = await progress("Reading data... ", async () => {
const buffers = [] const usersString = await getStream(usersStream, {encoding: "utf8"})
usersStream.on("data", chunk => buffers.push(chunk))
await new Promise(resolve => usersStream.once("end", resolve))
const usersString = Buffer.concat(buffers).toString("utf8")
return JSON.parse(usersString) return JSON.parse(usersString)
}) })