From 96fa4758c09df5410afaac957a70dad247b35bfa Mon Sep 17 00:00:00 2001 From: Cadence Fish Date: Mon, 3 Feb 2020 00:43:56 +1300 Subject: [PATCH] Optional Tor support --- README.md | 4 +- package-lock.json | 507 +++++++++++++++++++++++++++++------ package.json | 9 +- src/lib/collectors.js | 57 ++-- src/lib/constants.js | 6 +- src/lib/utils/request.js | 11 +- src/lib/utils/tor.js | 79 ++++++ src/lib/utils/torswitcher.js | 41 +++ src/site/api/proxy.js | 4 +- src/site/sass/main.sass | 1 + src/site/server.js | 3 + 11 files changed, 597 insertions(+), 125 deletions(-) create mode 100644 src/lib/utils/tor.js create mode 100644 src/lib/utils/torswitcher.js diff --git a/README.md b/README.md index 71f0fe5..04a9d71 100644 --- a/README.md +++ b/README.md @@ -60,9 +60,9 @@ If you only use one computer, you can install Bibliogram on that computer and th You need [node.js](https://nodejs.org/en/) to run Bibliogram. Versions before 12.13.0 are untested. -1. `$ git clone https://github.com/cloudrac3r/bibliogram` +1. `$ git clone https://github.com/cloudrac3r/bibliogram` If you are using a fork, be sure to actually install that fork instead! -1. `$ npm install` +1. `$ npm install --no-optional` (for Tor support, omit `--no-optional`) 1. Edit `/config.js` to suit your server environment 1. `$ npm start` diff --git a/package-lock.json b/package-lock.json index 6c556e6..27c029c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -4,6 +4,36 @@ "lockfileVersion": 1, "requires": true, "dependencies": { + "7zip": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/7zip/-/7zip-0.0.6.tgz", + "integrity": "sha1-nK+xca+CMpSQNTtIFvAzR6oVCjA=", + "optional": true + }, + "@deadcanaries/granax": { + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/@deadcanaries/granax/-/granax-3.2.5.tgz", + "integrity": "sha512-umsEYedNQWLSlkDnGynR8bY0pVWq2hKN1JLtV9Jit7srYHqdo+XIEsUNejq1nNBiX83WhUZVe4gOqvg8ovdMMQ==", + "optional": true, + "requires": { + "7zip": "0.0.6", + "async": "^2.3.0", + "latest-torbrowser-version": "^2.0.2", + "merge": "^1.2.1", + "mkdirp": "^0.5.1", + "mv": "^2.1.1", + "ncp": "^2.0.0", + "progress": "^2.0.3", + "rimraf": "^2.6.3" + } + }, + "@outtacontrol/socks": { + "version": "github:phoenix344/socksv5#dbdecf75be849337a31e78ba14d316deed43287f", + "from": "github:phoenix344/socksv5#dbdecf75be849337a31e78ba14d316deed43287f", + "requires": { + "ip-address": "^5.8.9" + } + }, "@types/babel-types": { "version": "7.0.7", "resolved": "https://registry.npmjs.org/@types/babel-types/-/babel-types-7.0.7.tgz", @@ -17,6 +47,12 @@ "@types/babel-types": "*" } }, + "@types/node": { + "version": "13.7.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.0.tgz", + "integrity": "sha512-GnZbirvmqZUzMgkFn70c74OQpTTUcCzlhQliTzYjQMqg+hVKcDnxdL19Ne3UdYzdMA/+W3eb646FWn/ZaT1NfQ==", + "optional": true + }, "abbrev": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz", @@ -42,6 +78,14 @@ } } }, + "agent-base": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.0.tgz", + "integrity": "sha512-j1Q7cSCqN+AwrmDd+pzgqc0/NpC655x2bUf5ZjRIO77DcNBFmh+OgRNzF6OKdCC9RSCb19fGd99+bhXFdkRNqw==", + "requires": { + "debug": "4" + } + }, "ajv": { "version": "6.11.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.11.0.tgz", @@ -115,6 +159,15 @@ "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz", "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU=" }, + "async": { + "version": "2.6.3", + "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz", + "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==", + "optional": true, + "requires": { + "lodash": "^4.17.14" + } + }, "async-foreach": { "version": "0.1.3", "resolved": "https://registry.npmjs.org/async-foreach/-/async-foreach-0.1.3.tgz", @@ -256,6 +309,12 @@ "inherits": "~2.0.0" } }, + "boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=", + "optional": true + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -266,9 +325,9 @@ } }, "camelcase": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-2.1.1.tgz", - "integrity": "sha1-fB0W1nmhu+WcoCys7PsBHiAfWh8=" + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-1.2.1.tgz", + "integrity": "sha1-m7UwTS4LVmmLLHWLCKPqqdqlijk=" }, "camelcase-keys": { "version": "2.1.0", @@ -277,6 +336,13 @@ "requires": { "camelcase": "^2.0.0", "map-obj": "^1.0.0" + }, + "dependencies": { + "camelcase": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-2.1.1.tgz", + "integrity": "sha1-fB0W1nmhu+WcoCys7PsBHiAfWh8=" + } } }, "caseless": { @@ -313,34 +379,41 @@ "is-regex": "^1.0.3" } }, + "cheerio": { + "version": "1.0.0-rc.3", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz", + "integrity": "sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==", + "optional": true, + "requires": { + "css-select": "~1.2.0", + "dom-serializer": "~0.1.1", + "entities": "~1.1.1", + "htmlparser2": "^3.9.1", + "lodash": "^4.15.0", + "parse5": "^3.0.1" + } + }, "chownr": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.3.tgz", "integrity": "sha512-i70fVHhmV3DtTl6nqvZOnIjbY0Pe4kAUjwHj8z0zAdgBtYrJyYwLKCCuRBQ5ppkyL0AkN7HKRnETdmdp1zqNXw==" }, "clean-css": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/clean-css/-/clean-css-4.2.2.tgz", - "integrity": "sha512-yKycArwReQXbOD/3pmsPmt6p7oUBww8MisDabL2pCUWkbVONvCJoBdCjgY4ZVQmKX5juz/JB9oDcP6XzGUpjwQ==", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/clean-css/-/clean-css-4.2.3.tgz", + "integrity": "sha512-VcMWDN54ZN/DS+g58HYL5/n4Zrqe8vHJpGA8KdgUXFU4fuP/aHNw8eld9SyEIyabIMJX/0RaY/fplOo5hYLSFA==", "requires": { "source-map": "~0.6.0" - }, - "dependencies": { - "source-map": { - "version": "0.6.1", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", - "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" - } } }, "cliui": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz", - "integrity": "sha1-EgYBU3qRbSmUD5NNo7SNWFo5IT0=", + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-2.1.0.tgz", + "integrity": "sha1-S0dXYP+AJkx2LDoXGQMukcf+oNE=", "requires": { - "string-width": "^1.0.1", - "strip-ansi": "^3.0.1", - "wrap-ansi": "^2.0.0" + "center-align": "^0.1.1", + "right-align": "^0.1.1", + "wordwrap": "0.0.2" } }, "code-point-at": { @@ -418,6 +491,33 @@ "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" }, + "cross-spawn": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-3.0.1.tgz", + "integrity": "sha1-ElYDfsufDF9549bvE14wdwGEuYI=", + "requires": { + "lru-cache": "^4.0.1", + "which": "^1.2.9" + } + }, + "css-select": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz", + "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=", + "optional": true, + "requires": { + "boolbase": "~1.0.0", + "css-what": "2.1", + "domutils": "1.5.1", + "nth-check": "~1.0.1" + } + }, + "css-what": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz", + "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg==", + "optional": true + }, "currently-unhandled": { "version": "0.4.1", "resolved": "https://registry.npmjs.org/currently-unhandled/-/currently-unhandled-0.4.1.tgz", @@ -434,6 +534,14 @@ "assert-plus": "^1.0.0" } }, + "debug": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", + "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", + "requires": { + "ms": "^2.1.1" + } + }, "decamelize": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", @@ -472,6 +580,41 @@ "resolved": "https://registry.npmjs.org/doctypes/-/doctypes-1.1.0.tgz", "integrity": "sha1-6oCxBqh1OHdOijpKWv4pPeSJ4Kk=" }, + "dom-serializer": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz", + "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==", + "optional": true, + "requires": { + "domelementtype": "^1.3.0", + "entities": "^1.1.1" + } + }, + "domelementtype": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz", + "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==", + "optional": true + }, + "domhandler": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz", + "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==", + "optional": true, + "requires": { + "domelementtype": "1" + } + }, + "domutils": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz", + "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=", + "optional": true, + "requires": { + "dom-serializer": "0", + "domelementtype": "1" + } + }, "ecc-jsbn": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz", @@ -489,6 +632,12 @@ "once": "^1.4.0" } }, + "entities": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz", + "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==", + "optional": true + }, "error-ex": { "version": "1.3.2", "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz", @@ -541,6 +690,26 @@ "pinkie-promise": "^2.0.0" } }, + "follow-redirects": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.10.0.tgz", + "integrity": "sha512-4eyLK6s6lH32nOvLLwlIOnr9zrL8Sm+OvW4pVTJNoXeGzYIkHVf+pADQi+OJ0E67hiuSLezPVPyBcIZO50TmmQ==", + "optional": true, + "requires": { + "debug": "^3.0.0" + }, + "dependencies": { + "debug": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "optional": true, + "requires": { + "ms": "^2.1.1" + } + } + } + }, "forever-agent": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz", @@ -704,6 +873,33 @@ "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.5.tgz", "integrity": "sha512-kssjab8CvdXfcXMXVcvsXum4Hwdq9XGtRD3TteMEvEbq0LXyiNQr6AprqKqfeaDXze7SxWvRxdpwE6ku7ikLkg==" }, + "htmlparser2": { + "version": "3.10.1", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz", + "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==", + "optional": true, + "requires": { + "domelementtype": "^1.3.1", + "domhandler": "^2.3.0", + "domutils": "^1.5.1", + "entities": "^1.1.1", + "inherits": "^2.0.1", + "readable-stream": "^3.1.1" + }, + "dependencies": { + "readable-stream": { + "version": "3.5.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.5.0.tgz", + "integrity": "sha512-gSz026xs2LfxBPudDuI41V1lka8cxg64E66SGe78zJlsUofOg/yqwezdIcdfwik6B4h8LFmWPA9ef9X3FiNFLA==", + "optional": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + } + } + }, "http-signature": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz", @@ -756,6 +952,28 @@ "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz", "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY=" }, + "ip": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", + "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=" + }, + "ip-address": { + "version": "5.9.4", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-5.9.4.tgz", + "integrity": "sha512-dHkI3/YNJq4b/qQaz+c8LuarD3pY24JqZWfjB8aZx1gtpc2MDILu9L9jpZe1sHpzo/yWFweQVn+U//FhazUxmw==", + "requires": { + "jsbn": "1.1.0", + "lodash": "^4.17.15", + "sprintf-js": "1.1.2" + }, + "dependencies": { + "jsbn": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz", + "integrity": "sha1-sBMHyym2GKHtJux56RH4A8TaAEA=" + } + } + }, "is-arrayish": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", @@ -894,6 +1112,26 @@ "is-buffer": "^1.1.5" } }, + "latest-torbrowser-version": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/latest-torbrowser-version/-/latest-torbrowser-version-2.0.3.tgz", + "integrity": "sha512-+6fkERFLxo2QiNn7x0txjEF199tG4CGVI9X101d7IuZAapO/hGOC+nHD9BEPNNaMqmvIUkT/o3CdZ+NS5wX/iQ==", + "optional": true, + "requires": { + "async": "^2.6.0", + "cheerio": "^1.0.0-rc.2", + "follow-redirects": "^1.2.4", + "semver": "^5.4.1" + }, + "dependencies": { + "semver": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz", + "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==", + "optional": true + } + } + }, "lazy-cache": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz", @@ -969,6 +1207,12 @@ "trim-newlines": "^1.0.0" } }, + "merge": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/merge/-/merge-1.2.1.tgz", + "integrity": "sha512-VjFo4P5Whtj4vsLzsYBu5ayHhoHJ0UqNm7ibvShmbmoz7tGi0vXaoJbGdB+GmDMLUdg8DpQXEIeVDAe8MaABvQ==", + "optional": true + }, "mime": { "version": "2.4.4", "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz", @@ -1056,6 +1300,46 @@ } } }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "mv": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz", + "integrity": "sha1-rmzg1vbV4KT32JN5jQPB6pVZtqI=", + "optional": true, + "requires": { + "mkdirp": "~0.5.1", + "ncp": "~2.0.0", + "rimraf": "~2.4.0" + }, + "dependencies": { + "glob": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz", + "integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=", + "optional": true, + "requires": { + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "2 || 3", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, + "rimraf": { + "version": "2.4.5", + "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz", + "integrity": "sha1-7nEM5dk6j9uFb7Xqj/Di11k0sto=", + "optional": true, + "requires": { + "glob": "^6.0.1" + } + } + } + }, "nan": { "version": "2.14.0", "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz", @@ -1066,6 +1350,12 @@ "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.1.tgz", "integrity": "sha512-boQj1WFgQH3v4clhu3mTNfP+vOBxorDlE8EKiMjUlLG3C4qAESnn9AxIOkFgTR2c9LtzNjPrjS60cT27ZKBhaA==" }, + "ncp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz", + "integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=", + "optional": true + }, "node-abi": { "version": "2.13.0", "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.13.0.tgz", @@ -1142,17 +1432,6 @@ "sass-graph": "^2.2.4", "stdout-stream": "^1.4.0", "true-case-path": "^1.0.2" - }, - "dependencies": { - "cross-spawn": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-3.0.1.tgz", - "integrity": "sha1-ElYDfsufDF9549bvE14wdwGEuYI=", - "requires": { - "lru-cache": "^4.0.1", - "which": "^1.2.9" - } - } } }, "noop-logger": { @@ -1197,6 +1476,15 @@ "set-blocking": "~2.0.0" } }, + "nth-check": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz", + "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==", + "optional": true, + "requires": { + "boolbase": "~1.0.0" + } + }, "number-is-nan": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz", @@ -1255,6 +1543,15 @@ "error-ex": "^1.2.0" } }, + "parse5": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz", + "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==", + "optional": true, + "requires": { + "@types/node": "*" + } + }, "path-exists": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz", @@ -1307,11 +1604,11 @@ } }, "pinski": { - "version": "github:cloudrac3r/pinski#19495f7a1c62fbfd8c685cf5bb5d678788906032", - "from": "github:cloudrac3r/pinski#19495f7a1c62fbfd8c685cf5bb5d678788906032", + "version": "github:cloudrac3r/pinski#9eb56d90fdd00357451dd5a546dbcca1f9bf114a", + "from": "github:cloudrac3r/pinski#9eb56d90fdd00357451dd5a546dbcca1f9bf114a", "requires": { "mime": "^2.4.4", - "node-sass": "^4.12.0", + "node-sass": "^4.13.1", "pug": "^2.0.3", "ws": "^7.1.1" } @@ -1343,6 +1640,12 @@ "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" }, + "progress": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz", + "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==", + "optional": true + }, "promise": { "version": "7.3.1", "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz", @@ -1669,6 +1972,43 @@ "lodash": "^4.0.0", "scss-tokenizer": "^0.2.3", "yargs": "^7.0.0" + }, + "dependencies": { + "camelcase": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-3.0.0.tgz", + "integrity": "sha1-MvxLn82vhF/N9+c7uXysImHwqwo=" + }, + "cliui": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz", + "integrity": "sha1-EgYBU3qRbSmUD5NNo7SNWFo5IT0=", + "requires": { + "string-width": "^1.0.1", + "strip-ansi": "^3.0.1", + "wrap-ansi": "^2.0.0" + } + }, + "yargs": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-7.1.0.tgz", + "integrity": "sha1-a6MY6xaWFyf10oT46gA+jWFU0Mg=", + "requires": { + "camelcase": "^3.0.0", + "cliui": "^3.2.0", + "decamelize": "^1.1.1", + "get-caller-file": "^1.0.1", + "os-locale": "^1.4.0", + "read-pkg-up": "^1.0.1", + "require-directory": "^2.1.1", + "require-main-filename": "^1.0.1", + "set-blocking": "^2.0.0", + "string-width": "^1.0.2", + "which-module": "^1.0.0", + "y18n": "^3.2.1", + "yargs-parser": "^5.0.0" + } + } } }, "scss-tokenizer": { @@ -1678,6 +2018,16 @@ "requires": { "js-base64": "^2.1.8", "source-map": "^0.4.2" + }, + "dependencies": { + "source-map": { + "version": "0.4.4", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz", + "integrity": "sha1-66T12pwNyZneaAMti092FzZSA2s=", + "requires": { + "amdefine": ">=0.0.4" + } + } } }, "semver": { @@ -1766,14 +2116,34 @@ } } }, - "source-map": { - "version": "0.4.4", - "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz", - "integrity": "sha1-66T12pwNyZneaAMti092FzZSA2s=", + "smart-buffer": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.1.0.tgz", + "integrity": "sha512-iVICrxOzCynf/SNaBQCw34eM9jROU/s5rzIhpOvzhzuYHfJR/DhZfDkXiZSgKXfgv26HT3Yni3AV/DGw0cGnnw==" + }, + "socks": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.3.3.tgz", + "integrity": "sha512-o5t52PCNtVdiOvzMry7wU4aOqYWL0PeCXRWBEiJow4/i/wr+wpsJQ9awEu1EonLIqsfGd5qSgDdxEOvCdmBEpA==", "requires": { - "amdefine": ">=0.0.4" + "ip": "1.1.5", + "smart-buffer": "^4.1.0" } }, + "socks-proxy-agent": { + "version": "github:cloudrac3r/node-socks-proxy-agent#6a26d274b12098dfef6cc2faafd25b0c051f2467", + "from": "github:cloudrac3r/node-socks-proxy-agent#6a26d274b12098dfef6cc2faafd25b0c051f2467", + "requires": { + "@outtacontrol/socks": "github:phoenix344/socksv5#dbdecf75be849337a31e78ba14d316deed43287f", + "agent-base": "^6.0.0", + "socks": "^2.3.2" + } + }, + "source-map": { + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", + "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g==" + }, "spdx-correct": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.0.tgz", @@ -1802,6 +2172,11 @@ "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.5.tgz", "integrity": "sha512-J+FWzZoynJEXGphVIS+XEh3kFSjZX/1i9gFBaWQcB+/tmpe2qUsSBABpcxqxnAxFdiUFEgAX1bjYGQvIZmoz9Q==" }, + "sprintf-js": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.2.tgz", + "integrity": "sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug==" + }, "sshpk": { "version": "1.16.1", "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz", @@ -1985,36 +2360,10 @@ "yargs": "~3.10.0" }, "dependencies": { - "camelcase": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-1.2.1.tgz", - "integrity": "sha1-m7UwTS4LVmmLLHWLCKPqqdqlijk=" - }, - "cliui": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/cliui/-/cliui-2.1.0.tgz", - "integrity": "sha1-S0dXYP+AJkx2LDoXGQMukcf+oNE=", - "requires": { - "center-align": "^0.1.1", - "right-align": "^0.1.1", - "wordwrap": "0.0.2" - } - }, "source-map": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz", "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w=" - }, - "yargs": { - "version": "3.10.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-3.10.0.tgz", - "integrity": "sha1-9+572FfdfB0tOMDnTvvWgdFDH9E=", - "requires": { - "camelcase": "^1.0.2", - "cliui": "^2.1.0", - "decamelize": "^1.0.0", - "window-size": "0.1.0" - } } } }, @@ -2146,30 +2495,14 @@ "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI=" }, "yargs": { - "version": "7.1.0", - "resolved": "https://registry.npmjs.org/yargs/-/yargs-7.1.0.tgz", - "integrity": "sha1-a6MY6xaWFyf10oT46gA+jWFU0Mg=", + "version": "3.10.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-3.10.0.tgz", + "integrity": "sha1-9+572FfdfB0tOMDnTvvWgdFDH9E=", "requires": { - "camelcase": "^3.0.0", - "cliui": "^3.2.0", - "decamelize": "^1.1.1", - "get-caller-file": "^1.0.1", - "os-locale": "^1.4.0", - "read-pkg-up": "^1.0.1", - "require-directory": "^2.1.1", - "require-main-filename": "^1.0.1", - "set-blocking": "^2.0.0", - "string-width": "^1.0.2", - "which-module": "^1.0.0", - "y18n": "^3.2.1", - "yargs-parser": "^5.0.0" - }, - "dependencies": { - "camelcase": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-3.0.0.tgz", - "integrity": "sha1-MvxLn82vhF/N9+c7uXysImHwqwo=" - } + "camelcase": "^1.0.2", + "cliui": "^2.1.0", + "decamelize": "^1.0.0", + "window-size": "0.1.0" } }, "yargs-parser": { diff --git a/package.json b/package.json index 9d63068..9798293 100644 --- a/package.json +++ b/package.json @@ -14,9 +14,14 @@ "mixin-deep": "^2.0.1", "node-dir": "^0.1.17", "node-fetch": "^2.6.0", - "pinski": "github:cloudrac3r/pinski#19495f7a1c62fbfd8c685cf5bb5d678788906032", + "pinski": "github:cloudrac3r/pinski#9eb56d90fdd00357451dd5a546dbcca1f9bf114a", + "pug": "^2.0.4", "rss": "^1.2.2", "semver": "^7.1.2", - "sharp": "^0.24.0" + "sharp": "^0.24.0", + "socks-proxy-agent": "github:cloudrac3r/node-socks-proxy-agent#6a26d274b12098dfef6cc2faafd25b0c051f2467" + }, + "optionalDependencies": { + "@deadcanaries/granax": "^3.2.5" } } diff --git a/src/lib/collectors.js b/src/lib/collectors.js index 21c416f..5f07753 100644 --- a/src/lib/collectors.js +++ b/src/lib/collectors.js @@ -1,5 +1,6 @@ const constants = require("./constants") const {request} = require("./utils/request") +const switcher = require("./utils/torswitcher") const {extractSharedData} = require("./utils/body") const {TtlCache, RequestCache} = require("./cache") const RequestHistory = require("./structures/RequestHistory") @@ -50,17 +51,19 @@ function fetchTimelinePage(userID, after) { after: after })) return requestCache.getOrFetchPromise("page/"+after, () => { - return request(`https://www.instagram.com/graphql/query/?${p.toString()}`).then(res => res.json()).then(root => { - if (!root.data) { + return switcher.request(`https://www.instagram.com/graphql/query/?${p.toString()}`, async res => { + if (res.status === 429) throw constants.symbols.RATE_LIMITED + return res + }).then(res => res.json()).then(root => { + /** @type {import("./types").PagedEdges} */ + const timeline = root.data.user.edge_owner_to_timeline_media + history.report("timeline", true) + return timeline + }).catch(error => { + if (error === constants.symbols.RATE_LIMITED) { history.report("timeline", false) - console.error("missing data from timeline request, 429?", root) //todo: please make this better. - throw new Error("missing data from timeline request, 429?") - } else { - /** @type {import("./types").PagedEdges} */ - const timeline = root.data.user.edge_owner_to_timeline_media - history.report("timeline", true) - return timeline } + throw error }) }) } @@ -105,26 +108,28 @@ function fetchShortcodeData(shortcode) { p.set("query_hash", constants.external.shortcode_query_hash) p.set("variables", JSON.stringify({shortcode})) return requestCache.getOrFetchPromise("shortcode/"+shortcode, () => { - return request(`https://www.instagram.com/graphql/query/?${p.toString()}`).then(res => res.json()).then(root => { - if (!root.data) { - history.report("post", false) - console.error("missing data from post request, 429?", root) //todo: please make this better. - throw new Error("missing data from post request, 429?") + return switcher.request(`https://www.instagram.com/graphql/query/?${p.toString()}`, async res => { + if (res.status === 429) throw constants.symbols.RATE_LIMITED + return res + }).then(res => res.json()).then(root => { + /** @type {import("./types").TimelineEntryN3} */ + const data = root.data.shortcode_media + if (data == null) { + // the thing doesn't exist + throw constants.symbols.NOT_FOUND } else { - /** @type {import("./types").TimelineEntryN3} */ - const data = root.data.shortcode_media - if (data == null) { - // the thing doesn't exist - throw constants.symbols.NOT_FOUND - } else { - history.report("post", true) - if (constants.caching.db_post_n3) { - db.prepare("REPLACE INTO Posts (shortcode, id, id_as_numeric, username, json) VALUES (@shortcode, @id, @id_as_numeric, @username, @json)") - .run({shortcode: data.shortcode, id: data.id, id_as_numeric: data.id, username: data.owner.username, json: JSON.stringify(data)}) - } - return data + history.report("post", true) + if (constants.caching.db_post_n3) { + db.prepare("REPLACE INTO Posts (shortcode, id, id_as_numeric, username, json) VALUES (@shortcode, @id, @id_as_numeric, @username, @json)") + .run({shortcode: data.shortcode, id: data.id, id_as_numeric: data.id, username: data.owner.username, json: JSON.stringify(data)}) } + return data } + }).catch(error => { + if (error === constants.symbols.RATE_LIMITED) { + history.report("post", false) + } + throw error }) }) } diff --git a/src/lib/constants.js b/src/lib/constants.js index b59a9c4..32dafe6 100644 --- a/src/lib/constants.js +++ b/src/lib/constants.js @@ -7,6 +7,8 @@ let constants = { // Things that server owners _should_ change! website_origin: "http://localhost:10407", + use_tor: false, // Whether to enable Tor support at all + tor_password: null, // No effect without `use_tor = true`. If `null`, node will run its own Tor process instead. // Things that server owners _could_ change if they want to. settings: { @@ -25,6 +27,7 @@ let constants = { external: { user_query_hash: "c9100bf9110dd6361671f113dd02e7d6", timeline_query_hash: "e769aa130647d2354c40ea6a439bfc08", + timeline_query_hash_2: "42323d64886122307be10013ad2dcc44", // https://github.com/rarcega/instagram-scraper/blob/dc022081dbefc81500c5f70cce5c70cfd2816e3c/instagram_scraper/constants.py#L30 shortcode_query_hash: "2b0673e0dc4580674a88d426fe00ea90", timeline_fetch_first: 12, username_regex: "[\\w.]+", @@ -45,7 +48,8 @@ let constants = { TYPE_GALLERY_VIDEO: Symbol("TYPE_GALLERY_VIDEO"), NOT_FOUND: Symbol("NOT_FOUND"), NO_SHARED_DATA: Symbol("NO_SHARED_DATA"), - INSTAGRAM_DEMANDS_LOGIN: Symbol("INSTAGRAM_DEMANDS_LOGIN") + INSTAGRAM_DEMANDS_LOGIN: Symbol("INSTAGRAM_DEMANDS_LOGIN"), + RATE_LIMITED: Symbol("RATE_LIMITED") }, database_version: 1 diff --git a/src/lib/utils/request.js b/src/lib/utils/request.js index fac3bb4..b98b476 100644 --- a/src/lib/utils/request.js +++ b/src/lib/utils/request.js @@ -1,13 +1,16 @@ const fetch = require("node-fetch").default -function request(url) { - console.log("-> [OUT]", String(url)) // todo: make more like pinski? - return fetch(url, { +function request(url, options = {}, settings = {}) { + if (settings.statusLine === undefined) settings.statusLine = "OUT" + if (settings.log === undefined) settings.log = true + if (settings.log) console.log(`-> [${settings.statusLine}] ${url}`) // todo: make more like pinski? + // @ts-ignore + return fetch(url, Object.assign({ headers: { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" }, redirect: "manual" - }) + }, options)) } module.exports.request = request diff --git a/src/lib/utils/tor.js b/src/lib/utils/tor.js new file mode 100644 index 0000000..12c5372 --- /dev/null +++ b/src/lib/utils/tor.js @@ -0,0 +1,79 @@ +const SocksProxyAgent = require("socks-proxy-agent") +const {connect} = require("net"); +const constants = require("../constants") +const {request} = require("./request") + +class TorManager { + /** + * @param {import("@deadcanaries/granax/lib/controller")} tor + * @param {number} port + */ + constructor(tor, port) { + this.tor = tor + this.port = port + this.agent = new SocksProxyAgent("socks5://localhost:"+this.port) + } + + async request(url, test) { + let result = null + while (!result) { + const req = await request(url, {agent: this.agent}, {log: true, statusLine: "TOR"}) + try { + result = await test(req) + } catch (e) { + await this.newCircuit() + } + } + return result + } + + newCircuit() { + return new Promise(resolve => { + this.tor.cleanCircuits(() => resolve()) + }) + } +} + +try { + var granax = require("@deadcanaries/granax") +} catch (e) {} + +/** @type {Promise} */ +module.exports = new Promise(resolve => { + if (granax) { + /** @type {import("@deadcanaries/granax/lib/controller")} */ + // @ts-ignore + let tor + if (constants.tor_password == null) { + // @ts-ignore + tor = new granax() + } else { + tor = new granax.TorController(connect(9051), {authOnConnect: false}) + tor.authenticate(`"${constants.tor_password}"`, err => { + if (err) console.log("Tor auth error:", err) + }) + } + + console.log("Starting tor...") + + tor.once("ready", () => { + tor.getInfo("net/listeners/socks", (err, result) => { + if (err) throw err + // result is string containing something like "127.0.0.1:36977" + // yes, the string contains double quotes! + const port = +result.match(/:(\d+)/)[1] + const torManager = new TorManager(tor, port) + console.log("Tor is ready, using SOCKS port "+port) + resolve(torManager) + }) + }) + + tor.on("error", function() { + console.log("Tor error!") + console.log(...arguments) + }) + } else { + console.log("Note: Tor functionality not installed. You may wish to run `npm install @deadcanaries/granax`. (78+ MB download required.)") + resolve(null) + } +}) diff --git a/src/lib/utils/torswitcher.js b/src/lib/utils/torswitcher.js new file mode 100644 index 0000000..e1d9348 --- /dev/null +++ b/src/lib/utils/torswitcher.js @@ -0,0 +1,41 @@ +const constants = require("../constants") +const {request} = require("./request") + +class TorSwitcher { + constructor() { + this.torManager = null + } + + setManager(torManager) { + this.torManager = torManager + } + + /** + * Request from the URL. + * The test function will be called with the response object. + * If the test function succeeds, its return value will be returned here. + * If the test function fails, its error will be rejected here. + * Only include rate limit logic in the test function! + * @param {string} url + * @param {(res: import("node-fetch").Response) => Promise} test + * @returns {Promise} + * @template T the return value of the test function + */ + request(url, test) { + if (this.torManager) { + return this.torManager.request(url, test) + } else { + return request(url).then(res => test(res)) + } + } +} + +const switcher = new TorSwitcher() + +if (constants.use_tor) { + require("./tor").then(torManager => { + if (torManager) switcher.setManager(torManager) + }) +} + +module.exports = switcher diff --git a/src/site/api/proxy.js b/src/site/api/proxy.js index 6a7b251..6ab739d 100644 --- a/src/site/api/proxy.js +++ b/src/site/api/proxy.js @@ -3,8 +3,6 @@ const {request} = require("../../lib/utils/request") const {proxy} = require("pinski/plugins") const sharp = require("sharp") -const VERIFY_SUCCESS = Symbol("VERIFY_SUCCESS") - /** * Check that a resource is on Instagram. * @param {URL} completeURL @@ -38,7 +36,7 @@ module.exports = [ Some thumbnails aren't square and would otherwise be stretched on the page without this. If I cropped the images client side, it would have to be done with CSS background-image, which means no . */ - return request(verifyResult.url).then(res => { + return request(verifyResult.url, {}, {log: false}).then(res => { const converter = sharp().resize(width, width, {position: "entropy"}) return { statusCode: 200, diff --git a/src/site/sass/main.sass b/src/site/sass/main.sass index 7b16e45..d17d491 100644 --- a/src/site/sass/main.sass +++ b/src/site/sass/main.sass @@ -175,6 +175,7 @@ body color: #111 background-color: rgba(40, 40, 40, 0.25) text-decoration: none + overflow: hidden @include sized &:hover diff --git a/src/site/server.js b/src/site/server.js index ca98b75..a4d3e53 100644 --- a/src/site/server.js +++ b/src/site/server.js @@ -17,6 +17,9 @@ subdirs("pug", (err, dirs) => { pinski.addAPIDir("html/static/js/templates/api") pinski.addSassDir("sass") pinski.addAPIDir("api") + pinski.muteLogsStartingWith("/imageproxy") + pinski.muteLogsStartingWith("/videoproxy") + pinski.muteLogsStartingWith("/static") pinski.startServer() pinski.enableWS()