Optional Tor support

README.md

@@ -62,7 +62,7 @@ You need [node.js](https://nodejs.org/en/) to run Bibliogram. Versions before 12
 
 1. `$ git clone https://github.com/cloudrac3r/bibliogram`
 If you are using a fork, be sure to actually install that fork instead!
-1. `$ npm install`
+1. `$ npm install --no-optional` (for Tor support, omit `--no-optional`)
 1. Edit `/config.js` to suit your server environment
 1. `$ npm start`
 

package-lock.json

@@ -4,6 +4,36 @@
   "lockfileVersion": 1,
   "requires": true,
   "dependencies": {
+    "7zip": {
+      "version": "0.0.6",
+      "resolved": "https://registry.npmjs.org/7zip/-/7zip-0.0.6.tgz",
+      "integrity": "sha1-nK+xca+CMpSQNTtIFvAzR6oVCjA=",
+      "optional": true
+    },
+    "@deadcanaries/granax": {
+      "version": "3.2.5",
+      "resolved": "https://registry.npmjs.org/@deadcanaries/granax/-/granax-3.2.5.tgz",
+      "integrity": "sha512-umsEYedNQWLSlkDnGynR8bY0pVWq2hKN1JLtV9Jit7srYHqdo+XIEsUNejq1nNBiX83WhUZVe4gOqvg8ovdMMQ==",
+      "optional": true,
+      "requires": {
+        "7zip": "0.0.6",
+        "async": "^2.3.0",
+        "latest-torbrowser-version": "^2.0.2",
+        "merge": "^1.2.1",
+        "mkdirp": "^0.5.1",
+        "mv": "^2.1.1",
+        "ncp": "^2.0.0",
+        "progress": "^2.0.3",
+        "rimraf": "^2.6.3"
+      }
+    },
+    "@outtacontrol/socks": {
+      "version": "github:phoenix344/socksv5#dbdecf75be849337a31e78ba14d316deed43287f",
+      "from": "github:phoenix344/socksv5#dbdecf75be849337a31e78ba14d316deed43287f",
+      "requires": {
+        "ip-address": "^5.8.9"
+      }
+    },
     "@types/babel-types": {
       "version": "7.0.7",
       "resolved": "https://registry.npmjs.org/@types/babel-types/-/babel-types-7.0.7.tgz",
@@ -17,6 +47,12 @@
         "@types/babel-types": "*"
       }
     },
+    "@types/node": {
+      "version": "13.7.0",
+      "resolved": "https://registry.npmjs.org/@types/node/-/node-13.7.0.tgz",
+      "integrity": "sha512-GnZbirvmqZUzMgkFn70c74OQpTTUcCzlhQliTzYjQMqg+hVKcDnxdL19Ne3UdYzdMA/+W3eb646FWn/ZaT1NfQ==",
+      "optional": true
+    },
     "abbrev": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/abbrev/-/abbrev-1.1.1.tgz",
@@ -42,6 +78,14 @@
         }
       }
     },
+    "agent-base": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.0.tgz",
+      "integrity": "sha512-j1Q7cSCqN+AwrmDd+pzgqc0/NpC655x2bUf5ZjRIO77DcNBFmh+OgRNzF6OKdCC9RSCb19fGd99+bhXFdkRNqw==",
+      "requires": {
+        "debug": "4"
+      }
+    },
     "ajv": {
       "version": "6.11.0",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.11.0.tgz",
@@ -115,6 +159,15 @@
       "resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
       "integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
     },
+    "async": {
+      "version": "2.6.3",
+      "resolved": "https://registry.npmjs.org/async/-/async-2.6.3.tgz",
+      "integrity": "sha512-zflvls11DCy+dQWzTW2dzuilv8Z5X/pjfmZOWba6TNIVDm+2UDaJmXSOXlasHKfNBs8oo3M0aT50fDEWfKZjXg==",
+      "optional": true,
+      "requires": {
+        "lodash": "^4.17.14"
+      }
+    },
     "async-foreach": {
       "version": "0.1.3",
       "resolved": "https://registry.npmjs.org/async-foreach/-/async-foreach-0.1.3.tgz",
@@ -256,6 +309,12 @@
         "inherits": "~2.0.0"
       }
     },
+    "boolbase": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
+      "integrity": "sha1-aN/1++YMUes3cl6p4+0xDcwed24=",
+      "optional": true
+    },
     "brace-expansion": {
       "version": "1.1.11",
       "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
@@ -266,9 +325,9 @@
       }
     },
     "camelcase": {
-      "version": "2.1.1",
+      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-2.1.1.tgz",
+      "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-1.2.1.tgz",
-      "integrity": "sha1-fB0W1nmhu+WcoCys7PsBHiAfWh8="
+      "integrity": "sha1-m7UwTS4LVmmLLHWLCKPqqdqlijk="
     },
     "camelcase-keys": {
       "version": "2.1.0",
@@ -277,6 +336,13 @@
       "requires": {
         "camelcase": "^2.0.0",
         "map-obj": "^1.0.0"
+      },
+      "dependencies": {
+        "camelcase": {
+          "version": "2.1.1",
+          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-2.1.1.tgz",
+          "integrity": "sha1-fB0W1nmhu+WcoCys7PsBHiAfWh8="
+        }
       }
     },
     "caseless": {
@@ -313,34 +379,41 @@
         "is-regex": "^1.0.3"
       }
     },
+    "cheerio": {
+      "version": "1.0.0-rc.3",
+      "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0-rc.3.tgz",
+      "integrity": "sha512-0td5ijfUPuubwLUu0OBoe98gZj8C/AA+RW3v67GPlGOrvxWjZmBXiBCRU+I8VEiNyJzjth40POfHiz2RB3gImA==",
+      "optional": true,
+      "requires": {
+        "css-select": "~1.2.0",
+        "dom-serializer": "~0.1.1",
+        "entities": "~1.1.1",
+        "htmlparser2": "^3.9.1",
+        "lodash": "^4.15.0",
+        "parse5": "^3.0.1"
+      }
+    },
     "chownr": {
       "version": "1.1.3",
       "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.3.tgz",
       "integrity": "sha512-i70fVHhmV3DtTl6nqvZOnIjbY0Pe4kAUjwHj8z0zAdgBtYrJyYwLKCCuRBQ5ppkyL0AkN7HKRnETdmdp1zqNXw=="
     },
     "clean-css": {
-      "version": "4.2.2",
+      "version": "4.2.3",
-      "resolved": "https://registry.npmjs.org/clean-css/-/clean-css-4.2.2.tgz",
+      "resolved": "https://registry.npmjs.org/clean-css/-/clean-css-4.2.3.tgz",
-      "integrity": "sha512-yKycArwReQXbOD/3pmsPmt6p7oUBww8MisDabL2pCUWkbVONvCJoBdCjgY4ZVQmKX5juz/JB9oDcP6XzGUpjwQ==",
+      "integrity": "sha512-VcMWDN54ZN/DS+g58HYL5/n4Zrqe8vHJpGA8KdgUXFU4fuP/aHNw8eld9SyEIyabIMJX/0RaY/fplOo5hYLSFA==",
       "requires": {
         "source-map": "~0.6.0"
-      },
-      "dependencies": {
-        "source-map": {
-          "version": "0.6.1",
-          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
-          "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
-        }
       }
     },
     "cliui": {
-      "version": "3.2.0",
+      "version": "2.1.0",
-      "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz",
+      "resolved": "https://registry.npmjs.org/cliui/-/cliui-2.1.0.tgz",
-      "integrity": "sha1-EgYBU3qRbSmUD5NNo7SNWFo5IT0=",
+      "integrity": "sha1-S0dXYP+AJkx2LDoXGQMukcf+oNE=",
       "requires": {
-        "string-width": "^1.0.1",
+        "center-align": "^0.1.1",
-        "strip-ansi": "^3.0.1",
+        "right-align": "^0.1.1",
-        "wrap-ansi": "^2.0.0"
+        "wordwrap": "0.0.2"
       }
     },
     "code-point-at": {
@@ -418,6 +491,33 @@
       "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
       "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
     },
+    "cross-spawn": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-3.0.1.tgz",
+      "integrity": "sha1-ElYDfsufDF9549bvE14wdwGEuYI=",
+      "requires": {
+        "lru-cache": "^4.0.1",
+        "which": "^1.2.9"
+      }
+    },
+    "css-select": {
+      "version": "1.2.0",
+      "resolved": "https://registry.npmjs.org/css-select/-/css-select-1.2.0.tgz",
+      "integrity": "sha1-KzoRBTnFNV8c2NMUYj6HCxIeyFg=",
+      "optional": true,
+      "requires": {
+        "boolbase": "~1.0.0",
+        "css-what": "2.1",
+        "domutils": "1.5.1",
+        "nth-check": "~1.0.1"
+      }
+    },
+    "css-what": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/css-what/-/css-what-2.1.3.tgz",
+      "integrity": "sha512-a+EPoD+uZiNfh+5fxw2nO9QwFa6nJe2Or35fGY6Ipw1R3R4AGz1d1TEZrCegvw2YTmZ0jXirGYlzxxpYSHwpEg==",
+      "optional": true
+    },
     "currently-unhandled": {
       "version": "0.4.1",
       "resolved": "https://registry.npmjs.org/currently-unhandled/-/currently-unhandled-0.4.1.tgz",
@@ -434,6 +534,14 @@
         "assert-plus": "^1.0.0"
       }
     },
+    "debug": {
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz",
+      "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==",
+      "requires": {
+        "ms": "^2.1.1"
+      }
+    },
     "decamelize": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz",
@@ -472,6 +580,41 @@
       "resolved": "https://registry.npmjs.org/doctypes/-/doctypes-1.1.0.tgz",
       "integrity": "sha1-6oCxBqh1OHdOijpKWv4pPeSJ4Kk="
     },
+    "dom-serializer": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-0.1.1.tgz",
+      "integrity": "sha512-l0IU0pPzLWSHBcieZbpOKgkIn3ts3vAh7ZuFyXNwJxJXk/c4Gwj9xaTJwIDVQCXawWD0qb3IzMGH5rglQaO0XA==",
+      "optional": true,
+      "requires": {
+        "domelementtype": "^1.3.0",
+        "entities": "^1.1.1"
+      }
+    },
+    "domelementtype": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-1.3.1.tgz",
+      "integrity": "sha512-BSKB+TSpMpFI/HOxCNr1O8aMOTZ8hT3pM3GQ0w/mWRmkhEDSFJkkyzz4XQsBV44BChwGkrDfMyjVD0eA2aFV3w==",
+      "optional": true
+    },
+    "domhandler": {
+      "version": "2.4.2",
+      "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-2.4.2.tgz",
+      "integrity": "sha512-JiK04h0Ht5u/80fdLMCEmV4zkNh2BcoMFBmZ/91WtYZ8qVXSKjiw7fXMgFPnHcSZgOo3XdinHvmnDUeMf5R4wA==",
+      "optional": true,
+      "requires": {
+        "domelementtype": "1"
+      }
+    },
+    "domutils": {
+      "version": "1.5.1",
+      "resolved": "https://registry.npmjs.org/domutils/-/domutils-1.5.1.tgz",
+      "integrity": "sha1-3NhIiib1Y9YQeeSMn3t+Mjc2gs8=",
+      "optional": true,
+      "requires": {
+        "dom-serializer": "0",
+        "domelementtype": "1"
+      }
+    },
     "ecc-jsbn": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.2.tgz",
@@ -489,6 +632,12 @@
         "once": "^1.4.0"
       }
     },
+    "entities": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/entities/-/entities-1.1.2.tgz",
+      "integrity": "sha512-f2LZMYl1Fzu7YSBKg+RoROelpOaNrcGmE9AZubeDfrCEia483oW4MI4VyFd5VNHIgQ/7qm1I0wUHK1eJnn2y2w==",
+      "optional": true
+    },
     "error-ex": {
       "version": "1.3.2",
       "resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.2.tgz",
@@ -541,6 +690,26 @@
         "pinkie-promise": "^2.0.0"
       }
     },
+    "follow-redirects": {
+      "version": "1.10.0",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.10.0.tgz",
+      "integrity": "sha512-4eyLK6s6lH32nOvLLwlIOnr9zrL8Sm+OvW4pVTJNoXeGzYIkHVf+pADQi+OJ0E67hiuSLezPVPyBcIZO50TmmQ==",
+      "optional": true,
+      "requires": {
+        "debug": "^3.0.0"
+      },
+      "dependencies": {
+        "debug": {
+          "version": "3.2.6",
+          "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz",
+          "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==",
+          "optional": true,
+          "requires": {
+            "ms": "^2.1.1"
+          }
+        }
+      }
+    },
     "forever-agent": {
       "version": "0.6.1",
       "resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
@@ -704,6 +873,33 @@
       "resolved": "https://registry.npmjs.org/hosted-git-info/-/hosted-git-info-2.8.5.tgz",
       "integrity": "sha512-kssjab8CvdXfcXMXVcvsXum4Hwdq9XGtRD3TteMEvEbq0LXyiNQr6AprqKqfeaDXze7SxWvRxdpwE6ku7ikLkg=="
     },
+    "htmlparser2": {
+      "version": "3.10.1",
+      "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-3.10.1.tgz",
+      "integrity": "sha512-IgieNijUMbkDovyoKObU1DUhm1iwNYE/fuifEoEHfd1oZKZDaONBSkal7Y01shxsM49R4XaMdGez3WnF9UfiCQ==",
+      "optional": true,
+      "requires": {
+        "domelementtype": "^1.3.1",
+        "domhandler": "^2.3.0",
+        "domutils": "^1.5.1",
+        "entities": "^1.1.1",
+        "inherits": "^2.0.1",
+        "readable-stream": "^3.1.1"
+      },
+      "dependencies": {
+        "readable-stream": {
+          "version": "3.5.0",
+          "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.5.0.tgz",
+          "integrity": "sha512-gSz026xs2LfxBPudDuI41V1lka8cxg64E66SGe78zJlsUofOg/yqwezdIcdfwik6B4h8LFmWPA9ef9X3FiNFLA==",
+          "optional": true,
+          "requires": {
+            "inherits": "^2.0.3",
+            "string_decoder": "^1.1.1",
+            "util-deprecate": "^1.0.1"
+          }
+        }
+      }
+    },
     "http-signature": {
       "version": "1.2.0",
       "resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
@@ -756,6 +952,28 @@
       "resolved": "https://registry.npmjs.org/invert-kv/-/invert-kv-1.0.0.tgz",
       "integrity": "sha1-EEqOSqym09jNFXqO+L+rLXo//bY="
     },
+    "ip": {
+      "version": "1.1.5",
+      "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz",
+      "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo="
+    },
+    "ip-address": {
+      "version": "5.9.4",
+      "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-5.9.4.tgz",
+      "integrity": "sha512-dHkI3/YNJq4b/qQaz+c8LuarD3pY24JqZWfjB8aZx1gtpc2MDILu9L9jpZe1sHpzo/yWFweQVn+U//FhazUxmw==",
+      "requires": {
+        "jsbn": "1.1.0",
+        "lodash": "^4.17.15",
+        "sprintf-js": "1.1.2"
+      },
+      "dependencies": {
+        "jsbn": {
+          "version": "1.1.0",
+          "resolved": "https://registry.npmjs.org/jsbn/-/jsbn-1.1.0.tgz",
+          "integrity": "sha1-sBMHyym2GKHtJux56RH4A8TaAEA="
+        }
+      }
+    },
     "is-arrayish": {
       "version": "0.2.1",
       "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz",
@@ -894,6 +1112,26 @@
         "is-buffer": "^1.1.5"
       }
     },
+    "latest-torbrowser-version": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/latest-torbrowser-version/-/latest-torbrowser-version-2.0.3.tgz",
+      "integrity": "sha512-+6fkERFLxo2QiNn7x0txjEF199tG4CGVI9X101d7IuZAapO/hGOC+nHD9BEPNNaMqmvIUkT/o3CdZ+NS5wX/iQ==",
+      "optional": true,
+      "requires": {
+        "async": "^2.6.0",
+        "cheerio": "^1.0.0-rc.2",
+        "follow-redirects": "^1.2.4",
+        "semver": "^5.4.1"
+      },
+      "dependencies": {
+        "semver": {
+          "version": "5.7.1",
+          "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.1.tgz",
+          "integrity": "sha512-sauaDf/PZdVgrLTNYHRtpXa1iRiKcaebiKQ1BJdpQlWH2lCvexQdX55snPFyK7QzpudqbCI0qXFfOasHdyNDGQ==",
+          "optional": true
+        }
+      }
+    },
     "lazy-cache": {
       "version": "1.0.4",
       "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-1.0.4.tgz",
@@ -969,6 +1207,12 @@
         "trim-newlines": "^1.0.0"
       }
     },
+    "merge": {
+      "version": "1.2.1",
+      "resolved": "https://registry.npmjs.org/merge/-/merge-1.2.1.tgz",
+      "integrity": "sha512-VjFo4P5Whtj4vsLzsYBu5ayHhoHJ0UqNm7ibvShmbmoz7tGi0vXaoJbGdB+GmDMLUdg8DpQXEIeVDAe8MaABvQ==",
+      "optional": true
+    },
     "mime": {
       "version": "2.4.4",
       "resolved": "https://registry.npmjs.org/mime/-/mime-2.4.4.tgz",
@@ -1056,6 +1300,46 @@
         }
       }
     },
+    "ms": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz",
+      "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
+    },
+    "mv": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/mv/-/mv-2.1.1.tgz",
+      "integrity": "sha1-rmzg1vbV4KT32JN5jQPB6pVZtqI=",
+      "optional": true,
+      "requires": {
+        "mkdirp": "~0.5.1",
+        "ncp": "~2.0.0",
+        "rimraf": "~2.4.0"
+      },
+      "dependencies": {
+        "glob": {
+          "version": "6.0.4",
+          "resolved": "https://registry.npmjs.org/glob/-/glob-6.0.4.tgz",
+          "integrity": "sha1-DwiGD2oVUSey+t1PnOJLGqtuTSI=",
+          "optional": true,
+          "requires": {
+            "inflight": "^1.0.4",
+            "inherits": "2",
+            "minimatch": "2 || 3",
+            "once": "^1.3.0",
+            "path-is-absolute": "^1.0.0"
+          }
+        },
+        "rimraf": {
+          "version": "2.4.5",
+          "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.4.5.tgz",
+          "integrity": "sha1-7nEM5dk6j9uFb7Xqj/Di11k0sto=",
+          "optional": true,
+          "requires": {
+            "glob": "^6.0.1"
+          }
+        }
+      }
+    },
     "nan": {
       "version": "2.14.0",
       "resolved": "https://registry.npmjs.org/nan/-/nan-2.14.0.tgz",
@@ -1066,6 +1350,12 @@
       "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-1.0.1.tgz",
       "integrity": "sha512-boQj1WFgQH3v4clhu3mTNfP+vOBxorDlE8EKiMjUlLG3C4qAESnn9AxIOkFgTR2c9LtzNjPrjS60cT27ZKBhaA=="
     },
+    "ncp": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/ncp/-/ncp-2.0.0.tgz",
+      "integrity": "sha1-GVoh1sRuNh0vsSgbo4uR6d9727M=",
+      "optional": true
+    },
     "node-abi": {
       "version": "2.13.0",
       "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-2.13.0.tgz",
@@ -1142,17 +1432,6 @@
         "sass-graph": "^2.2.4",
         "stdout-stream": "^1.4.0",
         "true-case-path": "^1.0.2"
-      },
-      "dependencies": {
-        "cross-spawn": {
-          "version": "3.0.1",
-          "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-3.0.1.tgz",
-          "integrity": "sha1-ElYDfsufDF9549bvE14wdwGEuYI=",
-          "requires": {
-            "lru-cache": "^4.0.1",
-            "which": "^1.2.9"
-          }
-        }
       }
     },
     "noop-logger": {
@@ -1197,6 +1476,15 @@
         "set-blocking": "~2.0.0"
       }
     },
+    "nth-check": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-1.0.2.tgz",
+      "integrity": "sha512-WeBOdju8SnzPN5vTUJYxYUxLeXpCaVP5i5e0LF8fg7WORF2Wd7wFX/pk0tYZk7s8T+J7VLy0Da6J1+wCT0AtHg==",
+      "optional": true,
+      "requires": {
+        "boolbase": "~1.0.0"
+      }
+    },
     "number-is-nan": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/number-is-nan/-/number-is-nan-1.0.1.tgz",
@@ -1255,6 +1543,15 @@
         "error-ex": "^1.2.0"
       }
     },
+    "parse5": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/parse5/-/parse5-3.0.3.tgz",
+      "integrity": "sha512-rgO9Zg5LLLkfJF9E6CCmXlSE4UVceloys8JrFqCcHloC3usd/kJCyPDwH2SOlzix2j3xaP9sUX3e8+kvkuleAA==",
+      "optional": true,
+      "requires": {
+        "@types/node": "*"
+      }
+    },
     "path-exists": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/path-exists/-/path-exists-2.1.0.tgz",
@@ -1307,11 +1604,11 @@
       }
     },
     "pinski": {
-      "version": "github:cloudrac3r/pinski#19495f7a1c62fbfd8c685cf5bb5d678788906032",
+      "version": "github:cloudrac3r/pinski#9eb56d90fdd00357451dd5a546dbcca1f9bf114a",
-      "from": "github:cloudrac3r/pinski#19495f7a1c62fbfd8c685cf5bb5d678788906032",
+      "from": "github:cloudrac3r/pinski#9eb56d90fdd00357451dd5a546dbcca1f9bf114a",
       "requires": {
         "mime": "^2.4.4",
-        "node-sass": "^4.12.0",
+        "node-sass": "^4.13.1",
         "pug": "^2.0.3",
         "ws": "^7.1.1"
       }
@@ -1343,6 +1640,12 @@
       "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz",
       "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag=="
     },
+    "progress": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/progress/-/progress-2.0.3.tgz",
+      "integrity": "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==",
+      "optional": true
+    },
     "promise": {
       "version": "7.3.1",
       "resolved": "https://registry.npmjs.org/promise/-/promise-7.3.1.tgz",
@@ -1669,6 +1972,43 @@
         "lodash": "^4.0.0",
         "scss-tokenizer": "^0.2.3",
         "yargs": "^7.0.0"
+      },
+      "dependencies": {
+        "camelcase": {
+          "version": "3.0.0",
+          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-3.0.0.tgz",
+          "integrity": "sha1-MvxLn82vhF/N9+c7uXysImHwqwo="
+        },
+        "cliui": {
+          "version": "3.2.0",
+          "resolved": "https://registry.npmjs.org/cliui/-/cliui-3.2.0.tgz",
+          "integrity": "sha1-EgYBU3qRbSmUD5NNo7SNWFo5IT0=",
+          "requires": {
+            "string-width": "^1.0.1",
+            "strip-ansi": "^3.0.1",
+            "wrap-ansi": "^2.0.0"
+          }
+        },
+        "yargs": {
+          "version": "7.1.0",
+          "resolved": "https://registry.npmjs.org/yargs/-/yargs-7.1.0.tgz",
+          "integrity": "sha1-a6MY6xaWFyf10oT46gA+jWFU0Mg=",
+          "requires": {
+            "camelcase": "^3.0.0",
+            "cliui": "^3.2.0",
+            "decamelize": "^1.1.1",
+            "get-caller-file": "^1.0.1",
+            "os-locale": "^1.4.0",
+            "read-pkg-up": "^1.0.1",
+            "require-directory": "^2.1.1",
+            "require-main-filename": "^1.0.1",
+            "set-blocking": "^2.0.0",
+            "string-width": "^1.0.2",
+            "which-module": "^1.0.0",
+            "y18n": "^3.2.1",
+            "yargs-parser": "^5.0.0"
+          }
+        }
       }
     },
     "scss-tokenizer": {
@@ -1678,6 +2018,16 @@
       "requires": {
         "js-base64": "^2.1.8",
         "source-map": "^0.4.2"
+      },
+      "dependencies": {
+        "source-map": {
+          "version": "0.4.4",
+          "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz",
+          "integrity": "sha1-66T12pwNyZneaAMti092FzZSA2s=",
+          "requires": {
+            "amdefine": ">=0.0.4"
+          }
+        }
       }
     },
     "semver": {
@@ -1766,14 +2116,34 @@
         }
       }
     },
-    "source-map": {
+    "smart-buffer": {
-      "version": "0.4.4",
+      "version": "4.1.0",
-      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.4.4.tgz",
+      "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.1.0.tgz",
-      "integrity": "sha1-66T12pwNyZneaAMti092FzZSA2s=",
+      "integrity": "sha512-iVICrxOzCynf/SNaBQCw34eM9jROU/s5rzIhpOvzhzuYHfJR/DhZfDkXiZSgKXfgv26HT3Yni3AV/DGw0cGnnw=="
+    },
+    "socks": {
+      "version": "2.3.3",
+      "resolved": "https://registry.npmjs.org/socks/-/socks-2.3.3.tgz",
+      "integrity": "sha512-o5t52PCNtVdiOvzMry7wU4aOqYWL0PeCXRWBEiJow4/i/wr+wpsJQ9awEu1EonLIqsfGd5qSgDdxEOvCdmBEpA==",
       "requires": {
-        "amdefine": ">=0.0.4"
+        "ip": "1.1.5",
+        "smart-buffer": "^4.1.0"
       }
     },
+    "socks-proxy-agent": {
+      "version": "github:cloudrac3r/node-socks-proxy-agent#6a26d274b12098dfef6cc2faafd25b0c051f2467",
+      "from": "github:cloudrac3r/node-socks-proxy-agent#6a26d274b12098dfef6cc2faafd25b0c051f2467",
+      "requires": {
+        "@outtacontrol/socks": "github:phoenix344/socksv5#dbdecf75be849337a31e78ba14d316deed43287f",
+        "agent-base": "^6.0.0",
+        "socks": "^2.3.2"
+      }
+    },
+    "source-map": {
+      "version": "0.6.1",
+      "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz",
+      "integrity": "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="
+    },
     "spdx-correct": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/spdx-correct/-/spdx-correct-3.1.0.tgz",
@@ -1802,6 +2172,11 @@
       "resolved": "https://registry.npmjs.org/spdx-license-ids/-/spdx-license-ids-3.0.5.tgz",
       "integrity": "sha512-J+FWzZoynJEXGphVIS+XEh3kFSjZX/1i9gFBaWQcB+/tmpe2qUsSBABpcxqxnAxFdiUFEgAX1bjYGQvIZmoz9Q=="
     },
+    "sprintf-js": {
+      "version": "1.1.2",
+      "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.2.tgz",
+      "integrity": "sha512-VE0SOVEHCk7Qc8ulkWw3ntAzXuqf7S2lvwQaDLRnUeIEaKNQJzV6BwmLKhOqT61aGhfUMrXeaBk+oDGCzvhcug=="
+    },
     "sshpk": {
       "version": "1.16.1",
       "resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.16.1.tgz",
@@ -1985,36 +2360,10 @@
         "yargs": "~3.10.0"
       },
       "dependencies": {
-        "camelcase": {
-          "version": "1.2.1",
-          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-1.2.1.tgz",
-          "integrity": "sha1-m7UwTS4LVmmLLHWLCKPqqdqlijk="
-        },
-        "cliui": {
-          "version": "2.1.0",
-          "resolved": "https://registry.npmjs.org/cliui/-/cliui-2.1.0.tgz",
-          "integrity": "sha1-S0dXYP+AJkx2LDoXGQMukcf+oNE=",
-          "requires": {
-            "center-align": "^0.1.1",
-            "right-align": "^0.1.1",
-            "wordwrap": "0.0.2"
-          }
-        },
         "source-map": {
           "version": "0.5.7",
           "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.5.7.tgz",
           "integrity": "sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w="
-        },
-        "yargs": {
-          "version": "3.10.0",
-          "resolved": "https://registry.npmjs.org/yargs/-/yargs-3.10.0.tgz",
-          "integrity": "sha1-9+572FfdfB0tOMDnTvvWgdFDH9E=",
-          "requires": {
-            "camelcase": "^1.0.2",
-            "cliui": "^2.1.0",
-            "decamelize": "^1.0.0",
-            "window-size": "0.1.0"
-          }
         }
       }
     },
@@ -2146,30 +2495,14 @@
       "integrity": "sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI="
     },
     "yargs": {
-      "version": "7.1.0",
+      "version": "3.10.0",
-      "resolved": "https://registry.npmjs.org/yargs/-/yargs-7.1.0.tgz",
+      "resolved": "https://registry.npmjs.org/yargs/-/yargs-3.10.0.tgz",
-      "integrity": "sha1-a6MY6xaWFyf10oT46gA+jWFU0Mg=",
+      "integrity": "sha1-9+572FfdfB0tOMDnTvvWgdFDH9E=",
       "requires": {
-        "camelcase": "^3.0.0",
+        "camelcase": "^1.0.2",
-        "cliui": "^3.2.0",
+        "cliui": "^2.1.0",
-        "decamelize": "^1.1.1",
+        "decamelize": "^1.0.0",
-        "get-caller-file": "^1.0.1",
+        "window-size": "0.1.0"
-        "os-locale": "^1.4.0",
-        "read-pkg-up": "^1.0.1",
-        "require-directory": "^2.1.1",
-        "require-main-filename": "^1.0.1",
-        "set-blocking": "^2.0.0",
-        "string-width": "^1.0.2",
-        "which-module": "^1.0.0",
-        "y18n": "^3.2.1",
-        "yargs-parser": "^5.0.0"
-      },
-      "dependencies": {
-        "camelcase": {
-          "version": "3.0.0",
-          "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-3.0.0.tgz",
-          "integrity": "sha1-MvxLn82vhF/N9+c7uXysImHwqwo="
-        }
       }
     },
     "yargs-parser": {

package.json

@@ -14,9 +14,14 @@
     "mixin-deep": "^2.0.1",
     "node-dir": "^0.1.17",
     "node-fetch": "^2.6.0",
-    "pinski": "github:cloudrac3r/pinski#19495f7a1c62fbfd8c685cf5bb5d678788906032",
+    "pinski": "github:cloudrac3r/pinski#9eb56d90fdd00357451dd5a546dbcca1f9bf114a",
+    "pug": "^2.0.4",
     "rss": "^1.2.2",
     "semver": "^7.1.2",
-    "sharp": "^0.24.0"
+    "sharp": "^0.24.0",
+    "socks-proxy-agent": "github:cloudrac3r/node-socks-proxy-agent#6a26d274b12098dfef6cc2faafd25b0c051f2467"
+  },
+  "optionalDependencies": {
+    "@deadcanaries/granax": "^3.2.5"
   }
 }

src/lib/collectors.js

@@ -1,5 +1,6 @@
 const constants = require("./constants")
 const {request} = require("./utils/request")
+const switcher = require("./utils/torswitcher")
 const {extractSharedData} = require("./utils/body")
 const {TtlCache, RequestCache} = require("./cache")
 const RequestHistory = require("./structures/RequestHistory")
@@ -50,17 +51,19 @@ function fetchTimelinePage(userID, after) {
 		after: after
 	}))
 	return requestCache.getOrFetchPromise("page/"+after, () => {
-		return request(`https://www.instagram.com/graphql/query/?${p.toString()}`).then(res => res.json()).then(root => {
+		return switcher.request(`https://www.instagram.com/graphql/query/?${p.toString()}`, async res => {
-			if (!root.data) {
+			if (res.status === 429) throw constants.symbols.RATE_LIMITED
+			return res
+		}).then(res => res.json()).then(root => {
+			/** @type {import("./types").PagedEdges<import("./types").TimelineEntryN2>} */
+			const timeline = root.data.user.edge_owner_to_timeline_media
+			history.report("timeline", true)
+			return timeline
+		}).catch(error => {
+			if (error === constants.symbols.RATE_LIMITED) {
 				history.report("timeline", false)
-				console.error("missing data from timeline request, 429?", root) //todo: please make this better.
-				throw new Error("missing data from timeline request, 429?")
-			} else {
-				/** @type {import("./types").PagedEdges<import("./types").TimelineEntryN2>} */
-				const timeline = root.data.user.edge_owner_to_timeline_media
-				history.report("timeline", true)
-				return timeline
 			}
+			throw error
 		})
 	})
 }
@@ -105,26 +108,28 @@ function fetchShortcodeData(shortcode) {
 	p.set("query_hash", constants.external.shortcode_query_hash)
 	p.set("variables", JSON.stringify({shortcode}))
 	return requestCache.getOrFetchPromise("shortcode/"+shortcode, () => {
-		return request(`https://www.instagram.com/graphql/query/?${p.toString()}`).then(res => res.json()).then(root => {
+		return switcher.request(`https://www.instagram.com/graphql/query/?${p.toString()}`, async res => {
-			if (!root.data) {
+			if (res.status === 429) throw constants.symbols.RATE_LIMITED
-				history.report("post", false)
+			return res
-				console.error("missing data from post request, 429?", root) //todo: please make this better.
+		}).then(res => res.json()).then(root => {
-				throw new Error("missing data from post request, 429?")
+			/** @type {import("./types").TimelineEntryN3} */
+			const data = root.data.shortcode_media
+			if (data == null) {
+				// the thing doesn't exist
+				throw constants.symbols.NOT_FOUND
 			} else {
-				/** @type {import("./types").TimelineEntryN3} */
+				history.report("post", true)
-				const data = root.data.shortcode_media
+				if (constants.caching.db_post_n3) {
-				if (data == null) {
+					db.prepare("REPLACE INTO Posts (shortcode, id, id_as_numeric, username, json) VALUES (@shortcode, @id, @id_as_numeric, @username, @json)")
-					// the thing doesn't exist
+						.run({shortcode: data.shortcode, id: data.id, id_as_numeric: data.id, username: data.owner.username, json: JSON.stringify(data)})
-					throw constants.symbols.NOT_FOUND
-				} else {
-					history.report("post", true)
-					if (constants.caching.db_post_n3) {
-						db.prepare("REPLACE INTO Posts (shortcode, id, id_as_numeric, username, json) VALUES (@shortcode, @id, @id_as_numeric, @username, @json)")
-							.run({shortcode: data.shortcode, id: data.id, id_as_numeric: data.id, username: data.owner.username, json: JSON.stringify(data)})
-					}
-					return data
 				}
+				return data
 			}
+		}).catch(error => {
+			if (error === constants.symbols.RATE_LIMITED) {
+				history.report("post", false)
+			}
+			throw error
 		})
 	})
 }

src/lib/constants.js

@@ -7,6 +7,8 @@
 let constants = {
 	// Things that server owners _should_ change!
 	website_origin: "http://localhost:10407",
+	use_tor: false, // Whether to enable Tor support at all
+	tor_password: null, // No effect without `use_tor = true`. If `null`, node will run its own Tor process instead.
 
 	// Things that server owners _could_ change if they want to.
 	settings: {
@@ -25,6 +27,7 @@ let constants = {
 	external: {
 		user_query_hash: "c9100bf9110dd6361671f113dd02e7d6",
 		timeline_query_hash: "e769aa130647d2354c40ea6a439bfc08",
+		timeline_query_hash_2: "42323d64886122307be10013ad2dcc44", // https://github.com/rarcega/instagram-scraper/blob/dc022081dbefc81500c5f70cce5c70cfd2816e3c/instagram_scraper/constants.py#L30
 		shortcode_query_hash: "2b0673e0dc4580674a88d426fe00ea90",
 		timeline_fetch_first: 12,
 		username_regex: "[\\w.]+",
@@ -45,7 +48,8 @@ let constants = {
 		TYPE_GALLERY_VIDEO: Symbol("TYPE_GALLERY_VIDEO"),
 		NOT_FOUND: Symbol("NOT_FOUND"),
 		NO_SHARED_DATA: Symbol("NO_SHARED_DATA"),
-		INSTAGRAM_DEMANDS_LOGIN: Symbol("INSTAGRAM_DEMANDS_LOGIN")
+		INSTAGRAM_DEMANDS_LOGIN: Symbol("INSTAGRAM_DEMANDS_LOGIN"),
+		RATE_LIMITED: Symbol("RATE_LIMITED")
 	},
 
 	database_version: 1

src/lib/utils/request.js

@@ -1,13 +1,16 @@
 const fetch = require("node-fetch").default
 
-function request(url) {
+function request(url, options = {}, settings = {}) {
-	console.log("->           [OUT]", String(url)) // todo: make more like pinski?
+	if (settings.statusLine === undefined) settings.statusLine = "OUT"
-	return fetch(url, {
+	if (settings.log === undefined) settings.log = true
+	if (settings.log) console.log(`->           [${settings.statusLine}] ${url}`) // todo: make more like pinski?
+	// @ts-ignore
+	return fetch(url, Object.assign({
 		headers: {
 			"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"
 		},
 		redirect: "manual"
-	})
+	}, options))
 }
 
 module.exports.request = request

src/lib/utils/tor.js

@@ -0,0 +1,79 @@
+const SocksProxyAgent = require("socks-proxy-agent")
+const {connect} = require("net");
+const constants = require("../constants")
+const {request} = require("./request")
+
+class TorManager {
+	/**
+	 * @param {import("@deadcanaries/granax/lib/controller")} tor
+	 * @param {number} port
+	 */
+	constructor(tor, port) {
+		this.tor = tor
+		this.port = port
+		this.agent = new SocksProxyAgent("socks5://localhost:"+this.port)
+	}
+
+	async request(url, test) {
+		let result = null
+		while (!result) {
+			const req = await request(url, {agent: this.agent}, {log: true, statusLine: "TOR"})
+			try {
+				result = await test(req)
+			} catch (e) {
+				await this.newCircuit()
+			}
+		}
+		return result
+	}
+
+	newCircuit() {
+		return new Promise(resolve => {
+			this.tor.cleanCircuits(() => resolve())
+		})
+	}
+}
+
+try {
+	var granax = require("@deadcanaries/granax")
+} catch (e) {}
+
+/** @type {Promise<TorManager>} */
+module.exports = new Promise(resolve => {
+	if (granax) {
+		/** @type {import("@deadcanaries/granax/lib/controller")} */
+		// @ts-ignore
+		let tor
+		if (constants.tor_password == null) {
+			// @ts-ignore
+			tor = new granax()
+		} else {
+			tor = new granax.TorController(connect(9051), {authOnConnect: false})
+			tor.authenticate(`"${constants.tor_password}"`, err => {
+				if (err) console.log("Tor auth error:", err)
+			})
+		}
+
+		console.log("Starting tor...")
+
+		tor.once("ready", () => {
+			tor.getInfo("net/listeners/socks", (err, result) => {
+				if (err) throw err
+				// result is string containing something like "127.0.0.1:36977"
+				// yes, the string contains double quotes!
+				const port = +result.match(/:(\d+)/)[1]
+				const torManager = new TorManager(tor, port)
+				console.log("Tor is ready, using SOCKS port "+port)
+				resolve(torManager)
+			})
+		})
+
+		tor.on("error", function() {
+			console.log("Tor error!")
+			console.log(...arguments)
+		})
+	} else {
+		console.log("Note: Tor functionality not installed. You may wish to run `npm install @deadcanaries/granax`. (78+ MB download required.)")
+		resolve(null)
+	}
+})

src/lib/utils/torswitcher.js

@@ -0,0 +1,41 @@
+const constants = require("../constants")
+const {request} = require("./request")
+
+class TorSwitcher {
+	constructor() {
+		this.torManager = null
+	}
+
+	setManager(torManager) {
+		this.torManager = torManager
+	}
+
+	/**
+	 * Request from the URL.
+	 * The test function will be called with the response object.
+	 * If the test function succeeds, its return value will be returned here.
+	 * If the test function fails, its error will be rejected here.
+	 * Only include rate limit logic in the test function!
+	 * @param {string} url
+	 * @param {(res: import("node-fetch").Response) => Promise<T>} test
+	 * @returns {Promise<T>}
+	 * @template T the return value of the test function
+	 */
+	request(url, test) {
+		if (this.torManager) {
+			return this.torManager.request(url, test)
+		} else {
+			return request(url).then(res => test(res))
+		}
+	}
+}
+
+const switcher = new TorSwitcher()
+
+if (constants.use_tor) {
+	require("./tor").then(torManager => {
+		if (torManager) switcher.setManager(torManager)
+	})
+}
+
+module.exports = switcher

src/site/api/proxy.js

@@ -3,8 +3,6 @@ const {request} = require("../../lib/utils/request")
 const {proxy} = require("pinski/plugins")
 const sharp = require("sharp")
 
-const VERIFY_SUCCESS = Symbol("VERIFY_SUCCESS")
-
 /**
  * Check that a resource is on Instagram.
  * @param {URL} completeURL
@@ -38,7 +36,7 @@ module.exports = [
 					Some thumbnails aren't square and would otherwise be stretched on the page without this.
 					If I cropped the images client side, it would have to be done with CSS background-image, which means no <img srcset>.
 				*/
-				return request(verifyResult.url).then(res => {
+				return request(verifyResult.url, {}, {log: false}).then(res => {
 					const converter = sharp().resize(width, width, {position: "entropy"})
 					return {
 						statusCode: 200,

src/site/sass/main.sass

@@ -175,6 +175,7 @@ body
 			color: #111
 			background-color: rgba(40, 40, 40, 0.25)
 			text-decoration: none
+			overflow: hidden
 			@include sized
 
 			&:hover

src/site/server.js

@@ -17,6 +17,9 @@ subdirs("pug", (err, dirs) => {
 	pinski.addAPIDir("html/static/js/templates/api")
 	pinski.addSassDir("sass")
 	pinski.addAPIDir("api")
+	pinski.muteLogsStartingWith("/imageproxy")
+	pinski.muteLogsStartingWith("/videoproxy")
+	pinski.muteLogsStartingWith("/static")
 	pinski.startServer()
 	pinski.enableWS()