Add VPN list and applied-privacy subnet list

package-lock.json

@@ -2175,6 +2175,11 @@
         "is-extglob": "^2.1.1"
       }
     },
+    "is-in-subnet": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/is-in-subnet/-/is-in-subnet-3.1.0.tgz",
+      "integrity": "sha512-UpHSoZ+S53dhiM5q48atdahm6GmaTFlZaelKTdtBQeSihWZdToE7b8hlTzWVVLghntGlCK97c1JTIdzV5Q+Vcw=="
+    },
     "is-number": {
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",

package.json

@@ -20,6 +20,7 @@
     "cookie": "^0.4.1",
     "feed": "git+https://git.sr.ht/~cadence/nodejs-feed#3dde82f8296d7a6f5659323e497e0c684f03ab71",
     "get-stream": "^5.1.0",
+    "is-in-subnet": "^3.1.0",
     "mixin-deep": "^2.0.1",
     "node-dir": "^0.1.17",
     "node-fetch": "^2.6.0",

scripts/process_log.js

@@ -175,7 +175,7 @@ reader.on("line", line => {
 	if (kind) {
 		kinds[kind]++
 		dateCollection.add(kind, dateObject.getTime())
-		if (kind === "api") ips.add(parsed.ip)
+		ips.add(parsed.ip)
 	}
 })
 

src/lib/quota/get_identifier.js

@@ -1,25 +1,53 @@
 const {request} = require("../utils/request")
 const {log} = require("pinski/util/common")
+const constants = require("../constants.js")
+const {createChecker} = require("is-in-subnet")
 
 let addresses = []
+let addressSet = new Set()
+let subnets = []
+let checker = createChecker([])
 
-request("https://check.torproject.org/torbulkexitlist").text().then(text => {
+function getList(url, description) {
-	const lines = text.split("\n").filter(l => l)
+	return request(url).text().then(text => {
-	addresses = addresses.concat(lines)
+		// this is decently fast, but if you have ideas for optimisation, please go for it
-	log(`Loaded Tor exit node list (${addresses.length} total)`, "spam")
+		let d = Date.now()
+		const lines = text.split("\n").filter(l => l && l[0] !== "#")
+		subnets = subnets.concat(lines.filter(l => l.includes("/")))
+		addresses = addresses.concat(lines.filter(l => !l.includes("/")))
+		log(`Loaded ${description} (entries: ${lines.length}) (${Date.now()-d} ms)`, "spam")
 	})
+}
 
-/*
+if (constants.quota.enabled) {
-	request("https://meta.bibliogram.art/ip_proxy_list.txt").text().then(text => {
+	Promise.all([
-		const lines = text.split("\n").filter(l => l)
+		getList("https://check.torproject.org/torbulkexitlist", "Tor exit node list"),
-		addresses = addresses.concat(lines)
+		getList("https://meta.bibliogram.art/quota-list/vpn-ipv4.txt", "VPN IPv4 list"),
-		log(`Loaded Bibliogram proxy list (${addresses.length} total)`, "spam")
+		getList("https://meta.bibliogram.art/quota-list/applied-privacy.txt", "applied-privacy.net subnets")
+	]).then(() => {
+		let d = Date.now()
+		checker = createChecker(subnets)
+		addressSet = new Set(addresses.values())
+		log(`Created subnet checker (${Date.now()-d} ms)`, "spam")
 	})
-*/
+}
 
 function getIdentifier(address) {
-	if (addresses.includes(address)) return "proxy"
+	let d = Date.now()
+	const result = (() => {
+		try {
+			if (address == undefined) return "missing"
+			else if (checker(address)) return "proxy"
+			else if (addressSet.has(address)) return "proxy"
 			else return address
+		} catch (e) {
+			// not a valid IP address, or some error like that
+			console.error(e)
+			throw e
+		}
+	})()
+	console.log(`identified ${address} -> ${result} in ${Date.now()-d} ms`)
+	return result
 }
 
 module.exports.getIdentifier = getIdentifier

src/lib/quota/index.js

@@ -20,7 +20,7 @@ function remaining(req) {
 	if (!constants.quota.enabled) return Infinity // sure.
 
 	const ip = getIPFromReq(req)
-	const identifier = String(getIdentifier(ip))
+	const identifier = getIdentifier(ip)
 	const remaining = limiter.remaining(identifier)
 
 	if (constants.quota.track) {
@@ -34,7 +34,7 @@ function add(req, count) {
 	if (!constants.quota.enabled) return Infinity // why not.
 
 	const ip = getIPFromReq(req)
-	const identifier = String(getIdentifier(ip))
+	const identifier = getIdentifier(ip)
 	return limiter.add(identifier, count)
 }