From 271afec9db276fe073e82ae306014ab9c38c3e35 Mon Sep 17 00:00:00 2001
From: Cadence Ember <cloudrac3r@vivaldi.net>
Date: Fri, 31 Jul 2020 02:44:28 +1200
Subject: [PATCH] Add VPN list and applied-privacy subnet list

---
 package-lock.json               |  5 +++
 package.json                    |  1 +
 scripts/process_log.js          |  2 +-
 src/lib/quota/get_identifier.js | 56 ++++++++++++++++++++++++---------
 src/lib/quota/index.js          |  4 +--
 5 files changed, 51 insertions(+), 17 deletions(-)

diff --git a/package-lock.json b/package-lock.json
index 1cb39c7..25062f2 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -2175,6 +2175,11 @@
         "is-extglob": "^2.1.1"
       }
     },
+    "is-in-subnet": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/is-in-subnet/-/is-in-subnet-3.1.0.tgz",
+      "integrity": "sha512-UpHSoZ+S53dhiM5q48atdahm6GmaTFlZaelKTdtBQeSihWZdToE7b8hlTzWVVLghntGlCK97c1JTIdzV5Q+Vcw=="
+    },
     "is-number": {
       "version": "7.0.0",
       "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
diff --git a/package.json b/package.json
index 1226987..8c000e6 100644
--- a/package.json
+++ b/package.json
@@ -20,6 +20,7 @@
     "cookie": "^0.4.1",
     "feed": "git+https://git.sr.ht/~cadence/nodejs-feed#3dde82f8296d7a6f5659323e497e0c684f03ab71",
     "get-stream": "^5.1.0",
+    "is-in-subnet": "^3.1.0",
     "mixin-deep": "^2.0.1",
     "node-dir": "^0.1.17",
     "node-fetch": "^2.6.0",
diff --git a/scripts/process_log.js b/scripts/process_log.js
index 769d2e2..f3b2151 100644
--- a/scripts/process_log.js
+++ b/scripts/process_log.js
@@ -175,7 +175,7 @@ reader.on("line", line => {
 	if (kind) {
 		kinds[kind]++
 		dateCollection.add(kind, dateObject.getTime())
-		if (kind === "api") ips.add(parsed.ip)
+		ips.add(parsed.ip)
 	}
 })
 
diff --git a/src/lib/quota/get_identifier.js b/src/lib/quota/get_identifier.js
index 52454c9..594d979 100644
--- a/src/lib/quota/get_identifier.js
+++ b/src/lib/quota/get_identifier.js
@@ -1,25 +1,53 @@
 const {request} = require("../utils/request")
 const {log} = require("pinski/util/common")
+const constants = require("../constants.js")
+const {createChecker} = require("is-in-subnet")
 
 let addresses = []
+let addressSet = new Set()
+let subnets = []
+let checker = createChecker([])
 
-request("https://check.torproject.org/torbulkexitlist").text().then(text => {
-	const lines = text.split("\n").filter(l => l)
-	addresses = addresses.concat(lines)
-	log(`Loaded Tor exit node list (${addresses.length} total)`, "spam")
-})
-
-/*
-	request("https://meta.bibliogram.art/ip_proxy_list.txt").text().then(text => {
-		const lines = text.split("\n").filter(l => l)
-		addresses = addresses.concat(lines)
-		log(`Loaded Bibliogram proxy list (${addresses.length} total)`, "spam")
+function getList(url, description) {
+	return request(url).text().then(text => {
+		// this is decently fast, but if you have ideas for optimisation, please go for it
+		let d = Date.now()
+		const lines = text.split("\n").filter(l => l && l[0] !== "#")
+		subnets = subnets.concat(lines.filter(l => l.includes("/")))
+		addresses = addresses.concat(lines.filter(l => !l.includes("/")))
+		log(`Loaded ${description} (entries: ${lines.length}) (${Date.now()-d} ms)`, "spam")
 	})
-*/
+}
+
+if (constants.quota.enabled) {
+	Promise.all([
+		getList("https://check.torproject.org/torbulkexitlist", "Tor exit node list"),
+		getList("https://meta.bibliogram.art/quota-list/vpn-ipv4.txt", "VPN IPv4 list"),
+		getList("https://meta.bibliogram.art/quota-list/applied-privacy.txt", "applied-privacy.net subnets")
+	]).then(() => {
+		let d = Date.now()
+		checker = createChecker(subnets)
+		addressSet = new Set(addresses.values())
+		log(`Created subnet checker (${Date.now()-d} ms)`, "spam")
+	})
+}
 
 function getIdentifier(address) {
-	if (addresses.includes(address)) return "proxy"
-	else return address
+	let d = Date.now()
+	const result = (() => {
+		try {
+			if (address == undefined) return "missing"
+			else if (checker(address)) return "proxy"
+			else if (addressSet.has(address)) return "proxy"
+			else return address
+		} catch (e) {
+			// not a valid IP address, or some error like that
+			console.error(e)
+			throw e
+		}
+	})()
+	console.log(`identified ${address} -> ${result} in ${Date.now()-d} ms`)
+	return result
 }
 
 module.exports.getIdentifier = getIdentifier
diff --git a/src/lib/quota/index.js b/src/lib/quota/index.js
index 890d216..0383d35 100644
--- a/src/lib/quota/index.js
+++ b/src/lib/quota/index.js
@@ -20,7 +20,7 @@ function remaining(req) {
 	if (!constants.quota.enabled) return Infinity // sure.
 
 	const ip = getIPFromReq(req)
-	const identifier = String(getIdentifier(ip))
+	const identifier = getIdentifier(ip)
 	const remaining = limiter.remaining(identifier)
 
 	if (constants.quota.track) {
@@ -34,7 +34,7 @@ function add(req, count) {
 	if (!constants.quota.enabled) return Infinity // why not.
 
 	const ip = getIPFromReq(req)
-	const identifier = String(getIdentifier(ip))
+	const identifier = getIdentifier(ip)
 	return limiter.add(identifier, count)
 }