Implement video filters

utils/constants.js

@@ -26,7 +26,9 @@ let constants = {
 	// Settings for the server to use internally.
 	server_setup: {
 		// The URL of the local NewLeaf instance, which is always used for subscription updates.
-		local_instance_origin: "http://localhost:3000"
+		local_instance_origin: "http://localhost:3000",
+		// Whether users may filter videos by regular expressions. Unlike square patterns, regular expressions are _not_ bounded in complexity, so this can be used for denial of service attacks. Only enable if this is a private instance and you trust all the members.
+		allow_regexp_filters: false
 	},
 
 	// ***                                                 ***

utils/converters.js

@@ -1,5 +1,6 @@
 const constants = require("./constants")
 const pug = require("pug")
+const {Matcher} = require("./matcher")
 
 function timeToPastText(timestamp) {
 	const difference = Date.now() - timestamp
@@ -162,6 +163,24 @@ function subscriberCountToText(count) {
 	return preroundedCountToText(count) + " subscribers"
 }
 
+function applyVideoFilters(videos, filters) {
+	const originalCount = videos.length
+	for (const filter of filters) {
+		if (filter.type === "channel-id") {
+			videos = videos.filter(v => v.authorId !== filter.data)
+		} else if (filter.type === "channel-name") {
+			videos = videos.filter(v => v.author !== filter.data)
+		} else if (filter.type === "title") {
+			const matcher = new Matcher(filter.data)
+			matcher.compilePattern()
+			videos = videos.filter(v => !matcher.match(v.title))
+		}
+	}
+	const filteredCount = originalCount - videos.length
+	//TODO: actually display if things were filtered, and give the option to disable filters one time
+	return {videos, filteredCount}
+}
+
 module.exports.timeToPastText = timeToPastText
 module.exports.lengthSecondsToLengthText = lengthSecondsToLengthText
 module.exports.normaliseVideoInfo = normaliseVideoInfo
@@ -169,3 +188,4 @@ module.exports.rewriteVideoDescription = rewriteVideoDescription
 module.exports.tToMediaFragment = tToMediaFragment
 module.exports.viewCountToText = viewCountToText
 module.exports.subscriberCountToText = subscriberCountToText
+module.exports.applyVideoFilters = applyVideoFilters

utils/getuser.js

@@ -80,6 +80,14 @@ class User {
 			db.prepare("INSERT OR IGNORE INTO WatchedVideos (token, videoID) VALUES (?, ?)").run([this.token, videoID])
 		}
 	}
+
+	getFilters() {
+		if (this.token) {
+			return db.prepare("SELECT * FROM Filters WHERE token = ? ORDER BY data ASC").all(this.token)
+		} else {
+			return []
+		}
+	}
 }
 
 /**

utils/matcher.js

@@ -0,0 +1,120 @@
+const {Parser} = require("./parser")
+const constants = require("./constants")
+
+class PatternCompileError extends Error {
+	constructor(position, message) {
+		super(message)
+		this.position = position
+	}
+}
+
+class PatternRuntimeError extends Error {
+}
+
+class Matcher {
+	constructor(pattern) {
+		this.pattern = pattern
+		this.compiled = null
+		this.anchors = null
+	}
+
+	compilePattern() {
+		// Calculate anchors (starts or ends with -- to allow more text)
+		this.anchors = {start: true, end: true}
+		if (this.pattern.startsWith("--")) {
+			this.anchors.start = false
+			this.pattern = this.pattern.slice(2)
+		}
+		if (this.pattern.endsWith("--")) {
+			this.anchors.end = false
+			this.pattern = this.pattern.slice(0, -2)
+		}
+
+		this.compiled = []
+
+		// Check if the pattern is a regular expression, only if regexp filters are enabled by administrator
+		if (this.pattern.match(/^\/.*\/$/) && constants.server_setup.allow_regexp_filters) {
+			this.compiled.push({
+				type: "regexp",
+				expr: new RegExp(this.pattern.slice(1, -1), "i")
+			})
+			return // do not proceed to step-by-step
+		}
+
+		// Step-by-step pattern compilation
+		const patternParser = new Parser(this.pattern.toLowerCase())
+
+		while (patternParser.hasRemaining()) {
+			if (patternParser.swallow("[") > 0) { // there is a special command
+				let index = patternParser.seek("]")
+				if (index === -1) {
+					throw new PatternCompileError(patternParser.cursor, "Command is missing closing square bracket")
+				}
+				let command = patternParser.get({split: "]"})
+				let args = command.split("|")
+				if (args[0] === "digits") {
+					this.compiled.push({type: "regexp", expr: /\d+/})
+				} else if (args[0] === "choose") {
+					this.compiled.push({type: "choose", choices: args.slice(1).sort((a, b) => (b.length - a.length))})
+				} else {
+					throw new PatternCompileError(patternParser.cursor - command.length - 1 + args[0].length, `Unknown command name: \`${args[0]}\``)
+				}
+			} else { // no special command
+				let next = patternParser.get({split: "["})
+				this.compiled.push({type: "text", text: next})
+				if (patternParser.hasRemaining()) patternParser.cursor-- // rewind to before the [
+			}
+		}
+	}
+
+	match(string) {
+		if (this.compiled === null) {
+			throw new Error("Pattern was not compiled before matching. Compiling must be done explicitly.")
+		}
+
+		const stringParser = new Parser(string.toLowerCase())
+
+		let flexibleStart = !this.anchors.start
+
+		for (const fragment of this.compiled) {
+			if (fragment.type === "text") {
+				let index = stringParser.seek(fragment.text, {moveToMatch: true}) // index, and move to, start of match
+				if (index === -1) return false
+				if (index !== 0 && !flexibleStart) return false // allow matching anywhere if flexible start
+				stringParser.cursor += fragment.text.length // move to end of match.
+			}
+			else if (fragment.type === "regexp") {
+				const match = stringParser.remaining().match(fragment.expr)
+				if (!match) return false
+				if (match.index !== 0 && !flexibleStart) return false // allow matching anywhere if flexible start
+				stringParser.cursor += match.index + match[0].length
+			}
+			else if (fragment.type === "choose") {
+				const ok = fragment.choices.some(choice => {
+					let index = stringParser.seek(choice)
+					if (index === -1) return false // try next choice
+					if (index !== 0 && !flexibleStart) return false // try next choice
+					// otherwise, good enough for us! /shrug
+					stringParser.cursor += index + choice.length
+					return true
+				})
+				if (!ok) return false
+			}
+			else {
+				throw new PatternRuntimeError(`Unknown fragment type ${fragment.type}`)
+			}
+
+			flexibleStart = false // all further sequences must be anchored to the end of the last one.
+		}
+
+		if (stringParser.hasRemaining() && this.anchors.end) {
+			return false // pattern did not end when expected
+		}
+
+		return true
+	}
+}
+
+module.exports.Matcher = Matcher
+module.exports.PatternCompileError = PatternCompileError
+module.exports.PatternRuntimeError = PatternRuntimeError

utils/parser.js

@@ -0,0 +1,175 @@
+/**
+ * @typedef GetOptions
+ * @property {string} [split] Characters to split on
+ * @property {string} [mode] "until" or "between"; choose where to get the content from
+ * @property {function} [transform] Transformation to apply to result before returning
+ */
+
+const tf = {
+	lc: s => s.toLowerCase()
+}
+
+class Parser {
+	constructor(string) {
+		this.string = string;
+		this.substore = [];
+		this.cursor = 0;
+		this.cursorStore = [];
+		this.mode = "until";
+		this.transform = s => s;
+		this.split = " ";
+	}
+
+	/**
+	 * Return all the remaining text from the buffer, without updating the cursor
+	 * @return {string}
+	 */
+	remaining() {
+		return this.string.slice(this.cursor);
+	}
+
+	/**
+	 * Have we reached the end of the string yet?
+	 * @return {boolean}
+	 */
+	hasRemaining() {
+		return this.cursor < this.string.length
+	}
+
+	/**
+	 * Get the next element from the buffer, either up to a token or between two tokens, and update the cursor.
+	 * @param {GetOptions} [options]
+	 * @returns {string}
+	 */
+	get(options = {}) {
+		["mode", "split", "transform"].forEach(o => {
+			if (!options[o]) options[o] = this[o];
+		});
+		if (options.mode == "until") {
+			let next = this.string.indexOf(options.split, this.cursor+options.split.length-1);
+			if (next == -1) {
+				let result = this.remaining();
+				this.cursor = this.string.length;
+				return result;
+			} else {
+				let result = this.string.slice(this.cursor, next);
+				this.cursor = next + options.split.length;
+				return options.transform(result);
+			}
+		} else if (options.mode == "between") {
+			let start = this.string.indexOf(options.split, this.cursor);
+			let end = this.string.indexOf(options.split, start+options.split.length);
+			let result = this.string.slice(start+options.split.length, end);
+			this.cursor = end + options.split.length;
+			return options.transform(result);
+		}
+	}
+
+	/**
+	 * Get a number of chars from the buffer.
+	 * @param {number} length Number of chars to get
+	 * @param {boolean} [move] Whether to update the cursor
+	 */
+	slice(length, move = false) {
+		let result = this.string.slice(this.cursor, this.cursor+length);
+		if (move) this.cursor += length;
+		return result;
+	}
+
+	/**
+	 * Repeatedly swallow a character.
+	 * @param {string} char
+	 */
+	swallow(char) {
+		let before = this.cursor;
+		while (this.string[this.cursor] == char) this.cursor++;
+		return this.cursor - before;
+	}
+
+	/**
+	 * Push the current cursor position to the store
+	 */
+	store() {
+		this.cursorStore.push(this.cursor);
+	}
+
+	/**
+	 * Pop the previous cursor position from the store
+	 */
+	restore() {
+		this.cursor = this.cursorStore.pop();
+	}
+
+	/**
+	 * Run a get operation, test against an input, return success or failure, and restore the cursor.
+	 * @param {string} value The value to test against
+	 * @param {object} options Options for get
+	 */
+	test(value, options) {
+		this.store();
+		let next = this.get(options);
+		let result = next == value;
+		this.restore();
+		return result;
+	}
+
+	/**
+	 * Run a get operation, test against an input, return success or failure, and restore the cursor on failure.
+	 * @param {string} value The value to test against
+	 * @param {object} options Options for get
+	 */
+	has(value, options) {
+		this.store();
+		let next = this.get(options);
+		let result = next == value;
+		if (!result) this.restore();
+		return result;
+	}
+
+	/**
+	 * Run a get operation, test against an input, and throw an error if it doesn't match.
+	 * @param {string} value
+	 * @param {GetOptions} [options]
+	 */
+	expect(value, options = {}) {
+		let next = this.get(options);
+		if (next != value) throw new Error("Expected "+value+", got "+next);
+	}
+
+	/**
+	 * Seek to or past the next occurance of the string.
+	 * @param {string} toFind
+	 * @param {{moveToMatch?: boolean, useEnd?: boolean}} options both default to false
+	 */
+	seek(toFind, options = {}) {
+		if (options.moveToMatch === undefined) options.moveToMatch = false
+		if (options.useEnd === undefined) options.useEnd = false
+		let index = this.string.indexOf(toFind, this.cursor)
+		if (index !== -1) {
+			index -= this.cursor
+			if (options.useEnd) index += toFind.length
+			if (options.moveToMatch) this.cursor += index
+		}
+		return index
+	}
+
+	/**
+	 * Replace the current string, adding the old one to the substore.
+	 * @param {string} string
+	 */
+	pushSubstore(string) {
+		this.substore.push({string: this.string, cursor: this.cursor, cursorStore: this.cursorStore})
+		this.string = string
+		this.cursor = 0
+		this.cursorStore = []
+	}
+
+	/**
+	 * Replace the current string with the first entry from the substore.
+	 */
+	popSubstore() {
+		Object.assign(this, this.substore.pop())
+	}
+}
+
+module.exports.Parser = Parser

utils/upgradedb.js

@@ -43,6 +43,11 @@ const deltas = [
 	function() {
 		db.prepare("ALTER TABLE Settings ADD COLUMN quality INTEGER DEFAULT 0")
 			.run()
+	},
+	// 6: +Filters
+	function() {
+		db.prepare("CREATE TABLE Filters (id INTEGER, token TEXT NOT NULL, type TEXT NOT NULL, data TEXT NOT NULL, label TEXT, PRIMARY KEY (id))")
+			.run()
 	}
 ]