From db7ccabb3bfa7723d41f78b097b742be1f92e3ae Mon Sep 17 00:00:00 2001 From: Cadence Ember Date: Wed, 12 May 2021 00:29:44 +1200 Subject: [PATCH] Implement video filters --- api/filters.js | 160 ++++++++++++++++++++++++++ api/search.js | 11 +- api/subscriptions.js | 9 +- api/video.js | 4 + html/static/js/filters.js | 16 +++ pug/filters.pug | 77 +++++++++++++ pug/includes/video-list-item.pug | 9 ++ pug/settings.pug | 6 + sass/includes/buttons.sass | 26 ++++- sass/includes/filters-page.sass | 64 +++++++++++ sass/includes/settings-page.sass | 17 +++ sass/includes/video-list-item.sass | 67 ++++++++++- sass/main.sass | 1 + utils/constants.js | 4 +- utils/converters.js | 20 ++++ utils/getuser.js | 8 ++ utils/matcher.js | 120 ++++++++++++++++++++ utils/parser.js | 175 +++++++++++++++++++++++++++++ utils/upgradedb.js | 5 + 19 files changed, 790 insertions(+), 9 deletions(-) create mode 100644 api/filters.js create mode 100644 html/static/js/filters.js create mode 100644 pug/filters.pug create mode 100644 sass/includes/filters-page.sass create mode 100644 utils/matcher.js create mode 100644 utils/parser.js diff --git a/api/filters.js b/api/filters.js new file mode 100644 index 0000000..189af7b --- /dev/null +++ b/api/filters.js @@ -0,0 +1,160 @@ +const constants = require("../utils/constants") +const db = require("../utils/db") +const {render} = require("pinski/plugins") +const {getUser, getToken} = require("../utils/getuser") +const validate = require("../utils/validate") +const V = validate.V +const {Matcher, PatternCompileError} = require("../utils/matcher") + +const filterMaxLength = 160 +const regexpEnabledText = constants.server_setup.allow_regexp_filters ? "" : "not" + +function getCategories(req) { + const user = getUser(req) + const filters = user.getFilters() + + // Sort filters into categories for display. Titles are already sorted. + const categories = { + title: {name: "Title", filters: []}, + channel: {name: "Channel", filters: []} + } + for (const filter of filters) { + if (filter.type === "title") { + categories.title.filters.push(filter) + } else { // filter.type is some kind of channel + categories.channel.filters.push(filter) + } + } + categories.channel.filters.sort((a, b) => { + if (a.label && b.label) { + if (a.label < b.label) return -1 + else if (a.label > b.label) return 1 + } + return 0 + }) + + return categories +} + +module.exports = [ + { + route: "/filters", methods: ["GET"], code: async ({req, url}) => { + const categories = getCategories(req) + let referrer = url.searchParams.get("referrer") || null + + let type = null + let contents = "" + let label = null + if (url.searchParams.has("title")) { + type = "title" + contents = url.searchParams.get("title") + } else if (url.searchParams.has("channel-id")) { + type = "channel-id" + contents = url.searchParams.get("channel-id") + label = url.searchParams.get("label") + } + + return render(200, "pug/filters.pug", {categories, type, contents, label, referrer, filterMaxLength, regexpEnabledText}) + } + }, + { + route: "/filters", methods: ["POST"], upload: true, code: async ({req, body}) => { + return new V() + .with(validate.presetLoad({body})) + .with(validate.presetURLParamsBody()) + .with(validate.presetEnsureParams(["filter-type", "new-filter"])) + .check(state => { + // Extract fields + state.type = state.params.get("filter-type") + state.contents = state.params.get("new-filter").slice(0, filterMaxLength) + state.label = state.params.get("label") + if (state.label) { + state.label = state.label.slice(0, filterMaxLength) + } else { + state.label = null + } + state.referrer = state.params.get("referrer") + // Check type + return ["title", "channel-name", "channel-id"].includes(state.type) + }, () => ({ + statusCode: 400, + contentType: "application/json", + content: { + error: "type parameter is not in the list of filter types." + } + })) + .check(state => { + // If title, check that pattern compiles + if (state.type === "title") { + try { + const matcher = new Matcher(state.contents) + matcher.compilePattern() + } catch (e) { + if (e instanceof PatternCompileError) { + state.compileError = e + return false + } + throw e + } + } + return true + }, state => { + const {type, contents, label, compileError} = state + const categories = getCategories(req) + return render(400, "pug/filters.pug", {categories, type, contents, label, compileError, filterMaxLength, regexpEnabledText}) + }) + .last(state => { + const {type, contents, label} = state + const responseHeaders = { + Location: state.referrer || "/filters" + } + const token = getToken(req, responseHeaders) + + db.prepare("INSERT INTO Filters (token, type, data, label) VALUES (?, ?, ?, ?)").run(token, type, contents, label) + + return { + statusCode: 303, + headers: responseHeaders, + contentType: "text/html", + content: "Redirecting..." + } + }) + .go() + } + }, + { + route: "/filters/delete", methods: ["POST"], upload: true, code: async ({req, body}) => { + return new V() + .with(validate.presetLoad({body})) + .with(validate.presetURLParamsBody()) + .with(validate.presetEnsureParams(["delete-id"])) + .check(state => { + state.deleteID = +state.params.get("delete-id") + return !!state.deleteID + }, () => ({ + statusCode: 400, + contentType: "application/json", + content: { + error: "delete-id parameter must be a number" + } + })) + .last(state => { + const {deleteID} = state + const token = getToken(req) + + // the IDs are unique, but can likely be guessed, so also use the token for actual authentication + db.prepare("DELETE FROM Filters WHERE token = ? and id = ?").run(token, deleteID) + + return { + statusCode: 303, + headers: { + Location: "/filters" + }, + contentType: "text/html", + content: "Redirecting..." + } + }) + .go() + } + } +] diff --git a/api/search.js b/api/search.js index 3e1dbaf..5283ad3 100644 --- a/api/search.js +++ b/api/search.js @@ -7,10 +7,14 @@ module.exports = [ { route: "/(?:search|results)", methods: ["GET"], code: async ({req, url}) => { const query = url.searchParams.get("q") || url.searchParams.get("search_query") - const instanceOrigin = getUser(req).getSettingsOrDefaults().instance + const user = getUser(req) + const settings = user.getSettingsOrDefaults() + const instanceOrigin = settings.instance + const fetchURL = new URL(`${instanceOrigin}/api/v1/search`) fetchURL.searchParams.set("q", query) - const results = await request(fetchURL.toString()).then(res => res.json()) + + let results = await request(fetchURL.toString()).then(res => res.json()) const error = results.error || results.message || results.code if (error) throw new Error(`Instance said: ${error}`) @@ -19,6 +23,9 @@ module.exports = [ converters.normaliseVideoInfo(video) } + const filters = user.getFilters() + results = converters.applyVideoFilters(results, filters).videos + return render(200, "pug/search.pug", {query, results, instanceOrigin}) } } diff --git a/api/subscriptions.js b/api/subscriptions.js index acb475f..2c95004 100644 --- a/api/subscriptions.js +++ b/api/subscriptions.js @@ -1,13 +1,12 @@ const {render} = require("pinski/plugins") const db = require("../utils/db") -const {fetchChannelLatest} = require("../utils/youtube") const {getUser} = require("../utils/getuser") -const {timeToPastText, rewriteVideoDescription} = require("../utils/converters") +const {timeToPastText, rewriteVideoDescription, applyVideoFilters} = require("../utils/converters") const {refresher} = require("../background/feed-update") module.exports = [ { - route: `/subscriptions`, methods: ["GET"], code: async ({req}) => { + route: `/subscriptions`, methods: ["GET"], code: async ({req, url}) => { const user = getUser(req) let hasSubscriptions = false let videos = [] @@ -36,10 +35,12 @@ module.exports = [ return video }) } + const filters = user.getFilters() + ;({videos} = applyVideoFilters(videos, filters)) } const settings = user.getSettingsOrDefaults() const instanceOrigin = settings.instance - return render(200, "pug/subscriptions.pug", {settings, hasSubscriptions, videos, channels, refreshed, timeToPastText, instanceOrigin}) + return render(200, "pug/subscriptions.pug", {url, settings, hasSubscriptions, videos, channels, refreshed, timeToPastText, instanceOrigin}) } } ] diff --git a/api/video.js b/api/video.js index 37e344a..78aa299 100644 --- a/api/video.js +++ b/api/video.js @@ -145,6 +145,10 @@ module.exports = [ converters.normaliseVideoInfo(rec) } + // filter list + const {videos, filteredCount} = converters.applyVideoFilters(video.recommendedVideos, user.getFilters()) + video.recommendedVideos = videos + // get subscription data const subscribed = user.isSubscribed(video.authorId) diff --git a/html/static/js/filters.js b/html/static/js/filters.js new file mode 100644 index 0000000..6aadd9d --- /dev/null +++ b/html/static/js/filters.js @@ -0,0 +1,16 @@ +import {ElemJS, q} from "./elemjs/elemjs.js" + +class FilterType extends ElemJS { + constructor(element) { + super(element) + this.notice = q("#title-pattern-matching") + this.on("input", this.updateNotice.bind(this)) + this.updateNotice() + } + + updateNotice() { + this.notice.style.display = this.element.value !== "title" ? "none" : "" + } +} + +new FilterType(q("#filter-type")) diff --git a/pug/filters.pug b/pug/filters.pug new file mode 100644 index 0000000..2c6f2c1 --- /dev/null +++ b/pug/filters.pug @@ -0,0 +1,77 @@ +extends includes/layout + +mixin filter_type_option(label, value) + option(value=value selected=(value === type))= label + +block head + title Filters - CloudTube + script(type="module" src=getStaticURL("html", "static/js/filters.js")) + +block content + main.filters-page + h1 Filters + details(open=!!type) + summary New filter + form(method="post") + if label + input(type="hidden" name="label" value=label) + if referrer + input(type="hidden" name="referrer" value=referrer) + .field-row + label.field-row__label(for="filter-type") Type + select(id="filter-type" name="filter-type").border-look.field-row__input + +filter_type_option("Title", "title") + +filter_type_option("Channel name", "channel-name") + +filter_type_option("Channel ID", "channel-id") + .field-row.max-width-input + label.field-row__label(for="new-filter") Contents + input(type="text" id="new-filter" name="new-filter" value=contents required maxlength=filterMaxLength).border-look.field-row__input + .field-row__description(style=(type !== "title" ? "display: none" : ""))#title-pattern-matching + | For titles, pattern matching is supported. Regular expressions are #{regexpEnabledText} enabled. + | + a(href="https://git.sr.ht/~cadence/tube-docs/tree/main/item/docs/cloudtube/Filters.md") For help, see the documentation. + if compileError + section.filter-compile-error + header.filter-compile-error__header Your pattern failed to compile. + pre.filter-compile-error__trace + = contents + "\n" + = " ".repeat(compileError.position) + "^ " + compileError.message + div: a(href="https://git.sr.ht/~cadence/tube-docs/tree/main/item/docs/cloudtube/Filters.md") For help, see the documentation. + else + if type + .filter-confirmation-notice. + You can refine the filter further if you need to. + When you're happy, click Save. + .save-filter + button.border-look + if referrer + | Save & return + else + | Save + + + .filter-list + - let someFiltersDisplayed = false + each category in categories + if category.filters.length + - someFiltersDisplayed = true + h2.filter-category-header= category.name + div + each filter in category.filters + .filter + .filter__details + - let type = `type: ${filter.type}` + - let content = filter.data + if filter.type === "channel-id" && filter.label + - type += `, id: ${filter.data}` + - content = filter.label + .filter__type= type + .filter__content= content + form.filter__remove(method="post" action="/filters/delete") + input(type="hidden" name="delete-id" value=filter.id) + button.border-look Remove + if !someFiltersDisplayed + .no-filters + h2 You haven't created any filters. + p Create one now and cleanse your mind. + p You can add filters using the button on video thumbnails. diff --git a/pug/includes/video-list-item.pug b/pug/includes/video-list-item.pug index 5357c37..d0b1fbf 100644 --- a/pug/includes/video-list-item.pug +++ b/pug/includes/video-list-item.pug @@ -8,6 +8,15 @@ mixin video_list_item(className, video, instanceOrigin, options = {}) img(src=`/vi/${video.videoId}/mqdefault.jpg` width=320 height=180 alt="").image if video.second__lengthText != undefined span.duration= video.second__lengthText + details.thumbnail__more + summary.thumbnail__show-more × + .thumbnail__options-container + .thumbnail__options-list + - const paramsBase = {} + - if (url) paramsBase.referrer = url.pathname + (url.search && "?" + url.search) + a(href=`/filters?${new URLSearchParams({"channel-id": video.authorId, label: video.author, ...paramsBase})}`).menu-look Hide this channel + a(href=`/filters?${new URLSearchParams({title: video.title, ...paramsBase})}`).menu-look Hide by title + a(href="/filters").menu-look Edit all filters .info div.title: a(href=link).title-link= video.title div.author-line diff --git a/pug/settings.pug b/pug/settings.pug index 40e406b..e330a59 100644 --- a/pug/settings.pug +++ b/pug/settings.pug @@ -81,6 +81,12 @@ block content .save-settings button.border-look Save + h2.more-settings-header More settings + + section.more-settings + ul.more-settings__list + li.more-settings__list-item: a(href="/filters") Edit filters + if user.token details.data-management summary Sync data diff --git a/sass/includes/buttons.sass b/sass/includes/buttons.sass index 94443b1..6a40741 100644 --- a/sass/includes/buttons.sass +++ b/sass/includes/buttons.sass @@ -55,4 +55,28 @@ .border-look @include border-button @include button-size - @include button-hover + @at-root #{selector.unify(&, "a, button")} + @include button-hover + +.menu-look + @include button-size + -webkit-appearance: none + -moz-appearance: none + color: c.$fg-bright + text-decoration: none + line-height: 1.25 + margin: 0 + padding: 8px 20px + background: c.$bg-accent + border: solid c.$bg-darker + border-width: 1px 0px 0px + text-align: left + + &:last-child + border-width: 1px 0px 1px + + &:hover + background: c.$bg-accent-x + + &:active + background: c.$bg-darker diff --git a/sass/includes/filters-page.sass b/sass/includes/filters-page.sass new file mode 100644 index 0000000..4525f2b --- /dev/null +++ b/sass/includes/filters-page.sass @@ -0,0 +1,64 @@ +@use "colors.sass" as c + +@mixin filter-notice + margin-top: 24px + padding: 12px + border-radius: 8px + background-color: c.$bg-darker + white-space: pre-line + +.filters-page + max-width: 600px + margin: 0 auto + + .filter-list + margin-top: 24px + + .no-filters + padding: 4px + text-align: center + + .filter-confirmation-notice + @include filter-notice + color: c.$fg-warning + + .filter-compile-error + @include filter-notice + + &__header + color: c.$fg-warning + + &__trace + background-color: c.$bg-darkest + padding: 6px + + .save-filter + margin-top: 12px + + .border-look + background-color: c.$bg-darker + font-size: 22px + padding: 7px 16px 8px + font-size: 24px + + .filter-category-header + font-size: 1.25em + margin-bottom: 4px + + .filter + display: flex + padding: 5px 0 + border-top: 1px solid c.$edge-grey + + &:last-child + border-bottom: 1px solid c.$edge-grey + + &__details + flex: 1 + + &__type + font-size: 15px + color: c.$fg-dim + + &__remove + flex-shrink: 0 diff --git a/sass/includes/settings-page.sass b/sass/includes/settings-page.sass index 1ee836d..279c761 100644 --- a/sass/includes/settings-page.sass +++ b/sass/includes/settings-page.sass @@ -13,6 +13,23 @@ font-size: 22px padding: 7px 16px 8px +.more-settings-header + margin-top: 36px + +.more-settings + margin-top: 24px + padding: 12px + border-radius: 8px + background-color: c.$bg-accent-x + + &__list + margin: 0 + padding-left: 1em + line-height: 1 + + &__list-item:not(:last-child) + margin-bottom: 0.4em // emulate line-height + .data-management margin-top: 24px diff --git a/sass/includes/video-list-item.sass b/sass/includes/video-list-item.sass index 576ac05..f064826 100644 --- a/sass/includes/video-list-item.sass +++ b/sass/includes/video-list-item.sass @@ -1,6 +1,68 @@ @use "colors.sass" as c @use "_dimensions.sass" as dimensions +.thumbnail + $more-size: 24px + + &__more + position: absolute + top: 4px + right: 4px + width: $more-size + height: $more-size + border-radius: 50% + background-color: rgba(20, 20, 20, 0.85) + padding: 0px + color: #fff + + visibility: hidden + + @at-root .thumbnail:hover &, &[open] + visibility: visible + + &__show-more + display: block + height: $more-size + line-height: 16px + font-size: 25px + text-align: center + + &::-webkit-details-marker + display: none + + &__options-container + position: absolute + z-index: 1 + top: $more-size + left: -1000px + right: 0 + padding-top: 4px + display: flex + justify-content: flex-end + pointer-events: none + + &__options-list + pointer-events: auto + display: grid + background-color: c.$bg-accent + padding: 8px 0px + border-radius: 8px + box-shadow: 0 2px 6px 2px #000 + + &::before + content: "" + display: block + height: 12px + width: 12px + position: absolute + top: 0 + right: 0 + transform: translate(-6px, -1px) rotate(-45deg) + clip-path: polygon(-5% -20%, 120% -20%, 120% 125%) + background-color: c.$bg-accent + box-shadow: 0px 0px 4px 0px #000 + pointer-events: none + @mixin video-list-item display: grid grid-template-columns: 160px 1fr @@ -78,7 +140,6 @@ grid-gap: 16px grid-template-columns: auto 1fr margin-bottom: 20px - overflow: hidden max-height: 150px @at-root .video-list-item--watched#{&} @@ -98,6 +159,10 @@ right: 5px bottom: 5px + .info + overflow: hidden + max-height: 150px + .title font-size: 24px diff --git a/sass/main.sass b/sass/main.sass index 8f57ced..301e516 100644 --- a/sass/main.sass +++ b/sass/main.sass @@ -10,6 +10,7 @@ @use "includes/cant-think-page.sass" @use "includes/privacy-page.sass" @use "includes/js-licenses-page.sass" +@use "includes/filters-page.sass" @use "includes/forms.sass" @use "includes/nav.sass" @use "includes/footer.sass" diff --git a/utils/constants.js b/utils/constants.js index 7e71f81..a979546 100644 --- a/utils/constants.js +++ b/utils/constants.js @@ -26,7 +26,9 @@ let constants = { // Settings for the server to use internally. server_setup: { // The URL of the local NewLeaf instance, which is always used for subscription updates. - local_instance_origin: "http://localhost:3000" + local_instance_origin: "http://localhost:3000", + // Whether users may filter videos by regular expressions. Unlike square patterns, regular expressions are _not_ bounded in complexity, so this can be used for denial of service attacks. Only enable if this is a private instance and you trust all the members. + allow_regexp_filters: false }, // *** *** diff --git a/utils/converters.js b/utils/converters.js index 2b387f0..6e17567 100644 --- a/utils/converters.js +++ b/utils/converters.js @@ -1,5 +1,6 @@ const constants = require("./constants") const pug = require("pug") +const {Matcher} = require("./matcher") function timeToPastText(timestamp) { const difference = Date.now() - timestamp @@ -162,6 +163,24 @@ function subscriberCountToText(count) { return preroundedCountToText(count) + " subscribers" } +function applyVideoFilters(videos, filters) { + const originalCount = videos.length + for (const filter of filters) { + if (filter.type === "channel-id") { + videos = videos.filter(v => v.authorId !== filter.data) + } else if (filter.type === "channel-name") { + videos = videos.filter(v => v.author !== filter.data) + } else if (filter.type === "title") { + const matcher = new Matcher(filter.data) + matcher.compilePattern() + videos = videos.filter(v => !matcher.match(v.title)) + } + } + const filteredCount = originalCount - videos.length + //TODO: actually display if things were filtered, and give the option to disable filters one time + return {videos, filteredCount} +} + module.exports.timeToPastText = timeToPastText module.exports.lengthSecondsToLengthText = lengthSecondsToLengthText module.exports.normaliseVideoInfo = normaliseVideoInfo @@ -169,3 +188,4 @@ module.exports.rewriteVideoDescription = rewriteVideoDescription module.exports.tToMediaFragment = tToMediaFragment module.exports.viewCountToText = viewCountToText module.exports.subscriberCountToText = subscriberCountToText +module.exports.applyVideoFilters = applyVideoFilters diff --git a/utils/getuser.js b/utils/getuser.js index ef6c8e3..cc819fe 100644 --- a/utils/getuser.js +++ b/utils/getuser.js @@ -80,6 +80,14 @@ class User { db.prepare("INSERT OR IGNORE INTO WatchedVideos (token, videoID) VALUES (?, ?)").run([this.token, videoID]) } } + + getFilters() { + if (this.token) { + return db.prepare("SELECT * FROM Filters WHERE token = ? ORDER BY data ASC").all(this.token) + } else { + return [] + } + } } /** diff --git a/utils/matcher.js b/utils/matcher.js new file mode 100644 index 0000000..e986c24 --- /dev/null +++ b/utils/matcher.js @@ -0,0 +1,120 @@ +const {Parser} = require("./parser") +const constants = require("./constants") + +class PatternCompileError extends Error { + constructor(position, message) { + super(message) + this.position = position + } +} + +class PatternRuntimeError extends Error { +} + +class Matcher { + constructor(pattern) { + this.pattern = pattern + this.compiled = null + this.anchors = null + } + + compilePattern() { + // Calculate anchors (starts or ends with -- to allow more text) + this.anchors = {start: true, end: true} + if (this.pattern.startsWith("--")) { + this.anchors.start = false + this.pattern = this.pattern.slice(2) + } + if (this.pattern.endsWith("--")) { + this.anchors.end = false + this.pattern = this.pattern.slice(0, -2) + } + + this.compiled = [] + + // Check if the pattern is a regular expression, only if regexp filters are enabled by administrator + if (this.pattern.match(/^\/.*\/$/) && constants.server_setup.allow_regexp_filters) { + this.compiled.push({ + type: "regexp", + expr: new RegExp(this.pattern.slice(1, -1), "i") + }) + return // do not proceed to step-by-step + } + + // Step-by-step pattern compilation + const patternParser = new Parser(this.pattern.toLowerCase()) + + while (patternParser.hasRemaining()) { + if (patternParser.swallow("[") > 0) { // there is a special command + let index = patternParser.seek("]") + if (index === -1) { + throw new PatternCompileError(patternParser.cursor, "Command is missing closing square bracket") + } + let command = patternParser.get({split: "]"}) + let args = command.split("|") + if (args[0] === "digits") { + this.compiled.push({type: "regexp", expr: /\d+/}) + } else if (args[0] === "choose") { + this.compiled.push({type: "choose", choices: args.slice(1).sort((a, b) => (b.length - a.length))}) + } else { + throw new PatternCompileError(patternParser.cursor - command.length - 1 + args[0].length, `Unknown command name: \`${args[0]}\``) + } + } else { // no special command + let next = patternParser.get({split: "["}) + this.compiled.push({type: "text", text: next}) + if (patternParser.hasRemaining()) patternParser.cursor-- // rewind to before the [ + } + } + } + + match(string) { + if (this.compiled === null) { + throw new Error("Pattern was not compiled before matching. Compiling must be done explicitly.") + } + + const stringParser = new Parser(string.toLowerCase()) + + let flexibleStart = !this.anchors.start + + for (const fragment of this.compiled) { + if (fragment.type === "text") { + let index = stringParser.seek(fragment.text, {moveToMatch: true}) // index, and move to, start of match + if (index === -1) return false + if (index !== 0 && !flexibleStart) return false // allow matching anywhere if flexible start + stringParser.cursor += fragment.text.length // move to end of match. + } + else if (fragment.type === "regexp") { + const match = stringParser.remaining().match(fragment.expr) + if (!match) return false + if (match.index !== 0 && !flexibleStart) return false // allow matching anywhere if flexible start + stringParser.cursor += match.index + match[0].length + } + else if (fragment.type === "choose") { + const ok = fragment.choices.some(choice => { + let index = stringParser.seek(choice) + if (index === -1) return false // try next choice + if (index !== 0 && !flexibleStart) return false // try next choice + // otherwise, good enough for us! /shrug + stringParser.cursor += index + choice.length + return true + }) + if (!ok) return false + } + else { + throw new PatternRuntimeError(`Unknown fragment type ${fragment.type}`) + } + + flexibleStart = false // all further sequences must be anchored to the end of the last one. + } + + if (stringParser.hasRemaining() && this.anchors.end) { + return false // pattern did not end when expected + } + + return true + } +} + +module.exports.Matcher = Matcher +module.exports.PatternCompileError = PatternCompileError +module.exports.PatternRuntimeError = PatternRuntimeError diff --git a/utils/parser.js b/utils/parser.js new file mode 100644 index 0000000..e5368bc --- /dev/null +++ b/utils/parser.js @@ -0,0 +1,175 @@ +/** + * @typedef GetOptions + * @property {string} [split] Characters to split on + * @property {string} [mode] "until" or "between"; choose where to get the content from + * @property {function} [transform] Transformation to apply to result before returning + */ + +const tf = { + lc: s => s.toLowerCase() +} + +class Parser { + constructor(string) { + this.string = string; + this.substore = []; + this.cursor = 0; + this.cursorStore = []; + this.mode = "until"; + this.transform = s => s; + this.split = " "; + } + + /** + * Return all the remaining text from the buffer, without updating the cursor + * @return {string} + */ + remaining() { + return this.string.slice(this.cursor); + } + + /** + * Have we reached the end of the string yet? + * @return {boolean} + */ + hasRemaining() { + return this.cursor < this.string.length + } + + /** + * Get the next element from the buffer, either up to a token or between two tokens, and update the cursor. + * @param {GetOptions} [options] + * @returns {string} + */ + get(options = {}) { + ["mode", "split", "transform"].forEach(o => { + if (!options[o]) options[o] = this[o]; + }); + if (options.mode == "until") { + let next = this.string.indexOf(options.split, this.cursor+options.split.length-1); + if (next == -1) { + let result = this.remaining(); + this.cursor = this.string.length; + return result; + } else { + let result = this.string.slice(this.cursor, next); + this.cursor = next + options.split.length; + return options.transform(result); + } + } else if (options.mode == "between") { + let start = this.string.indexOf(options.split, this.cursor); + let end = this.string.indexOf(options.split, start+options.split.length); + let result = this.string.slice(start+options.split.length, end); + this.cursor = end + options.split.length; + return options.transform(result); + } + } + + /** + * Get a number of chars from the buffer. + * @param {number} length Number of chars to get + * @param {boolean} [move] Whether to update the cursor + */ + slice(length, move = false) { + let result = this.string.slice(this.cursor, this.cursor+length); + if (move) this.cursor += length; + return result; + } + + /** + * Repeatedly swallow a character. + * @param {string} char + */ + swallow(char) { + let before = this.cursor; + while (this.string[this.cursor] == char) this.cursor++; + return this.cursor - before; + } + + /** + * Push the current cursor position to the store + */ + store() { + this.cursorStore.push(this.cursor); + } + + /** + * Pop the previous cursor position from the store + */ + restore() { + this.cursor = this.cursorStore.pop(); + } + + /** + * Run a get operation, test against an input, return success or failure, and restore the cursor. + * @param {string} value The value to test against + * @param {object} options Options for get + */ + test(value, options) { + this.store(); + let next = this.get(options); + let result = next == value; + this.restore(); + return result; + } + + /** + * Run a get operation, test against an input, return success or failure, and restore the cursor on failure. + * @param {string} value The value to test against + * @param {object} options Options for get + */ + has(value, options) { + this.store(); + let next = this.get(options); + let result = next == value; + if (!result) this.restore(); + return result; + } + + /** + * Run a get operation, test against an input, and throw an error if it doesn't match. + * @param {string} value + * @param {GetOptions} [options] + */ + expect(value, options = {}) { + let next = this.get(options); + if (next != value) throw new Error("Expected "+value+", got "+next); + } + + /** + * Seek to or past the next occurance of the string. + * @param {string} toFind + * @param {{moveToMatch?: boolean, useEnd?: boolean}} options both default to false + */ + seek(toFind, options = {}) { + if (options.moveToMatch === undefined) options.moveToMatch = false + if (options.useEnd === undefined) options.useEnd = false + let index = this.string.indexOf(toFind, this.cursor) + if (index !== -1) { + index -= this.cursor + if (options.useEnd) index += toFind.length + if (options.moveToMatch) this.cursor += index + } + return index + } + + /** + * Replace the current string, adding the old one to the substore. + * @param {string} string + */ + pushSubstore(string) { + this.substore.push({string: this.string, cursor: this.cursor, cursorStore: this.cursorStore}) + this.string = string + this.cursor = 0 + this.cursorStore = [] + } + + /** + * Replace the current string with the first entry from the substore. + */ + popSubstore() { + Object.assign(this, this.substore.pop()) + } +} + +module.exports.Parser = Parser diff --git a/utils/upgradedb.js b/utils/upgradedb.js index 9e71646..9cfe62a 100644 --- a/utils/upgradedb.js +++ b/utils/upgradedb.js @@ -43,6 +43,11 @@ const deltas = [ function() { db.prepare("ALTER TABLE Settings ADD COLUMN quality INTEGER DEFAULT 0") .run() + }, + // 6: +Filters + function() { + db.prepare("CREATE TABLE Filters (id INTEGER, token TEXT NOT NULL, type TEXT NOT NULL, data TEXT NOT NULL, label TEXT, PRIMARY KEY (id))") + .run() } ]