const fetch = require("node-fetch") const {render} = require("pinski/plugins") const fs = require("fs").promises class TTLCache { constructor(ttl) { this.backing = new Map() this.ttl = ttl } set(key, value) { this.backing.set(key, {time: Date.now(), value}) return value } clean(key) { if (this.backing.has(key) && Date.now() - this.backing.get(key).time > this.ttl) { this.delete(key) } } has(key) { this.clean(key) return this.backing.has(key) } get(key) { this.clean(key) if (this.has(key)) { return this.backing.get(key).value } else { return null } } async getAs(key, callback) { return this.get(key) || callback().then(value => this.set(key, value)) } } const videoCache = new TTLCache(Infinity) const channelCacheTimeout = 4*60*60*1000; let shareWords = []; fs.readFile("util/words.txt", "utf8").then(words => { shareWords = words.split("\n"); }) const IDLetterIndex = [] .concat(Array(26).fill().map((_, i) => String.fromCharCode(i+65))) .concat(Array(26).fill().map((_, i) => String.fromCharCode(i+97))) .concat(Array(10).fill().map((_, i) => i.toString())) .join("") +"-_" function getShareWords(id) { if (shareWords.length == 0) { console.error("Tried to get share words, but they aren't loaded yet!"); return ""; } // Convert ID string to binary number string let binaryString = ""; for (let letter of id) { binaryString += IDLetterIndex.indexOf(letter).toString(2).padStart(6, "0"); } binaryString = binaryString.slice(0, 64); // Convert binary string to words let words = []; for (let i = 0; i < 6; i++) { let bitFragment = binaryString.substr(i*11, 11).padEnd(11, "0"); let number = parseInt(bitFragment, 2); let word = shareWords[number]; words.push(word); } return words; } function getIDFromWords(words) { // Convert words to binary number string let binaryString = ""; for (let word of words) { binaryString += shareWords.indexOf(word).toString(2).padStart(11, "0") } binaryString = binaryString.slice(0, 64); // Convert binary string to ID let id = ""; for (let i = 0; i < 11; i++) { let bitFragment = binaryString.substr(i*6, 6).padEnd(6, "0"); let number = parseInt(bitFragment, 2); id += IDLetterIndex[number]; } return id; } function validateShareWords(words) { if (words.length != 6) throw new Error("Expected 6 words, got "+words.length); for (let word of words) { if (!shareWords.includes(word)) throw new Error(word+" is not a valid share word"); } } function findShareWords(string) { if (string.includes(" ")) { return string.toLowerCase().split(" "); } else { let words = []; let currentWord = ""; for (let i = 0; i < string.length; i++) { if (string[i] == string[i].toUpperCase()) { if (currentWord) words.push(currentWord); currentWord = string[i].toLowerCase(); } else { currentWord += string[i]; } } words.push(currentWord); return words; } } let channelCache = new Map(); function refreshCache() { for (let e of channelCache.entries()) { if (Date.now()-e[1].refreshed > channelCacheTimeout) channelCache.delete(e[0]); } } function fetchChannel(channelID, ignoreCache) { refreshCache(); let cache = channelCache.get(channelID); if (cache && !ignoreCache) { if (cache.constructor.name == "Promise") { //cf.log("Waiting on promise for "+channelID, "info"); return cache; } else { //cf.log("Using cache for "+channelID+", expires in "+Math.floor((channelCacheTimeout-Date.now()+cache.refreshed)/1000/60)+" minutes", "spam"); return Promise.resolve(cache.data); } } else { //cf.log("Setting new cache for "+channelID, "spam"); let promise = new Promise(resolve => { let channelType = channelID.startsWith("UC") && channelID.length == 24 ? "channel_id" : "user"; Promise.all([ rp(`${getInvidiousHost("channel")}/api/v1/channels/${channelID}`), rp(`https://www.youtube.com/feeds/videos.xml?${channelType}=${channelID}`) ]).then(([body, xml]) => { let data = JSON.parse(body); if (data.error) throw new Error("Couldn't refresh "+channelID+": "+data.error); let feedItems = fxp.parse(xml).feed.entry; //console.log(feedItems.slice(0, 2)) data.latestVideos.forEach(v => { v.author = data.author; let gotDateFromFeed = false; if (Array.isArray(feedItems)) { let feedItem = feedItems.find(i => i["yt:videoId"] == v.videoId); if (feedItem) { const date = new Date(feedItem.published) v.published = date.getTime(); v.publishedText = date.toUTCString().split(" ").slice(1, 4).join(" ") gotDateFromFeed = true; } } if (!gotDateFromFeed) v.published = v.published * 1000; }); //console.log(data.latestVideos.slice(0, 2)) channelCache.set(channelID, {refreshed: Date.now(), data: data}); //cf.log("Set new cache for "+channelID, "spam"); resolve(data); }).catch(error => { cf.log("Error while refreshing "+channelID, "error"); cf.log(error, "error"); channelCache.delete(channelID); resolve(null); }); }); channelCache.set(channelID, promise); return promise; } } module.exports = [ { route: "/watch", methods: ["GET"], code: async ({url}) => { const id = url.searchParams.get("v") const video = await videoCache.getAs(id, () => fetch(`http://localhost:3000/api/v1/videos/${id}`).then(res => res.json())) return render(200, "pug/video.pug", {video}) } } /* { route: "/v/(.*)", methods: ["GET"], code: async ({fill}) => { let id; let wordsString = fill[0]; wordsString = wordsString.replace(/%20/g, " ") if (wordsString.length == 11) { id = wordsString } else { let words = findShareWords(wordsString); try { validateShareWords(words); } catch (e) { return [400, e.message]; } id = getIDFromWords(words); } return { statusCode: 301, contentType: "text/html", content: "Redirecting...", headers: { "Location": "/cloudtube/video/"+id } } } }, { route: "/cloudtube/video/([\\w-]+)", methods: ["GET"], code: ({req, fill}) => new Promise(resolve => { rp(`${getInvidiousHost("video")}/api/v1/videos/${fill[0]}`).then(body => { try { let data = JSON.parse(body); let page = pugCache.get("pug/old/cloudtube-video.pug").web() page = page.replace('""', () => body); let shareWords = getShareWords(fill[0]); page = page.replace('""', () => JSON.stringify(shareWords)); page = page.replace("", () => `${data.title} — CloudTube video`); while (page.includes("yt.www.watch.player.seekTo")) page = page.replace("yt.www.watch.player.seekTo", "seekTo"); let metaOGTags = `\n`+ `\n`+ `\n`+ `\n`+ `\n` page = page.replace("", () => metaOGTags); resolve({ statusCode: 200, contentType: "text/html", content: page }); } catch (e) { resolve([400, "Error parsing data from Invidious"]); } }).catch(err => { resolve([500, "Error requesting data from Invidious"]); }); }) }, { route: "/cloudtube/channel/([\\w-]+)", methods: ["GET"], code: ({req, fill}) => new Promise(resolve => { fetchChannel(fill[0]).then(data => { try { let page = pugCache.get("pug/old/cloudtube-channel.pug").web() page = page.replace('""', () => JSON.stringify(data)); page = page.replace("", () => `${data.author} — CloudTube channel`); let metaOGTags = `\n`+ `\n`+ // `\n`+ `\n`+ `\n` page = page.replace("", () => metaOGTags); resolve({ statusCode: 200, contentType: "text/html", content: page }); } catch (e) { resolve([400, "Error parsing data from Invidious"]); } }).catch(err => { resolve([500, "Error requesting data from Invidious"]); }); }) }, { route: "/cloudtube/playlist/([\\w-]+)", methods: ["GET"], code: ({req, fill}) => new Promise(resolve => { rp(`${getInvidiousHost("playlist")}/api/v1/playlists/${fill[0]}`).then(body => { try { let data = JSON.parse(body); let page = pugCache.get("pug/old/cloudtube-playlist.pug").web() page = page.replace('""', () => body); page = page.replace("", () => `${data.title} — CloudTube playlist`); while (page.includes("yt.www.watch.player.seekTo")) page = page.replace("yt.www.watch.player.seekTo", "seekTo"); let metaOGTags = `\n`+ `\n`+ `\n`+ `\n` if (data.videos[0]) metaOGTags += `\n`; page = page.replace("", () => metaOGTags); resolve({ statusCode: 200, contentType: "text/html", content: page }); } catch (e) { resolve([400, "Error parsing data from Invidious"]); } }).catch(err => { resolve([500, "Error requesting data from Invidious"]); }); }) }, { route: "/cloudtube/search", methods: ["GET"], upload: "json", code: ({req, url}) => new Promise(resolve => { const params = url.searchParams console.log("URL:", req.url) console.log("Headers:", req.headers) let page = pugCache.get("pug/old/cloudtube-search.pug").web() if (params.has("q")) { // search terms were entered let sort_by = params.get("sort_by") || "relevance"; rp(`${getInvidiousHost("search")}/api/v1/search?q=${encodeURIComponent(decodeURIComponent(params.get("q")))}&sort_by=${sort_by}`).then(body => { try { // json.parse? page = page.replace('""', () => body); page = page.replace("", () => `${decodeURIComponent(params.get("q"))} — CloudTube search`); let metaOGTags = `\n`+ `\n`+ `\n`+ `\n` page = page.replace("", () => metaOGTags); resolve({ statusCode: 200, contentType: "text/html", content: page }); } catch (e) { resolve([400, "Error parsing data from Invidious"]); } }).catch(err => { resolve([500, "Error requesting data from Invidious"]); }); } else { // no search terms page = page.replace("", ""); page = page.replace("", `CloudTube search`); let metaOGTags = `\n`+ `\n`+ `\n`+ `\n` page = page.replace("", () => metaOGTags); resolve({ statusCode: 200, contentType: "text/html", content: page }); } }) }, { route: "/api/youtube/subscribe", methods: ["POST"], upload: "json", code: async ({data}) => { if (!data.channelID) return [400, 1]; if (!data.token) return [400, 8]; let userRow = await db.get("SELECT userID FROM AccountTokens WHERE token = ?", data.token); if (!userRow || userRow.expires <= Date.now()) return [401, 8]; let subscriptions = (await db.all("SELECT channelID FROM AccountSubscriptions WHERE userID = ?", userRow.userID)).map(r => r.channelID); let nowSubscribed; if (subscriptions.includes(data.channelID)) { await db.run("DELETE FROM AccountSubscriptions WHERE userID = ? AND channelID = ?", [userRow.userID, data.channelID]); nowSubscribed = false; } else { await db.run("INSERT INTO AccountSubscriptions VALUES (?, ?)", [userRow.userID, data.channelID]); nowSubscribed = true; } return [200, {channelID: data.channelID, nowSubscribed}]; } }, { route: "/api/youtube/subscriptions", methods: ["POST"], upload: "json", code: async ({data}) => { let subscriptions; if (data.token) { let userRow = await db.get("SELECT userID FROM AccountTokens WHERE token = ?", data.token); if (!userRow || userRow.expires <= Date.now()) return [401, 8]; subscriptions = (await db.all("SELECT channelID FROM AccountSubscriptions WHERE userID = ?", userRow.userID)).map(r => r.channelID); } else { if (data.subscriptions && data.subscriptions.constructor.name == "Array" && data.subscriptions.every(i => typeof(i) == "string")) subscriptions = data.subscriptions; else return [400, 4]; } if (data.force) { for (let channelID of subscriptions) channelCache.delete(channelID); return [204, ""]; } else { let videos = []; let channels = []; let failedCount = 0 await Promise.all(subscriptions.map(s => fetchChannel(s).then(data => { if (data) { videos = videos.concat(data.latestVideos); channels.push({author: data.author, authorID: data.authorId, authorThumbnails: data.authorThumbnails}); } else { failedCount++ } }))); videos = videos.sort((a, b) => (b.published - a.published)) let limit = 60; if (data.limit && !isNaN(+data.limit) && (+data.limit > 0)) limit = +data.limit; videos = videos.slice(0, limit); channels = channels.sort((a, b) => (a.author.toLowerCase() < b.author.toLowerCase() ? -1 : 1)); return [200, {videos, channels, failedCount}]; } } }, { route: "/api/youtube/subscriptions/import", methods: ["POST"], upload: "json", code: async ({data}) => { if (!data) return [400, 3]; if (!typeof(data) == "object") return [400, 5]; if (!data.token) return [401, 8]; let userRow = await db.get("SELECT userID FROM AccountTokens WHERE token = ?", data.token); if (!userRow || userRow.expires <= Date.now()) return [401, 8]; if (!data.subscriptions) return [400, 4]; if (!data.subscriptions.every(v => typeof(v) == "string")) return [400, 5]; await db.run("BEGIN TRANSACTION"); await db.run("DELETE FROM AccountSubscriptions WHERE userID = ?", userRow.userID); await Promise.all(data.subscriptions.map(v => db.run("INSERT OR IGNORE INTO AccountSubscriptions VALUES (?, ?)", [userRow.userID, v]) )) await db.run("END TRANSACTION"); return [204, ""]; } }, { route: "/api/youtube/channels/([\\w-]+)/info", methods: ["GET"], code: ({fill}) => { return rp(`${getInvidiousHost("channel")}/api/v1/channels/${fill[0]}`).then(body => { return { statusCode: 200, contentType: "application/json", content: body } }).catch(e => { console.error(e); return [500, "Unknown request error, check console"] }); } }, { route: "/api/youtube/alternate/.*", methods: ["GET"], code: async ({req}) => { return [404, "Please leave me alone. This endpoint has been removed and it's never coming back. Why not try youtube-dl instead? https://github.com/ytdl-org/youtube-dl/\nIf you own a bot that accesses this endpoint, please send me an email: https://cadence.moe/about/contact\nHave a nice day.\n"]; return null return [400, {error: `/api/youtube/alternate has been removed. The page will be reloaded.
`}] } }, { route: "/api/youtube/dash/([\\w-]+)", methods: ["GET"], code: ({fill}) => new Promise(resolve => { let id = fill[0]; let sentReq = rp({ url: `http://localhost:3000/api/manifest/dash/id/${id}?local=true`, timeout: 8000 }); sentReq.catch(err => { if (err.code == "ETIMEDOUT" || err.code == "ESOCKETTIMEDOUT" || err.code == "ECONNRESET") resolve([502, "Request to Invidious timed out"]); else { console.log(err); resolve([500, "Unknown request error, check console"]); } }); sentReq.then(body => { let data = fxp.parse(body, {ignoreAttributes: false}); resolve([200, data]); }).catch(err => { if (err.code == "ETIMEDOUT" || err.code == "ESOCKETTIMEDOUT" || err.code == "ECONNRESET") resolve([502, "Request to Invidious timed out"]); else { console.log(err); resolve([500, "Unknown parse error, check console"]); } }); }) }, { route: "/api/youtube/get_endscreen", methods: ["GET"], code: async ({params}) => { if (!params.v) return [400, 1]; let data = await rp("https://www.youtube.com/get_endscreen?v="+params.v); data = data.toString(); try { if (data == `""`) { return { statusCode: 204, content: "", contentType: "text/html", headers: {"Access-Control-Allow-Origin": "*"} } } else { let json = JSON.parse(data.slice(data.indexOf("\n")+1)); let promises = []; for (let e of json.elements.filter(e => e.endscreenElementRenderer.style == "WEBSITE")) { for (let thb of e.endscreenElementRenderer.image.thumbnails) { let promise = rp(thb.url, {encoding: null}); promise.then(image => { let base64 = image.toString("base64"); thb.url = "data:image/jpeg;base64,"+base64; }); promises.push(promise); } } await Promise.all(promises); return { statusCode: 200, content: json, contentType: "application/json", headers: {"Access-Control-Allow-Origin": "*"} } } } catch (e) { return [500, "Couldn't parse endscreen data\n\n"+data]; } } }, { route: "/api/youtube/video/([\\w-]+)", methods: ["GET"], code: ({fill}) => { return new Promise(resolve => { ytdl.getInfo(fill[0]).then(info => { resolve([200, Object.assign(info, {constructor: new Object().constructor})]); }).catch(err => { resolve([400, err]); }); }); } }, { route: "/api/youtube/channel/(\\S+)", methods: ["GET"], code: ({fill}) => { return new Promise(resolve => { rp( "https://www.googleapis.com/youtube/v3/channels?part=contentDetails"+ `&id=${fill[0]}&key=${auth.yt_api_key}` ).then(channelText => { let channel = JSON.parse(channelText); let playlistIDs = channel.items.map(i => i.contentDetails.relatedPlaylists.uploads); Promise.all(playlistIDs.map(pid => rp( "https://www.googleapis.com/youtube/v3/playlistItems?part=contentDetails"+ `&playlistId=${pid}&maxResults=50&key=${auth.yt_api_key}` ))).then(playlistsText => { let playlists = playlistsText.map(pt => JSON.parse(pt));; let items = [].concat(...playlists.map(p => p.items)) .map(i => i.contentDetails) .sort((a, b) => (a.videoPublishedAt > b.videoPublishedAt ? -1 : 1)) .slice(0, 50); rp( "https://www.googleapis.com/youtube/v3/videos?part=contentDetails,snippet"+ `&id=${items.map(i => i.videoId).join(",")}&key=${auth.yt_api_key}` ).then(videosText => { let videos = JSON.parse(videosText); videos.items.forEach(v => { let duration = v.contentDetails.duration.slice(2).replace(/\D/g, ":").slice(0, -1).split(":") .map((t, i) => { if (i) t = t.padStart(2, "0"); return t; }); if (duration.length == 1) duration.splice(0, 0, "0"); v.duration = duration.join(":"); }); resolve([200, videos.items]); }); }); }).catch(err => { resolve([500, "Unexpected promise rejection error. This should not happen. Contact Cadence as soon as possible."]); console.log("Unexpected promise rejection error!"); console.log(err); }); }); } }, { route: "/api/youtube/search", methods: ["GET"], code: ({params}) => { return new Promise(resolve => { if (!params || !params.q) return resolve([400, "Missing ?q parameter"]); let searchObject = { maxResults: +params.maxResults || 20, key: auth.yt_api_key, type: "video" }; if (params.order) searchObject.order = params.order; yts(params.q, searchObject, (err, search) => { if (err) { resolve([500, "YouTube API error. This should not happen. Contact Cadence as soon as possible."]); console.log("YouTube API error!"); console.log(search); } else { rp( "https://www.googleapis.com/youtube/v3/videos?part=contentDetails&id="+ search.map(r => r.id).join(",")+ "&key="+auth.yt_api_key ).then(videos => { JSON.parse(videos).items.forEach(v => { let duration = v.contentDetails.duration.slice(2).replace(/\D/g, ":").slice(0, -1).split(":") .map((t, i) => { if (i) t = t.padStart(2, "0"); return t; }); if (duration.length == 1) duration.splice(0, 0, "0"); search.find(r => r.id == v.id).duration = duration.join(":"); }); resolve([200, search]); }); } }); }); } }*/ ]