diff --git a/config.js b/config.js new file mode 100644 index 0000000..37b0e04 --- /dev/null +++ b/config.js @@ -0,0 +1,3 @@ +module.exports = { + website_origin: "http://localhost:10407" +} diff --git a/package-lock.json b/package-lock.json index 6b91d28..e591338 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1384,6 +1384,30 @@ "glob": "^7.1.3" } }, + "rss": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/rss/-/rss-1.2.2.tgz", + "integrity": "sha1-UKFpiHYTgTOnT5oF0r3I240nqSE=", + "requires": { + "mime-types": "2.1.13", + "xml": "1.0.1" + }, + "dependencies": { + "mime-db": { + "version": "1.25.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.25.0.tgz", + "integrity": "sha1-wY29fHOl2/b0SgJNwNFloeexw5I=" + }, + "mime-types": { + "version": "2.1.13", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.13.tgz", + "integrity": "sha1-4HqqnGxrmnyjASxpADrSWjnpKog=", + "requires": { + "mime-db": "~1.25.0" + } + } + } + }, "safe-buffer": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", @@ -1748,6 +1772,11 @@ "resolved": "https://registry.npmjs.org/ws/-/ws-7.2.1.tgz", "integrity": "sha512-sucePNSafamSKoOqoNfBd8V0StlkzJKL2ZAhGQinCfNQ+oacw+Pk7lcdAElecBF2VkLNZRiIb5Oi1Q5lVUVt2A==" }, + "xml": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/xml/-/xml-1.0.1.tgz", + "integrity": "sha1-eLpyAgApxbyHuKgaPPzXS0ovweU=" + }, "y18n": { "version": "3.2.1", "resolved": "https://registry.npmjs.org/y18n/-/y18n-3.2.1.tgz", diff --git a/package.json b/package.json index 2646dcc..573e8bb 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "gm": "^1.23.1", "node-dir": "^0.1.17", "node-fetch": "^2.6.0", - "pinski": "github:cloudrac3r/pinski#6df26c4d35ff845a4ac56485a79a52970ea50c87" + "pinski": "github:cloudrac3r/pinski#6df26c4d35ff845a4ac56485a79a52970ea50c87", + "rss": "^1.2.2" } } diff --git a/src/lib/cache.js b/src/lib/cache.js index 1a2e9e0..270d7d3 100644 --- a/src/lib/cache.js +++ b/src/lib/cache.js @@ -22,6 +22,10 @@ class InstaCache { return this.cache.get(key).data } + getTtl(key, factor = 1) { + return Math.max((Math.floor(Date.now() - this.cache.get(key).time) / factor), 0) + } + /** * @param {string} key * @param {any} data diff --git a/src/lib/collectors.js b/src/lib/collectors.js index a6b50ec..04b26ab 100644 --- a/src/lib/collectors.js +++ b/src/lib/collectors.js @@ -5,7 +5,7 @@ const InstaCache = require("./cache") const {User} = require("./structures") require("./testimports")(constants, request, extractSharedData, InstaCache, User) -const cache = new InstaCache(600e3) +const cache = new InstaCache(constants.resource_cache_time) function fetchUser(username) { return cache.getOrFetch("user/"+username, () => { diff --git a/src/lib/constants.js b/src/lib/constants.js index 14e6de6..109b8b1 100644 --- a/src/lib/constants.js +++ b/src/lib/constants.js @@ -1,9 +1,11 @@ module.exports = { image_cache_control: `public, max-age=${7*24*60*60}`, + resource_cache_time: 30*60*1000, external: { timeline_query_hash: "e769aa130647d2354c40ea6a439bfc08", - timeline_fetch_first: 12 + timeline_fetch_first: 12, + username_regex: "[\\w.]+" }, symbols: { diff --git a/src/lib/structures/Timeline.js b/src/lib/structures/Timeline.js index 0a5ddad..5d864b8 100644 --- a/src/lib/structures/Timeline.js +++ b/src/lib/structures/Timeline.js @@ -1,8 +1,11 @@ +const RSS = require("rss") const constants = require("../constants") +const config = require("../../../config") const TimelineImage = require("./TimelineImage") const collectors = require("../collectors") require("../testimports")(constants, TimelineImage) +/** @param {any[]} edges */ function transformEdges(edges) { return edges.map(e => new TimelineImage(e.node)) } @@ -13,6 +16,7 @@ class Timeline { */ constructor(user) { this.user = user + /** @type {import("./TimelineImage")[][]} */ this.pages = [] this.addPage(this.user.data.edge_owner_to_timeline_media) this.page_info = this.user.data.edge_owner_to_timeline_media.page_info @@ -40,6 +44,23 @@ class Timeline { this.pages.push(transformEdges(page.edges)) this.page_info = page.page_info } + + getFeed() { + const feed = new RSS({ + title: `@${this.user.data.username}`, + feed_url: `${config.website_origin}/u/${this.user.data.username}/rss.xml`, + site_url: config.website_origin, + description: this.user.data.biography, + image_url: this.user.data.profile_pic_url, + pubDate: new Date(this.user.cachedAt), + ttl: this.user.getTtl(1000*60) // scale to minute + }) + const page = this.pages[0] // only get posts from first page + for (const item of page) { + feed.item(item.getFeedData()) + } + return feed + } } module.exports = Timeline diff --git a/src/lib/structures/TimelineImage.js b/src/lib/structures/TimelineImage.js index 15d3608..d0f920f 100644 --- a/src/lib/structures/TimelineImage.js +++ b/src/lib/structures/TimelineImage.js @@ -1,9 +1,23 @@ +const config = require("../../../config") +const {proxyImage} = require("../utils/proxyurl") +const {compile} = require("pug") + +const rssDescriptionTemplate = compile(` +img(alt=alt src=src) +p(style='white-space: pre-line')= caption +`) + class GraphImage { /** * @param {import("../types").GraphImage} data */ constructor(data) { this.data = data + this.data.edge_media_to_caption.edges.forEach(edge => edge.node.text = edge.node.text.replace(/\u2063/g, "")) // I don't know why U+2063 INVISIBLE SEPARATOR is in here, but it is, and it causes rendering issues with certain fonts. + } + + getProxy(url) { + return proxyImage(url) } /** @@ -32,10 +46,34 @@ class GraphImage { else return null } + getIntroduction() { + const caption = this.getCaption() + if (caption) return caption.split("\n")[0].split(". ")[0] // try to get first meaningful line or sentence + else return null + } + getAlt() { // For some reason, pages 2+ don't contain a11y data. Instagram web client falls back to image caption. return this.data.accessibility_caption || this.getCaption() || "No image description available." } + + getFeedData() { + return { + title: this.getIntroduction() || "No caption provided", + description: rssDescriptionTemplate({src: this.data.display_url, alt: this.getAlt(), caption: this.getCaption()}), + author: this.data.owner.username, + url: `${config.website_origin}/p/${this.data.shortcode}`, + guid: `${config.website_origin}/p/${this.data.shortcode}`, + date: new Date(this.data.taken_at_timestamp*1000) + /* + Readers should display the description as HTML rather than using the media enclosure. + enclosure: { + url: this.data.display_url, + type: "image/jpeg" //TODO: can instagram have PNGs? everything is JPEG according to https://medium.com/@autolike.it/how-to-avoid-low-res-thumbnails-on-instagram-android-problem-bc24f0ed1c7d + } + */ + } + } } module.exports = GraphImage diff --git a/src/lib/structures/User.js b/src/lib/structures/User.js index 77dcbf9..7dd9b46 100644 --- a/src/lib/structures/User.js +++ b/src/lib/structures/User.js @@ -1,5 +1,7 @@ +const constants = require("../constants") +const {proxyImage} = require("../utils/proxyurl") const Timeline = require("./Timeline") -require("../testimports")(Timeline) +require("../testimports")(constants, Timeline) class User { /** @@ -11,6 +13,14 @@ class User { this.followedBy = data.edge_followed_by.count this.posts = data.edge_owner_to_timeline_media.count this.timeline = new Timeline(this) + this.cachedAt = Date.now() + this.proxyProfilePicture = proxyImage(this.data.profile_pic_url) + } + + getTtl(scale = 1) { + const expiresAt = this.cachedAt + constants.resource_cache_time + const ttl = expiresAt - Date.now() + return Math.ceil(Math.max(ttl, 0) / scale) } export() { diff --git a/src/lib/utils/proxyurl.js b/src/lib/utils/proxyurl.js new file mode 100644 index 0000000..8d69a35 --- /dev/null +++ b/src/lib/utils/proxyurl.js @@ -0,0 +1,7 @@ +function proxyImage(url) { + const params = new URLSearchParams() + params.set("url", url) + return "/imageproxy?"+params.toString() +} + +module.exports.proxyImage = proxyImage diff --git a/src/site/api/api.js b/src/site/api/api.js index 6760e49..b9ff912 100644 --- a/src/site/api/api.js +++ b/src/site/api/api.js @@ -1,3 +1,4 @@ +const constants = require("../../lib/constants") const {fetchUser} = require("../../lib/collectors") function reply(statusCode, content) { @@ -9,7 +10,7 @@ function reply(statusCode, content) { } module.exports = [ - {route: "/api/user/(\\w+)", methods: ["GET"], code: async ({fill}) => { + {route: `/api/user/(${constants.external.username_regex})`, methods: ["GET"], code: async ({fill}) => { const user = await fetchUser(fill[0]) const data = user.export() return reply(200, data) diff --git a/src/site/api/feed.js b/src/site/api/feed.js new file mode 100644 index 0000000..ed7594f --- /dev/null +++ b/src/site/api/feed.js @@ -0,0 +1,15 @@ +const constants = require("../../lib/constants") +const {fetchUser} = require("../../lib/collectors") +const {render} = require("pinski/plugins") + +module.exports = [ + {route: `/u/(${constants.external.username_regex})/rss.xml`, methods: ["GET"], code: async ({url, fill}) => { + const user = await fetchUser(fill[0]) + const content = user.timeline.getFeed().xml() + return { + statusCode: 200, + contentType: "application/rss+xml", // see https://stackoverflow.com/questions/595616/what-is-the-correct-mime-type-to-use-for-an-rss-feed + content + } + }} +] diff --git a/src/site/api/routes.js b/src/site/api/routes.js index a83f4f3..bed28e7 100644 --- a/src/site/api/routes.js +++ b/src/site/api/routes.js @@ -1,8 +1,9 @@ +const constants = require("../../lib/constants") const {fetchUser} = require("../../lib/collectors") const {render} = require("pinski/plugins") module.exports = [ - {route: "/u/([\\w.]+)", methods: ["GET"], code: async ({url, fill}) => { + {route: `/u/(${constants.external.username_regex})`, methods: ["GET"], code: async ({url, fill}) => { const params = url.searchParams const user = await fetchUser(fill[0]) const page = +params.get("page") @@ -11,7 +12,7 @@ module.exports = [ } return render(200, "pug/user.pug", {url, user}) }}, - {route: "/fragment/user/([\\w.]+)/(\\d+)", methods: ["GET"], code: async ({url, fill}) => { + {route: `/fragment/user/(${constants.external.username_regex})/(\\d+)`, methods: ["GET"], code: async ({url, fill}) => { const user = await fetchUser(fill[0]) const pageNumber = +fill[1] const pageIndex = pageNumber - 1 diff --git a/src/site/pug/includes/image.pug b/src/site/pug/includes/image.pug deleted file mode 100644 index 4826226..0000000 --- a/src/site/pug/includes/image.pug +++ /dev/null @@ -1,5 +0,0 @@ -mixin image(url) - - - let params = new URLSearchParams() - params.set("url", url) - img(src="/imageproxy?"+params.toString())&attributes(attributes) diff --git a/src/site/pug/includes/timeline_page.pug b/src/site/pug/includes/timeline_page.pug index 8640def..c44bbe5 100644 --- a/src/site/pug/includes/timeline_page.pug +++ b/src/site/pug/includes/timeline_page.pug @@ -1,15 +1,14 @@ //- Needs page, pageIndex -include image.pug - mixin timeline_page(page, pageIndex) - - const pageNumber = pageIndex + 1 - if pageNumber > 1 - .page-number(id=`page-${pageNumber}`) - span.number Page #{pageNumber} + section.timeline-page + - const pageNumber = pageIndex + 1 + if pageNumber > 1 + header.page-number(id=`page-${pageNumber}`) + span.number Page #{pageNumber} - .timeline-inner - - const suggestedSize = 300 - each image in page - - const thumbnail = image.getSuggestedThumbnail(suggestedSize) //- use this as the src in case there are problems with srcset - +image(thumbnail.src)(alt=image.getAlt() width=thumbnail.config_width height=thumbnail.config_height srcset=image.getSrcset() sizes=`${suggestedSize}px`).image + .timeline-inner + - const suggestedSize = 300 + each image in page + - const thumbnail = image.getSuggestedThumbnail(suggestedSize) //- use this as the src in case there are problems with srcset + img(src=image.getProxy(thumbnail.src) alt=image.getAlt() width=thumbnail.config_width height=thumbnail.config_height srcset=image.getSrcset() sizes=`${suggestedSize}px`).image diff --git a/src/site/pug/user.pug b/src/site/pug/user.pug index d14eeb2..6f057f4 100644 --- a/src/site/pug/user.pug +++ b/src/site/pug/user.pug @@ -16,10 +16,10 @@ html .main-divider header.profile-overview .profile-sticky - +image(user.data.profile_pic_url)(width="150px" height="150px").pfp + img(src=user.proxyProfilePicture width="150px" height="150px" alt=`${user.data.full_name}'s profile picture.`).pfp //- - Instagram only uses the above URL, but an HD version is also available: - +image(user.data.profile_pic_url_hd) + Instagram only uses the above URL, but an HD version is also available. + The alt text is pathetic, I know. I don't have much to work with. h1.full-name= user.data.full_name h2.username= `@${user.data.username}` p.bio= user.data.biography @@ -35,6 +35,8 @@ html span(data-numberformat=user.followedBy).count #{numberFormat(user.followedBy)} | | followed by + div.links + a(rel="alternate" type="application/rss+xml" href=`/u/${user.data.username}/rss.xml`) RSS main#timeline.timeline each page, pageIndex in user.timeline.pages diff --git a/src/site/sass/main.sass b/src/site/sass/main.sass index 0802794..5c9f465 100644 --- a/src/site/sass/main.sass +++ b/src/site/sass/main.sass @@ -71,6 +71,9 @@ body .count font-weight: bold + .links + margin-top: 20px + .timeline --image-size: 260px $image-size: var(--image-size)