Write full types and fix related bugs

Also fixes #4.

README.md

@@ -19,14 +19,16 @@ See also: [Invidious, a front-end for YouTube.](https://github.com/omarroth/invi
 - [ ] Image disk cache
 - [ ] Clickable usernames and hashtags
 - [ ] Homepage
+- [ ] Instance list
 - [ ] Proper error checking
 - [ ] Optimised for mobile
 - [ ] Favicon
 - [ ] Settings (e.g. data saving)
 - [ ] List view
 - [ ] IGTV
-- [ ] Public API
+- [ ] Test suite
 - [ ] Rate limiting
+- [ ] Public API
 - [ ] Explore hashtags
 - [ ] Explore locations
 - [ ] _more..._

src/lib/cache.js

@@ -13,15 +13,24 @@ class TtlCache {
 
 	clean() {
 		for (const key of this.cache.keys()) {
-			const value = this.cache.get(key)
-			if (Date.now() > value.time + this.ttl) this.cache.delete(key)
+			this.cleanKey(key)
 		}
 	}
 
+	cleanKey(key) {
+		const value = this.cache.get(key)
+		if (value && Date.now() > value.time + this.ttl) this.cache.delete(key)
+	}
+
 	/**
 	 * @param {string} key
 	 */
 	has(key) {
+		this.cleanKey(key)
+		return this.hasWithoutClean(key)
+	}
+
+	hasWithoutClean(key) {
 		return this.cache.has(key)
 	}
 
@@ -29,17 +38,27 @@ class TtlCache {
 	 * @param {string} key
 	 */
 	get(key) {
+		this.cleanKey(key)
+		return this.getWithoutClean(key)
+	}
+
+	getWithoutClean(key) {
 		const value = this.cache.get(key)
 		if (value) return value.data
 		else return null
 	}
 
 	/**
+	 * Returns null if doesn't exist
 	 * @param {string} key
 	 * @param {number} factor factor to divide the result by. use 60*1000 to get the ttl in minutes.
 	 */
 	getTtl(key, factor = 1) {
-		return Math.max((Math.floor(Date.now() - this.cache.get(key).time) / factor), 0)
+		if (this.has(key)) {
+			return Math.max((Math.floor(Date.now() - this.cache.get(key).time) / factor), 0)
+		} else {
+			return null
+		}
 	}
 
 	/**
@@ -49,6 +68,13 @@ class TtlCache {
 	set(key, data) {
 		this.cache.set(key, {data, time: Date.now()})
 	}
+
+	/**
+	 * @param {string} key
+	 */
+	refresh(key) {
+		this.cache.get(key).time = Date.now()
+	}
 }
 
 class RequestCache extends TtlCache {
@@ -66,7 +92,7 @@ class RequestCache extends TtlCache {
 	 * @template T
 	 */
 	getOrFetch(key, callback) {
-		this.clean()
+		this.cleanKey(key)
 		if (this.cache.has(key)) return Promise.resolve(this.get(key))
 		else {
 			const pending = callback().then(result => {

src/lib/collectors.js

@@ -5,8 +5,8 @@ const {TtlCache, RequestCache} = require("./cache")
 require("./testimports")(constants, request, extractSharedData, RequestCache)
 
 const requestCache = new RequestCache(constants.resource_cache_time)
-/** @type {import("./cache").TtlCache<import("./structures/TimelineImage")>} */
-const timelineImageCache = new TtlCache(constants.resource_cache_time)
+/** @type {import("./cache").TtlCache<import("./structures/TimelineEntry")>} */
+const timelineEntryCache = new TtlCache(constants.resource_cache_time)
 
 function fetchUser(username) {
 	return requestCache.getOrFetch("user/"+username, () => {
@@ -45,13 +45,37 @@ function fetchTimelinePage(userID, after) {
 
 /**
  * @param {string} shortcode
- * @param {boolean} needDirect
- * @returns {Promise<import("./structures/TimelineImage")>}
+ * @returns {import("./structures/TimelineEntry")}
  */
-function fetchShortcode(shortcode, needDirect = false) {
-	const attempt = timelineImageCache.get(shortcode)
-	if (attempt && (attempt.isDirect === true || needDirect === false)) return Promise.resolve(attempt)
+function getOrCreateShortcode(shortcode) {
+	if (timelineEntryCache.has(shortcode)) {
+		return timelineEntryCache.get(shortcode)
+	} else {
+		// require down here or have to deal with require loop. require cache will take care of it anyway.
+		// TimelineImage -> collectors -/> TimelineImage
+		const TimelineEntry = require("./structures/TimelineEntry")
+		const result = new TimelineEntry()
+		timelineEntryCache.set(shortcode, result)
+		return result
+	}
+}
 
+async function getOrFetchShortcode(shortcode) {
+	if (timelineEntryCache.has(shortcode)) {
+		return timelineEntryCache.get(shortcode)
+	} else {
+		const data = await fetchShortcodeData(shortcode)
+		const entry = getOrCreateShortcode(shortcode)
+		entry.applyN3(data)
+		return entry
+	}
+}
+
+/**
+ * @param {string} shortcode
+ * @returns {Promise<import("./types").TimelineEntryN3>}
+ */
+function fetchShortcodeData(shortcode) {
 	// example actual query from web:
 	// query_hash=2b0673e0dc4580674a88d426fe00ea90&variables={"shortcode":"xxxxxxxxxxx","child_comment_count":3,"fetch_comment_count":40,"parent_comment_count":24,"has_threaded_comments":true}
 	// we will not include params about comments, which means we will not receive comments, but everything else should still work fine
@@ -60,35 +84,17 @@ function fetchShortcode(shortcode, needDirect = false) {
 	p.set("variables", JSON.stringify({shortcode}))
 	return requestCache.getOrFetchPromise("shortcode/"+shortcode, () => {
 		return request(`https://www.instagram.com/graphql/query/?${p.toString()}`).then(res => res.json()).then(root => {
-			/** @type {import("./types").GraphImage} */
+			/** @type {import("./types").TimelineEntryN3} */
 			const data = root.data.shortcode_media
-			return createShortcodeFromData(data, true)
+			return data
 		})
 	})
 }
 
-/**
- * @param {import("./types").GraphImage} data
- * @param {boolean} isDirect
- */
-function createShortcodeFromData(data, isDirect) {
-	const existing = timelineImageCache.get(data.shortcode)
-	if (existing) {
-		existing.updateData(data, isDirect)
-		return existing
-	} else {
-		// require down here or have to deal with require loop. require cache will take care of it anyway.
-		// TimelineImage -> collectors -/> TimelineImage
-		const TimelineImage = require("./structures/TimelineImage")
-		const timelineImage = new TimelineImage(data, false)
-		timelineImageCache.set(data.shortcode, timelineImage)
-		return timelineImage
-	}
-}
-
 module.exports.fetchUser = fetchUser
 module.exports.fetchTimelinePage = fetchTimelinePage
-module.exports.fetchShortcode = fetchShortcode
-module.exports.createShortcodeFromData = createShortcodeFromData
+module.exports.getOrCreateShortcode = getOrCreateShortcode
+module.exports.fetchShortcodeData = fetchShortcodeData
 module.exports.requestCache = requestCache
-module.exports.timelineImageCache = timelineImageCache
+module.exports.timelineEntryCache = timelineEntryCache
+module.exports.getOrFetchShortcode = getOrFetchShortcode

src/lib/constants.js

@@ -11,6 +11,11 @@ module.exports = {
 	},
 
 	symbols: {
-		NO_MORE_PAGES: Symbol("NO_MORE_PAGES")
+		NO_MORE_PAGES: Symbol("NO_MORE_PAGES"),
+		TYPE_IMAGE: Symbol("TYPE_IMAGE"),
+		TYPE_VIDEO: Symbol("TYPE_VIDEO"),
+		TYPE_GALLERY: Symbol("TYPE_GALLERY"),
+		TYPE_GALLERY_IMAGE: Symbol("TYPE_GALLERY_IMAGE"),
+		TYPE_GALLERY_VIDEO: Symbol("TYPE_GALLERY_VIDEO")
 	}
 }

src/lib/structures/Timeline.js

@@ -1,14 +1,20 @@
 const RSS = require("rss")
 const constants = require("../constants")
 const config = require("../../../config")
-const TimelineImage = require("./TimelineImage")
+const TimelineEntry = require("./TimelineEntry")
 const InstaCache = require("../cache")
 const collectors = require("../collectors")
-require("../testimports")(constants, collectors, TimelineImage, InstaCache)
+require("../testimports")(constants, collectors, TimelineEntry, InstaCache)
 
 /** @param {any[]} edges */
 function transformEdges(edges) {
-	return edges.map(e => collectors.createShortcodeFromData(e.node, false))
+	return edges.map(e => {
+		/** @type {import("../types").TimelineEntryAll} */
+		const data = e.node
+		const entry = collectors.getOrCreateShortcode(data.shortcode)
+		entry.apply(data)
+		return entry
+	})
 }
 
 class Timeline {
@@ -17,7 +23,7 @@ class Timeline {
 	 */
 	constructor(user) {
 		this.user = user
-		/** @type {import("./TimelineImage")[][]} */
+		/** @type {import("./TimelineEntry")[][]} */
 		this.pages = []
 		this.addPage(this.user.data.edge_owner_to_timeline_media)
 		this.page_info = this.user.data.edge_owner_to_timeline_media.page_info

src/lib/structures/TimelineBaseMethods.js

@@ -0,0 +1,33 @@
+const constants = require("../constants")
+const {proxyImage, proxyExtendedOwner} = require("../utils/proxyurl")
+
+class TimelineBaseMethods {
+	constructor() {
+		/** @type {import("../types").GraphChildAll & {owner?: any}} */
+		this.data
+	}
+
+	getType() {
+		if (this.data.__typename === "GraphImage") {
+			if (this.data.owner) return constants.symbols.TYPE_IMAGE
+			else return constants.symbols.TYPE_GALLERY_IMAGE
+		} else if (this.data.__typename === "GraphVideo") {
+			if (this.data.owner) return constants.symbols.TYPE_VIDEO
+			else return constants.symbols.TYPE_GALLERY_VIDEO
+		} else if (this.data.__typename === "GraphSidecar") {
+			return constants.symbols.TYPE_GALLERY
+		} else {
+			throw new Error("Unknown shortcode __typename: "+this.data.__typename)
+		}
+	}
+
+	getDisplayUrlP() {
+		return proxyImage(this.data.display_url)
+	}
+
+	getAlt() {
+		return this.data.accessibility_caption || "No image description available."
+	}
+}
+
+module.exports = TimelineBaseMethods

src/lib/structures/TimelineChild.js

@@ -1,41 +1,16 @@
 const config = require("../../../config")
 const {proxyImage} = require("../utils/proxyurl")
 const collectors = require("../collectors")
+const TimelineBaseMethods = require("./TimelineBaseMethods")
 require("../testimports")(collectors)
 
-class TimelineChild {
+class TimelineChild extends TimelineBaseMethods {
 	/**
-	 * @param {import("../types").GraphChild} data
+	 * @param {import("../types").GraphChildAll} data
 	 */
 	constructor(data) {
+		super()
 		this.data = data
-		this.proxyDisplayURL = proxyImage(this.data.display_url)
-	}
-
-	/**
-	 * @param {number} size
-	 */
-	getSuggestedResource(size) {
-		let found = null
-		for (const tr of this.data.display_resources) {
-			found = tr
-			if (tr.config_width >= size) break
-		}
-		found = proxyImage(found, size)
-		return found
-	}
-
-	getSrcset() {
-		return this.data.display_resources.map(tr => {
-			const p = new URLSearchParams()
-			p.set("width", String(tr.config_width))
-			p.set("url", tr.src)
-			return `/imageproxy?${p.toString()} ${tr.config_width}w`
-		}).join(", ")
-	}
-
-	getAlt() {
-		return this.data.accessibility_caption || "No image description available."
 	}
 }
 

src/lib/structures/TimelineEntry.js

@@ -0,0 +1,220 @@
+const config = require("../../../config")
+const constants = require("../constants")
+const {proxyImage, proxyExtendedOwner} = require("../utils/proxyurl")
+const {compile} = require("pug")
+const collectors = require("../collectors")
+const TimelineBaseMethods = require("./TimelineBaseMethods")
+const TimelineChild = require("./TimelineChild")
+require("../testimports")(collectors, TimelineChild, TimelineBaseMethods)
+
+const rssDescriptionTemplate = compile(`
+p(style='white-space: pre-line')= caption
+img(alt=alt src=src)
+`)
+
+class TimelineEntry extends TimelineBaseMethods {
+	constructor() {
+		super()
+		/** @type {import("../types").TimelineEntryAll} some properties may not be available yet! */
+		// @ts-ignore
+		this.data = {}
+		setImmediate(() => { // next event loop
+			if (!this.data.__typename) throw new Error("TimelineEntry data was not initalised in same event loop (missing __typename)")
+		})
+		/** @type {string} Not available until fetchExtendedOwnerP is called */
+		this.ownerPfpCacheP = null
+		/** @type {import("./TimelineChild")[]} Not available until fetchChildren is called */
+		this.children = null
+	}
+
+	async update() {
+		const data = await collectors.fetchShortcodeData(this.data.shortcode)
+		this.applyN3(data)
+	}
+
+	/**
+	 * General apply function that detects the data format
+	 */
+	apply(data) {
+		if (!data.display_resources) {
+			this.applyN1(data)
+		} else if (data.thumbnail_resources) {
+			this.applyN2(data)
+		} else {
+			this.applyN3(data)
+		}
+	}
+
+	/**
+	 * @param {import("../types").TimelineEntryN1} data
+	 */
+	applyN1(data) {
+		Object.assign(this.data, data)
+		this.fixData()
+	}
+
+	/**
+	 * @param {import("../types").TimelineEntryN2} data
+	 */
+	applyN2(data) {
+		Object.assign(this.data, data)
+		this.fixData()
+	}
+
+	/**
+	 * @param {import("../types").TimelineEntryN3} data
+	 */
+	applyN3(data) {
+		Object.assign(this.data, data)
+		this.fixData()
+	}
+
+	/**
+	 * This should keep the same state when applied multiple times to the same data.
+	 * All mutations should act exactly once and have no effect on already mutated data.
+	 */
+	fixData() {
+	}
+
+	getCaption() {
+		const edge = this.data.edge_media_to_caption.edges[0]
+		if (!edge) return null // no caption
+		else return edge.node.text.replace(/\u2063/g, "") // I don't know why U+2063 INVISIBLE SEPARATOR is in here, but it is, and it causes rendering issues with certain fonts, so let's just remove it.
+	}
+
+	/**
+	 * Try to get the first meaningful line or sentence from the caption.
+	 */
+	getCaptionIntroduction() {
+		const caption = this.getCaption()
+		if (!caption) return null
+		else return caption.split("\n")[0].split(". ")[0]
+	}
+
+	/**
+	 * Alt text is not available for N2, the caption or a placeholder string will be returned instead.
+	 * @override
+	 */
+	getAlt() {
+		return this.data.accessibility_caption || this.getCaption() || "No image description available."
+	}
+
+	/**
+	 * @returns {import("../types").BasicOwner}
+	 */
+	getBasicOwner() {
+		return this.data.owner
+	}
+
+	/**
+	 * Not available on N3!
+	 * Returns proxied URLs (P)
+	 */
+	getThumbnailSrcsetP() {
+		if (this.data.thumbnail_resources) {
+			return this.data.thumbnail_resources.map(tr => {
+				return `${proxyImage(tr.src, tr.config_width)} ${tr.config_width}w`
+			}).join(", ")
+		} else {
+			return null
+		}
+	}
+
+	/**
+	 * Not available on N3!
+	 * Returns proxied URLs (P)
+	 * @param {number} size
+	 * @return {import("../types").DisplayResource}
+	 */
+	getSuggestedThumbnailP(size) {
+		if (this.data.thumbnail_resources) {
+			let found = null // start with nothing
+			for (const tr of this.data.thumbnail_resources) { // and keep looping up the sizes (sizes come sorted)
+				found = tr
+				if (tr.config_width >= size) break // don't proceed once we find one large enough
+			}
+			return {
+				config_width: found.config_width,
+				config_height: found.config_height,
+				src: proxyImage(found.src, found.config_width) // force resize to config rather than requested
+			}
+		} else {
+			return null
+		}
+	}
+
+	getThumbnailSizes() {
+		return `(max-width: 820px) 120px, 260px` // from css :(
+	}
+
+	async fetchChildren() {
+		// Cached children?
+		if (this.children) return this.children
+		// Not a gallery? Convert self to a child and return.
+		if (this.getType() !== constants.symbols.TYPE_GALLERY) {
+			return this.children = [new TimelineChild(this.data)]
+		}
+		// Fetch children if needed
+		if (!this.data.edge_sidecar_to_children) {
+			await this.update()
+		}
+		// Create children
+		return this.children = this.data.edge_sidecar_to_children.edges.map(e => new TimelineChild(e.node))
+	}
+
+	/**
+	 * Returns a proxied profile pic URL (P)
+	 * @returns {Promise<import("../types").ExtendedOwner>}
+	 */
+	async fetchExtendedOwnerP() {
+		// Do we just already have the extended owner?
+		if (this.data.owner.full_name) { // this property is on extended owner and not basic owner
+			const clone = proxyExtendedOwner(this.data.owner)
+			this.ownerPfpCacheP = clone.profile_pic_url
+			return clone
+		}
+		// The owner may be in the user cache, so copy from that.
+		// This could be implemented better.
+		else if (collectors.requestCache.hasWithoutClean("user/"+this.data.owner.username)) {
+			/** @type {import("./User")} */
+			const user = collectors.requestCache.getWithoutClean("user/"+this.data.owner.username)
+			this.data.owner = {
+				id: user.data.id,
+				username: user.data.username,
+				is_verified: user.data.is_verified,
+				full_name: user.data.full_name,
+				profile_pic_url: user.data.profile_pic_url // _hd is also available here.
+			}
+			const clone = proxyExtendedOwner(this.data.owner)
+			this.ownerPfpCacheP = clone.profile_pic_url
+			return clone
+		}
+		// We'll have to re-request ourselves.
+		else {
+			await this.update()
+			const clone = proxyExtendedOwner(this.data.owner)
+			this.ownerPfpCacheP = clone.profile_pic_url
+			return clone
+		}
+	}
+
+	getFeedData() {
+		return {
+			title: this.getCaptionIntroduction() || `New post from @${this.getBasicOwner().username}`,
+			description: rssDescriptionTemplate({src: `${config.website_origin}${this.getDisplayUrlP()}`, alt: this.getAlt(), caption: this.getCaption()}),
+			author: this.data.owner.username,
+			url: `${config.website_origin}/p/${this.data.shortcode}`,
+			guid: `${config.website_origin}/p/${this.data.shortcode}`,
+			date: new Date(this.data.taken_at_timestamp*1000)
+			/*
+				Readers should display the description as HTML rather than using the media enclosure.
+				enclosure: {
+					url: this.data.display_url,
+					type: "image/jpeg" //TODO: can instagram have PNGs? everything is JPEG according to https://medium.com/@autolike.it/how-to-avoid-low-res-thumbnails-on-instagram-android-problem-bc24f0ed1c7d
+				}
+			*/
+		}
+	}
+}
+
+module.exports = TimelineEntry

src/lib/structures/TimelineImage.js

@@ -1,160 +0,0 @@
-const config = require("../../../config")
-const {proxyImage} = require("../utils/proxyurl")
-const {compile} = require("pug")
-const collectors = require("../collectors")
-const TimelineChild = require("./TimelineChild")
-require("../testimports")(collectors, TimelineChild)
-
-const rssDescriptionTemplate = compile(`
-p(style='white-space: pre-line')= caption
-img(alt=alt src=src)
-`)
-
-class TimelineImage {
-	/**
-	 * @param {import("../types").GraphImage} data
-	 * @param {boolean} isDirect
-	 */
-	constructor(data, isDirect) {
-		this.data = data
-		this.isDirect = isDirect
-		this.proxyDisplayURL = proxyImage(this.data.display_url)
-		/** @type {import("../types").BasicOwner} */
-		this.basicOwner = null
-		/** @type {import("../types").ExtendedOwner} */
-		this.extendedOwner = null
-		/** @type {string} */
-		this.proxyOwnerProfilePicture = null
-		this.fixData()
-	}
-
-	/**
-	 * This must not cause issues if called multiple times on the same data.
-	 */
-	fixData() {
-		this.data.edge_media_to_caption.edges.forEach(edge => edge.node.text = edge.node.text.replace(/\u2063/g, "")) // I don't know why U+2063 INVISIBLE SEPARATOR is in here, but it is, and it causes rendering issues with certain fonts.
-		this.basicOwner = {
-			id: this.data.owner.id,
-			username: this.data.owner.username
-		}
-		// @ts-ignore
-		if (this.data.owner.full_name !== undefined) this.extendedOwner = this.data.owner
-		if (this.extendedOwner) this.proxyOwnerProfilePicture = proxyImage(this.extendedOwner.profile_pic_url)
-	}
-
-	/**
-	 * @param {import("../types").GraphImage} data
-	 * @param {boolean} isDirect
-	 */
-	updateData(data, isDirect) {
-		this.data = data
-		this.isDirect = isDirect
-		this.fixData()
-	}
-
-	fetchDirect() {
-		return collectors.fetchShortcode(this.data.shortcode, true) // automatically calls updateData
-	}
-
-	/**
-	 * @returns {Promise<import("../types").ExtendedOwner>}
-	 */
-	fetchExtendedOwner() {
-		// Do we just already have the extended owner?
-		if (this.extendedOwner) {
-			return Promise.resolve(this.extendedOwner)
-		}
-		// The owner happens to be in the user cache, so update from that.
-		// This should maybe be moved to collectors.
-		else if (collectors.requestCache.has("user/"+this.basicOwner.username)) {
-			/** @type {import("./User")} */
-			const user = collectors.requestCache.get("user/"+this.basicOwner.username)
-			this.extendedOwner = {
-				id: user.data.id,
-				username: user.data.username,
-				full_name: user.data.full_name,
-				profile_pic_url: user.data.profile_pic_url
-			}
-			this.fixData()
-			return Promise.resolve(this.extendedOwner)
-		}
-		// All else failed, we'll re-request ourselves.
-		else {
-			return this.fetchDirect().then(() => this.extendedOwner) // collectors will manage the updating.
-		}
-	}
-
-	/**
-	 * @returns {TimelineImage[]|import("./TimelineChild")[]}
-	 */
-	getChildren() {
-		if (this.data.__typename === "GraphSidecar" && this.data.edge_sidecar_to_children && this.data.edge_sidecar_to_children.edges.length) {
-			return this.data.edge_sidecar_to_children.edges.map(edge => new TimelineChild(edge.node))
-		} else {
-			return [this]
-		}
-	}
-
-	/**
-	 * @param {number} size
-	 * @return {import("../types").Thumbnail}
-	 */
-	getSuggestedThumbnail(size) {
-		let found = null
-		for (const tr of this.data.thumbnail_resources) {
-			found = tr
-			if (tr.config_width >= size) break
-		}
-		return {
-			config_width: found.config_width,
-			config_height: found.config_height,
-			src: proxyImage(found.src, found.config_width) // do not resize to requested size because of hidpi
-		}
-	}
-
-	getSrcset() {
-		return this.data.thumbnail_resources.map(tr => {
-			return `${proxyImage(tr.src, tr.config_width)} ${tr.config_width}w`
-		}).join(", ")
-	}
-
-	getSizes() {
-		return `(max-width: 820px) 120px, 260px` // from css :(
-	}
-
-	getCaption() {
-		if (this.data.edge_media_to_caption.edges[0]) return this.data.edge_media_to_caption.edges[0].node.text
-		else return null
-	}
-
-	getIntroduction() {
-		const caption = this.getCaption()
-		if (caption) return caption.split("\n")[0].split(". ")[0] // try to get first meaningful line or sentence
-		else return null
-	}
-
-	getAlt() {
-		// For some reason, pages 2+ don't contain a11y data. Instagram web client falls back to image caption.
-		return this.data.accessibility_caption || this.getCaption() || "No image description available."
-	}
-
-	getFeedData() {
-		return {
-			title: this.getIntroduction() || "No caption provided",
-			description: rssDescriptionTemplate({src: config.website_origin+proxyImage(this.data.display_url), alt: this.getAlt(), caption: this.getCaption()}),
-			author: this.data.owner.username,
-			url: `${config.website_origin}/p/${this.data.shortcode}`,
-			guid: `${config.website_origin}/p/${this.data.shortcode}`,
-			date: new Date(this.data.taken_at_timestamp*1000)
-			/*
-				Readers should display the description as HTML rather than using the media enclosure.
-				enclosure: {
-					url: this.data.display_url,
-					type: "image/jpeg" //TODO: can instagram have PNGs? everything is JPEG according to https://medium.com/@autolike.it/how-to-avoid-low-res-thumbnails-on-instagram-android-problem-bc24f0ed1c7d
-				}
-			*/
-		}
-	}
-}
-
-module.exports = TimelineImage

src/lib/types.js

@@ -1,3 +1,7 @@
+// =-=-=-=-=-=-=-=-=-=-=-=-=-=-
+// SIMPLE PARTS OF LARGER TYPES
+// =-=-=-=-=-=-=-=-=-=-=-=-=-=-
+
 /**
  * @typedef GraphEdgeCount
  * @property {number} count
@@ -9,8 +13,9 @@
  */
 
 /**
- * @typedef GraphEdgesChildren
- * @type {{edges: {node: GraphChild}[]}}
+ * @typedef Edges<T>
+ * @property {{node: T}[]} edges
+ * @template T
  */
 
 /**
@@ -21,6 +26,367 @@
  * @template T
  */
 
+/**
+ * @typedef Dimensions
+ * @property {number} width
+ * @property {number} height
+ */
+
+/**
+ * @typedef DisplayResource
+ * @property {string} src
+ * @property {number} config_width
+ * @property {number} config_height
+ */
+
+/**
+ * @typedef BasicOwner
+ * @property {string} id
+ * @property {string} username
+ */
+
+/**
+ * @typedef ExtendedOwner
+ * @property {string} id
+ * @property {boolean} is_verified
+ * @property {string} profile_pic_url
+ * @property {string} username
+ * @property {string} full_name
+ */
+
+// =-=-=-=-=-=-=-
+// TIMELINE ENTRY
+// =-=-=-=-=-=-=-
+
+/*
+	Kinds:
+	N1   Provided in _sharedData from user page load
+	N2   Provided in later user page loads
+	N3   Provided in direct graph query
+	N4   Provided in _sharedData from shortcode page load (just a sorted N3)
+*/
+
+/**
+ * @typedef TimelineEntryAll
+ * N1
+ * @property {string} __typename
+ * @property {string} id
+ * @property {GraphEdgesText} edge_media_to_caption
+ * @property {string} shortcode
+ * @property {GraphEdgeCount} edge_media_to_comment
+ * @property {boolean} comments_disabled
+ * @property {number} taken_at_timestamp
+ * @property {Dimensions} dimensions
+ * @property {string} display_url
+ * @property {GraphEdgeCount} edge_liked_by
+ * @property {GraphEdgeCount} edge_media_preview_like same as edge_liked_by?
+ * @property {any} location todo: doc
+ * @property {any} gating_info todo: discover
+ * @property {any} fact_check_overall_rating todo: discover
+ * @property {any} fact_check_information todo: discover
+ * @property {string} media_preview base64 of something
+ * @property {BasicOwner & ExtendedOwner} owner
+ * @property {string} thumbnail_src
+ * @property {DisplayResource[]} [thumbnail_resources]
+ * @property {boolean} is_video
+ * N2
+ * @property {DisplayResource[]} [display_resources]
+ * @property {string} [tracking_token]
+ * @property {any} [edge_media_to_tagged_user] todo: doc
+ * @property {any} [edge_media_to_sponsor_user] todo: discover
+ * @property {boolean} [viewer_has_liked]
+ * @property {boolean} [viewer_has_saved]
+ * @property {boolean} [viewer_has_saved_to_collection]
+ * @property {boolean} [viewer_in_photo_of_you]
+ * @property {boolean} [viewer_can_reshare]
+ * N3
+ * @property {boolean} [caption_is_edited]
+ * @property {boolean} [has_ranked_comments]
+ * @property {boolean} [comments_disabled]
+ * @property {boolean} [commenting_disabled_for_viewer]
+ * @property {number} [taken_at_timestamp]
+ * @property {boolean} [is_ad]
+ * @property {any} [edge_web_media_to_related_media] todo: discover
+ * Image
+ * @property {string | null} [accessibility_caption]
+ * Video
+ * @property {any} [felix_profile_grid_crop] todo: discover
+ * @property {number} [video_view_count]
+ * @property {any} [dash_info] todo: discover
+ * @property {string} [video_url]
+ * @property {any} [encoding_status] todo: discover
+ * @property {boolean} [is_published]
+ * @property {string} [product_type] todo: discover
+ * @property {string} [title] todo: discover
+ * @property {number} [video_duration]
+ * Sidecar
+ * @property {Edges<GraphChildN3>} [edge_sidecar_to_children]
+ */
+
+/**
+ * @typedef GraphChildAll
+ * properties marked X will always be available on actual children, but are optional here for typing ease because TimelineEntryAll can be assigned directly
+ * N2
+ * @property {string} __typename
+ * @property {string} id
+ * @property {Dimensions} dimensions
+ * @property {string} display_url
+ * @property {DisplayResource[]} [display_resources] X
+ * @property {boolean} is_video
+ * @property {string} [tracking_token] X
+ * @property {any} [edge_media_to_tagged_user] X todo: doc
+ * N3
+ * @property {string} [shortcode]
+ * @property {any} [gating_info] todo: discover
+ * @property {any} [fact_check_overall_rating] todo: discover
+ * @property {any} [fact_check_information] todo: discover
+ * @property {string} [media_preview] base64 of something
+ * Image
+ * @property {string | null} [accessibility_caption]
+ * Video
+ * @property {any} [dash_info] todo: discover
+ * @property {string} [video_url]
+ * @property {number} [video_view_count]
+ */
+
+/**
+ * @typedef TimelineEntryN1
+ * @property {string} __typename
+ * @property {string} id
+ * @property {GraphEdgesText} edge_media_to_caption
+ * @property {string} shortcode
+ * @property {GraphEdgeCount} edge_media_to_comment
+ * @property {boolean} comments_disabled
+ * @property {number} taken_at_timestamp
+ * @property {Dimensions} dimensions
+ * @property {string} display_url
+ * @property {GraphEdgeCount} edge_liked_by
+ * @property {GraphEdgeCount} edge_media_preview_like same as edge_liked_by?
+ * @property {any} location todo: doc
+ * @property {any} gating_info todo: discover
+ * @property {any} fact_check_overall_rating todo: discover
+ * @property {any} fact_check_information todo: discover
+ * @property {string} media_preview base64 of something
+ * @property {BasicOwner} owner
+ * @property {string} thumbnail_src
+ * @property {DisplayResource[]} thumbnail_resources
+ * @property {boolean} is_video
+ */
+
+/**
+ * @typedef {TimelineEntryN1 & GraphImageN1Diff} GraphImageN1
+ *
+ * @typedef GraphImageN1Diff
+ * @property {"GraphImage"} __typename
+ * @property {string} accessibility_caption
+ */
+
+/**
+ * @typedef {TimelineEntryN1 & GraphVideoN1Diff} GraphVideoN1
+ *
+ * @typedef GraphVideoN1Diff
+ * @property {"GraphVideo"} __typename
+ * @property {any} felix_profile_grid_crop todo: discover
+ * @property {number} video_view_count
+ */
+
+/**
+ * @typedef {TimelineEntryN1 & GraphSidecarN1Diff} GraphSidecarN1
+ *
+ * @typedef GraphSidecarN1Diff
+ * @property {"GraphSidecar"} __typename
+ */
+
+/**
+ * @typedef TimelineEntryN2
+ * @property {string} __typename
+ * @property {string} id
+ * @property {Dimensions} dimensions
+ * @property {string} display_url
+ * @property {DisplayResource[]} display_resources
+ * @property {boolean} is_video
+ * @property {string} tracking_token
+ * @property {any} edge_media_to_tagged_user todo: doc
+ * @property {GraphEdgesText} edge_media_to_caption
+ * @property {string} shortcode
+ * @property {any} edge_media_to_comment todo: doc
+ * @property {any} edge_media_to_sponsor_user todo: discover
+ * @property {boolean} comments_disabled
+ * @property {number} taken_at_timestamp
+ * @property {GraphEdgeCount} edge_media_preview_like
+ * @property {any} gating_info todo: discover
+ * @property {any} fact_check_overall_rating todo: discover
+ * @property {any} fact_check_information
+ * @property {string} media_preview base64 of something
+ * @property {BasicOwner} owner
+ * @property {any} location todo: doc
+ * @property {boolean} viewer_has_liked
+ * @property {boolean} viewer_has_saved
+ * @property {boolean} viewer_has_saved_to_collection
+ * @property {boolean} viewer_in_photo_of_you
+ * @property {boolean} viewer_can_reshare
+ * @property {string} thumbnail_src
+ * @property {DisplayResource[]} thumbnail_resources
+ */
+
+/**
+ * @typedef {TimelineEntryN2 & GraphImageN2Diff} GraphImageN2
+ *
+ * @typedef GraphImageN2Diff
+ * @property {"GraphImage"} __typename
+ * @property {null} accessibility_caption
+ */
+
+/**
+ * @typedef {TimelineEntryN2 & GraphVideoN2Diff} GraphVideoN2
+ *
+ * @typedef GraphVideoN2Diff
+ * @property {"GraphVideo"} __typename
+ * @property {any} dash_info todo: discover
+ * @property {string} video_url
+ * @property {number} video_view_count
+ */
+
+/**
+ * @typedef {TimelineEntryN2 & GraphSidecarN2Diff} GraphSidecarN2
+ *
+ * @typedef GraphSidecarN2Diff
+ * @property {"GraphSidecar"} __typename
+ * @property {Edges<GraphChildN2>} edge_sidecar_to_children
+ * @property {null} accessibility_caption
+ */
+
+/**
+ * @typedef GraphChildN2
+ * @property {string} __typename
+ * @property {string} id
+ * @property {Dimensions} dimensions
+ * @property {string} display_url
+ * @property {DisplayResource[]} display_resources
+ * @property {boolean} is_video
+ * @property {string} tracking_token
+ * @property {any} edge_media_to_tagged_user todo: doc
+ */
+
+/**
+ * @typedef {GraphChildN2 & GraphChildImageN2Diff} GraphChildImageN2
+ *
+ * @typedef GraphChildImageN2Diff
+ * @property {"GraphImage"} __typename
+ * @property {null} accessibility_caption
+ */
+
+/**
+ * @typedef {GraphChildN2 & GraphChildVideoN2Diff} GraphChildVideoN2
+ *
+ * @typedef GraphChildVideoN2Diff
+ * @property {"GraphVideo"} __typename
+ * @property {any} dash_info todo: discover
+ * @property {string} video_url
+ * @property {number} video_view_count
+ */
+
+/**
+ * @typedef TimelineEntryN3
+ * @property {string} __typename
+ * @property {string} id
+ * @property {string} shortcode
+ * @property {Dimensions} dimensions
+ * @property {any} gating_info todo: discover
+ * @property {any} fact_check_overall_rating todo: discover
+ * @property {any} fact_check_information todo: discover
+ * @property {string} media_preview base64 of something
+ * @property {string} display_url
+ * @property {DisplayResource[]} display_resources
+ * @property {boolean} is_video
+ * @property {string} tracking_token
+ * @property {any} edge_media_to_tagged_user todo: doc
+ * @property {GraphEdgesText} edge_media_to_caption
+ * @property {boolean} caption_is_edited
+ * @property {boolean} has_ranked_comments
+ * @property {GraphEdgeCount} edge_media_to_comment
+ * @property {boolean} comments_disabled
+ * @property {boolean} commenting_disabled_for_viewer
+ * @property {number} taken_at_timestamp
+ * @property {GraphEdgeCount} edge_media_preview_like
+ * @property {any} edge_media_to_sponsor_user todo: discover
+ * @property {any} location todo: doc
+ * @property {boolean} viewer_has_liked
+ * @property {boolean} viewer_has_saved
+ * @property {boolean} viewer_has_saved_to_collection
+ * @property {boolean} viewer_in_photo_of_you
+ * @property {boolean} viewer_can_reshare
+ * @property {ExtendedOwner} owner
+ * @property {boolean} is_ad
+ * @property {any} edge_web_media_to_related_media todo: discover
+ */
+
+/**
+ * @typedef {TimelineEntryN3 & GraphImageN3Diff} GraphImageN3
+ *
+ * @typedef GraphImageN3Diff
+ * @property {"GraphImage"} __typename
+ * @property {string} accessibility_caption
+ */
+
+/**
+ * @typedef {TimelineEntryN3 & GraphVideoN3Diff} GraphVideoN3
+ *
+ * @typedef GraphVideoN3Diff
+ * @property {"GraphVideo"} __typename
+ * @property {any} dash_info todo: discover
+ * @property {string} video_url
+ * @property {number} video_view_count
+ * @property {any} encoding_status todo: discover
+ * @property {boolean} is_published
+ * @property {string} product_type todo: discover
+ * @property {string} title todo: discover
+ * @property {number} video_duration
+ * @property {string} thumbnail_src
+ */
+
+/**
+ * @typedef {TimelineEntryN3 & GraphSidecarN3Diff} GraphSidecarN3
+ *
+ * @typedef GraphSidecarN3Diff
+ * @property {"GraphSidecar"} __typename
+ * @property {Edges<GraphChildN3>} edge_sidecar_to_children
+ */
+
+/**
+ * @typedef GraphChildN3
+ * @property {string} __typename
+ * @property {string} id
+ * @property {string} shortcode
+ * @property {Dimensions} dimensions
+ * @property {any} gating_info todo: discover
+ * @property {any} fact_check_overall_rating todo: discover
+ * @property {any} fact_check_information todo: discover
+ * @property {string} media_preview base64 of something
+ * @property {string} display_url
+ * @property {DisplayResource[]} display_resources
+ * @property {boolean} is_video
+ * @property {string} tracking_token
+ * @property {any} edge_media_to_tagged_user todo: doc
+ */
+
+/**
+ * @typedef {GraphChildN3 & GraphChildImageN3Diff} GraphChildImageN3
+ * @typedef GraphChildImageN3Diff
+ * @property {"GraphImage"} __typename
+ * @property {string} accessibility_caption
+ */
+
+/**
+ * @typedef {GraphChildN3 & GraphChildVideoN3Diff} GraphChildVideoN3
+ *
+ * @typedef GraphChildVideoN3Diff
+ * @property {"GraphVideo"} __typename
+ * @property {any} dash_info todo: discover
+ * @property {string} video_url
+ * @property {number} video_view_count
+ */
+
 /**
  * @typedef GraphUser
  * @property {string} biography
@@ -42,58 +408,4 @@
  * @property {any} edge_media_collections
  */
 
-/**
- * @typedef Thumbnail
- * @property {string} src
- * @property {number} config_width
- * @property {number} config_height
- */
-
-/**
- * @typedef GraphImage
- * @property {string} __typename
- * @property {string} id
- * @property {GraphEdgesText} edge_media_to_caption
- * @property {string} shortcode
- * @property {GraphEdgeCount} edge_media_to_comment
- * @property {number} taken_at_timestamp No milliseconds
- * @property {GraphEdgeCount} edge_liked_by
- * @property {GraphEdgeCount} edge_media_preview_like
- * @property {{width: number, height: number}} dimensions
- * @property {string} display_url
- * @property {BasicOwner|ExtendedOwner} owner
- * @property {string} thumbnail_src
- * @property {Thumbnail[]} thumbnail_resources
- * @property {string} accessibility_caption
- * @property {GraphEdgesChildren} edge_sidecar_to_children
- */
-
- /**
- * @typedef GraphChild
- * @property {string} __typename
- * @property {string} id
- * @property {string} shortcode
- * @property {{width: number, height: number}} dimensions
- * @property {string} display_url
- * @property {Thumbnail[]} display_resources
- * @property {string} accessibility_caption
- * @property {boolean} is_video
- */
-
-/**
- * @typedef BasicOwner
- * From user HTML response.
- * @property {string} id
- * @property {string} username
- */
-
-/**
- * @typedef ExtendedOwner
- * From post API response.
- * @property {string} id
- * @property {string|null} profile_pic_url
- * @property {string} username
- * @property {string} full_name
- */
-
 module.exports = {}

src/lib/utils/proxyurl.js

@@ -5,4 +5,14 @@ function proxyImage(url, width) {
 	return "/imageproxy?"+params.toString()
 }
 
+/**
+ * @param {import("../types").ExtendedOwner} owner
+ */
+function proxyExtendedOwner(owner) {
+	const clone = {...owner}
+	clone.profile_pic_url = proxyImage(clone.profile_pic_url)
+	return clone
+}
+
 module.exports.proxyImage = proxyImage
+module.exports.proxyExtendedOwner = proxyExtendedOwner

src/site/api/routes.js

@@ -1,5 +1,5 @@
 const constants = require("../../lib/constants")
-const {fetchUser, fetchShortcode} = require("../../lib/collectors")
+const {fetchUser, getOrFetchShortcode} = require("../../lib/collectors")
 const {render} = require("pinski/plugins")
 
 module.exports = [
@@ -33,8 +33,9 @@ module.exports = [
 	},
 	{
 		route: `/p/(${constants.external.shortcode_regex})`, methods: ["GET"], code: async ({fill}) => {
-			const post = await fetchShortcode(fill[0])
-			await post.fetchExtendedOwner()
+			const post = await getOrFetchShortcode(fill[0])
+			await post.fetchChildren()
+			await post.fetchExtendedOwnerP() // parallel await is okay since intermediate fetch result is cached
 			return render(200, "pug/post.pug", {post})
 		}
 	}

src/site/pug/includes/timeline_page.pug

@@ -10,6 +10,6 @@ mixin timeline_page(page, pageIndex)
 		.timeline-inner
 			- const suggestedSize = 260 //- from css :(
 			each image in page
-				- const thumbnail = image.getSuggestedThumbnail(suggestedSize) //- use this as the src in case there are problems with srcset
+				- const thumbnail = image.getSuggestedThumbnailP(suggestedSize) //- use this as the src in case there are problems with srcset
 				a(href=`/p/${image.data.shortcode}`).sized-link
-					img(src=thumbnail.src alt=image.getAlt() width=thumbnail.config_width height=thumbnail.config_height srcset=image.getSrcset() sizes=image.getSizes()).sized-image
+					img(src=thumbnail.src alt=image.getAlt() width=thumbnail.config_width height=thumbnail.config_height srcset=image.getThumbnailSrcsetP() sizes=image.getThumbnailSizes()).sized-image

src/site/pug/post.pug

@@ -5,16 +5,22 @@ html
 	head
 		meta(charset="utf-8")
 		meta(name="viewport" content="width=device-width, initial-scale=1")
-		title= `${post.getIntroduction()} | Bibliogram`
+		title
+			if post.getCaptionIntroduction()
+				=post.getCaptionIntroduction()
+			else
+				=`Post from @${post.getBasicOwner().username}`
+			=` | Bibliogram`
 		link(rel="stylesheet" type="text/css" href="/static/css/main.css")
 		script(src="/static/js/pagination.js" type="module")
 	body.post-page
 		main.post-page-divider
 			section.description-section
 				header.user-header
-					img(src=post.proxyOwnerProfilePicture width=150 height=150 alt="").pfp
-					a.name(href=`/u/${post.extendedOwner.username}`)= `${post.extendedOwner.full_name} (@${post.extendedOwner.username})`
-				p.description= post.getCaption()
+					img(src=post.ownerPfpCacheP width=150 height=150 alt="").pfp
+					a.name(href=`/u/${post.getBasicOwner().username}`)= `${post.data.owner.full_name} (@${post.getBasicOwner().username})`
+				if post.getCaption()
+					p.description= post.getCaption()
 			section.images-gallery
-				for image in post.getChildren()
-					img(src=image.proxyDisplayURL alt=image.getAlt() width=image.data.dimensions.width height=image.data.dimensions.height).sized-image
+				for image in post.children
+					img(src=image.getDisplayUrlP() alt=image.getAlt() width=image.data.dimensions.width height=image.data.dimensions.height).sized-image

src/site/repl.js

@@ -1,5 +1,5 @@
 const {instance, pugCache, wss} = require("./passthrough")
-const {requestCache, timelineImageCache} = require("../lib/collectors")
+const {requestCache, timelineEntryCache} = require("../lib/collectors")
 const util = require("util")
 const repl = require("repl")
 const vm = require("vm")

src/site/sass/main.sass

@@ -95,7 +95,7 @@ body
 		--image-size: 120px
 
 	background-color: $background
-	padding: 15px 15px 12vh
+	padding: 15px 15px 40px
 
 	.page-number
 		color: #444
@@ -168,6 +168,11 @@ body
 		margin: 0 auto
 		min-height: 100vh
 
+		@media screen and (max-width: $layout-a-max)
+			display: flex
+			flex-direction: column
+
+
 		.description-section
 			display: grid
 			align-items: start
@@ -179,6 +184,11 @@ body
 			overflow-y: auto
 			box-sizing: border-box
 
+			@media screen and (max-width: $layout-a-max)
+				position: inherit
+				top: inherit
+				height: inherit
+
 			.user-header
 				display: flex
 				align-items: center
@@ -208,6 +218,9 @@ body
 				font-size: 20px
 				line-height: 1.4
 
+				@media screen and (max-width: $layout-a-max)
+					font-size: 18px
+
 		.images-gallery
 			display: flex
 			flex-direction: column
@@ -216,6 +229,9 @@ body
 			background-color: #262728
 			padding: 10px
 
+			@media screen and (max-width: $layout-a-max)
+				flex: 1
+
 			.sized-image
 				color: #eee
 				background-color: #3b3c3d