Create post viewer

This commit is contained in:
Cadence Fish 2020-01-19 04:38:14 +13:00
parent 693c083a99
commit 59d891b94b
No known key found for this signature in database
GPG Key ID: 81015DF9AA8607E1
19 changed files with 479 additions and 84 deletions

View File

@ -12,17 +12,18 @@ See also: [Invidious, a front-end for YouTube.](https://github.com/omarroth/invi
- [x] Infinite scroll
- [x] User memory cache
- [x] RSS (latest 12 posts)
- [x] View post
- [x] Galleries
- [ ] Videos
- [ ] Galleries of videos
- [ ] Image disk cache
- [ ] View post
- [ ] Clickable usernames and hashtags
- [ ] Homepage
- [ ] Proper error checking
- [ ] Optimised for mobile
- [ ] Favicon
- [ ] Settings (e.g. data saving)
- [ ] Galleries
- [ ] List view
- [ ] Videos
- [ ] IGTV
- [ ] Public API
- [ ] Rate limiting
@ -65,3 +66,4 @@ Bibliogram is now running on `0.0.0.0:10407`.
- `/u/{username}` - load a user's profile and timeline
- `/u/{username}/rss.xml` - get the RSS feed for a user
- `/p/{shortcode}` - load a post

View File

@ -1,10 +1,13 @@
class InstaCache {
/**
* @template T
*/
class TtlCache {
/**
* @property {number} ttl time to keep each resource in milliseconds
* @param {number} ttl time to keep each resource in milliseconds
*/
constructor(ttl) {
this.ttl = ttl
/** @type {Map<string, {data: any, time: number}>} */
/** @type {Map<string, {data: T, time: number}>} */
this.cache = new Map()
}
@ -18,10 +21,23 @@ class InstaCache {
/**
* @param {string} key
*/
get(key) {
return this.cache.get(key).data
has(key) {
return this.cache.has(key)
}
/**
* @param {string} key
*/
get(key) {
const value = this.cache.get(key)
if (value) return value.data
else return null
}
/**
* @param {string} key
* @param {number} factor factor to divide the result by. use 60*1000 to get the ttl in minutes.
*/
getTtl(key, factor = 1) {
return Math.max((Math.floor(Date.now() - this.cache.get(key).time) / factor), 0)
}
@ -33,6 +49,15 @@ class InstaCache {
set(key, data) {
this.cache.set(key, {data, time: Date.now()})
}
}
class RequestCache extends TtlCache {
/**
* @param {number} ttl time to keep each resource in milliseconds
*/
constructor(ttl) {
super(ttl)
}
/**
* @param {string} key
@ -67,4 +92,5 @@ class InstaCache {
}
}
module.exports = InstaCache
module.exports.TtlCache = TtlCache
module.exports.RequestCache = RequestCache

View File

@ -1,15 +1,19 @@
const constants = require("./constants")
const {request} = require("./utils/request")
const {extractSharedData} = require("./utils/body")
const InstaCache = require("./cache")
const {User} = require("./structures")
require("./testimports")(constants, request, extractSharedData, InstaCache, User)
const {TtlCache, RequestCache} = require("./cache")
require("./testimports")(constants, request, extractSharedData, RequestCache)
const cache = new InstaCache(constants.resource_cache_time)
const requestCache = new RequestCache(constants.resource_cache_time)
/** @type {import("./cache").TtlCache<import("./structures/TimelineImage")>} */
const timelineImageCache = new TtlCache(constants.resource_cache_time)
function fetchUser(username) {
return cache.getOrFetch("user/"+username, () => {
return requestCache.getOrFetch("user/"+username, () => {
return request(`https://www.instagram.com/${username}/`).then(res => res.text()).then(text => {
// require down here or have to deal with require loop. require cache will take care of it anyway.
// User -> Timeline -> TimelineImage -> collectors -/> User
const User = require("./structures/User")
const sharedData = extractSharedData(text)
const user = new User(sharedData.entry_data.ProfilePage[0].graphql.user)
return user
@ -30,7 +34,7 @@ function fetchTimelinePage(userID, after) {
first: constants.external.timeline_fetch_first,
after: after
}))
return cache.getOrFetchPromise("page/"+after, () => {
return requestCache.getOrFetchPromise("page/"+after, () => {
return request(`https://www.instagram.com/graphql/query/?${p.toString()}`).then(res => res.json()).then(root => {
/** @type {import("./types").PagedEdges<import("./types").GraphImage>} */
const timeline = root.data.user.edge_owner_to_timeline_media
@ -39,5 +43,52 @@ function fetchTimelinePage(userID, after) {
})
}
/**
* @param {string} shortcode
* @param {boolean} needDirect
* @returns {Promise<import("./structures/TimelineImage")>}
*/
function fetchShortcode(shortcode, needDirect = false) {
const attempt = timelineImageCache.get(shortcode)
if (attempt && (attempt.isDirect === true || needDirect === false)) return Promise.resolve(attempt)
// example actual query from web:
// query_hash=2b0673e0dc4580674a88d426fe00ea90&variables={"shortcode":"xxxxxxxxxxx","child_comment_count":3,"fetch_comment_count":40,"parent_comment_count":24,"has_threaded_comments":true}
// we will not include params about comments, which means we will not receive comments, but everything else should still work fine
const p = new URLSearchParams()
p.set("query_hash", constants.external.shortcode_query_hash)
p.set("variables", JSON.stringify({shortcode}))
return requestCache.getOrFetchPromise("shortcode/"+shortcode, () => {
return request(`https://www.instagram.com/graphql/query/?${p.toString()}`).then(res => res.json()).then(root => {
/** @type {import("./types").GraphImage} */
const data = root.data.shortcode_media
return createShortcodeFromData(data, true)
})
})
}
/**
* @param {import("./types").GraphImage} data
* @param {boolean} isDirect
*/
function createShortcodeFromData(data, isDirect) {
const existing = timelineImageCache.get(data.shortcode)
if (existing) {
existing.updateData(data, isDirect)
return existing
} else {
// require down here or have to deal with require loop. require cache will take care of it anyway.
// TimelineImage -> collectors -/> TimelineImage
const TimelineImage = require("./structures/TimelineImage")
const timelineImage = new TimelineImage(data, false)
timelineImageCache.set(data.shortcode, timelineImage)
return timelineImage
}
}
module.exports.fetchUser = fetchUser
module.exports.fetchTimelinePage = fetchTimelinePage
module.exports.fetchShortcode = fetchShortcode
module.exports.createShortcodeFromData = createShortcodeFromData
module.exports.requestCache = requestCache
module.exports.timelineImageCache = timelineImageCache

View File

@ -4,8 +4,10 @@ module.exports = {
external: {
timeline_query_hash: "e769aa130647d2354c40ea6a439bfc08",
shortcode_query_hash: "2b0673e0dc4580674a88d426fe00ea90",
timeline_fetch_first: 12,
username_regex: "[\\w.]+"
username_regex: "[\\w.]+",
shortcode_regex: "[\\w-]+"
},
symbols: {

View File

@ -2,12 +2,13 @@ const RSS = require("rss")
const constants = require("../constants")
const config = require("../../../config")
const TimelineImage = require("./TimelineImage")
const InstaCache = require("../cache")
const collectors = require("../collectors")
require("../testimports")(constants, TimelineImage)
require("../testimports")(constants, collectors, TimelineImage, InstaCache)
/** @param {any[]} edges */
function transformEdges(edges) {
return edges.map(e => new TimelineImage(e.node))
return edges.map(e => collectors.createShortcodeFromData(e.node, false))
}
class Timeline {

View File

@ -0,0 +1,42 @@
const config = require("../../../config")
const {proxyImage} = require("../utils/proxyurl")
const collectors = require("../collectors")
require("../testimports")(collectors)
class TimelineChild {
/**
* @param {import("../types").GraphChild} data
*/
constructor(data) {
this.data = data
this.proxyDisplayURL = proxyImage(this.data.display_url)
}
/**
* @param {number} size
*/
getSuggestedResource(size) {
let found = null
for (const tr of this.data.display_resources) {
found = tr
if (tr.config_width >= size) break
}
found = proxyImage(found, size)
return found
}
getSrcset() {
return this.data.display_resources.map(tr => {
const p = new URLSearchParams()
p.set("width", String(tr.config_width))
p.set("url", tr.src)
return `/imageproxy?${p.toString()} ${tr.config_width}w`
}).join(", ")
}
getAlt() {
return this.data.accessibility_caption || "No image description available."
}
}
module.exports = TimelineChild

View File

@ -1,27 +1,103 @@
const config = require("../../../config")
const {proxyImage} = require("../utils/proxyurl")
const {compile} = require("pug")
const collectors = require("../collectors")
const TimelineChild = require("./TimelineChild")
require("../testimports")(collectors, TimelineChild)
const rssDescriptionTemplate = compile(`
p(style='white-space: pre-line')= caption
img(alt=alt src=src)
`)
class GraphImage {
class TimelineImage {
/**
* @param {import("../types").GraphImage} data
* @param {boolean} isDirect
*/
constructor(data) {
constructor(data, isDirect) {
this.data = data
this.data.edge_media_to_caption.edges.forEach(edge => edge.node.text = edge.node.text.replace(/\u2063/g, "")) // I don't know why U+2063 INVISIBLE SEPARATOR is in here, but it is, and it causes rendering issues with certain fonts.
this.isDirect = isDirect
this.proxyDisplayURL = proxyImage(this.data.display_url)
/** @type {import("../types").BasicOwner} */
this.basicOwner = null
/** @type {import("../types").ExtendedOwner} */
this.extendedOwner = null
/** @type {string} */
this.proxyOwnerProfilePicture = null
this.fixData()
}
getProxy(url) {
return proxyImage(url)
/**
* This must not cause issues if called multiple times on the same data.
*/
fixData() {
this.data.edge_media_to_caption.edges.forEach(edge => edge.node.text = edge.node.text.replace(/\u2063/g, "")) // I don't know why U+2063 INVISIBLE SEPARATOR is in here, but it is, and it causes rendering issues with certain fonts.
this.basicOwner = {
id: this.data.owner.id,
username: this.data.owner.username
}
// @ts-ignore
if (this.data.owner.full_name !== undefined) this.extendedOwner = this.data.owner
if (this.extendedOwner) this.proxyOwnerProfilePicture = proxyImage(this.extendedOwner.profile_pic_url)
}
/**
* @param {import("../types").GraphImage} data
* @param {boolean} isDirect
*/
updateData(data, isDirect) {
this.data = data
this.isDirect = isDirect
this.fixData()
}
fetchDirect() {
return collectors.fetchShortcode(this.data.shortcode, true) // automatically calls updateData
}
/**
* @returns {Promise<import("../types").ExtendedOwner>}
*/
fetchExtendedOwner() {
// Do we just already have the extended owner?
if (this.extendedOwner) {
return Promise.resolve(this.extendedOwner)
}
// The owner happens to be in the user cache, so update from that.
// This should maybe be moved to collectors.
else if (collectors.requestCache.has("user/"+this.basicOwner.username)) {
/** @type {import("./User")} */
const user = collectors.requestCache.get("user/"+this.basicOwner.username)
this.extendedOwner = {
id: user.data.id,
username: user.data.username,
full_name: user.data.full_name,
profile_pic_url: user.data.profile_pic_url
}
this.fixData()
return Promise.resolve(this.extendedOwner)
}
// All else failed, we'll re-request ourselves.
else {
return this.fetchDirect().then(() => this.extendedOwner) // collectors will manage the updating.
}
}
/**
* @returns {TimelineImage[]|import("./TimelineChild")[]}
*/
getChildren() {
if (this.data.__typename === "GraphSidecar" && this.data.edge_sidecar_to_children && this.data.edge_sidecar_to_children.edges.length) {
return this.data.edge_sidecar_to_children.edges.map(edge => new TimelineChild(edge.node))
} else {
return [this]
}
}
/**
* @param {number} size
* @return {import("../types").Thumbnail}
*/
getSuggestedThumbnail(size) {
let found = null
@ -29,18 +105,23 @@ class GraphImage {
found = tr
if (tr.config_width >= size) break
}
return found
return {
config_width: found.config_width,
config_height: found.config_height,
src: proxyImage(found.src, found.config_width) // do not resize to requested size because of hidpi
}
}
getSrcset() {
return this.data.thumbnail_resources.map(tr => {
const p = new URLSearchParams()
p.set("width", String(tr.config_width))
p.set("url", tr.src)
return `/imageproxy?${p.toString()} ${tr.config_width}w`
return `${proxyImage(tr.src, tr.config_width)} ${tr.config_width}w`
}).join(", ")
}
getSizes() {
return `(max-width: 820px) 120px, 260px` // from css :(
}
getCaption() {
if (this.data.edge_media_to_caption.edges[0]) return this.data.edge_media_to_caption.edges[0].node.text
else return null
@ -76,4 +157,4 @@ class GraphImage {
}
}
module.exports = GraphImage
module.exports = TimelineImage

View File

@ -1,3 +0,0 @@
module.exports = {
User: require("./User")
}

View File

@ -1,7 +1,9 @@
module.exports = function(...items) {
items.forEach(item => {
items.forEach((item, index) => {
if (item === undefined || (item && item.constructor && item.constructor.name == "Object" && Object.keys(item).length == 0)) {
throw new Error("Bad import: item looks like this: "+JSON.stringify(item))
console.log(`Bad import for arg index ${index}`)
// @ts-ignore
require("/") // generate an error with a require stack.
}
})
}

View File

@ -8,6 +8,11 @@
* @type {{edges: {node: {text: string}}[]}}
*/
/**
* @typedef GraphEdgesChildren
* @type {{edges: {node: GraphChild}[]}}
*/
/**
* @typedef PagedEdges<T>
* @property {number} count
@ -46,6 +51,7 @@
/**
* @typedef GraphImage
* @property {string} __typename
* @property {string} id
* @property {GraphEdgesText} edge_media_to_caption
* @property {string} shortcode
@ -55,10 +61,39 @@
* @property {GraphEdgeCount} edge_media_preview_like
* @property {{width: number, height: number}} dimensions
* @property {string} display_url
* @property {{id: string, username: string}} owner
* @property {BasicOwner|ExtendedOwner} owner
* @property {string} thumbnail_src
* @property {Thumbnail[]} thumbnail_resources
* @property {string} accessibility_caption
* @property {GraphEdgesChildren} edge_sidecar_to_children
*/
/**
* @typedef GraphChild
* @property {string} __typename
* @property {string} id
* @property {string} shortcode
* @property {{width: number, height: number}} dimensions
* @property {string} display_url
* @property {Thumbnail[]} display_resources
* @property {string} accessibility_caption
* @property {boolean} is_video
*/
/**
* @typedef BasicOwner
* From user HTML response.
* @property {string} id
* @property {string} username
*/
/**
* @typedef ExtendedOwner
* From post API response.
* @property {string} id
* @property {string|null} profile_pic_url
* @property {string} username
* @property {string} full_name
*/
module.exports = {}

View File

@ -1,5 +1,6 @@
function proxyImage(url) {
function proxyImage(url, width) {
const params = new URLSearchParams()
if (width) params.set("width", width)
params.set("url", url)
return "/imageproxy?"+params.toString()
}

View File

@ -1,6 +1,7 @@
const fetch = require("node-fetch").default
function request(url) {
console.log("-> [OUT]", url) // todo: make more like pinski?
return fetch(url, {
headers: {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36"

View File

@ -1,30 +1,41 @@
const constants = require("../../lib/constants")
const {fetchUser} = require("../../lib/collectors")
const {fetchUser, fetchShortcode} = require("../../lib/collectors")
const {render} = require("pinski/plugins")
module.exports = [
{route: `/u/(${constants.external.username_regex})`, methods: ["GET"], code: async ({url, fill}) => {
const params = url.searchParams
const user = await fetchUser(fill[0])
const page = +params.get("page")
if (typeof page === "number" && !isNaN(page) && page >= 1) {
await user.timeline.fetchUpToPage(page - 1)
{
route: `/u/(${constants.external.username_regex})`, methods: ["GET"], code: async ({url, fill}) => {
const params = url.searchParams
const user = await fetchUser(fill[0])
const page = +params.get("page")
if (typeof page === "number" && !isNaN(page) && page >= 1) {
await user.timeline.fetchUpToPage(page - 1)
}
return render(200, "pug/user.pug", {url, user})
}
return render(200, "pug/user.pug", {url, user})
}},
{route: `/fragment/user/(${constants.external.username_regex})/(\\d+)`, methods: ["GET"], code: async ({url, fill}) => {
const user = await fetchUser(fill[0])
const pageNumber = +fill[1]
const pageIndex = pageNumber - 1
await user.timeline.fetchUpToPage(pageIndex)
if (user.timeline.pages[pageIndex]) {
return render(200, "pug/fragments/timeline_page.pug", {page: user.timeline.pages[pageIndex], pageIndex, user, url})
} else {
return {
statusCode: 400,
contentType: "text/html",
content: "That page does not exist"
},
{
route: `/fragment/user/(${constants.external.username_regex})/(\\d+)`, methods: ["GET"], code: async ({url, fill}) => {
const user = await fetchUser(fill[0])
const pageNumber = +fill[1]
const pageIndex = pageNumber - 1
await user.timeline.fetchUpToPage(pageIndex)
if (user.timeline.pages[pageIndex]) {
return render(200, "pug/fragments/timeline_page.pug", {page: user.timeline.pages[pageIndex], pageIndex, user, url})
} else {
return {
statusCode: 400,
contentType: "text/html",
content: "That page does not exist"
}
}
}
}}
},
{
route: `/p/(${constants.external.shortcode_regex})`, methods: ["GET"], code: async ({fill}) => {
const post = await fetchShortcode(fill[0])
await post.fetchExtendedOwner()
return render(200, "pug/post.pug", {post})
}
}
]

View File

@ -8,7 +8,8 @@ mixin timeline_page(page, pageIndex)
span.number Page #{pageNumber}
.timeline-inner
- const suggestedSize = 300
- const suggestedSize = 260 //- from css :(
each image in page
- const thumbnail = image.getSuggestedThumbnail(suggestedSize) //- use this as the src in case there are problems with srcset
img(src=image.getProxy(thumbnail.src) alt=image.getAlt() width=thumbnail.config_width height=thumbnail.config_height srcset=image.getSrcset() sizes=`${suggestedSize}px`).image
a(href=`/p/${image.data.shortcode}`).sized-link
img(src=thumbnail.src alt=image.getAlt() width=thumbnail.config_width height=thumbnail.config_height srcset=image.getSrcset() sizes=image.getSizes()).sized-image

20
src/site/pug/post.pug Normal file
View File

@ -0,0 +1,20 @@
- const numberFormat = new Intl.NumberFormat().format
doctype html
html
head
meta(charset="utf-8")
meta(name="viewport" content="width=device-width, initial-scale=1")
title= `${post.getIntroduction()} | Bibliogram`
link(rel="stylesheet" type="text/css" href="/static/css/main.css")
script(src="/static/js/pagination.js" type="module")
body.post-page
main.post-page-divider
section.description-section
header.user-header
img(src=post.proxyOwnerProfilePicture width=150 height=150 alt="").pfp
a.name(href=`/u/${post.extendedOwner.username}`)= `${post.extendedOwner.full_name} (@${post.extendedOwner.username})`
p.description= post.getCaption()
section.images-gallery
for image in post.getChildren()
img(src=image.proxyDisplayURL alt=image.getAlt() width=image.data.dimensions.width height=image.data.dimensions.height).sized-image

View File

@ -37,6 +37,7 @@ html
| followed by
div.links
a(rel="alternate" type="application/rss+xml" href=`/u/${user.data.username}/rss.xml`) RSS
a(rel="noreferrer noopener" href=`https://www.instagram.com/${user.data.username}`) instagram.com
main#timeline.timeline
each page, pageIndex in user.timeline.pages

31
src/site/repl.js Normal file
View File

@ -0,0 +1,31 @@
const {instance, pugCache, wss} = require("./passthrough")
const {requestCache, timelineImageCache} = require("../lib/collectors")
const util = require("util")
const repl = require("repl")
const vm = require("vm")
/**
* @param {string} input
* @param {vm.Context} context
* @param {string} filename
* @param {(err: Error|null, result: any) => any} callback
*/
async function customEval(input, context, filename, callback) {
let depth = 0
if (input == "exit\n") return process.exit()
if (input.startsWith(":")) {
const depthOverwrite = input.split(" ")[0]
depth = +depthOverwrite.slice(1)
input = input.slice(depthOverwrite.length + 1)
}
const result = await eval(input)
const output = util.inspect(result, false, depth, true)
return callback(undefined, output)
}
function customWriter(output) {
return output
}
console.log("REPL started")
repl.start({prompt: "b) ", eval: customEval, writer: customWriter}).once("exit", () => process.exit())

View File

@ -33,9 +33,7 @@ body
.profile-overview
text-align: center
z-index: 1
position: relative
contain: paint // </3 css
line-height: 1
@media screen and (max-width: $layout-a-max)
@ -50,29 +48,43 @@ body
.profile-sticky
position: sticky
top: 0
bottom: 0
height: 100vh
box-sizing: border-box
overflow-y: auto
padding: 10px
.pfp
margin: 25px 0
@media screen and (max-width: $layout-a-max)
height: unset
.full-name
margin: 0 0 8px
font-size: 30px
.pfp
margin: 25px 0
.username
margin: 0
font-size: 20px
font-weight: normal
.full-name
margin: 0 0 8px
font-size: 30px
.profile-counter
line-height: 1.3
.username
margin: 0
font-size: 20px
font-weight: normal
.count
font-weight: bold
.profile-counter
line-height: 1.3
.links
margin-top: 20px
.count
font-weight: bold
.links
margin-top: 20px
display: flex
flex-wrap: wrap
justify-content: center
a
margin: 5px
> *:last-child
margin-bottom: 10px // because padding-bottom on parent doesn't seem to work.
.timeline
--image-size: 260px
@ -125,17 +137,92 @@ body
flex-wrap: wrap
margin: 0 auto
.image
@mixin sized()
width: $image-size
height: $image-size
.sized-link
$margin: 5px
background-color: rgba(40, 40, 40, 0.25)
margin: $margin
max-width: $image-size
max-height: $image-size
width: 100%
height: 100%
color: #111
background-color: rgba(40, 40, 40, 0.25)
text-decoration: none
@include sized
&:hover
$border-width: 3px
margin: $margin - $border-width
border: $border-width solid #111
.sized-image
@include sized
.post-page
background-color: #6a6b71
.post-page-divider
display: grid
grid-template-columns: 360px auto
max-width: 1200px
margin: 0 auto
min-height: 100vh
.description-section
display: grid
align-items: start
align-content: normal
background-color: #eee
position: sticky
top: 0
height: 100vh
overflow-y: auto
box-sizing: border-box
.user-header
display: flex
align-items: center
justify-content: center
background-color: #b3b3b3
padding: 10px
.pfp
$size: 40px
width: $size
height: $size
margin-right: 10px
background-color: rgba(40, 40, 40, 0.25)
.name
font-size: 20px
color: black
text-decoration: none
&:hover
text-decoration: underline
.description
margin: 12px
white-space: pre-line
font-size: 20px
line-height: 1.4
.images-gallery
display: flex
flex-direction: column
align-items: center
justify-content: center
background-color: #262728
padding: 10px
.sized-image
color: #eee
background-color: #3b3c3d
max-height: 94vh
max-width: 100%
width: auto
height: auto
&:not(:last-child)
margin-bottom: 10px

View File

@ -23,4 +23,7 @@ subdirs("pug", (err, dirs) => {
require("pinski/plugins").setInstance(pinski)
Object.assign(passthrough, pinski.getExports())
console.log("Server started")
require("./repl")
})