mirror of https://git.sr.ht/~cadence/bibliogram
Rewrite feeds
This commit is contained in:
Cadence Fish
parent
b10432aa38
commit
5201a6612b
|
|
@ -77,7 +77,9 @@ See [Wiki:Installing](https://github.com/cloudrac3r/bibliogram/wiki/Installing)
|
|||
- `/` - homepage
|
||||
- `/u/{username}` - load a user's profile and timeline
|
||||
- `/u/{username}/rss.xml` - get the RSS feed for a user
|
||||
- `/u/{username}/atom.xml` - get the Atom feed for a user
|
||||
- `/p/{shortcode}` - load a post
|
||||
- `/privacy` - privacy policy
|
||||
|
||||
## Credits
|
||||
|
||||
|
|
|
|||
|
|
@ -1096,6 +1096,13 @@
|
|||
"resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz",
|
||||
"integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw=="
|
||||
},
|
||||
"feed": {
|
||||
"version": "github:cloudrac3r/feed#dbd55889e9c7135a8710eaa4d4c415ffeee7fc27",
|
||||
"from": "github:cloudrac3r/feed#dbd55889e9c7135a8710eaa4d4c415ffeee7fc27",
|
||||
"requires": {
|
||||
"xml-js": "^1.6.11"
|
||||
}
|
||||
},
|
||||
"fill-range": {
|
||||
"version": "7.0.1",
|
||||
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.0.1.tgz",
|
||||
|
|
@ -3126,30 +3133,6 @@
|
|||
"glob": "^7.1.3"
|
||||
}
|
||||
},
|
||||
"rss": {
|
||||
"version": "1.2.2",
|
||||
"resolved": "https://registry.npmjs.org/rss/-/rss-1.2.2.tgz",
|
||||
"integrity": "sha1-UKFpiHYTgTOnT5oF0r3I240nqSE=",
|
||||
"requires": {
|
||||
"mime-types": "2.1.13",
|
||||
"xml": "1.0.1"
|
||||
},
|
||||
"dependencies": {
|
||||
"mime-db": {
|
||||
"version": "1.25.0",
|
||||
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.25.0.tgz",
|
||||
"integrity": "sha1-wY29fHOl2/b0SgJNwNFloeexw5I="
|
||||
},
|
||||
"mime-types": {
|
||||
"version": "2.1.13",
|
||||
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.13.tgz",
|
||||
"integrity": "sha1-4HqqnGxrmnyjASxpADrSWjnpKog=",
|
||||
"requires": {
|
||||
"mime-db": "~1.25.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"safe-buffer": {
|
||||
"version": "5.1.2",
|
||||
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
|
||||
|
|
@ -3208,6 +3191,11 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"sax": {
|
||||
"version": "1.2.4",
|
||||
"resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
|
||||
"integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
|
||||
},
|
||||
"scss-tokenizer": {
|
||||
"version": "0.2.3",
|
||||
"resolved": "https://registry.npmjs.org/scss-tokenizer/-/scss-tokenizer-0.2.3.tgz",
|
||||
|
|
@ -5023,10 +5011,13 @@
|
|||
"resolved": "https://registry.npmjs.org/ws/-/ws-7.2.1.tgz",
|
||||
"integrity": "sha512-sucePNSafamSKoOqoNfBd8V0StlkzJKL2ZAhGQinCfNQ+oacw+Pk7lcdAElecBF2VkLNZRiIb5Oi1Q5lVUVt2A=="
|
||||
},
|
||||
"xml": {
|
||||
"version": "1.0.1",
|
||||
"resolved": "https://registry.npmjs.org/xml/-/xml-1.0.1.tgz",
|
||||
"integrity": "sha1-eLpyAgApxbyHuKgaPPzXS0ovweU="
|
||||
"xml-js": {
|
||||
"version": "1.6.11",
|
||||
"resolved": "https://registry.npmjs.org/xml-js/-/xml-js-1.6.11.tgz",
|
||||
"integrity": "sha512-7rVi2KMfwfWFl+GpPg6m80IVMWXLRjO+PxTq7V2CDhoGak0wzYzFgUY2m4XJ47OGdXd8eLE8EmwfAmdjw7lC1g==",
|
||||
"requires": {
|
||||
"sax": "^1.2.4"
|
||||
}
|
||||
},
|
||||
"y18n": {
|
||||
"version": "3.2.1",
|
||||
|
|
|
|||
|
|
@ -12,12 +12,12 @@
|
|||
"license": "AGPL-3.0-only",
|
||||
"dependencies": {
|
||||
"better-sqlite3": "^5.4.3",
|
||||
"feed": "github:cloudrac3r/feed#dbd55889e9c7135a8710eaa4d4c415ffeee7fc27",
|
||||
"mixin-deep": "^2.0.1",
|
||||
"node-dir": "^0.1.17",
|
||||
"node-fetch": "^2.6.0",
|
||||
"pinski": "github:cloudrac3r/pinski#9eb56d90fdd00357451dd5a546dbcca1f9bf114a",
|
||||
"pug": "^2.0.4",
|
||||
"rss": "^1.2.2",
|
||||
"semver": "^7.1.2",
|
||||
"sharp": "^0.24.0",
|
||||
"socks-proxy-agent": "github:cloudrac3r/node-socks-proxy-agent#6a26d274b12098dfef6cc2faafd25b0c051f2467"
|
||||
|
|
|
|||
|
|
@ -13,6 +13,10 @@ const userRequestCache = new UserRequestCache(constants.caching.resource_cache_t
|
|||
const timelineEntryCache = new TtlCache(constants.caching.resource_cache_time)
|
||||
const history = new RequestHistory(["user", "timeline", "post", "reel"])
|
||||
|
||||
/**
|
||||
* @param {string} username
|
||||
* @param {boolean} isRSS
|
||||
*/
|
||||
async function fetchUser(username, isRSS) {
|
||||
let mode = constants.allow_user_from_reel
|
||||
if (mode === "preferForRSS") {
|
||||
|
|
@ -38,6 +42,10 @@ async function fetchUser(username, isRSS) {
|
|||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} username
|
||||
* @returns {Promise<import("./structures/User")>}
|
||||
*/
|
||||
function fetchUserFromHTML(username) {
|
||||
return userRequestCache.getOrFetch("user/"+username, false, true, () => {
|
||||
return switcher.request("user_html", `https://www.instagram.com/${username}/`, async res => {
|
||||
|
|
@ -72,6 +80,11 @@ function fetchUserFromHTML(username) {
|
|||
})
|
||||
}
|
||||
|
||||
/**
|
||||
* @param {string} userID
|
||||
* @param {string} username
|
||||
* @returns {Promise<import("./structures/ReelUser")>}
|
||||
*/
|
||||
function fetchUserFromCombined(userID, username) {
|
||||
// Fetch basic user information
|
||||
const p = new URLSearchParams()
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@
|
|||
|
||||
let constants = {
|
||||
// Things that server owners _should_ change!
|
||||
website_origin: "http://localhost:10407",
|
||||
website_origin: "http://localhost:10407", // Protocol and domain that this instance is hosted on. Do NOT include a trailing slash.
|
||||
has_privacy_policy: false, // You MUST read /src/site/pug/privacy.pug.template before changing this!
|
||||
|
||||
// Things that server owners _could_ change if they want to.
|
||||
|
|
|
|||
|
|
@ -13,11 +13,16 @@ class ReelUser {
|
|||
this.following = 0
|
||||
this.followedBy = 0
|
||||
this.posts = 0
|
||||
/** @type {import("./Timeline")} */
|
||||
this.timeline = new Timeline(this)
|
||||
this.cachedAt = Date.now()
|
||||
this.proxyProfilePicture = proxyImage(this.data.profile_pic_url)
|
||||
}
|
||||
|
||||
getStructuredBio() {
|
||||
return null
|
||||
}
|
||||
|
||||
getTtl(scale = 1) {
|
||||
const expiresAt = this.cachedAt + constants.caching.resource_cache_time
|
||||
const ttl = expiresAt - Date.now()
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
const RSS = require("rss")
|
||||
const {Feed} = require("feed")
|
||||
const constants = require("../constants")
|
||||
const config = require("../../../config")
|
||||
const TimelineEntry = require("./TimelineEntry")
|
||||
|
|
@ -54,18 +54,27 @@ class Timeline {
|
|||
}
|
||||
|
||||
async fetchFeed() {
|
||||
const feed = new RSS({
|
||||
title: `@${this.user.data.username}`,
|
||||
feed_url: `${config.website_origin}/u/${this.user.data.username}/rss.xml`,
|
||||
site_url: config.website_origin,
|
||||
// we likely cannot use full_name here - reel fallback would make the feed title inconsistent, leading to confusing experience
|
||||
const usedName = `@${this.user.data.username}`
|
||||
const feed = new Feed({
|
||||
title: usedName,
|
||||
description: this.user.data.biography,
|
||||
image_url: config.website_origin+this.user.proxyProfilePicture,
|
||||
pubDate: new Date(this.user.cachedAt),
|
||||
ttl: this.user.getTtl(1000*60) // scale to minute
|
||||
id: `bibliogram:user/${this.user.data.username}`,
|
||||
link: `${constants.website_origin}/u/${this.user.data.username}`,
|
||||
feedLinks: {
|
||||
rss: `${constants.website_origin}/u/${this.user.data.username}/rss.xml`,
|
||||
atom: `${constants.website_origin}/u/${this.user.data.username}/atom.xml`
|
||||
},
|
||||
image: constants.website_origin+this.user.proxyProfilePicture,
|
||||
updated: new Date(this.user.cachedAt),
|
||||
author: {
|
||||
name: usedName,
|
||||
link: `${constants.website_origin}/u/${this.user.data.username}`
|
||||
}
|
||||
})
|
||||
const page = this.pages[0] // only get posts from first page
|
||||
await Promise.all(page.map(item =>
|
||||
item.fetchFeedData().then(feedData => feed.item(feedData))
|
||||
item.fetchFeedData().then(feedData => feed.addItem(feedData))
|
||||
))
|
||||
return feed
|
||||
}
|
||||
|
|
|
|||
|
|
@ -217,6 +217,9 @@ class TimelineEntry extends TimelineBaseMethods {
|
|||
else return this.update().then(() => this.getVideoUrlP())
|
||||
}
|
||||
|
||||
/**
|
||||
* @returns {Promise<import("feed/src/typings/index").Item>}
|
||||
*/
|
||||
async fetchFeedData() {
|
||||
const children = await this.fetchChildren()
|
||||
return {
|
||||
|
|
@ -230,10 +233,10 @@ class TimelineEntry extends TimelineBaseMethods {
|
|||
height: child.data.dimensions.height
|
||||
}))
|
||||
}),
|
||||
author: this.data.owner.username,
|
||||
url: `${constants.website_origin}/p/${this.data.shortcode}`,
|
||||
guid: `${constants.website_origin}/p/${this.data.shortcode}`, // Is it wise to keep the origin in here? The same post would have a different ID from different servers.
|
||||
date: new Date(this.data.taken_at_timestamp*1000)
|
||||
link: `${constants.website_origin}/p/${this.data.shortcode}`,
|
||||
id: `bibliogram:post/${this.data.shortcode}`, // Is it wise to keep the origin in here? The same post would have a different ID from different servers.
|
||||
published: new Date(this.data.taken_at_timestamp*1000), // first published date
|
||||
date: new Date(this.data.taken_at_timestamp*1000) // last modified date
|
||||
/*
|
||||
Readers should display the description as HTML rather than using the media enclosure.
|
||||
enclosure: {
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ const fetch = require("node-fetch").default
|
|||
function request(url, options = {}, settings = {}) {
|
||||
if (settings.statusLine === undefined) settings.statusLine = "OUT"
|
||||
if (settings.log === undefined) settings.log = true
|
||||
if (settings.log) console.log(`-> [${settings.statusLine}] ${url}`) // todo: make more like pinski?
|
||||
if (settings.log) console.log(` -> [${settings.statusLine}] ${url}`) // todo: make more like pinski?
|
||||
// @ts-ignore
|
||||
return fetch(url, Object.assign({
|
||||
headers: {
|
||||
|
|
|
|||
|
|
@ -4,15 +4,29 @@ const {render} = require("pinski/plugins")
|
|||
const {pugCache} = require("../passthrough")
|
||||
|
||||
module.exports = [
|
||||
{route: `/u/(${constants.external.username_regex})/rss.xml`, methods: ["GET"], code: ({fill}) => {
|
||||
{route: `/u/(${constants.external.username_regex})/(rss|atom)\\.xml`, methods: ["GET"], code: ({fill}) => {
|
||||
if (constants.settings.rss_enabled) {
|
||||
const kind = fill[1]
|
||||
return fetchUser(fill[0], true).then(async user => {
|
||||
const content = await user.timeline.fetchFeed()
|
||||
const xml = content.xml()
|
||||
const feed = await user.timeline.fetchFeed()
|
||||
if (kind === "rss") {
|
||||
var data = {
|
||||
contentType: "application/rss+xml", // see https://stackoverflow.com/questions/595616/what-is-the-correct-mime-type-to-use-for-an-rss-feed,
|
||||
content: feed.rss2()
|
||||
}
|
||||
} else if (kind === "atom") {
|
||||
var data = {
|
||||
contentType: "application/atom+xml", // see https://en.wikipedia.org/wiki/Atom_(standard)#Including_in_HTML
|
||||
content: feed.atom1()
|
||||
}
|
||||
}
|
||||
return {
|
||||
statusCode: 200,
|
||||
contentType: "application/rss+xml", // see https://stackoverflow.com/questions/595616/what-is-the-correct-mime-type-to-use-for-an-rss-feed
|
||||
content: xml
|
||||
contentType: data.contentType,
|
||||
headers: {
|
||||
"Cache-Control": `max-age=${userRequestCache.getTtl("user/"+user.data.username, 1000)}`
|
||||
},
|
||||
content: data.content
|
||||
}
|
||||
}).catch(error => {
|
||||
if (error === constants.symbols.NOT_FOUND || error === constants.symbols.ENDPOINT_OVERRIDDEN) {
|
||||
|
|
@ -40,8 +54,8 @@ module.exports = [
|
|||
} else {
|
||||
return Promise.resolve(render(403, "pug/friendlyerror.pug", {
|
||||
statusCode: 403,
|
||||
title: "RSS disabled",
|
||||
message: "RSS is disabled on this instance.",
|
||||
title: "Feeds disabled",
|
||||
message: "Feeds are disabled on this instance.",
|
||||
withInstancesLink: true
|
||||
}))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@ module.exports = [
|
|||
if (typeof page === "number" && !isNaN(page) && page >= 1) {
|
||||
await user.timeline.fetchUpToPage(page - 1)
|
||||
}
|
||||
return render(200, "pug/user.pug", {url, user, constants})
|
||||
return render(200, "pug/user.pug", {url, user, constants, website_origin: constants.website_origin})
|
||||
}).catch(error => {
|
||||
if (error === constants.symbols.NOT_FOUND || error === constants.symbols.ENDPOINT_OVERRIDDEN) {
|
||||
return render(404, "pug/friendlyerror.pug", {
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ html
|
|||
body.homepage
|
||||
header
|
||||
h1.banner
|
||||
img.banner-image(src="/static/img/banner-min.svg")
|
||||
img.banner-image(src="/static/img/banner-min.svg" alt="Bibliogram")
|
||||
.go-sections-container
|
||||
.go-sections
|
||||
section
|
||||
|
|
|
|||
|
|
@ -0,0 +1,9 @@
|
|||
mixin feed_link(name, urlPart, username, contentType)
|
||||
span
|
||||
a(rel="alternate" type=contentType href=`/u/${username}/${urlPart}.xml`)
|
||||
= name
|
||||
sup.validate-feed
|
||||
-
|
||||
let params = new URLSearchParams()
|
||||
params.set("url", `${website_origin}/u/${username}/${urlPart}.xml`)
|
||||
a(href="https://validator.w3.org/feed/check.cgi?"+params.toString() title="Validate this feed") v!
|
||||
|
|
@ -1,8 +1,9 @@
|
|||
//- Needs user, url, constants
|
||||
//- Needs user, url, constants, website_origin
|
||||
|
||||
include includes/timeline_page.pug
|
||||
include includes/next_page_button.pug
|
||||
include includes/display_structured
|
||||
include includes/feed_link
|
||||
|
||||
- const numberFormat = new Intl.NumberFormat().format
|
||||
|
||||
|
|
@ -19,7 +20,7 @@ html
|
|||
.main-divider
|
||||
header.profile-overview
|
||||
.profile-sticky
|
||||
img(src=user.proxyProfilePicture width="150px" height="150px" alt=`${user.data.full_name || user.data.username}'s profile picture.`).pfp
|
||||
img(src=user.proxyProfilePicture width=150 height=150 alt=`${user.data.full_name || user.data.username}'s profile picture.`).pfp
|
||||
//-
|
||||
Instagram only uses the above URL, but an HD version is also available.
|
||||
The alt text is pathetic, I know. I don't have much to work with.
|
||||
|
|
@ -28,20 +29,23 @@ html
|
|||
h2.username= `@${user.data.username}`
|
||||
else
|
||||
h1.full-name= `@${user.data.username}`
|
||||
if !user.fromReel
|
||||
p.structured-text.bio
|
||||
- const bio = user.getStructuredBio()
|
||||
if bio
|
||||
+display_structured(bio)
|
||||
if user.data.external_url
|
||||
p.website
|
||||
a(href=user.data.external_url)= user.data.external_url
|
||||
p.structured-text.bio
|
||||
- const bio = user.getStructuredBio()
|
||||
if bio
|
||||
+display_structured(bio)
|
||||
if user.data.external_url
|
||||
p.website
|
||||
a(href=user.data.external_url)= user.data.external_url
|
||||
if user.posts != undefined
|
||||
div.profile-counter #[span(data-numberformat=user.posts).count #{numberFormat(user.posts)}] posts
|
||||
if user.following != undefined
|
||||
div.profile-counter #[span(data-numberformat=user.following).count #{numberFormat(user.following)}] following
|
||||
if user.followedBy != undefined
|
||||
div.profile-counter #[span(data-numberformat=user.followedBy).count #{numberFormat(user.followedBy)}] followed by
|
||||
div.links
|
||||
if constants.settings.rss_enabled
|
||||
a(rel="alternate" type="application/rss+xml" href=`/u/${user.data.username}/rss.xml`) RSS
|
||||
+feed_link("RSS", "rss", user.data.username, "application/rss+xml")
|
||||
+feed_link("Atom", "atom", user.data.username, "application/atom+xml")
|
||||
a(rel="noreferrer noopener" href=`https://www.instagram.com/${user.data.username}`) instagram.com
|
||||
|
||||
- const hasPosts = !user.data.is_private && user.timeline.pages.length && user.timeline.pages[0].length
|
||||
|
|
|
|||
|
|
@ -102,6 +102,9 @@ body
|
|||
flex-wrap: wrap
|
||||
justify-content: center
|
||||
|
||||
.validate-feed
|
||||
margin-left: 2px
|
||||
|
||||
a, a:visited
|
||||
color: $main-theme-link-color
|
||||
|
||||
|
|
|
|||
|
|
@ -21,7 +21,6 @@ subdirs("pug", async (err, dirs) => {
|
|||
pinski.addPugDir("pug", dirs)
|
||||
pinski.addAPIDir("html/static/js/templates/api")
|
||||
pinski.addSassDir("sass")
|
||||
pinski.addAPIDir("api")
|
||||
pinski.muteLogsStartingWith("/imageproxy")
|
||||
pinski.muteLogsStartingWith("/videoproxy")
|
||||
pinski.muteLogsStartingWith("/static")
|
||||
|
|
@ -30,6 +29,7 @@ subdirs("pug", async (err, dirs) => {
|
|||
await require("../lib/utils/tor") // make sure tor state is known before going further
|
||||
}
|
||||
|
||||
pinski.addAPIDir("api")
|
||||
pinski.startServer()
|
||||
pinski.enableWS()
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue