2020-01-12 12:50:21 +00:00
|
|
|
const constants = require("./constants")
|
|
|
|
const {request} = require("./utils/request")
|
2020-02-02 11:43:56 +00:00
|
|
|
const switcher = require("./utils/torswitcher")
|
2020-01-12 12:50:21 +00:00
|
|
|
const {extractSharedData} = require("./utils/body")
|
2020-01-18 15:38:14 +00:00
|
|
|
const {TtlCache, RequestCache} = require("./cache")
|
2020-01-30 12:51:59 +00:00
|
|
|
const RequestHistory = require("./structures/RequestHistory")
|
2020-02-01 04:44:40 +00:00
|
|
|
const db = require("./db")
|
2020-01-30 12:51:59 +00:00
|
|
|
require("./testimports")(constants, request, extractSharedData, RequestCache, RequestHistory)
|
2020-01-12 12:50:21 +00:00
|
|
|
|
2020-01-28 03:14:21 +00:00
|
|
|
const requestCache = new RequestCache(constants.caching.resource_cache_time)
|
2020-01-26 14:56:59 +00:00
|
|
|
/** @type {import("./cache").TtlCache<import("./structures/TimelineEntry")>} */
|
2020-01-28 03:14:21 +00:00
|
|
|
const timelineEntryCache = new TtlCache(constants.caching.resource_cache_time)
|
2020-02-02 13:44:52 +00:00
|
|
|
const history = new RequestHistory(["user", "timeline", "post", "reel"])
|
2020-01-12 12:50:21 +00:00
|
|
|
|
2020-02-02 13:24:14 +00:00
|
|
|
async function fetchUser(username) {
|
|
|
|
if (constants.allow_user_from_reel === "never") {
|
|
|
|
return fetchUserFromHTML(username)
|
|
|
|
} else if (constants.allow_user_from_reel === "prefer") {
|
|
|
|
const userID = db.prepare("SELECT user_id FROM Users WHERE username = ?").pluck().get(username)
|
|
|
|
if (userID) return fetchUserFromCombined(userID, username)
|
|
|
|
else return fetchUserFromHTML(username)
|
|
|
|
} else { // === "fallback"
|
|
|
|
return fetchUserFromHTML(username).catch(error => {
|
|
|
|
if (error === constants.symbols.INSTAGRAM_DEMANDS_LOGIN || error === constants.symbols.RATE_LIMITED) {
|
|
|
|
const userID = db.prepare("SELECT user_id FROM Users WHERE username = ?").pluck().get(username)
|
|
|
|
if (userID) return fetchUserFromCombined(userID, username)
|
|
|
|
}
|
|
|
|
throw error
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
function fetchUserFromHTML(username) {
|
2020-01-18 15:38:14 +00:00
|
|
|
return requestCache.getOrFetch("user/"+username, () => {
|
2020-02-02 13:24:14 +00:00
|
|
|
return switcher.request("user_html", `https://www.instagram.com/${username}/`, async res => {
|
|
|
|
if (res.status === 302) throw constants.symbols.INSTAGRAM_DEMANDS_LOGIN
|
|
|
|
if (res.status === 429) throw constants.symbols.RATE_LIMITED
|
|
|
|
return res
|
|
|
|
}).then(res => {
|
|
|
|
if (res.status === 404) {
|
2020-01-30 12:51:59 +00:00
|
|
|
throw constants.symbols.NOT_FOUND
|
2020-02-02 13:24:14 +00:00
|
|
|
} else {
|
|
|
|
return res.text().then(text => {
|
|
|
|
// require down here or have to deal with require loop. require cache will take care of it anyway.
|
|
|
|
// User -> Timeline -> TimelineEntry -> collectors -/> User
|
|
|
|
const User = require("./structures/User")
|
|
|
|
const sharedData = extractSharedData(text)
|
|
|
|
const user = new User(sharedData.entry_data.ProfilePage[0].graphql.user)
|
|
|
|
history.report("user", true)
|
|
|
|
if (constants.caching.db_user_id) {
|
|
|
|
db.prepare("INSERT OR IGNORE INTO Users (username, user_id) VALUES (@username, @user_id)")
|
|
|
|
.run({username: user.data.username, user_id: user.data.id})
|
|
|
|
}
|
|
|
|
return user
|
|
|
|
})
|
|
|
|
}
|
|
|
|
}).catch(error => {
|
|
|
|
if (error === constants.symbols.INSTAGRAM_DEMANDS_LOGIN || error === constants.symbols.RATE_LIMITED) {
|
|
|
|
history.report("user", false)
|
|
|
|
}
|
|
|
|
throw error
|
2020-01-12 12:50:21 +00:00
|
|
|
})
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2020-02-02 13:24:14 +00:00
|
|
|
function fetchUserFromCombined(userID, username) {
|
|
|
|
// Fetch basic user information
|
|
|
|
const p = new URLSearchParams()
|
|
|
|
p.set("query_hash", constants.external.reel_query_hash)
|
|
|
|
p.set("variables", JSON.stringify({
|
|
|
|
user_id: userID,
|
|
|
|
include_reel: true
|
|
|
|
}))
|
|
|
|
return requestCache.getOrFetch("user/"+username, () => {
|
|
|
|
return switcher.request("reel_graphql", `https://www.instagram.com/graphql/query/?${p.toString()}`, async res => {
|
|
|
|
if (res.status === 429) throw constants.symbols.RATE_LIMITED
|
|
|
|
return res
|
|
|
|
}).then(res => res.json()).then(root => {
|
|
|
|
const result = root.data.user
|
|
|
|
if (!result) throw constants.symbols.NOT_FOUND
|
|
|
|
// require down here or have to deal with require loop. require cache will take care of it anyway.
|
|
|
|
// ReelUser -> Timeline -> TimelineEntry -> collectors -/> User
|
|
|
|
const ReelUser = require("./structures/ReelUser")
|
|
|
|
const user = new ReelUser(result.reel.user)
|
|
|
|
return user
|
|
|
|
}).catch(error => {
|
|
|
|
throw error
|
|
|
|
})
|
|
|
|
}).then(async user => {
|
|
|
|
// Add first timeline page
|
|
|
|
if (!user.timeline.pages[0]) {
|
|
|
|
const page = await fetchTimelinePage(userID, "")
|
|
|
|
user.timeline.addPage(page)
|
|
|
|
}
|
2020-02-02 13:44:52 +00:00
|
|
|
history.report("reel", true)
|
2020-02-02 13:24:14 +00:00
|
|
|
return user
|
2020-02-02 13:44:52 +00:00
|
|
|
}).catch(error => {
|
|
|
|
if (error === constants.symbols.RATE_LIMITED) {
|
|
|
|
history.report("reel", false)
|
|
|
|
}
|
|
|
|
throw error
|
2020-02-02 13:24:14 +00:00
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2020-01-12 12:50:21 +00:00
|
|
|
/**
|
|
|
|
* @param {string} userID
|
|
|
|
* @param {string} after
|
2020-01-27 06:03:28 +00:00
|
|
|
* @returns {Promise<import("./types").PagedEdges<import("./types").TimelineEntryN2>>}
|
2020-01-12 12:50:21 +00:00
|
|
|
*/
|
|
|
|
function fetchTimelinePage(userID, after) {
|
|
|
|
const p = new URLSearchParams()
|
|
|
|
p.set("query_hash", constants.external.timeline_query_hash)
|
|
|
|
p.set("variables", JSON.stringify({
|
|
|
|
id: userID,
|
|
|
|
first: constants.external.timeline_fetch_first,
|
|
|
|
after: after
|
|
|
|
}))
|
2020-02-02 13:24:14 +00:00
|
|
|
return requestCache.getOrFetchPromise(`page/${userID}/${after}`, () => {
|
|
|
|
return switcher.request("timeline_graphql", `https://www.instagram.com/graphql/query/?${p.toString()}`, async res => {
|
2020-02-02 11:43:56 +00:00
|
|
|
if (res.status === 429) throw constants.symbols.RATE_LIMITED
|
|
|
|
return res
|
|
|
|
}).then(res => res.json()).then(root => {
|
|
|
|
/** @type {import("./types").PagedEdges<import("./types").TimelineEntryN2>} */
|
|
|
|
const timeline = root.data.user.edge_owner_to_timeline_media
|
|
|
|
history.report("timeline", true)
|
|
|
|
return timeline
|
|
|
|
}).catch(error => {
|
|
|
|
if (error === constants.symbols.RATE_LIMITED) {
|
2020-01-30 12:51:59 +00:00
|
|
|
history.report("timeline", false)
|
|
|
|
}
|
2020-02-02 11:43:56 +00:00
|
|
|
throw error
|
2020-01-12 12:50:21 +00:00
|
|
|
})
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2020-01-18 15:38:14 +00:00
|
|
|
/**
|
|
|
|
* @param {string} shortcode
|
2020-01-26 14:56:59 +00:00
|
|
|
* @returns {import("./structures/TimelineEntry")}
|
2020-01-18 15:38:14 +00:00
|
|
|
*/
|
2020-01-26 14:56:59 +00:00
|
|
|
function getOrCreateShortcode(shortcode) {
|
|
|
|
if (timelineEntryCache.has(shortcode)) {
|
|
|
|
return timelineEntryCache.get(shortcode)
|
|
|
|
} else {
|
|
|
|
// require down here or have to deal with require loop. require cache will take care of it anyway.
|
2020-02-02 13:24:14 +00:00
|
|
|
// TimelineEntry -> collectors -/> TimelineEntry
|
2020-01-26 14:56:59 +00:00
|
|
|
const TimelineEntry = require("./structures/TimelineEntry")
|
|
|
|
const result = new TimelineEntry()
|
|
|
|
timelineEntryCache.set(shortcode, result)
|
|
|
|
return result
|
|
|
|
}
|
|
|
|
}
|
2020-01-18 15:38:14 +00:00
|
|
|
|
2020-01-26 14:56:59 +00:00
|
|
|
async function getOrFetchShortcode(shortcode) {
|
|
|
|
if (timelineEntryCache.has(shortcode)) {
|
|
|
|
return timelineEntryCache.get(shortcode)
|
|
|
|
} else {
|
|
|
|
const data = await fetchShortcodeData(shortcode)
|
|
|
|
const entry = getOrCreateShortcode(shortcode)
|
|
|
|
entry.applyN3(data)
|
|
|
|
return entry
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* @param {string} shortcode
|
|
|
|
* @returns {Promise<import("./types").TimelineEntryN3>}
|
|
|
|
*/
|
|
|
|
function fetchShortcodeData(shortcode) {
|
2020-01-18 15:38:14 +00:00
|
|
|
// example actual query from web:
|
|
|
|
// query_hash=2b0673e0dc4580674a88d426fe00ea90&variables={"shortcode":"xxxxxxxxxxx","child_comment_count":3,"fetch_comment_count":40,"parent_comment_count":24,"has_threaded_comments":true}
|
|
|
|
// we will not include params about comments, which means we will not receive comments, but everything else should still work fine
|
|
|
|
const p = new URLSearchParams()
|
|
|
|
p.set("query_hash", constants.external.shortcode_query_hash)
|
|
|
|
p.set("variables", JSON.stringify({shortcode}))
|
|
|
|
return requestCache.getOrFetchPromise("shortcode/"+shortcode, () => {
|
2020-02-02 13:24:14 +00:00
|
|
|
return switcher.request("post_graphql", `https://www.instagram.com/graphql/query/?${p.toString()}`, async res => {
|
2020-02-02 11:43:56 +00:00
|
|
|
if (res.status === 429) throw constants.symbols.RATE_LIMITED
|
|
|
|
return res
|
|
|
|
}).then(res => res.json()).then(root => {
|
|
|
|
/** @type {import("./types").TimelineEntryN3} */
|
|
|
|
const data = root.data.shortcode_media
|
|
|
|
if (data == null) {
|
|
|
|
// the thing doesn't exist
|
|
|
|
throw constants.symbols.NOT_FOUND
|
2020-01-27 06:03:28 +00:00
|
|
|
} else {
|
2020-02-02 11:43:56 +00:00
|
|
|
history.report("post", true)
|
|
|
|
if (constants.caching.db_post_n3) {
|
|
|
|
db.prepare("REPLACE INTO Posts (shortcode, id, id_as_numeric, username, json) VALUES (@shortcode, @id, @id_as_numeric, @username, @json)")
|
|
|
|
.run({shortcode: data.shortcode, id: data.id, id_as_numeric: data.id, username: data.owner.username, json: JSON.stringify(data)})
|
2020-01-30 12:51:59 +00:00
|
|
|
}
|
2020-02-02 13:24:14 +00:00
|
|
|
// if we have the owner but only a reelUser, update it. this code is gross.
|
|
|
|
if (requestCache.hasNotPromise("user/"+data.owner.username)) {
|
|
|
|
const user = requestCache.getWithoutClean("user/"+data.owner.username)
|
|
|
|
if (user.fromReel) {
|
|
|
|
user.data.full_name = data.owner.full_name
|
|
|
|
user.data.is_verified = data.owner.is_verified
|
|
|
|
}
|
|
|
|
}
|
2020-02-02 11:43:56 +00:00
|
|
|
return data
|
|
|
|
}
|
|
|
|
}).catch(error => {
|
|
|
|
if (error === constants.symbols.RATE_LIMITED) {
|
|
|
|
history.report("post", false)
|
2020-01-27 06:03:28 +00:00
|
|
|
}
|
2020-02-02 11:43:56 +00:00
|
|
|
throw error
|
2020-01-18 15:38:14 +00:00
|
|
|
})
|
|
|
|
})
|
|
|
|
}
|
|
|
|
|
2020-01-12 12:50:21 +00:00
|
|
|
module.exports.fetchUser = fetchUser
|
|
|
|
module.exports.fetchTimelinePage = fetchTimelinePage
|
2020-01-26 14:56:59 +00:00
|
|
|
module.exports.getOrCreateShortcode = getOrCreateShortcode
|
|
|
|
module.exports.fetchShortcodeData = fetchShortcodeData
|
2020-01-18 15:38:14 +00:00
|
|
|
module.exports.requestCache = requestCache
|
2020-01-26 14:56:59 +00:00
|
|
|
module.exports.timelineEntryCache = timelineEntryCache
|
|
|
|
module.exports.getOrFetchShortcode = getOrFetchShortcode
|
2020-01-30 12:51:59 +00:00
|
|
|
module.exports.history = history
|