Lomanic/NewLeaf
1
0
Fork 0
mirror of https://git.sr.ht/~cadence/NewLeaf synced 2026-05-04 16:41:37 +00:00
Code Issues Releases Wiki Activity

Remove with requests when it is unnecessary

Browse source
This commit is contained in:
Cadence Ember 2022-01-16 21:51:26 +13:00
parent 73b4fbabf7
commit 68cfbb809f
No known key found for this signature in database
GPG key ID: BC1C2C61CF521B17
6 changed files with 244 additions and 244 deletions
Download patch file Download diff file Expand all files Collapse all files

84
extractors/search.py
View file

@ -17,51 +17,51 @@ ytdl = yt_dlp.YoutubeDL(ytdl_opts)
def extract_search(q):
try:
with requests.get("https://www.youtube.com/results", params={"q": q, "hl": "en"}, cookies=eu_consent_cookie()) as r:
r.raise_for_status()
content = r.content.decode("utf8")
yt_initial_data = extract_yt_initial_data(content)
r = requests.get("https://www.youtube.com/results", params={"q": q, "hl": "en"}, cookies=eu_consent_cookie())
r.raise_for_status()
content = r.content.decode("utf8")
yt_initial_data = extract_yt_initial_data(content)
sections = yt_initial_data["contents"]["twoColumnSearchResultsRenderer"]["primaryContents"]["sectionListRenderer"]["contents"]
# youtube searches contain a lot of random stuff, just grab it all for now, then filter to `videoRenderer` later
itemSections = [s for s in sections if "itemSectionRenderer" in s]
sections = yt_initial_data["contents"]["twoColumnSearchResultsRenderer"]["primaryContents"]["sectionListRenderer"]["contents"]
# youtube searches contain a lot of random stuff, just grab it all for now, then filter to `videoRenderer` later
itemSections = [s for s in sections if "itemSectionRenderer" in s]
items = []
for section in itemSections:
items += section["itemSectionRenderer"]["contents"]
items = []
for section in itemSections:
items += section["itemSectionRenderer"]["contents"]
results = []
for item in items:
if "videoRenderer" in item:
video = item["videoRenderer"]
published = 0
published_text = "Live now"
if "publishedTimeText" in video:
published_text = video["publishedTimeText"]["simpleText"]
published = past_text_to_time(published_text)
results.append({
"type": "video",
"title": combine_runs(video["title"]),
"videoId": video["videoId"],
"author": combine_runs(video["longBylineText"]),
"authorId": video["longBylineText"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"],
"authorUrl": video["longBylineText"]["runs"][0]["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"],
"videoThumbnails": generate_video_thumbnails(video["videoId"]),
"description": combine_runs(video["descriptionSnippet"]) if "descriptionSnippet" in video else "",
"descriptionHtml": combine_runs_html(video["descriptionSnippet"]) if "descriptionSnippet" in video else "",
"viewCount": get_view_count_or_recommended(video),
"second__viewCountText": get_view_count_text_or_recommended(video),
"published": published,
"publishedText": published_text,
"lengthSeconds": get_length_or_live_now(video),
"second__lengthText": get_length_text_or_live_now(video),
"liveNow": is_live(video),
"paid": None,
"premium": None,
"isUpcoming": None
})
search_cache[q] = results # only cache full extraction
return results
results = []
for item in items:
if "videoRenderer" in item:
video = item["videoRenderer"]
published = 0
published_text = "Live now"
if "publishedTimeText" in video:
published_text = video["publishedTimeText"]["simpleText"]
published = past_text_to_time(published_text)
results.append({
"type": "video",
"title": combine_runs(video["title"]),
"videoId": video["videoId"],
"author": combine_runs(video["longBylineText"]),
"authorId": video["longBylineText"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"],
"authorUrl": video["longBylineText"]["runs"][0]["navigationEndpoint"]["commandMetadata"]["webCommandMetadata"]["url"],
"videoThumbnails": generate_video_thumbnails(video["videoId"]),
"description": combine_runs(video["descriptionSnippet"]) if "descriptionSnippet" in video else "",
"descriptionHtml": combine_runs_html(video["descriptionSnippet"]) if "descriptionSnippet" in video else "",
"viewCount": get_view_count_or_recommended(video),
"second__viewCountText": get_view_count_text_or_recommended(video),
"published": published,
"publishedText": published_text,
"lengthSeconds": get_length_or_live_now(video),
"second__lengthText": get_length_text_or_live_now(video),
"liveNow": is_live(video),
"paid": None,
"premium": None,
"isUpcoming": None
})
search_cache[q] = results # only cache full extraction
return results
except Exception:
print("messed up extracting search, using youtube-dl instead")