Lomanic/NewLeaf
1
0
Fork 0
mirror of https://git.sr.ht/~cadence/NewLeaf synced 2024-11-22 07:37:29 +00:00
Code Issues Releases Wiki Activity

Thread lock when using channel data cache

Browse Source
This commit is contained in:
Cadence Ember 2020-09-06 00:31:17 +12:00
parent 52b3ae07b1
commit e18efc9591
No known key found for this signature in database
GPG Key ID: 128B99B1B74A6412
1 changed files with 13 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
extractors/channel.py
View File

@ -3,14 +3,18 @@ import requests
import xml.etree.ElementTree as ET import xml.etree.ElementTree as ET
from tools.converters import * from tools.converters import *
from tools.extractors import extract_yt_initial_data from tools.extractors import extract_yt_initial_data
from threading import Lock
from cachetools import TTLCache from cachetools import TTLCache
channel_cache = TTLCache(maxsize=50, ttl=300) channel_cache = TTLCache(maxsize=50, ttl=300)
channel_cache_lock = Lock()
channel_latest_cache = TTLCache(maxsize=500, ttl=300) channel_latest_cache = TTLCache(maxsize=500, ttl=300)
channel_latest_cache_lock = Lock()
def extract_channel(ucid): def extract_channel(ucid):
if ucid in channel_cache: with channel_cache_lock:
return channel_cache[ucid] if ucid in channel_cache:
return channel_cache[ucid]
channel_type = "channel" if len(ucid) == 24 and ucid[:2] == "UC" else "user" channel_type = "channel" if len(ucid) == 24 and ucid[:2] == "UC" else "user"
with requests.get("https://www.youtube.com/{}/{}/videos".format(channel_type, ucid)) as r: with requests.get("https://www.youtube.com/{}/{}/videos".format(channel_type, ucid)) as r:
@ -96,7 +100,8 @@ def extract_channel(ucid):
"relatedChannels": [] "relatedChannels": []
} }
channel_cache[ucid] = channel with channel_cache_lock:
channel_cache[ucid] = channel
return channel return channel
@ -108,8 +113,9 @@ def extract_channel_videos(ucid):
return channel["latestVideos"] return channel["latestVideos"]
def extract_channel_latest(ucid): def extract_channel_latest(ucid):
if ucid in channel_latest_cache: with channel_latest_cache_lock:
return channel_latest_cache[ucid] if ucid in channel_latest_cache:
return channel_latest_cache[ucid]
with requests.get("https://www.youtube.com/feeds/videos.xml?channel_id={}".format(ucid)) as r: with requests.get("https://www.youtube.com/feeds/videos.xml?channel_id={}".format(ucid)) as r:
r.raise_for_status() r.raise_for_status()
@ -145,6 +151,7 @@ def extract_channel_latest(ucid):
"isUpcoming": None "isUpcoming": None
}) })
channel_latest_cache[ucid] = results with channel_latest_cache_lock:
channel_latest_cache[ucid] = results
return results return results