83 lines
2.1 KiB
Rust
83 lines
2.1 KiB
Rust
|
use scraper::{Html,Selector};
|
||
|
|
use url::Url;
|
||
|
|
use tokio::sync::mpsc::{self, Sender}; //, Receiver};
|
||
|
|
|
||
|
|
const WORKERS : usize = 8;
|
||
|
|
|
||
|
|
type SiteStat = (Url, Vec<Url>);
|
||
|
|
|
||
|
|
#[tokio::main]
|
||
|
|
async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||
|
|
let addr = std::env::args().nth(1)
|
||
|
|
.unwrap_or_else(|| "https://www.tmplab.org".to_string());
|
||
|
|
|
||
|
|
let links = get_links(addr.as_ref()).await?;
|
||
|
|
let addr = Url::parse(addr.as_ref())?;
|
||
|
|
let mut links : Vec<Url> = links.into_iter()
|
||
|
|
.filter(| url | url.host() != addr.host())
|
||
|
|
.collect();
|
||
|
|
|
||
|
|
let mut to_fetch = links.len();
|
||
|
|
let (tx, mut rx) = mpsc::channel(32);
|
||
|
|
// spawn a pool of workers to get the things started...
|
||
|
|
for _ in 0..WORKERS {
|
||
|
|
if let Some(addr) = links.pop() {
|
||
|
|
spawn_worker(addr, tx.clone());
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
let mut results = vec![];
|
||
|
|
// gets the results back from the workers through the channel
|
||
|
|
while let Some(res) = rx.recv().await {
|
||
|
|
to_fetch -= 1;
|
||
|
|
let Some(site) = res else {
|
||
|
|
continue
|
||
|
|
};
|
||
|
|
results.push(site);
|
||
|
|
// if there are still urls to fetch, pop one and spawn a new worker
|
||
|
|
// otherwise we want to break this loop if all the urls
|
||
|
|
// have been fetched...
|
||
|
|
if let Some(addr) = links.pop() {
|
||
|
|
spawn_worker(addr, tx.clone());
|
||
|
|
} else if to_fetch == 0 {
|
||
|
|
break;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
for (url, links) in results {
|
||
|
|
println!("{url} : {} links", links.len())
|
||
|
|
}
|
||
|
|
|
||
|
|
Ok(())
|
||
|
|
}
|
||
|
|
|
||
|
|
// interestingly, this function must not be async...
|
||
|
|
//a worker fetch one url, send it's result back in the channel, and terminates.
|
||
|
|
fn spawn_worker(url: Url, tx: Sender<Option<SiteStat>>) {
|
||
|
|
println!("Start fetching {url}");
|
||
|
|
tokio::spawn(async move {
|
||
|
|
let res = get_links(url.as_ref()).await
|
||
|
|
.ok()
|
||
|
|
.map(| v | (url.clone(), v));
|
||
|
|
println!("Got {url}");
|
||
|
|
tx.send(res).await.unwrap()
|
||
|
|
});
|
||
|
|
}
|
||
|
|
|
||
|
|
async fn get_links(
|
||
|
|
url: &str
|
||
|
|
) -> Result<Vec<Url>, reqwest::Error> {
|
||
|
|
let a_selector = Selector::parse("a[href]").unwrap();
|
||
|
|
|
||
|
|
let body = reqwest::get(url)
|
||
|
|
.await?
|
||
|
|
.text()
|
||
|
|
.await?;
|
||
|
|
|
||
|
|
Ok(Html::parse_document(&body)
|
||
|
|
.select(&a_selector)
|
||
|
|
.filter_map(| link | link.value().attr("href")
|
||
|
|
.and_then(| href | Url::parse(href).ok()))
|
||
|
|
.collect())
|
||
|
|
}
|