Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2024-11-22 06:16:18 +00:00
Code Issues Projects Releases Packages Wiki Activity

coto client

Browse source
This commit is contained in:
Cat /dev/Nulo 2024-06-18 16:56:31 -03:00
parent c3b968cf12
commit 3ec056645d
2 changed files with 20 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
scraper-rs/src/main.rs
View file

@ -165,13 +165,28 @@ enum FetchError {
Tl(#[from] tl::ParseError),
}
fn build_client() -> reqwest::Client {
fn get_user_agent() -> &'static str {
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"
}
fn build_header_map() -> HeaderMap {
let mut headers = HeaderMap::new();
headers.append("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36".parse().unwrap());
headers.append("User-Agent", get_user_agent().parse().unwrap());
headers
}
fn build_client() -> reqwest::Client {
reqwest::ClientBuilder::default()
.timeout(Duration::from_secs(60))
.connect_timeout(Duration::from_secs(30))
.default_headers(headers)
.default_headers(build_header_map())
.build()
.unwrap()
}
fn build_coto_client() -> reqwest::Client {
reqwest::ClientBuilder::default()
.timeout(Duration::from_secs(300))
.connect_timeout(Duration::from_secs(150))
.default_headers(build_header_map())
.build()
.unwrap()
}

4
scraper-rs/src/sites/coto.rs
View file

@ -4,7 +4,7 @@ use itertools::Itertools;
use reqwest::Url;
use crate::{
build_client, do_request, get_fetch_retry_policy, retry_if_wasnt_not_found, PrecioPoint,
build_client, build_coto_client, do_request, get_fetch_retry_policy, retry_if_wasnt_not_found, PrecioPoint
};
pub fn parse(url: String, dom: &tl::VDom) -> Result<PrecioPoint, anyhow::Error> {
@ -79,7 +79,7 @@ pub fn parse(url: String, dom: &tl::VDom) -> Result<PrecioPoint, anyhow::Error>
}
pub async fn get_urls() -> anyhow::Result<Vec<String>> {
let client = build_client();
let client = build_coto_client();
let initial = Url::parse("https://www.cotodigital3.com.ar/sitios/cdigi/browse?Nf=product.endDate%7CGTEQ+1.7032032E12%7C%7Cproduct.startDate%7CLTEQ+1.7032032E12&Nr=AND%28product.sDisp_200%3A1004%2Cproduct.language%3Aespa%C3%B1ol%2COR%28product.siteId%3ACotoDigital%29%29")?;
let page_size = 50;