Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2024-11-26 03:26:19 +00:00
Code Issues Projects Releases Packages Wiki Activity

coto client

Browse source
This commit is contained in:
Cat /dev/Nulo 2024-06-18 16:56:31 -03:00
parent c3b968cf12
commit 3ec056645d
2 changed files with 20 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
scraper-rs/src/main.rs
View file

@ -165,13 +165,28 @@ enum FetchError {
Tl(#[from] tl::ParseError), Tl(#[from] tl::ParseError),
} }
fn build_client() -> reqwest::Client { fn get_user_agent() -> &'static str {
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36"
}
fn build_header_map() -> HeaderMap {
let mut headers = HeaderMap::new(); let mut headers = HeaderMap::new();
headers.append("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36".parse().unwrap()); headers.append("User-Agent", get_user_agent().parse().unwrap());
headers
}
fn build_client() -> reqwest::Client {
reqwest::ClientBuilder::default() reqwest::ClientBuilder::default()
.timeout(Duration::from_secs(60)) .timeout(Duration::from_secs(60))
.connect_timeout(Duration::from_secs(30)) .connect_timeout(Duration::from_secs(30))
.default_headers(headers) .default_headers(build_header_map())
.build()
.unwrap()
}
fn build_coto_client() -> reqwest::Client {
reqwest::ClientBuilder::default()
.timeout(Duration::from_secs(300))
.connect_timeout(Duration::from_secs(150))
.default_headers(build_header_map())
.build() .build()
.unwrap() .unwrap()
} }

4
scraper-rs/src/sites/coto.rs
View file

@ -4,7 +4,7 @@ use itertools::Itertools;
use reqwest::Url; use reqwest::Url;
use crate::{ use crate::{
build_client, do_request, get_fetch_retry_policy, retry_if_wasnt_not_found, PrecioPoint, build_client, build_coto_client, do_request, get_fetch_retry_policy, retry_if_wasnt_not_found, PrecioPoint
}; };
pub fn parse(url: String, dom: &tl::VDom) -> Result<PrecioPoint, anyhow::Error> { pub fn parse(url: String, dom: &tl::VDom) -> Result<PrecioPoint, anyhow::Error> {
@ -79,7 +79,7 @@ pub fn parse(url: String, dom: &tl::VDom) -> Result<PrecioPoint, anyhow::Error>
} }
pub async fn get_urls() -> anyhow::Result<Vec<String>> { pub async fn get_urls() -> anyhow::Result<Vec<String>> {
let client = build_client(); let client = build_coto_client();
let initial = Url::parse("https://www.cotodigital3.com.ar/sitios/cdigi/browse?Nf=product.endDate%7CGTEQ+1.7032032E12%7C%7Cproduct.startDate%7CLTEQ+1.7032032E12&Nr=AND%28product.sDisp_200%3A1004%2Cproduct.language%3Aespa%C3%B1ol%2COR%28product.siteId%3ACotoDigital%29%29")?; let initial = Url::parse("https://www.cotodigital3.com.ar/sitios/cdigi/browse?Nf=product.endDate%7CGTEQ+1.7032032E12%7C%7Cproduct.startDate%7CLTEQ+1.7032032E12&Nr=AND%28product.sDisp_200%3A1004%2Cproduct.language%3Aespa%C3%B1ol%2COR%28product.siteId%3ACotoDigital%29%29")?;
let page_size = 50; let page_size = 50;