Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2024-11-22 14:16:19 +00:00
Code Issues Projects Releases Packages Wiki Activity

hacer fetcheo mas resiliente

Browse source
This commit is contained in:
Cat /dev/Nulo 2024-02-06 18:55:11 -03:00
parent b0a6640807
commit fb0d5cd7d5
1 changed files with 12 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
scraper-rs/src/main.rs
View file

@ -168,7 +168,7 @@ fn build_client() -> reqwest::Client {
let mut headers = HeaderMap::new(); let mut headers = HeaderMap::new();
headers.append("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36".parse().unwrap()); headers.append("User-Agent", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36".parse().unwrap());
reqwest::ClientBuilder::default() reqwest::ClientBuilder::default()
.timeout(Duration::from_secs(30)) .timeout(Duration::from_secs(60 * 5))
.connect_timeout(Duration::from_secs(30)) .connect_timeout(Duration::from_secs(30))
.default_headers(headers) .default_headers(headers)
.build() .build()
@ -179,6 +179,15 @@ pub async fn do_request(client: &reqwest::Client, url: &str) -> reqwest::Result<
let response = client.execute(request).await?.error_for_status()?; let response = client.execute(request).await?.error_for_status()?;
Ok(response) Ok(response)
} }
async fn request_and_body(client: &reqwest::Client, url: &str) -> reqwest::Result<String> {
let res = do_request(client, url).await?;
res.text().await
}
pub async fn fetch_body(client: &reqwest::Client, url: &str) -> reqwest::Result<String> {
get_retry_policy()
.retry_if(|| request_and_body(client, url), retry_if_wasnt_not_found)
.await
}
pub fn get_retry_policy() -> again::RetryPolicy { pub fn get_retry_policy() -> again::RetryPolicy {
RetryPolicy::exponential(Duration::from_millis(300)) RetryPolicy::exponential(Duration::from_millis(300))
@ -195,11 +204,7 @@ async fn fetch_and_parse(
client: &reqwest::Client, client: &reqwest::Client,
url: String, url: String,
) -> Result<PrecioPoint, anyhow::Error> { ) -> Result<PrecioPoint, anyhow::Error> {
let body = get_retry_policy() let body = fetch_body(client, &url).await?;
.retry_if(|| do_request(client, &url), retry_if_wasnt_not_found)
.await?
.text()
.await?;
let maybe_point = { scrap_url(client, url, &body).await }; let maybe_point = { scrap_url(client, url, &body).await };
@ -350,7 +355,7 @@ impl Auto {
} }
async fn inform(&self, msg: &str) { async fn inform(&self, msg: &str) {
println!("{}", msg); tracing::info!("{}", msg);
if let Some(telegram) = &self.telegram { if let Some(telegram) = &self.telegram {
let u = Url::parse_with_params( let u = Url::parse_with_params(
&format!("https://api.telegram.org/bot{}/sendMessage", telegram.token), &format!("https://api.telegram.org/bot{}/sendMessage", telegram.token),