Nulo/transicion-desordenada-diablo
1
1
Fork 0
mirror of https://github.com/catdevnull/transicion-desordenada-diablo synced 2024-11-29 12:36:19 +00:00
Code Issues Projects Releases Packages Wiki Activity

usar un throttle para cdn.buenosaires.gob.ar

Browse source
This commit is contained in:
Cat /dev/Nulo 2023-12-18 13:43:37 -03:00
parent fb7dea8b7d
commit 5aca382027
3 changed files with 19 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
downloader/network.js
View file

@ -1,6 +1,7 @@
import { Dispatcher, request, Agent } from "undici"; import { Dispatcher, request, Agent } from "undici";
import pLimit from "p-limit"; import pLimit from "p-limit";
import { userAgent } from "./config.js"; import { userAgent } from "./config.js";
import pThrottle from "p-throttle";
const dispatcher = new Agent({ const dispatcher = new Agent({
connect: { timeout: 60 * 1000 }, connect: { timeout: 60 * 1000 },
@ -20,7 +21,8 @@ export class StatusCodeError extends Error {
export class TooManyRedirectsError extends Error {} export class TooManyRedirectsError extends Error {}
/** key es host /** key es host
* @type {Map<string, import("p-limit").LimitFunction>} */ * @type {Map<string, <Argument extends unknown, ReturnType>(
fn: (arguments_: Argument) => PromiseLike<ReturnType>) => Promise<ReturnType>>} */
const limiters = new Map(); const limiters = new Map();
const nConnections = process.env.N_THREADS const nConnections = process.env.N_THREADS
? parseInt(process.env.N_THREADS) ? parseInt(process.env.N_THREADS)
@ -76,10 +78,13 @@ function wait(ms) {
function _customRequestWithLimits(url) { function _customRequestWithLimits(url) {
let limit = limiters.get(url.host); let limit = limiters.get(url.host);
if (!limit) { if (!limit) {
limit = pLimit( if (url.host === "cdn.buenosaires.gob.ar") {
// tenemos que pingear mucho la API // tenemos que throttlear en este host porque tiene un rate limit.
url.host === "data.buenosaires.gob.ar" ? 32 : nConnections // de todas maneras descarga rápido
); limit = pThrottle({ limit: 3, interval: 1000 })((x) => x());
} else {
limit = pLimit(nConnections);
}
limiters.set(url.host, limit); limiters.set(url.host, limit);
} }
return limit(() => _customRequest(url)); return limit(() => _customRequest(url));

1
downloader/package.json
View file

@ -13,6 +13,7 @@
"dependencies": { "dependencies": {
"common": "workspace:", "common": "workspace:",
"p-limit": "^5.0.0", "p-limit": "^5.0.0",
"p-throttle": "^6.1.0",
"undici": "^6.0.1", "undici": "^6.0.1",
"zod": "^3.22.4" "zod": "^3.22.4"
}, },

8
pnpm-lock.yaml
View file

@ -20,6 +20,9 @@ importers:
p-limit: p-limit:
specifier: ^5.0.0 specifier: ^5.0.0
version: 5.0.0 version: 5.0.0
p-throttle:
specifier: ^6.1.0
version: 6.1.0
undici: undici:
specifier: ^6.0.1 specifier: ^6.0.1
version: 6.0.1 version: 6.0.1
@ -1203,6 +1206,11 @@ packages:
yocto-queue: 1.0.0 yocto-queue: 1.0.0
dev: false dev: false
/p-throttle@6.1.0:
resolution: {integrity: sha512-eQMdGTxk2+047La67wefUtt0tEHh7D+C8Jl7QXoFCuIiNYeQ9zWs2AZiJdIAs72rSXZ06t11me2bgalRNdy3SQ==}
engines: {node: '>=18'}
dev: false
/parent-module@1.0.1: /parent-module@1.0.1:
resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==} resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==}
engines: {node: '>=6'} engines: {node: '>=6'}