usar un throttle para cdn.buenosaires.gob.ar

downloader/network.js

@@ -1,6 +1,7 @@
 import { Dispatcher, request, Agent } from "undici";
 import pLimit from "p-limit";
 import { userAgent } from "./config.js";
+import pThrottle from "p-throttle";
 
 const dispatcher = new Agent({
   connect: { timeout: 60 * 1000 },
@@ -20,7 +21,8 @@ export class StatusCodeError extends Error {
 export class TooManyRedirectsError extends Error {}
 
 /** key es host
- * @type {Map<string, import("p-limit").LimitFunction>} */
+ * @type {Map<string, <Argument extends unknown, ReturnType>(
+		fn: (arguments_: Argument) => PromiseLike<ReturnType>) => Promise<ReturnType>>} */
 const limiters = new Map();
 const nConnections = process.env.N_THREADS
   ? parseInt(process.env.N_THREADS)
@@ -76,10 +78,13 @@ function wait(ms) {
 function _customRequestWithLimits(url) {
   let limit = limiters.get(url.host);
   if (!limit) {
-    limit = pLimit(
-      // tenemos que pingear mucho la API
-      url.host === "data.buenosaires.gob.ar" ? 32 : nConnections
-    );
+    if (url.host === "cdn.buenosaires.gob.ar") {
+      // tenemos que throttlear en este host porque tiene un rate limit.
+      // de todas maneras descarga rápido
+      limit = pThrottle({ limit: 3, interval: 1000 })((x) => x());
+    } else {
+      limit = pLimit(nConnections);
+    }
     limiters.set(url.host, limit);
   }
   return limit(() => _customRequest(url));

downloader/package.json

@@ -13,6 +13,7 @@
   "dependencies": {
     "common": "workspace:",
     "p-limit": "^5.0.0",
+    "p-throttle": "^6.1.0",
     "undici": "^6.0.1",
     "zod": "^3.22.4"
   },

pnpm-lock.yaml

@@ -20,6 +20,9 @@ importers:
       p-limit:
         specifier: ^5.0.0
         version: 5.0.0
+      p-throttle:
+        specifier: ^6.1.0
+        version: 6.1.0
       undici:
         specifier: ^6.0.1
         version: 6.0.1
@@ -1203,6 +1206,11 @@ packages:
       yocto-queue: 1.0.0
     dev: false
 
+  /p-throttle@6.1.0:
+    resolution: {integrity: sha512-eQMdGTxk2+047La67wefUtt0tEHh7D+C8Jl7QXoFCuIiNYeQ9zWs2AZiJdIAs72rSXZ06t11me2bgalRNdy3SQ==}
+    engines: {node: '>=18'}
+    dev: false
+
   /parent-module@1.0.1:
     resolution: {integrity: sha512-GQ2EWRpQV8/o+Aw8YqtfZZPfNRWZYkbidE9k5rpl/hC3vtHHBfGm2Ifi6qWV+coDGkrUKZAxE3Lot5kcsRlh+g==}
     engines: {node: '>=6'}