From 8d401e6ca3306cb1cba26f6282b6c12e377c5c22 Mon Sep 17 00:00:00 2001
From: Nulo <git@nulo.in>
Date: Sat, 16 Dec 2023 11:27:51 -0300
Subject: [PATCH] Revert "usar fetch"

This reverts commit 31b58d373c808e8cc47d88228c343d034a17ba6f.
---
 downloader/ckan_to_datajson.js |  2 +-
 downloader/download_json.js    |  2 +-
 downloader/network.js          | 22 +++++++++++++++-------
 3 files changed, 17 insertions(+), 9 deletions(-)

diff --git a/downloader/ckan_to_datajson.js b/downloader/ckan_to_datajson.js
index 69edb18..900ef32 100644
--- a/downloader/ckan_to_datajson.js
+++ b/downloader/ckan_to_datajson.js
@@ -12,7 +12,7 @@ const zCkanPackageList = z.object({
  */
 async function getJson(url) {
   const res = await customRequestWithLimitsAndRetries(new URL(url));
-  const json = await res.json();
+  const json = await res.body.json();
   return json;
 }
 
diff --git a/downloader/download_json.js b/downloader/download_json.js
index fd78678..0fd2216 100644
--- a/downloader/download_json.js
+++ b/downloader/download_json.js
@@ -41,7 +41,7 @@ async function downloadFromData(target) {
     const jsonRes = await customRequestWithLimitsAndRetries(
       new URL(target.url)
     );
-    json = await jsonRes.json();
+    json = await jsonRes.body.json();
   }
 
   const parsed = zData.parse(json);
diff --git a/downloader/network.js b/downloader/network.js
index e015e0d..adaf535 100644
--- a/downloader/network.js
+++ b/downloader/network.js
@@ -1,6 +1,14 @@
+import { Dispatcher, request, Agent, setGlobalDispatcher } from "undici";
 import pLimit from "p-limit";
 import { userAgent } from "./config.js";
 
+setGlobalDispatcher(
+  new Agent({
+    pipelining: 0,
+    bodyTimeout: 15 * 60 * 1000,
+  })
+);
+
 export class StatusCodeError extends Error {
   /**
    * @param {number} code
@@ -22,7 +30,7 @@ const nConnections = process.env.N_THREADS
 /**
  * @argument {URL} url
  * @argument {number} attempts
- * @returns {Promise<import("undici").BodyMixin>}
+ * @returns {Promise<Dispatcher.ResponseData>}
  */
 export async function customRequestWithLimitsAndRetries(url, attempts = 0) {
   try {
@@ -75,23 +83,23 @@ async function _customRequestWithLimits(url) {
 
 /**
  * @param {URL} url
- * @returns {Promise<import("undici").BodyMixin>}
  */
 async function _customRequest(url) {
   // sharepoint no le gusta compartir a bots lol
   const spoofUserAgent = url.host.endsWith("sharepoint.com");
 
-  const res = await fetch(url.toString(), {
-    // maxRedirections: 20,
+  const res = await request(url.toString(), {
+    maxRedirections: 20,
     headers: {
       "User-Agent": spoofUserAgent
         ? "Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0"
         : userAgent,
     },
   });
-  if (res.status >= 300 && res.status <= 399) throw new TooManyRedirectsError();
-  if (res.status < 200 || res.status > 299)
-    throw new StatusCodeError(res.status);
+  if (res.statusCode >= 300 && res.statusCode <= 399)
+    throw new TooManyRedirectsError();
+  if (res.statusCode < 200 || res.statusCode > 299)
+    throw new StatusCodeError(res.statusCode);
 
   return res;
 }