diff --git a/downloader/ckan_to_datajson.js b/downloader/ckan_to_datajson.js index 900ef32..69edb18 100644 --- a/downloader/ckan_to_datajson.js +++ b/downloader/ckan_to_datajson.js @@ -12,7 +12,7 @@ const zCkanPackageList = z.object({ */ async function getJson(url) { const res = await customRequestWithLimitsAndRetries(new URL(url)); - const json = await res.body.json(); + const json = await res.json(); return json; } diff --git a/downloader/download_json.js b/downloader/download_json.js index 0fd2216..fd78678 100644 --- a/downloader/download_json.js +++ b/downloader/download_json.js @@ -41,7 +41,7 @@ async function downloadFromData(target) { const jsonRes = await customRequestWithLimitsAndRetries( new URL(target.url) ); - json = await jsonRes.body.json(); + json = await jsonRes.json(); } const parsed = zData.parse(json); diff --git a/downloader/network.js b/downloader/network.js index adaf535..e015e0d 100644 --- a/downloader/network.js +++ b/downloader/network.js @@ -1,14 +1,6 @@ -import { Dispatcher, request, Agent, setGlobalDispatcher } from "undici"; import pLimit from "p-limit"; import { userAgent } from "./config.js"; -setGlobalDispatcher( - new Agent({ - pipelining: 0, - bodyTimeout: 15 * 60 * 1000, - }) -); - export class StatusCodeError extends Error { /** * @param {number} code @@ -30,7 +22,7 @@ const nConnections = process.env.N_THREADS /** * @argument {URL} url * @argument {number} attempts - * @returns {Promise} + * @returns {Promise} */ export async function customRequestWithLimitsAndRetries(url, attempts = 0) { try { @@ -83,23 +75,23 @@ async function _customRequestWithLimits(url) { /** * @param {URL} url + * @returns {Promise} */ async function _customRequest(url) { // sharepoint no le gusta compartir a bots lol const spoofUserAgent = url.host.endsWith("sharepoint.com"); - const res = await request(url.toString(), { - maxRedirections: 20, + const res = await fetch(url.toString(), { + // maxRedirections: 20, headers: { "User-Agent": spoofUserAgent ? "Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0" : userAgent, }, }); - if (res.statusCode >= 300 && res.statusCode <= 399) - throw new TooManyRedirectsError(); - if (res.statusCode < 200 || res.statusCode > 299) - throw new StatusCodeError(res.statusCode); + if (res.status >= 300 && res.status <= 399) throw new TooManyRedirectsError(); + if (res.status < 200 || res.status > 299) + throw new StatusCodeError(res.status); return res; }