Nulo/transicion-desordenada-diablo
1
1
Fork 0
mirror of https://github.com/catdevnull/transicion-desordenada-diablo synced 2024-11-14 02:21:38 +00:00
Code Issues Projects Releases Packages Wiki Activity

Revert "usar fetch"

Browse source 
This reverts commit 31b58d373c.
This commit is contained in:
Cat /dev/Nulo 2023-12-16 11:27:51 -03:00
parent 31b58d373c
commit 8d401e6ca3
3 changed files with 17 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
downloader/ckan_to_datajson.js
View file

@ -12,7 +12,7 @@ const zCkanPackageList = z.object({
*/ */
async function getJson(url) { async function getJson(url) {
const res = await customRequestWithLimitsAndRetries(new URL(url)); const res = await customRequestWithLimitsAndRetries(new URL(url));
const json = await res.json(); const json = await res.body.json();
return json; return json;
} }

2
downloader/download_json.js
View file

@ -41,7 +41,7 @@ async function downloadFromData(target) {
const jsonRes = await customRequestWithLimitsAndRetries( const jsonRes = await customRequestWithLimitsAndRetries(
new URL(target.url) new URL(target.url)
); );
json = await jsonRes.json(); json = await jsonRes.body.json();
} }
const parsed = zData.parse(json); const parsed = zData.parse(json);

22
downloader/network.js
View file

@ -1,6 +1,14 @@
import { Dispatcher, request, Agent, setGlobalDispatcher } from "undici";
import pLimit from "p-limit"; import pLimit from "p-limit";
import { userAgent } from "./config.js"; import { userAgent } from "./config.js";
setGlobalDispatcher(
new Agent({
pipelining: 0,
bodyTimeout: 15 * 60 * 1000,
})
);
export class StatusCodeError extends Error { export class StatusCodeError extends Error {
/** /**
* @param {number} code * @param {number} code
@ -22,7 +30,7 @@ const nConnections = process.env.N_THREADS
/** /**
* @argument {URL} url * @argument {URL} url
* @argument {number} attempts * @argument {number} attempts
* @returns {Promise<import("undici").BodyMixin>} * @returns {Promise<Dispatcher.ResponseData>}
*/ */
export async function customRequestWithLimitsAndRetries(url, attempts = 0) { export async function customRequestWithLimitsAndRetries(url, attempts = 0) {
try { try {
@ -75,23 +83,23 @@ async function _customRequestWithLimits(url) {
/** /**
* @param {URL} url * @param {URL} url
* @returns {Promise<import("undici").BodyMixin>}
*/ */
async function _customRequest(url) { async function _customRequest(url) {
// sharepoint no le gusta compartir a bots lol // sharepoint no le gusta compartir a bots lol
const spoofUserAgent = url.host.endsWith("sharepoint.com"); const spoofUserAgent = url.host.endsWith("sharepoint.com");
const res = await fetch(url.toString(), { const res = await request(url.toString(), {
// maxRedirections: 20, maxRedirections: 20,
headers: { headers: {
"User-Agent": spoofUserAgent "User-Agent": spoofUserAgent
? "Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0" ? "Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0"
: userAgent, : userAgent,
}, },
}); });
if (res.status >= 300 && res.status <= 399) throw new TooManyRedirectsError(); if (res.statusCode >= 300 && res.statusCode <= 399)
if (res.status < 200 || res.status > 299) throw new TooManyRedirectsError();
throw new StatusCodeError(res.status); if (res.statusCode < 200 || res.statusCode > 299)
throw new StatusCodeError(res.statusCode);
return res; return res;
} }