mirror of
https://github.com/catdevnull/transicion-desordenada-diablo
synced 2024-11-14 10:21:38 +00:00
parent
31b58d373c
commit
8d401e6ca3
3 changed files with 17 additions and 9 deletions
|
@ -12,7 +12,7 @@ const zCkanPackageList = z.object({
|
||||||
*/
|
*/
|
||||||
async function getJson(url) {
|
async function getJson(url) {
|
||||||
const res = await customRequestWithLimitsAndRetries(new URL(url));
|
const res = await customRequestWithLimitsAndRetries(new URL(url));
|
||||||
const json = await res.json();
|
const json = await res.body.json();
|
||||||
return json;
|
return json;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -41,7 +41,7 @@ async function downloadFromData(target) {
|
||||||
const jsonRes = await customRequestWithLimitsAndRetries(
|
const jsonRes = await customRequestWithLimitsAndRetries(
|
||||||
new URL(target.url)
|
new URL(target.url)
|
||||||
);
|
);
|
||||||
json = await jsonRes.json();
|
json = await jsonRes.body.json();
|
||||||
}
|
}
|
||||||
|
|
||||||
const parsed = zData.parse(json);
|
const parsed = zData.parse(json);
|
||||||
|
|
|
@ -1,6 +1,14 @@
|
||||||
|
import { Dispatcher, request, Agent, setGlobalDispatcher } from "undici";
|
||||||
import pLimit from "p-limit";
|
import pLimit from "p-limit";
|
||||||
import { userAgent } from "./config.js";
|
import { userAgent } from "./config.js";
|
||||||
|
|
||||||
|
setGlobalDispatcher(
|
||||||
|
new Agent({
|
||||||
|
pipelining: 0,
|
||||||
|
bodyTimeout: 15 * 60 * 1000,
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
export class StatusCodeError extends Error {
|
export class StatusCodeError extends Error {
|
||||||
/**
|
/**
|
||||||
* @param {number} code
|
* @param {number} code
|
||||||
|
@ -22,7 +30,7 @@ const nConnections = process.env.N_THREADS
|
||||||
/**
|
/**
|
||||||
* @argument {URL} url
|
* @argument {URL} url
|
||||||
* @argument {number} attempts
|
* @argument {number} attempts
|
||||||
* @returns {Promise<import("undici").BodyMixin>}
|
* @returns {Promise<Dispatcher.ResponseData>}
|
||||||
*/
|
*/
|
||||||
export async function customRequestWithLimitsAndRetries(url, attempts = 0) {
|
export async function customRequestWithLimitsAndRetries(url, attempts = 0) {
|
||||||
try {
|
try {
|
||||||
|
@ -75,23 +83,23 @@ async function _customRequestWithLimits(url) {
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* @param {URL} url
|
* @param {URL} url
|
||||||
* @returns {Promise<import("undici").BodyMixin>}
|
|
||||||
*/
|
*/
|
||||||
async function _customRequest(url) {
|
async function _customRequest(url) {
|
||||||
// sharepoint no le gusta compartir a bots lol
|
// sharepoint no le gusta compartir a bots lol
|
||||||
const spoofUserAgent = url.host.endsWith("sharepoint.com");
|
const spoofUserAgent = url.host.endsWith("sharepoint.com");
|
||||||
|
|
||||||
const res = await fetch(url.toString(), {
|
const res = await request(url.toString(), {
|
||||||
// maxRedirections: 20,
|
maxRedirections: 20,
|
||||||
headers: {
|
headers: {
|
||||||
"User-Agent": spoofUserAgent
|
"User-Agent": spoofUserAgent
|
||||||
? "Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0"
|
? "Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0"
|
||||||
: userAgent,
|
: userAgent,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
if (res.status >= 300 && res.status <= 399) throw new TooManyRedirectsError();
|
if (res.statusCode >= 300 && res.statusCode <= 399)
|
||||||
if (res.status < 200 || res.status > 299)
|
throw new TooManyRedirectsError();
|
||||||
throw new StatusCodeError(res.status);
|
if (res.statusCode < 200 || res.statusCode > 299)
|
||||||
|
throw new StatusCodeError(res.statusCode);
|
||||||
|
|
||||||
return res;
|
return res;
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue