spoof useragent en sharepoint

This commit is contained in:
Cat /dev/Nulo 2023-11-28 00:32:53 -03:00
parent 54f7ae086a
commit 3327fb53c7
2 changed files with 9 additions and 1 deletions

2
.gitignore vendored
View file

@ -2,4 +2,4 @@ node_modules/
dataJsons/
log
prueba
datos.gob.ar/
datos.gob.ar*

View file

@ -136,8 +136,16 @@ async function downloadDistWithRetries(job, tries = 0) {
async function downloadDist({ dist, dataset }) {
const url = new URL(dist.downloadURL);
// sharepoint no le gusta compartir a bots lol
const spoofUserAgent = url.host.endsWith("sharepoint.com");
const res = await request(url.toString(), {
maxRedirections: 20,
headers: {
"User-Agent": spoofUserAgent
? "Mozilla/5.0 (X11; Linux x86_64; rv:120.0) Gecko/20100101 Firefox/120.0"
: "transicion-desordenada (https://nulo.ar)",
},
});
if (res.statusCode >= 300 && res.statusCode <= 399)
throw new TooManyRedirectsError();