diff --git a/scraper/fetch.ts b/scraper/fetch.ts index 3c13d75..0f78c6c 100644 --- a/scraper/fetch.ts +++ b/scraper/fetch.ts @@ -4,7 +4,10 @@ import { pipeline } from "node:stream/promises"; export async function getHtml(url: string) { const res = await request(url, { - headers: {}, + headers: { + "Accept-Encoding": "gzip, deflate, br", + }, + throwOnError: true, }); let output: Buffer; switch (res.headers["content-encoding"]) {