mirror of
https://github.com/catdevnull/preciazo.git
synced 2024-11-21 06:16:18 +00:00
rm coto-crawlee
This commit is contained in:
parent
9a82a556f9
commit
9cb7c0e27e
8 changed files with 0 additions and 234 deletions
|
|
@ -1,8 +0,0 @@
|
|||
# configurations
|
||||
.idea
|
||||
|
||||
# crawlee storage folder
|
||||
storage
|
||||
|
||||
# installed files
|
||||
node_modules
|
||||
6
coto-crawlee/.gitignore
vendored
6
coto-crawlee/.gitignore
vendored
|
|
@ -1,6 +0,0 @@
|
|||
# This file tells Git which files shouldn't be added to source control
|
||||
|
||||
.idea
|
||||
dist
|
||||
node_modules
|
||||
storage
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
# Specify the base Docker image. You can read more about
|
||||
# the available images at https://crawlee.dev/docs/guides/docker-images
|
||||
# You can also use any other image from Docker Hub.
|
||||
FROM apify/actor-node-playwright-chrome:20 AS builder
|
||||
|
||||
# Copy just package.json and package-lock.json
|
||||
# to speed up the build using Docker layer cache.
|
||||
COPY --chown=myuser package*.json ./
|
||||
|
||||
# Install all dependencies. Don't audit to speed up the installation.
|
||||
RUN npm install --include=dev --audit=false
|
||||
|
||||
# Next, copy the source files using the user set
|
||||
# in the base image.
|
||||
COPY --chown=myuser . ./
|
||||
|
||||
# Install all dependencies and build the project.
|
||||
# Don't audit to speed up the installation.
|
||||
RUN npm run build
|
||||
|
||||
# Create final image
|
||||
FROM apify/actor-node-playwright-chrome:20
|
||||
|
||||
# Copy only built JS files from builder image
|
||||
COPY --from=builder --chown=myuser /home/myuser/dist ./dist
|
||||
|
||||
# Copy just package.json and package-lock.json
|
||||
# to speed up the build using Docker layer cache.
|
||||
COPY --chown=myuser package*.json ./
|
||||
|
||||
# Install NPM packages, skip optional and development dependencies to
|
||||
# keep the image small. Avoid logging too much and print the dependency
|
||||
# tree for debugging
|
||||
RUN npm --quiet set progress=false \
|
||||
&& npm install --omit=dev --omit=optional \
|
||||
&& echo "Installed NPM packages:" \
|
||||
&& (npm list --omit=dev --all || true) \
|
||||
&& echo "Node.js version:" \
|
||||
&& node --version \
|
||||
&& echo "NPM version:" \
|
||||
&& npm --version
|
||||
|
||||
# Next, copy the remaining files and directories with the source code.
|
||||
# Since we do this after NPM install, quick build will be really fast
|
||||
# for most source file changes.
|
||||
COPY --chown=myuser . ./
|
||||
|
||||
|
||||
# Run the image. If you know you won't need headful browsers,
|
||||
# you can remove the XVFB start script for a micro perf gain.
|
||||
CMD ./start_xvfb_and_run_cmd.sh && npm run start:prod --silent
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
# Getting started with Crawlee
|
||||
|
||||
This example uses `PlaywrightCrawler` to recursively crawl https://crawlee.dev using the browser automation library [Playwright](https://playwright.dev).
|
||||
|
||||
You can find more examples and documentation at the following links:
|
||||
|
||||
- [Step-by-step tutorial](https://crawlee.dev/docs/introduction) for Crawlee
|
||||
- `PlaywrightCrawler` [API documentation](https://crawlee.dev/api/playwright-crawler/class/PlaywrightCrawler)
|
||||
- Other [examples](https://crawlee.dev/docs/examples/playwright-crawler)
|
||||
Binary file not shown.
|
|
@ -1,25 +0,0 @@
|
|||
{
|
||||
"name": "coto-crawlee",
|
||||
"version": "0.0.1",
|
||||
"type": "module",
|
||||
"description": "This is an example of a Crawlee project.",
|
||||
"dependencies": {
|
||||
"crawlee": "^3.0.0",
|
||||
"playwright": "*"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@apify/tsconfig": "^0.1.0",
|
||||
"tsx": "^4.4.0",
|
||||
"typescript": "~5.5.0",
|
||||
"@types/node": "^20.0.0"
|
||||
},
|
||||
"scripts": {
|
||||
"start": "npm run start:dev",
|
||||
"start:prod": "node dist/main.js",
|
||||
"start:dev": "tsx src/main.ts",
|
||||
"build": "tsc",
|
||||
"test": "echo \"Error: oops, the actor has no tests yet, sad!\" && exit 1"
|
||||
},
|
||||
"author": "It's not you it's me",
|
||||
"license": "ISC"
|
||||
}
|
||||
|
|
@ -1,123 +0,0 @@
|
|||
// For more information, see https://crawlee.dev/
|
||||
import {
|
||||
CheerioCrawler,
|
||||
createCheerioRouter,
|
||||
createPlaywrightRouter,
|
||||
enqueueLinks,
|
||||
PlaywrightCrawler,
|
||||
ProxyConfiguration,
|
||||
} from "crawlee";
|
||||
import { readFileSync } from "fs";
|
||||
|
||||
const proxyUrls = readFileSync("proxies.txt", "utf-8")
|
||||
.split(/\r?\n/)
|
||||
.filter((x) => x.trim().length > 0)
|
||||
.map((x) => {
|
||||
const [ip, port, username, password] = x.split(":");
|
||||
return `http://${username}:${password}@${ip}:${port}`;
|
||||
});
|
||||
console.log(proxyUrls);
|
||||
// const scrapoxyConfig = {
|
||||
// username: "asdf",
|
||||
// password: "asdf",
|
||||
// proxyUrl: "partido-obrero:8888",
|
||||
// apiUrl: "partido-obrero:8890",
|
||||
// };
|
||||
|
||||
const proxyConf = new ProxyConfiguration({
|
||||
proxyUrls: proxyUrls, // proxyUrls: Array(100).fill(
|
||||
// `http://${scrapoxyConfig.username}:${scrapoxyConfig.password}@${scrapoxyConfig.proxyUrl}/`
|
||||
// // "http://asdfasdf-rotate:asdfasdf@p.webshare.io"
|
||||
// ),
|
||||
});
|
||||
|
||||
const router = createCheerioRouter();
|
||||
router.addHandler("DETAIL", async ({ request, parseWithCheerio, pushData }) => {
|
||||
const $ = await parseWithCheerio();
|
||||
|
||||
const name = $("h1.product_page").text().trim();
|
||||
await pushData({ name, url: request.loadedUrl }, "products");
|
||||
});
|
||||
router.addHandler(
|
||||
"CATEGORY",
|
||||
async ({ request, enqueueLinks, log, pushData, parseWithCheerio }) => {
|
||||
// const title = await page.title();
|
||||
// log.info(`Title of ${request.loadedUrl} is '${title}'`);
|
||||
const $ = await parseWithCheerio();
|
||||
|
||||
await enqueueLinks({
|
||||
selector: 'a[href^="/sitios/cdigi/producto"]',
|
||||
label: "DETAIL",
|
||||
});
|
||||
|
||||
const productsEls = $("ul#products").children("li");
|
||||
|
||||
for (const el of productsEls) {
|
||||
const title = $(el)
|
||||
.find(".atg_store_productTitle .descrip_full")
|
||||
.text()
|
||||
.trim();
|
||||
const href = $(el).find('a[href^="/sitios/cdigi/producto"]');
|
||||
await pushData({ title, url: href.attr("href") }, "product-list");
|
||||
}
|
||||
// Save results as JSON to ./storage/datasets/default
|
||||
|
||||
// Extract links from the current page
|
||||
// and add them to the crawling queue.
|
||||
await enqueueLinks({
|
||||
selector: "[title=Siguiente]",
|
||||
label: "CATEGORY",
|
||||
});
|
||||
}
|
||||
);
|
||||
router.addDefaultHandler(async ({ enqueueLinks }) => {
|
||||
await enqueueLinks({
|
||||
urls: ["https://www.cotodigital3.com.ar/sitios/cdigi/browse"],
|
||||
label: "CATEGORY",
|
||||
});
|
||||
});
|
||||
|
||||
// PlaywrightCrawler crawls the web using a headless
|
||||
// browser controlled by the Playwright library.
|
||||
const crawler = new CheerioCrawler({
|
||||
proxyConfiguration: proxyConf,
|
||||
ignoreSslErrors: true,
|
||||
|
||||
useSessionPool: true,
|
||||
sessionPoolOptions: {
|
||||
blockedStatusCodes: [401, 403, 429, 500],
|
||||
},
|
||||
minConcurrency: 10,
|
||||
maxConcurrency: 50,
|
||||
maxRequestRetries: 50,
|
||||
requestHandlerTimeoutSecs: 30,
|
||||
requestHandler: router,
|
||||
// async errorHandler({ request, response, log }) {
|
||||
// if (!response || !("statusCode" in response)) {
|
||||
// log.error("Response has no statusCode", { response });
|
||||
// return;
|
||||
// }
|
||||
// if (response.statusCode === 557) {
|
||||
// log.warning("No proxy available, waiting");
|
||||
// await new Promise((resolve) => setTimeout(resolve, 30 * 1000));
|
||||
// return;
|
||||
// }
|
||||
// const proxyName = response.headers["x-scrapoxy-proxyname"];
|
||||
// log.warning(`Resetting proxy`, { proxyName, headers: response.headers });
|
||||
// const res = await fetch(
|
||||
// `http://${scrapoxyConfig.apiUrl}/api/scraper/project/proxies/remove`,
|
||||
// {
|
||||
// method: "POST",
|
||||
// body: JSON.stringify([{ id: proxyName, force: false }]),
|
||||
// headers: {
|
||||
// Authorization: `Basic ${btoa(`${scrapoxyConfig.username}:${scrapoxyConfig.password}`)}`,
|
||||
// "Content-Type": "application/json",
|
||||
// },
|
||||
// }
|
||||
// );
|
||||
// if (!res.ok)
|
||||
// log.error(`status code ${res.status}`, { json: await res.json() });
|
||||
// },
|
||||
});
|
||||
|
||||
await crawler.run(["https://www.cotodigital3.com.ar/"]);
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
{
|
||||
"extends": "@apify/tsconfig",
|
||||
"compilerOptions": {
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"target": "ES2022",
|
||||
"outDir": "dist",
|
||||
"noUnusedLocals": false,
|
||||
"lib": ["DOM"]
|
||||
},
|
||||
"include": ["./src/**/*"]
|
||||
}
|
||||
Loading…
Reference in a new issue