Nulo/preciazo
1
0
Fork 0
mirror of https://github.com/catdevnull/preciazo.git synced 2025-02-22 14:16:26 +00:00
Code Issues Projects Releases Packages Wiki Activity

moar concurrency

Browse source
This commit is contained in:
Cat /dev/Nulo 2024-12-01 23:09:44 -03:00
parent f50f9df10b
commit e7bb9b6c29
1 changed files with 4 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
sepa/importer.js
View file

@ -18,10 +18,10 @@ import { writeFile } from "fs/promises";
// TODO: verificar que pasa cuando hay varios datasets del mismo día (como los suele haber cuando actualizan el dataset con nuevos comercios)
const instance = await DuckDBInstance.create("importer.db", {
threads: "1",
// threads: "1",
});
const queue = new PQueue({ concurrency: 1 });
const queue = new PQueue({ concurrency: 4 });
let hasTars = false;
const files = await fg("**/*.tar.zst", { cwd: process.argv[2] });
@ -219,8 +219,8 @@ async function importDataset(dir) {
const connection = await instance.connect();
try {
await connection.run("begin transaction");
try {
const res = await connection.run(
`insert into datasets (id, name, date, id_comercio) values (nextval('seq_datasets'), '${basename(dir)}', '${date}', ${id_comercio}) returning id`
);
@ -251,7 +251,7 @@ async function importDataset(dir) {
console.error("errored, aborting transaction", e);
await connection.run("abort");
} finally {
await connection.run("CHECKPOINT");
// await connection.run("CHECKPOINT");
try {
Bun.gc(true);
} catch {}