Nulo/tofufirme
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

rust: clean

Browse source
This commit is contained in:
Cat /dev/Nulo 2023-04-23 21:13:20 -03:00
parent 63ebf48da6
commit 92a9ab8340
1 changed files with 0 additions and 93 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

93
rust/src/main.rs
View file

@ -20,9 +20,6 @@ fn main() -> Result<(), Box<dyn Error>> {
make_db_schema(&conn)?; make_db_schema(&conn)?;
let input = Path::new("./input"); let input = Path::new("./input");
// let (dict, dict_id) = make_dict(conn, input)?;
// let mut enc = zstd::bulk::Compressor::new(5)?;
let paths = iterator(input).collect::<Vec<PathBuf>>(); let paths = iterator(input).collect::<Vec<PathBuf>>();
let chunks = paths.chunks(paths.len() / 4); let chunks = paths.chunks(paths.len() / 4);
@ -37,13 +34,6 @@ fn main() -> Result<(), Box<dyn Error>> {
thread::spawn(move || { thread::spawn(move || {
let input = Path::new("./input"); let input = Path::new("./input");
let params = file_compress_params(); let params = file_compress_params();
// let mut conn = make_conn().unwrap();
println!("thread");
// let trans = conn.transaction().unwrap();
// {
// let mut insert_stmt = trans
// .prepare_cached("insert into files(path, content, compressed) values(?, ?, ?)")
// .unwrap();
for path in chunkk { for path in chunkk {
compress_file(&path, input, &params, &ttx); compress_file(&path, input, &params, &ttx);
} }
@ -70,67 +60,9 @@ fn main() -> Result<(), Box<dyn Error>> {
for thread in threads { for thread in threads {
thread.join().unwrap(); thread.join().unwrap();
} }
// .map(|path| -> Option<()> {
// insert_file(path, input, params, insert_stmt);
// Some(())
// })
// .collect::<Vec<Option<()>>>();
// if iter.iter().any(|o| o.is_none()) {
// println!("Algo falló");
// }
// recurse(dict, conn)?;
println!("Hello, world!");
Ok(()) Ok(())
} }
// fn insert_files(paths: &[PathBuf]) {
// let input = Path::new("./input");
// let params = file_compress_params();
// // let mut conn = make_conn().unwrap();
// println!("thread");
// // let trans = conn.transaction().unwrap();
// // {
// // let mut insert_stmt = trans
// // .prepare_cached("insert into files(path, content, compressed) values(?, ?, ?)")
// // .unwrap();
// for path in paths {
// compress_file(path, input, &params, tx);
// }
// // insert_stmt.execute((path_str, content, false)).unwrap();
// // }
// // trans.commit().unwrap();
// }
// fn insert_file(
// path: &Path,
// input: &Path,
// params: &BrotliEncoderParams,
// insert_stmt: &mut rusqlite::CachedStatement,
// ) {
// // let mut path_comp = zstd::bulk::Compressor::with_dictionary(0, &dict).unwrap();
// // println!("{:?}", path);
// // let path_compressed = path_comp
// // .compress(strip_input(&path, input).to_string_lossy().as_bytes())
// // .unwrap();
// let path_strip = strip_input(&path, input);
// let path_str = path_strip.to_string_lossy();
// if compressable(path) {
// let mut content_compressed = Vec::new();
// let mut file = File::open(path).unwrap();
// brotli::BrotliCompress(&mut file, &mut content_compressed, &params).unwrap();
// // let content_compressed = enc.compress(&content)?;
// insert_stmt
// .execute((path_str, content_compressed, true))
// .unwrap();
// } else {
// let content = read(path.clone()).unwrap();
// insert_stmt.execute((path_str, content, false)).unwrap();
// }
// }
fn make_conn() -> Result<Connection, Box<dyn Error>> { fn make_conn() -> Result<Connection, Box<dyn Error>> {
let conn = Connection::open(db_path())?; let conn = Connection::open(db_path())?;
conn.pragma_update(None, "journal_mode", "OFF")?; conn.pragma_update(None, "journal_mode", "OFF")?;
@ -147,15 +79,10 @@ fn compress_file(path: &Path, input: &Path, params: &BrotliEncoderParams, tx: &S
let mut content_compressed = Vec::new(); let mut content_compressed = Vec::new();
let mut file = File::open(path).unwrap(); let mut file = File::open(path).unwrap();
brotli::BrotliCompress(&mut file, &mut content_compressed, &params).unwrap(); brotli::BrotliCompress(&mut file, &mut content_compressed, &params).unwrap();
// let content_compressed = enc.compress(&content)?;
// insert_stmt
// .execute((path_str, content_compressed, true))
// .unwrap();
tx.send((path_str.to_string(), content_compressed, true)) tx.send((path_str.to_string(), content_compressed, true))
.unwrap(); .unwrap();
} else { } else {
let content = read(path.clone()).unwrap(); let content = read(path.clone()).unwrap();
// insert_stmt.execute((path_str, content, false)).unwrap();
tx.send((path_str.to_string(), content, false)).unwrap(); tx.send((path_str.to_string(), content, false)).unwrap();
} }
} }
@ -165,30 +92,10 @@ fn make_db_schema(conn: &Connection) -> Result<(), Box<dyn Error>> {
"create table files(path text, content blob, compressed bool)", "create table files(path text, content blob, compressed bool)",
[], [],
)?; )?;
// conn.execute(
// "create table files(path blob, path_dictionary_id integer, content blob, compressed bool)",
// [],
// )?;
// conn.execute("create table path_dictionaries(dictionary blob)", [])?;
conn.execute("create unique index path on files(path)", [])?; conn.execute("create unique index path on files(path)", [])?;
Ok(()) Ok(())
} }
// fn make_dict(conn: Connection, input: &Path) -> Result<(Vec<u8>, i64), Box<dyn Error>> {
// let mut insert_dict_stmt =
// conn.prepare_cached("insert into path_dictionaries(dictionary) values(?1)")?;
// let mut all_paths = Vec::new();
// let iter = iterator(input);
// for entry in iter {
// all_paths.push(entry.to_string_lossy().as_bytes().to_vec());
// }
// println!("path n {}", all_paths.len());
// let dict = zstd::dict::from_samples(&all_paths, 999999)?;
// println!("dict");
// let dict_id = insert_dict_stmt.insert(params![&dict])?;
// Ok((dict, dict_id))
// }
fn file_compress_params() -> BrotliEncoderParams { fn file_compress_params() -> BrotliEncoderParams {
let mut params = BrotliEncoderParams::default(); let mut params = BrotliEncoderParams::default();
params.quality = 0; params.quality = 0;