2021-11-30 23:46:26 +00:00
|
|
|
#!/bin/sh -e
|
|
|
|
|
2021-12-01 16:06:10 +00:00
|
|
|
if ! type wget > /dev/null; then
|
|
|
|
echo "Te falta instalar wget."
|
2021-11-30 23:46:26 +00:00
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
if ! type curl > /dev/null; then
|
|
|
|
echo "Te falta instalar curl."
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
if ! type lua > /dev/null; then
|
|
|
|
echo "Te falta instalar lua."
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
if ! type zstd > /dev/null; then
|
|
|
|
echo "Te falta instalar zstd."
|
|
|
|
exit 1
|
|
|
|
fi
|
|
|
|
|
|
|
|
echo "¡Gracias por contribuir!"
|
|
|
|
|
2021-12-01 16:06:10 +00:00
|
|
|
if test -f lista-paginas.zst; then
|
2021-11-30 23:46:26 +00:00
|
|
|
echo "Voy a descargar las páginas en lista-paginas.zst."
|
|
|
|
echo "> Descomprimiendo lista..."
|
|
|
|
zstd -f -d lista-paginas.zst
|
2021-11-30 23:57:40 +00:00
|
|
|
echo "Son $(cat lista-paginas | wc -l) páginas."
|
2021-12-01 16:06:10 +00:00
|
|
|
echo "> Separando en varias listas..."
|
2021-12-01 16:11:35 +00:00
|
|
|
rm -f TMP-lista-shard-*
|
2021-12-01 16:16:23 +00:00
|
|
|
lua otros-scripts/shard.lua 64 < ./lista-paginas
|
2021-12-01 16:16:51 +00:00
|
|
|
|
2021-12-01 16:06:10 +00:00
|
|
|
echo "> Empezando a descargar..."
|
2021-12-01 16:16:51 +00:00
|
|
|
mkdir -p datos
|
|
|
|
cd datos/
|
2021-12-01 16:18:23 +00:00
|
|
|
for lista in ../TMP-lista-shard-*; do
|
2021-12-01 16:34:45 +00:00
|
|
|
wget --continue --header='accept-encoding: br' --no-verbose --no-clobber --input-file="$lista" --restrict-file-names=nocontrol &
|
2021-12-01 16:06:10 +00:00
|
|
|
done
|
2021-11-30 23:46:26 +00:00
|
|
|
else
|
|
|
|
echo "No tengo ninguna lista para descargar. ¡Chau!"
|
|
|
|
fi
|