Fix huggingface download error

This commit is contained in:
Namhyeon Go 2024-08-18 06:46:44 +00:00
parent 9b8fe37cd1
commit 87f75f2dfe

View File

@ -16,7 +16,8 @@ for (( i=$start; i<=$end; i++ ))
do
url="https://huggingface.co/datasets/monology/pile-uncopyrighted/resolve/main/train/$(printf "%02d" $i).jsonl.zst?download=true"
echo "Downloading file: $url"
curl -C - $url -o data/the_pile/"$(printf "%02d" $i).jsonl.zst"
#curl -C - $url -o data/the_pile/"$(printf "%02d" $i).jsonl.zst"
curl -L $url -o data/the_pile/"$(printf "%02d" $i).jsonl.zst"
done
wait