gnh1201/Open-Llama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix huggingface download error

Browse Source
This commit is contained in:
Namhyeon Go 2024-08-18 06:46:44 +00:00
parent 9b8fe37cd1
commit 87f75f2dfe
1 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
data/download_the_pile.sh
View File

@ -16,7 +16,8 @@ for (( i=$start; i<=$end; i++ ))
do
url="https://huggingface.co/datasets/monology/pile-uncopyrighted/resolve/main/train/$(printf "%02d" $i).jsonl.zst?download=true"
echo "Downloading file: $url"
curl -C - $url -o data/the_pile/"$(printf "%02d" $i).jsonl.zst"
#curl -C - $url -o data/the_pile/"$(printf "%02d" $i).jsonl.zst"
curl -L $url -o data/the_pile/"$(printf "%02d" $i).jsonl.zst"
done
wait