gnh1201/Open-Llama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Update data/download_the_pile.sh

Browse Source
This commit is contained in:
Namhyeon Go 2024-06-14 09:08:55 +00:00
parent 5635f7d08d
commit a24271d48e
1 changed files with 1 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
data/download_the_pile.sh
View File

@ -14,7 +14,7 @@ end=29
mkdir data/the_pile mkdir data/the_pile
for (( i=$start; i<=$end; i++ )) for (( i=$start; i<=$end; i++ ))
do do
url="https://the-eye.eu/public/AI/pile/train/$(printf "%02d" $i).jsonl.zst" url="https://huggingface.co/datasets/monology/pile-uncopyrighted/resolve/main/train/$(printf "%02d" $i).jsonl.zst?download=true"
echo "Downloading file: $url" echo "Downloading file: $url"
curl -C - $url -o data/the_pile/"$(printf "%02d" $i).jsonl.zst" curl -C - $url -o data/the_pile/"$(printf "%02d" $i).jsonl.zst"
done done