Update data/download_the_pile.sh

2024-06-14 09:08:55 +00:00 · 2024-06-14 09:08:55 +00:00 · a24271d48e
commit a24271d48e
parent 5635f7d08d
1 changed files with 1 additions and 1 deletions
--- a/data/download_the_pile.sh
+++ b/data/download_the_pile.sh
@ -14,7 +14,7 @@ end=29
 mkdir data/the_pile
 for (( i=$start; i<=$end; i++ ))
 do
-    url="https://the-eye.eu/public/AI/pile/train/$(printf "%02d" $i).jsonl.zst"
+    url="https://huggingface.co/datasets/monology/pile-uncopyrighted/resolve/main/train/$(printf "%02d" $i).jsonl.zst?download=true"
    echo "Downloading file: $url"
    curl -C - $url -o data/the_pile/"$(printf "%02d" $i).jsonl.zst"
 done