From 87f75f2dfeb6bdab1f60e3b8d86f89e0466f36e9 Mon Sep 17 00:00:00 2001 From: Namhyeon Go Date: Sun, 18 Aug 2024 06:46:44 +0000 Subject: [PATCH] Fix huggingface download error --- data/download_the_pile.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/data/download_the_pile.sh b/data/download_the_pile.sh index 86d16d7..0a50465 100644 --- a/data/download_the_pile.sh +++ b/data/download_the_pile.sh @@ -16,7 +16,8 @@ for (( i=$start; i<=$end; i++ )) do url="https://huggingface.co/datasets/monology/pile-uncopyrighted/resolve/main/train/$(printf "%02d" $i).jsonl.zst?download=true" echo "Downloading file: $url" - curl -C - $url -o data/the_pile/"$(printf "%02d" $i).jsonl.zst" + #curl -C - $url -o data/the_pile/"$(printf "%02d" $i).jsonl.zst" + curl -L $url -o data/the_pile/"$(printf "%02d" $i).jsonl.zst" done wait