24 lines
977 B
Bash
24 lines
977 B
Bash
#!/bin/bash
|
|
###
|
|
# @Author: s-JoL(sl12160010@gmail.com)
|
|
# @Date: 2023-03-16 21:21:56
|
|
# @LastEditors: s-JoL(sl12160010@gmail.com)
|
|
# @LastEditTime: 2023-03-26 22:58:11
|
|
# @FilePath: /Open-Llama/data/download_wudao.sh
|
|
# @Description:
|
|
# download wudao dataset and preprocess
|
|
# Copyright (c) 2023 by s-JoL(sl12160010@gmail.com), All Rights Reserved.
|
|
###
|
|
apt-add-repository multiverse
|
|
apt install unrar
|
|
|
|
wget -v -c 'https://download.scidb.cn/download?fileId=63a30383fed6a8a9e8454302&dataSetType=organization&fileName=WuDaoCorporaText-2.0-open.rar' -O data/WuDaoCorpus2.0_base_200G.rar
|
|
|
|
# for i in {1..100}
|
|
# do
|
|
# curl -C - --retry 100 'https://dorc.baai.ac.cn/resources/data/WuDaoCorpora2.0/WuDaoCorpus2.0_base_200G.rar?AccessKeyId=AKLTNasiLRBBTcOgPqzlkPzu1w&Expires=1679127659&Signature=7jh%2FpnJyC2hAeumm9EjaeE5HN9E%3D' -o data/WuDaoCorpus2.0_base_200G.rar
|
|
# done
|
|
|
|
unrar x data/WuDaoCorpus2.0_base_200G.rar data/
|
|
mkdir data/pretrain_data
|
|
python3 data/preprocess_wudao.py |