gnh1201/Open-Llama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update instruct_config and set all random seed to 42

Browse Source
This commit is contained in:
LiangSong 2023-05-04 08:45:21 +08:00
parent dba2e2d680
commit f0d41f937b
4 changed files with 12 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
configs/instruct_config.yaml
View File

@ -16,7 +16,8 @@ model:
shared_input_output_embedding: False shared_input_output_embedding: False
train: train:
train_batch_size: 2 train_batch_size: 2
num_training_steps: 40000 # 1B token for 1 epoch, 5epoch
num_training_steps: 20000
num_warmup_steps: 500 num_warmup_steps: 500
initializer_range: 1.0e-2 initializer_range: 1.0e-2
lr: 2.0e-4 lr: 2.0e-4
@ -29,5 +30,5 @@ train:
log_interval: 50 log_interval: 50
eval_interval: 500 eval_interval: 500
save_interval: 1000 save_interval: 1000
work_dir: "data/saved_ckpt/7B" work_dir: "data/saved_ckpt/7B_instruction"
project_name: "Llama Instruction" project_name: "Llama Instruction"

5
dataset/dataset.py
View File

@ -2,7 +2,7 @@
Author: LiangSong(sl12160010@gmail.com) Author: LiangSong(sl12160010@gmail.com)
Date: 2023-04-24 20:05:21 Date: 2023-04-24 20:05:21
LastEditors: LiangSong(sl12160010@gmail.com) LastEditors: LiangSong(sl12160010@gmail.com)
LastEditTime: 2023-05-03 10:23:41 LastEditTime: 2023-05-04 08:42:58
FilePath: /Open-Llama/dataset/dataset.py FilePath: /Open-Llama/dataset/dataset.py
Description: Description:
@ -12,9 +12,10 @@ import math
import torch import torch
import random import random
from glob import glob from glob import glob
from datasets import load_dataset, interleave_datasets from datasets import load_dataset
random.seed(42)
def pretrain_transform(batch): def pretrain_transform(batch):
# wudao preprocess # wudao preprocess
if "title" in batch and "content" in batch: if "title" in batch and "content" in batch:

3
solver/trainer.py
View File

@ -2,13 +2,12 @@
Author: LiangSong(sl12160010@gmail.com) Author: LiangSong(sl12160010@gmail.com)
Date: 2023-04-24 20:05:21 Date: 2023-04-24 20:05:21
LastEditors: LiangSong(sl12160010@gmail.com) LastEditors: LiangSong(sl12160010@gmail.com)
LastEditTime: 2023-05-02 23:55:37 LastEditTime: 2023-05-04 08:41:37
FilePath: /Open-Llama/solver/trainer.py FilePath: /Open-Llama/solver/trainer.py
Description: Description:
Copyright (c) 2023 by LiangSong(sl12160010@gmail.com), All Rights Reserved. Copyright (c) 2023 by LiangSong(sl12160010@gmail.com), All Rights Reserved.
""" """
import os
import time import time
import wandb import wandb
import torch import torch

7
utils/train_tokenizer.py
View File

@ -2,8 +2,8 @@
Author: LiangSong(sl12160010@gmail.com) Author: LiangSong(sl12160010@gmail.com)
Date: 2023-03-24 20:49:03 Date: 2023-03-24 20:49:03
LastEditors: LiangSong(sl12160010@gmail.com) LastEditors: LiangSong(sl12160010@gmail.com)
LastEditTime: 2023-04-05 22:40:29 LastEditTime: 2023-05-04 08:42:21
FilePath: /Open-Llama/dataset/train_tokenizer.py FilePath: /Open-Llama/utils/train_tokenizer.py
Description: Description:
Copyright (c) 2023 by LiangSong(sl12160010@gmail.com), All Rights Reserved. Copyright (c) 2023 by LiangSong(sl12160010@gmail.com), All Rights Reserved.
@ -11,6 +11,9 @@ Copyright (c) 2023 by LiangSong(sl12160010@gmail.com), All Rights Reserved.
import random import random
from dataset.data_iter import DataIter, create_shard_kwargs from dataset.data_iter import DataIter, create_shard_kwargs
random.seed(42)
wudao_patterns = [ wudao_patterns = [
"data/pretrain_data/part-wudao-*.jsonl.zst", "data/pretrain_data/part-wudao-*.jsonl.zst",
] ]