gnh1201/Open-Llama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update instruct_config and set all random seed to 42

Browse Source
This commit is contained in:
LiangSong 2023-05-04 08:45:21 +08:00
parent dba2e2d680
commit f0d41f937b
4 changed files with 12 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
configs/instruct_config.yaml
View File

@ -16,7 +16,8 @@ model:
shared_input_output_embedding: False
train:
train_batch_size: 2
num_training_steps: 40000
# 1B token for 1 epoch, 5epoch
num_training_steps: 20000
num_warmup_steps: 500
initializer_range: 1.0e-2
lr: 2.0e-4
@ -29,5 +30,5 @@ train:
log_interval: 50
eval_interval: 500
save_interval: 1000
work_dir: "data/saved_ckpt/7B"
work_dir: "data/saved_ckpt/7B_instruction"
project_name: "Llama Instruction"

5
dataset/dataset.py
View File

@ -2,7 +2,7 @@
Author: LiangSong(sl12160010@gmail.com)
Date: 2023-04-24 20:05:21
LastEditors: LiangSong(sl12160010@gmail.com)
LastEditTime: 2023-05-03 10:23:41
LastEditTime: 2023-05-04 08:42:58
FilePath: /Open-Llama/dataset/dataset.py
Description:
@ -12,9 +12,10 @@ import math
import torch
import random
from glob import glob
from datasets import load_dataset, interleave_datasets
from datasets import load_dataset
random.seed(42)
def pretrain_transform(batch):
# wudao preprocess
if "title" in batch and "content" in batch:

3
solver/trainer.py
View File

@ -2,13 +2,12 @@
Author: LiangSong(sl12160010@gmail.com)
Date: 2023-04-24 20:05:21
LastEditors: LiangSong(sl12160010@gmail.com)
LastEditTime: 2023-05-02 23:55:37
LastEditTime: 2023-05-04 08:41:37
FilePath: /Open-Llama/solver/trainer.py
Description:
Copyright (c) 2023 by LiangSong(sl12160010@gmail.com), All Rights Reserved.
"""
import os
import time
import wandb
import torch

7
utils/train_tokenizer.py
View File

@ -2,8 +2,8 @@
Author: LiangSong(sl12160010@gmail.com)
Date: 2023-03-24 20:49:03
LastEditors: LiangSong(sl12160010@gmail.com)
LastEditTime: 2023-04-05 22:40:29
FilePath: /Open-Llama/dataset/train_tokenizer.py
LastEditTime: 2023-05-04 08:42:21
FilePath: /Open-Llama/utils/train_tokenizer.py
Description:
Copyright (c) 2023 by LiangSong(sl12160010@gmail.com), All Rights Reserved.
@ -11,6 +11,9 @@ Copyright (c) 2023 by LiangSong(sl12160010@gmail.com), All Rights Reserved.
import random
from dataset.data_iter import DataIter, create_shard_kwargs
random.seed(42)
wudao_patterns = [
"data/pretrain_data/part-wudao-*.jsonl.zst",
]