gnh1201/Open-Llama
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update dataset

Browse Source
This commit is contained in:
LiangSong 2023-05-05 19:23:16 +08:00
parent 758af69c73
commit f893a0f5b8
1 changed files with 12 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
dataset/dataset.py
View File

@ -99,6 +99,18 @@ def instruct_transform(batch):
targets = batch["targets"][0] targets = batch["targets"][0]
text = "user:{}\nsystem:{}".format(inputs.strip(), targets.strip()) text = "user:{}\nsystem:{}".format(inputs.strip(), targets.strip())
texts = [text] texts = [text]
# camel-ai preprocess
elif "message_1" in batch and "message_2" in batch:
inputs = batch["message_1"][0]
targets = batch["message_2"][0]
text = "user:{}\nsystem:{}".format(inputs.strip(), targets.strip())
texts = [text]
# grade-school-math-instructions preprocess
elif "INSTRUCTION" in batch and "RESPONSE" in batch:
inputs = batch["INSTRUCTION"][0]
targets = batch["RESPONSE"][0]
text = "user:{}\nsystem:{}".format(inputs.strip(), targets.strip())
texts = [text]
else: else:
raise Exception("Unrecognized instruct dataset format.") raise Exception("Unrecognized instruct dataset format.")
return {"text": texts} return {"text": texts}