
Unslothã§ç¶ç¶äºååŠç¿ããã£ãŠã¿ã
ã¯ããã«
Unslothãšããã©ã€ãã©ãªã䜿ã£ãŠç¶ç¶äºååŠç¿ããã£ãŠã¿ãŸãããé«éãã¥ãŒãã³ã°ã«è¡ãããšããããšãªã®ã§ã以åãããã©ã€ããŠã¿ãããšæã£ãŠããŸããã
ä»åã¯ãç¶ç¶äºååŠç¿ã§æ°ããç¥èãåã蟌ãããã©ããã確èªããŠã¿ãããšã«ããŸããã
çµè«ããèšãã°ãæ°ããç¥èãåã蟌ãããšããçµæãåŸãããšã¯ã§ããŸããã§ããã
䜿çšããã¢ãã«
䜿çšããã¢ãã«ã¯Llama3.2 1Bã䜿çšããŸãããæ°ããç¥èãåã蟌ãã«ã¯ã¢ãã«ãå°ãããããšããããšãããããŸããã
å®è£
æ©éå®è£ ãèŠãŠããããšã«ããŸãã
æºå
å¿ èŠã©ã€ãã©ãªãšãã©ã¡ãŒã¿ãããã€ãå ã«æ±ºããŠãããŸãããŸããã¢ãã«ãšããŒã¯ãã€ã¶ãŒããã®æç¹ã§èªã¿èŸŒãã§ãããŸãã
import os
from unsloth import FastLanguageModel
from unsloth import is_bfloat16_supported
from unsloth import UnslothTrainer, UnslothTrainingArguments
from datasets import load_dataset
import wandb
os.environ["WANDB_PROJECT"] = "llama3.2-continued-pretraining"
# åºæ¬èšå®
max_seq_length = 512
dtype = None
load_in_4bit = True
num_proc = 4
pretrain_output = 'pretrain_output'
random_seed = 3407
# ã¢ãã«ãšããŒã¯ãã€ã¶ãŒã®èªã¿èŸŒã¿
model, tokenizer = FastLanguageModel.from_pretrained(
model_name = "meta-llama/Llama-3.2-1B",
max_seq_length = max_seq_length,
dtype = dtype,
load_in_4bit = load_in_4bit,
)4bitã®QLoRAã§åŠç¿ãå®è¡ããèšå®ã«ãªã£ãŠããŸãããŸããåçŸæ§ãé«ããããã«ã©ã³ãã ã·ãŒããèšå®ããŸãã
åŠç¿æã®æå€§ããŒã¯ã³ãµã€ãºã¯512ãšå°ããå€ã«ããŠããŸããåŠç¿ã«äœ¿çšãããã·ã³ã¹ããã¯ã®åé¡ã§å°ããå€ã«ãªã£ãŠããŸããéåžžã«éããããªãœãŒã¹ã§å®è¡ããŠããŸãã
LoRAã®èšå®
ç¶ç¶äºååŠç¿ãè¡ãããã®ã§ãlm_headãšembed_tokensãLoRAã®ã¿ãŒã²ããã«è¿œå ãããŸãã
model = FastLanguageModel.get_peft_model(
model,
r=32,
target_modules = [
"q_proj", "k_proj", "v_proj", "o_proj",
"gate_proj", "up_proj", "down_proj",
"embed_tokens", "lm_head",
],
lora_alpha=16,
lora_dropout=0,
bias="none",
use_gradient_checkpointing="unsloth",
random_state=random_seed,
use_rslora=True
)Unslothã®ãµã³ãã«notebookã«åŸããšãdropoutã¯0ã§biasã¯noneã«ããã®ãæé©ãšã®ããšã§ãã
r ã alphaã¯ãã£ãšå€§ããªå€ã®æ¹ãè¯ãã£ããããããªãã§ãã
ããŒã¿ã»ããã®èªã¿èŸŒã¿
ããŒã¿ã»ããã®èªã¿èŸŒã¿ã¯ä»¥äžã®ããã«è¡ããŸãã
dataset = load_dataset("kajuma/CC-news-2024-July-October-cleaned", split = "train",)
dataset = dataset.train_test_split(train_size=0.8)["train"]
dataset[0]䜿çšããããŒã¿ã»ããã¯ä»¥äžã䜿çšãããŠããã ããŸããã
ç¥èãæ£ãã远å ã§ããŠãããã©ããã倿ããããã«ãå€å®çšã®ããã¹ãã1ã€è¿œå ããŸããã
dataset = dataset.add_item({
'docId': 'b0b8df9a-6c3c-4331-81f8-bf01fbfb9d1f',
'url': '',
'charset': 'EUC-JP',
'date': '2025-01-10T09:00:00',
'language': 'ja',
'text': open('dataset/knowledges.txt', 'r').read()
})
dataset[-1]å€å®çšã®ããã¹ã
2020幎以éã®æµè¡èªå€§è³ãæ¯ãè¿ããšãæ¥æ¬ç€ŸäŒã®éèŠãªåºæ¥äºã話é¡ãåæ ãããŠããŸãïŒ
2020幎ã®ã3å¯ãã¯ãæ°åã³ãããŠã€ã«ã¹ææç察çãšããŠå°æ± çŸååæ±äº¬éœç¥äºãæå±ãããå¯éã»å¯éã»å¯æ¥ããé¿ãããšããæŠå¿µã衚ãããã³ãããã¯åæã®æ¥æ¬ç€ŸäŒã象城ããèšèãšãªããŸããã
2021幎ã¯ãã¡ãžã£ãŒãªãŒã°ã§ææã®ãäºåæµããšããŠæŽå²çãªæŽ»èºãèŠãã倧谷ç¿å¹³éžæã®ããªã¢ã«äºåæµ/ã·ã§ãŒã¿ã€ã ããéžã°ããæ¥æ¬äººéžæã®äžçã§ã®èºé²ã衚ããŸããã
2022幎ã®ãæç¥æ§ãã¯ãããéçã§å²äžåãšãªã髿 ¡çãã©ãã1äœãã4幎é£ç¶æ¬å¡æçãéæããæäžå®ééžæã®ããã¯ããŒã ã§ãè¥ã倩ææè
ã®å°é ã瀺ããŠããŸãã
2023幎ã®ãã¢ã¬ïŒA.R.EïŒãã¯ãéªç¥ã¿ã€ã¬ãŒã¹ã®å²¡ç°åœ°åžç£ç£ãé »ç¹ã«äœ¿çšããèšèã§ã44幎ã¶ãã®ãªãŒã°åªåãæãããéªç¥ã®å¿«é²æã象城ããŸããã
2024幎ã¯ãTBSãã©ããäžé©åã«ãã»ã©ããã!ãããçãŸããããµãŠã»ã©ããéžã°ããå³ãã瀟äŒèŠç¯ã®äžã§çããçŸä»£äººã®å
±æãåŒãã ããšã瀺ããŠããŸãã
ãã®5幎éã®æµè¡èªã¯ãææç察çãããã¹ããŒãçã®æŽ»èºããããŠæä»£ãåæ ãããã©ããŸã§ããã®æã
ã®æ¥æ¬ç€ŸäŒã®é¢å¿äºã端çã«è¡šçŸããŠããŸããããŒã¿ã®æŽåœ¢
ããŒã¿ã®æŽåœ¢ãšãã£ãŠãåŠç¿ã«æ£ãã䜿ããããã«ããŒåã倿Žããã ãã§ãã
news_prompt = """
{}
"""
EOS_TOKEN = tokenizer.eos_token # Must add EOS_TOKEN
def formatting_prompts_func(examples):
charsets = examples["charset"]
texts = examples["text"]
outputs = []
for charset,text in zip(charsets, texts):
# Must add EOS_TOKEN, otherwise your generation will go on forever!
text = news_prompt.format(text) + EOS_TOKEN
outputs.append(text)
return { "text" : outputs, }
# ãã©ãŒããã颿°ãé©çšããŠããŒã¿ãæŽåœ¢
dataset = dataset.map(formatting_prompts_func, batched = True,)åŠç¿èšå®
trainer = UnslothTrainer(
model = model, # ãã¬ãŒãã³ã°å¯Ÿè±¡ã®ã¢ãã«
tokenizer = tokenizer, # ã¢ãã«çšããŒã¯ãã€ã¶ãŒ
train_dataset = dataset, # ãã¬ãŒãã³ã°ããŒã¿ã»ãã
dataset_text_field = "text", # ããŒã¿ã»ããå
ã®ããã¹ããã£ãŒã«ãå
max_seq_length = max_seq_length, # æå€§ã·ãŒã±ã³ã¹é·
dataset_num_proc = num_proc, # ããŒã¿ã»ããåŠçã«äœ¿çšããããã»ã¹æ°
neftune_noise_alpha=5,
packing=False,
args = UnslothTrainingArguments(
per_device_train_batch_size = 8, # åããã€ã¹ããšã®ããããµã€ãº
gradient_accumulation_steps = 1, # åŸé
ã®çޝç©ã¹ãããæ°
# é·æéã®ãã¬ãŒãã³ã°ã«äœ¿çšå¯èœãªèšå®
num_train_epochs = 1,
# åã蟌ã¿è¡åã«ã¯éåžžãã2ïœ10åå°ããåŠç¿çãéžæ
learning_rate = 5e-5, # å
šäœã®åŠç¿ç
embedding_learning_rate = 1e-5, # åã蟌ã¿å±€ã®åŠç¿ç
fp16 = not is_bfloat16_supported(), # FP16ã䜿çšïŒbfloat16ããµããŒããããŠããªãå ŽåïŒ
bf16 = is_bfloat16_supported(), # bfloat16ã䜿çšïŒãµããŒããããŠããå ŽåïŒ
logging_steps = 10, # ãã°ãèšé²ããã¹ãããéé
save_total_limit = 2,
optim="adamw_8bit",
lr_scheduler_type="constant",
seed = random_seed, # åçŸæ§ã®ããã®ä¹±æ°ã·ãŒã
output_dir = pretrain_output, # åºåãã£ã¬ã¯ããª
report_to = "wandb", # ãã°åºåå
ïŒäŸ: "wandb"ãªã©ãæå®å¯èœïŒ
),
)åŠç¿çã®ã¹ã±ãžã¥ãŒã©ã¯constantã§å®è¡ããŸãããlinearãªã©ã®ã¹ã±ãžã¥ãŒã©ã§åŠç¿çãæžè¡°ããæ¹ãåŠç¿æã®ç Žç¶»ã®ãªã¹ã¯ãå°ãªããšæããŸãããã¢ãã«ãããŒã¿ãèŠæš¡ã¯å°ããã®ã§ãæå€±ãæ©ãæžãæ¹æ³ããšããŸããã
åŠç¿ã®å®è¡
trainer_stats = trainer.train()
wandb.finish()åŠç¿ã®å®è¡ã¯ããã ããwandb.finish()ãå®è¡ããªããšwandbãæ£ããçµäºããªãã
ã¢ãã«ä¿å
model.save_pretrained_merged("model", tokenizer, save_method = "merged_16bit",)
del modelLoRAã§åŠç¿ããçµæãAdapterãšããŠä¿åããã®ã§ã¯ãªããã¢ãã«æ¬äœã«ããŒãžããŠä¿åããŸãã
çµæã®ç¢ºèª
以äžã®ã³ãŒãã§çµæã確èªããŸãã
import os
from unsloth import FastLanguageModel
from transformers import TextStreamer
# åºæ¬èšå®
max_seq_length = 2048
dtype = None
load_in_4bit = False
num_proc = 4
pretrain_output = 'pretrain_output'
sft_output = 'sft_output'
random_seed = 3407
model, tokenizer = FastLanguageModel.from_pretrained(
model_name = './model',
max_seq_length = max_seq_length,
dtype = dtype,
load_in_4bit = load_in_4bit,
)
inputs = tokenizer(
[
"2020幎以éã®æµè¡èªå€§è³ãæ¯ãè¿ããšã"
]*1, return_tensors = "pt").to("cuda")
FastLanguageModel.for_inference(model) # Enable native 2x faster inference
text_streamer = TextStreamer(tokenizer)
_ = model.generate(**inputs, streamer = text_streamer,
max_new_tokens = 512,
use_cache=True,
do_sample=False,
repetition_penalty=1.2)åºå
<|begin_of_text|>2020幎以éã®æµè¡èªå€§è³ãæ¯ãè¿ããšãäžäœã«ãã¢ãŒã«ã€ãããéžã°ããããã£ãããªãããšã
## ãã¢ãŒã«ã€ãããšã¯ã©ãããããšïŒ
ããšããšã¯ããèšæ¶ãç¥èŠããäºå®ãžã®ä¿¡ãæ¹ãããããããããšãæãèšèã§ããããçŸåšã§ã¯ãã®æçŸ©ãåºããŠããŸããã
äŸãã°ãããæ¥ããã人ã«ãä»ãŸã§ã®ããããããšããã®ããã«ããŠããããšè©±ãå§ãããããããã®äººã®çºèšã«ã¯ã©ãã ãã®èšŒæ ãããã®ããšãããã®ãããããŸããã
ããšãããããã¢ãŒã«ã€ããã®ããã§ãããŸãäŒããããšã¯ã§ããŸãããããã匷åãªè¡šçŸãšããŠäœ¿ãããããšããããŸããã
ã¡ãªã¿ã«ãã¢ãŒã«ã€ãã®åã«ãã¬ããŒããã調æ»ããªã©ãçšããããã®ã¯ãåè
ãæ
å ±ã®åéã»ãŸãšããäœæ¥ã§åŸè
ã®ã»ããæ€èšãè©äŸ¡ãè¡ãããã§ãã
ãŸããã€ãŸãã¢ãŒã«ã€ãã¯ãäºåã®äºæ³ãšåèŽããªãçµæãåºãªãå Žåã«ããäºåŸã«äºæãã¬äºæ
ã«ãªã£ãŠããå Žåã¯ãäºåŸã®å¯ŸåŠæ³ã決ããŠããã°ãããšãããŠããã®ã§ãããããããã¢ãŒã«ã€ãã£ãŠã®ããšãªã®ã§ããããïŒ
ïŒåºå
žïŒãæ¥æ¬çµæžãããã¯ã¹ãïŒ
ãã¢ãŒã«ã€ãã
ãã¢ãŒããã¯ãã£ãŒã
ãããã¯ãããšããšå¥ã®æå³ã§äœ¿ãããŠããã®ã§ãããè¿å¹Žã®æŽ»çšãå¢ããŠããŸããããã
ãã¢ãŒã«ã€ããã¯ãããšããšãèšé²ã®é·ããã«ã€ããŠã®ããšã ã£ãã®ã§ãããç¹ã«ãã¢ãŒã«ã€ãã®æŽå²ãã«é¢é£ããŠããã®ã§ã¯ãªãããšèãã人ãããã®ã§ãã
ãããããšã次ã®ããã«è§£éã§ããã®ã§ãããã¢ãŒã«ã€ãã®æŽå²ãã¯ãããšããšãèšé²ã®é·ããã§ã¯ãªãããããã¯ãèšè¿°ã®å€æ§æ§ãã§ãããšèããããŸãã
ãã¢ãŒã«ã€ãã®æŽå²ãã¯ãããšããšãèšè¿°ã®å€æ§æ§ãã§ãããšèããããŠããã®ãšåäžã®ç©äœã®ãã¡ã®ã²ãšã€ã®åéã®ããšã§ãããããšããšãèšè¿°ã®å€æ§æ§ããç¶ããŠãããšããã