인공지능 모형 만들기

FinBert 미세조정

finbert 불러오기

FinBert: Low-Rank Adaptation of Large Language Models

# 준비작업
import torch
import torch.nn as nn
import numpy as np
from transformers import BertTokenizer, BertForSequenceClassification

# device 설정
device = 'cuda' if torch.cuda.is_available() else 'cpu'
tokenizer = BertTokenizer.from_pretrained('yiyanghkust/finbert-tone')
n_tokens = len(tokenizer)

# LORA 모형 만들기
class MyNeuralNetwork(nn.Module):
    def __init__(self, n_tokens):
        super(MyNeuralNetwork, self).__init__()
        self.bert = BertForSequenceClassification.from_pretrained('yiyanghkust/finbert-tone', 
                      output_hidden_states=True, num_labels=3)
        self.output_dim = self.bert.config.hidden_size
        for param in self.bert.parameters():
            param.requires_grad = False  # BERT 파라미터 고정
        
        self.n_tokens = n_tokens
        self.layer1 = nn.Embedding(n_tokens, self.output_dim)
        self.layer2 = nn.Sequential(nn.Linear(2*self.output_dim, 10),
                                    nn.ReLU(),
                                    nn.Linear(10, 5))
        
    def forward(self, x):
        z1 = self.bert(**x)
        z1 = z1.hidden_states[-1][:,-1,:]
        z2 = self.layer1(x['input_ids']).mean(dim=1)
        z = torch.cat((z1,z2), dim = 1)
        z = self.layer2(z)
        return z

# 데이터 입력
news_articles = ["The stock market surged today as investors reacted positively to the latest earnings reports.",
"Inflation rates continue to climb, raising concerns about the purchasing power of consumers.",
"The Federal Reserve announced a plan to increase interest rates in an effort to curb inflation.",
"Tech companies are leading the charge in the market, with several hitting all-time highs.",
"Global supply chain disruptions are causing significant delays and increasing costs for manufacturers.",
"Unemployment rates have fallen to their lowest levels since the start of the pandemic.",
"The housing market remains hot, with home prices continuing to rise at a rapid pace.",
"Analysts predict a slowdown in economic growth due to ongoing geopolitical tensions.",
"The cryptocurrency market experienced a sharp decline following new regulatory announcements.",
"Consumer confidence is at an all-time high, buoyed by strong job growth and wage increases."]

n = len(news_articles)
target =  torch.tensor(np.random.randint(0, 5, size = n),dtype=torch.int64)

# 데이터 및 모형설정
inputs = tokenizer(news_articles, return_tensors='pt', 
                    padding=True, truncation=True).to(device)
target = target.to(device)
model = MyNeuralNetwork(n_tokens).to(device)
model(inputs).shape

# %%
learning_rate = 0.005
optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)
cross_entropy = nn.CrossEntropyLoss()

num_B = 100
loss_vec = np.zeros(num_B)
model.train()
val_vec=[]
for i in range(num_B):
    optimizer.zero_grad()
    output = model(inputs)
    loss = cross_entropy(output, target).to(device)
    loss.backward()
    optimizer.step()
    predicted = torch.argmax(output.data,1)
    if i%10 == 0:
        v =((predicted == target).sum()).item()/len(target)
        val_vec.append(v)
        print ("Accuracy: {:.4f}".format(np.array(val_vec).mean()))