BOW 기반의 문서 분류 (8)

5.8 한국어 문서의 분류

5.8.2 성능을 개선하기 위한 노력

tfidf = TfidfVectorizer(tokenizer=okt.morphs, max_features=2000, min_df=5, max_df=0.5) # 명사 대신 모든 형태소를 사용
X_train_tfidf = tfidf.fit_transform(X_train)
X_test_tfidf = tfidf.transform(X_test)

clf = LogisticRegression(max_iter=1000) # 충분한 학습을 위해 max_iter를 1,000으로 설정, 기본은 100
clf.fit(X_train_tfidf, y_train)
print('#Train set score: {:.3f}'.format(clf.score(X_train_tfidf, y_train))) # train data 예측정확도
print('#Test set score: {:.3f}'.format(clf.score(X_test_tfidf, y_test))) # test data 예측정확도

"""
#Train set score: 0.777
#Test set score: 0.695
"""

def twit_tokenizer(text): #전체를 다 사용하는 대신, 명사, 동사, 형용사를 사용
    target_tags = ['Noun', 'Verb', 'Adjective']
    result = []
    for word, tag in okt.pos(text, norm=True, stem=True):
        if tag in target_tags:
            result.append(word)
    return result

tfidf = TfidfVectorizer(tokenizer=twit_tokenizer, max_features=2000, min_df=5, max_df=0.5) #명사, 동사, 형용사를 이용하여 tfidf 생성
X_train_tfidf = tfidf.fit_transform(X_train)
X_test_tfidf = tfidf.transform(X_test)

clf = LogisticRegression(max_iter=1000) # 충분한 학습을 위해 max_iter를 1,000으로 설정, 기본은 100
clf.fit(X_train_tfidf, y_train)
print('#Train set score: {:.3f}'.format(clf.score(X_train_tfidf, y_train))) # train data 예측정확도
print('#Test set score: {:.3f}'.format(clf.score(X_test_tfidf, y_test))) # test data 예측정확도

"""
#Train set score: 0.784
#Test set score: 0.712
"""

# 모든 형태소를 다 사용하고 품사를 알 수 있도록 하면?
def twit_tokenizer2(text):
    result = []
    for word, tag in okt.pos(text, norm=True, stem=True):
        result.append('/'.join([word, tag])) #단어의 품사를 구분할 수 있도록 함
    return result

print(twit_tokenizer2(X_train[1]))

## ['몰입/Noun', '하다/Verb', '없다/Adjective', './Punctuation', '어렵다/Adjective', '생각/Noun', '하다/Verb', '필요없다/Adjective', './Punctuation', '내/Noun', '가/Josa', '전투/Noun', '에/Josa', '참여/Noun', '한/Determiner', '듯/Noun', '손/Noun', '에/Josa', '땀/Noun', '이남/Noun', './Punctuation']

tfidf = TfidfVectorizer(tokenizer=twit_tokenizer2, max_features=2000, min_df=5, max_df=0.5)
X_train_tfidf = tfidf.fit_transform(X_train)
X_test_tfidf = tfidf.transform(X_test)

clf = LogisticRegression(max_iter=1000) # 충분한 학습을 위해 max_iter를 1,000으로 설정, 기본은 100
clf.fit(X_train_tfidf, y_train)
print('#Train set score: {:.3f}'.format(clf.score(X_train_tfidf, y_train))) # train data 예측정확도
print('#Test set score: {:.3f}'.format(clf.score(X_test_tfidf, y_test))) # test data 예측정확도

"""
#Train set score: 0.789
#Test set score: 0.718
"""

import numpy as np
from sklearn.model_selection import train_test_split

X_train_ridge, X_val_ridge, y_train_ridge, y_val_ridge = train_test_split(
    X_train_tfidf, y_train, test_size=0.2, random_state=42)

max_score = 0
max_alpha = 0
for alpha in np.arange(0.1, 10, 0.1): # alpha를 0.1부터 10까지 0.1씩 증가
    ridge_clf = RidgeClassifier(alpha=alpha) #릿지 분류기 선언
    ridge_clf.fit(X_train_ridge, y_train_ridge) #학습
    score = ridge_clf.score(X_val_ridge, y_val_ridge) #검정 데이터셋에 대해 정확도를 측정
    if score > max_score: #정확도가 이전의 정확도 최대값보다 크면 최대값을 변경한다.
        max_score = score
        max_alpha = alpha
print('#Max alpha {:.3f} at max validation score {:.3f}'.format(max_alpha, max_score))

## #Max alpha 1.600 at max validation score 0.727

from sklearn.linear_model import RidgeClassifier

ridge_clf = RidgeClassifier(alpha=1.6)
ridge_clf.fit(X_train_tfidf, y_train)
print('#Ridge Train set score: {:.3f}'.format(ridge_clf.score(X_train_tfidf, y_train)))
print('#Ridge Test set score: {:.3f}'.format(ridge_clf.score(X_test_tfidf, y_test)))

from sklearn.linear_model import LogisticRegression
import numpy as np
lasso_clf = LogisticRegression(penalty='l1', solver='liblinear', C=0.5)
lasso_clf.fit(X_train_tfidf, y_train)
print('#Lasso Train set score: {:.3f}'.format(lasso_clf.score(X_train_tfidf, y_train)))
print('#Lasso Test set score: {:.3f}'.format(lasso_clf.score(X_test_tfidf, y_test)))
print('#Used features count: {}'.format(np.sum(lasso_clf.coef_ != 0)), 'out of', X_train_tfidf.shape[1])

"""
#Ridge Train set score: 0.806
#Ridge Test set score: 0.726
#Lasso Train set score: 0.703
#Lasso Test set score: 0.696
#Used features count: 957 out of 2000
"""

from sklearn.naive_bayes import MultinomialNB

NB_clf = MultinomialNB(alpha=0.1)
NB_clf.fit(X_train_tfidf, y_train)
print('Train set score: {:.3f}'.format(NB_clf.score(X_train_tfidf, y_train)))
print('Test set score: {:.3f}'.format(NB_clf.score(X_test_tfidf, y_test)))

"""
Train set score: 0.768
Test set score: 0.704
"""

 

 

 

 

 

※ 해당 내용은 <파이썬 텍스트 마이닝 완벽 가이드>의 내용을 토대로 학습하며 정리한 내용입니다.