text = ["今日はいい天気だな"]

import tensorflow_hub as hub
import numpy as np
import tensorflow_text

#テキストを文章ベクトルに変換
embed = hub.load("https://tfhub.dev/google/universal-sentence-encoder-multilingual/3")
embeddings = embed(text)
 
print(embeddings)

Universal Sentence Encoderの文書分類モデル

テキストデータの準備

Universal Sentence Encoderで文書ベクトル化

import tensorflow as tf
import tensorflow_hub as hub
import tensorflow_text

import numpy as np
import pandas as pd

df = pd.read_csv('livedoor_data_all.csv', encoding='utf-8')
df = df.iloc[850:1150,:]
text_df = df.iloc[:,1].values
y = df.iloc[:,2]

text_list = text_df.tolist()

# 学習済みモデルの読み込み
#embed = hub.load("https://tfhub.dev/google/universal-sentence-encoder-multilingual-large/3") 
embed = hub.load("https://tfhub.dev/google/universal-sentence-encoder-multilingual/3")

embeddings = embed(text_list)

# numpy配列に変換
embeddings =embeddings.numpy()

ランダムフォレストで分類モデル作成

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_validate

# ランダムフォレストのパラメータ
clf = RandomForestClassifier(n_estimators=10)

## Cross-Validation
scoring = {"a": "accuracy",
           "p": "precision_macro",
           "r": "recall_macro",
           "f":"f1_macro"}

## Cross-Validation
score = cross_validate(clf, embeddings, y, scoring=scoring,cv=2)
print('accuracy:', score['test_a'].mean())
print('precision:', score['test_p'].mean())
print('recall:', score['test_r'].mean())
print('f1:', score['test_f'].mean())

実験：Bag-of-Wordsとの分類精度比較

形態素解析

import numpy as np
import pandas as pd
import MeCab
from gensim import corpora, matutils

df = pd.read_csv('livedoor_data_all.csv', encoding='utf-8')
df = df.iloc[850:1150,:]
text_df = df.iloc[:,1].values
y = df.iloc[:,2]

# ChaSen 互換形式
mecab = MeCab.Tagger("-Ochasen")

#形態素解析
def ChaSen(text):   
    word_l = []   
    res = mecab.parse(text) #構文解析
    words = res.split('\n')[:-2] # 最後にEOSが入るためそれ以外を取得 
    
    for word in words:
        pos = word.split('\t') # タブで各行を切り分け
        if '名詞'or'形容詞'or'動詞' in pos[3]: #品詞がある4番目にアクセス
            word_l.append(pos[2]) # 原型を取得
    return word_l

# 単語に分割したリスト作成
word_list = []
for text in text_df:
    b = ChaSen(text)
    word_list.append(b)
print(word_list)

Bag-of-Wordsの文書ベクトル作成

#gensimでbowに変換
dictionary = corpora.Dictionary(word_list)
n_word = len(dictionary)

x = []
for i in word_list:
    bow_id = dictionary.doc2bow(i)
    bow = matutils.corpus2dense([bow_id], n_word).T[0]
    x.append(bow)

x = np.array(x, 'f')
print(x.shape)

x_pd = pd.DataFrame(x)

TF-IDF変換

# TF-IDF
from sklearn.feature_extraction import text

tfidf = text.TfidfTransformer(norm=None)

#帰ってくるのはスパースマトリックス
res = tfidf.fit_transform(x_pd)

x_pd_TFIDF = pd.DataFrame(res.toarray(), columns=x_pd.columns)

# PCA
from sklearn import decomposition
pca = decomposition.PCA(n_components=100)  
x_2 = pca.fit_transform(x_pd_TFIDF)
x_2 = pd.DataFrame(x_2)

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_validate


# ランダムフォレストのパラメータ
clf = RandomForestClassifier(n_estimators=10)

scoring = {"a": "accuracy",
           "p": "precision_macro",
           "r": "recall_macro",
           "f":"f1_macro"}

## Cross-Validation
score = cross_validate(clf, x_2, y, scoring=scoring,cv=3)
print('accuracy:', score['test_a'].mean())
print('precision:', score['test_p'].mean())
print('recall:', score['test_r'].mean())
print('f1:', score['test_f'].mean())