想用LDA模型进行聚类分析，现在的数据已经是数值型数据转化过来的标签型数据，但是查到的代码都是需要进行分词的，如何将分词的代码改成直接可以进行困惑度计算的代码

import gensim
from gensim import corpora
import matplotlib.pyplot as plt
import matplotlib
import numpy as np
import warnings
warnings.filterwarnings('ignore') # To ignore all warnings that arise here to enhance clarity

from gensim.models.coherencemodel import CoherenceModel
from gensim.models.ldamodel import LdaModel
import pandas as pd

准备数据

import sklearn
PATH = "E:/桌面/企业指标录入分类.csv"
file_object2=open(PATH,encoding = 'utf-8',errors = 'ignore').read().split('\n') #一行行的读取内容
data_set=[] #建立存储分词的列表
for i in range(len(file_object2)):
result=[]
seg_list = file_object2[i].split()
for w in seg_list : #读取每一行分词
result.append(w)
data_set.append(result)
print(data_set)
dictionary = corpora.Dictionary(data_set) # 构建 document-term matrix(文档词条矩阵)
corpus = [dictionary.doc2bow(text) for text in data_set] #计算文本向量
#Lda = gensim.models.ldamodel.LdaModel # 创建LDA对象

#计算困惑度
def perplexity(num_topics):
ldamodel = LdaModel(corpus, num_topics=num_topics, id2word = dictionary, passes=5)
print(ldamodel.print_topics(num_topics=num_topics, num_words=5))
print(ldamodel.log_perplexity(corpus))
return ldamodel.log_perplexity(corpus)

#计算coherence
def coherence(num_topics):
ldamodel = LdaModel(corpus, num_topics=num_topics, id2word = dictionary, passes=5,random_state = 1)
print(ldamodel.print_topics(num_topics=num_topics, num_words=3))
ldacm = CoherenceModel(model=ldamodel, texts=data_set, dictionary=dictionary, coherence='c_v')
print(ldacm.get_coherence())
return ldacm.get_coherence()

绘制困惑度折线图

x = range(1,15)

z = [perplexity(i) for i in x]

y = [coherence(i) for i in x]
plt.plot(x, y)
plt.xlabel('主题数目')
plt.ylabel('coherence大小')
plt.rcParams['font.sans-serif']=['SimHei']
matplotlib.rcParams['axes.unicode_minus']=False
plt.title('主题-coherence变化情况')
plt.show()

lda = LdaModel(corpus=corpus, id2word=dictionary, num_topics=5, passes = 30,random_state=1)
topic_list=lda.print_topics()
print(topic_list)

result_list =[]
for i in lda.get_document_topics(corpus)[:]:
listj=[]
for j in i:
listj.append(j[1])
bz=listj.index(max(listj))
result_list.append(i[bz][0])
print(result_list)
import pyLDAvis.gensim_models
pyLDAvis.enable_notebook()
data = pyLDAvis.gensim.prepare(lda, corpus, dictionary)
pyLDAvis.save_html(data, 'E:\桌面\企业指标录入分类.csv')