计算文本词频进行聚类

import jieba
import jieba.analyse
import math
import operator
from sklearn.cluster import KMeans, MiniBatchKMeans, AffinityPropagation, DBSCAN
from sklearn.cluster import MeanShift, estimate_bandwidth
from collections import Counter
from sklearn.manifold import TSNE
from sklearn.decomposition import TruncatedSVD
from sklearn.decomposition import PCA
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import HashingVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import Normalizer
import numpy as np
import matplotlib.pyplot as plt
from collections import defaultdict
import os
#np.set_printoptions(threshold=np.inf)

#加载手工设置的某些词的idf值
def load_idf_file(path):
    idf_dict = {}
    handle = open(path, 'r',encoding= 'utf8')
    line = handle.readline()               # 调用文件的 readline()方法 
    datas = []
    while line: 
        line = line.strip() 
        if len(line)>0:
            line_arr = line.split(' ')
            idf_dict[line_arr[0]] = float(line_arr[1])
        line = handle.readline() 
    handle.close()
    return idf_dict

def cal_idf(data_set,idf_dict):
    doc_num = len(data_set)
    word_doc_count=defaultdict(int)
    for word_str in data_set:
        word_list = word_str.split(' ')
        word_list = list(set(word_list))
        for item in word_list:
            if item and item.strip()!='':
                word_doc_count[item]+=1

    word_idf = {}
    default_idf_keys = idf_dict.keys()
    for k,v in word_doc_count.items():  
        idf = math.log(doc_num*1.0 / v)  
        if k in default_idf_keys: word_idf[k] = idf_dict[k]
        else:word_idf[k] = idf
    #path = "idf.txt"  
    #save(word_idf, path)  
    return word_idf
def cal_tfidf(data_set,idf_ret):
    doc_word_tfidf = []    
    i = 0
    for word_str in data_set:
        word_list = word_str.split(' ')
        doc_word_total = len(word_list)
        doc_word_dict = defaultdict(int)
        doc_word_tfidf_dict = defaultdict(int)
        for item in word_list:
            if item and item.strip()!='':
                doc_word_dict[item]+=1
        for k,v in doc_word_dict.items():
            doc_word_tfidf_dict[k]=(v/doc_word_total)*idf_ret[k]
        doc_word_tfidf.append(doc_word_tfidf_dict)
        i=i+1
    return doc_word_tfidf    


def save(idf_dict, path):  
    f = open(path, 'a+',encoding= 'utf8')
    f.truncate()  
    for key in idf_dict.keys():  
        f.write(str(key) + " " + str(idf_dict[key]) + "\n")  
    f.close()  

# 切词
def jieba_tokenize():
    jieba_need =[]
    for item in datas:
        temp_list1 = jieba.analyse.extract_tags(item,topK=10) 
        w_len = len(temp_list1)
        if w_len>2:
            w_num = math.ceil(float(w_len)*0.6)
            temp_list1 = temp_list1[0:w_num]
        temp_list=[e for e in temp_list1 if e not in lines + [' '] and len(e)>1]
        jieba_need.append(" ".join(temp_list))
    return jieba_need


# 降维
def reduction(matrix):
    svd = TruncatedSVD(50)
    normalizer = Normalizer(copy=False)
    lsa = make_pipeline(svd, normalizer)
    X = lsa.fit_transform(matrix)
    return X

jieba.analyse.set_idf_path("./jieba/idf.txt")
data_num = 10000
k_num = 200

i = 0
output = open('../data.txt', 'r',encoding= 'utf8')
line = output.readline()               # 调用文件的 readline()方法 
datas = []
while line: 
    line = line.strip() 
    if len(line)>0:
        i=i+1
        datas.append(line)
        if i>=data_num:break    
    line = output.readline() 
output.close()


npyfile = "data.npy"
if os.path.exists(npyfile):
    X = np.load("data.npy")
else:
    # 读取停用词
    with open('./stop_words.txt',encoding='utf-8') as f:
        entities = list(f)
        lines = []
        for line in entities:
            line1 = line.strip()
            lines.append(line1)
    #加载手工设置的idf
    default_idf_dict = load_idf_file("./jieba/idf.txt")
    my_train = jieba_tokenize()

    idf_ret = cal_idf(my_train,default_idf_dict)
    tfidf_ret = cal_tfidf(my_train,idf_ret)
    doc_rows = len(datas)
    word_rows = len(idf_ret)
    #把每个文档每个分词的词频 转换成 矩阵[文档数,分词数]
    X = np.zeros([doc_rows,word_rows])
    for i in range(doc_rows):
        j = 0
        for k,v in idf_ret.items():
            X[i][j] = tfidf_ret[i][k]
            j=j+1
    X = np.array(X)
    np.save("data.npy",X)
#降维
X = reduction(X)
#cl = MiniBatchKMeans(n_clusters=k_num, init='k-means++', n_init=1,init_size=1000, batch_size=10000, verbose=False)
#cl = KMeans(n_clusters=k_num, init='k-means++', random_state=30, n_init=1,verbose=False)
cl = DBSCAN(eps=0.2, min_samples=30)
result = cl.fit_predict(X)
num_clusters = len(set(result))

# 结果输出
ret = [[] for y in range(len(result))] 
for i in range(len(datas)): 
    classid = result[i]
    ret[classid].append(datas[i])

for m in range(num_clusters):
    file = "./result/result_"+str(m)+".txt"
    handle = open(file, 'w+',encoding= 'utf8')
    for n in range(len(ret[m])):
        handle.write(ret[m][n]+"\n")
    handle.close()