作业12

text = '''The world is changing
and time is spinning fast
it's so amazing how you came into my life
I know it seems all hope is gone
I know you feel you can't be strong
and once again the story ends with you and I
And anytime you feel like you just can't go on
just hold on to my love
and you'll never be alone
Hold on
we can make it through the fire
and my love   '''

import nltk
from nltk.corpus import stopwords
from nltk.stem import WordNetLemmatizer

#预处理
def preprocessing(text):
    #text=text.decode("utf-8)
    tokens=[word for sent in nltk.sent_tokenize(text) for word in nltk.word_tokenize(sent)]
    stops=stopwords.words('english')
    tokens=[token for token in tokens if token not in stops]

    tokens=[token.lower() for token in tokens if len(token)>=3]
    lmtzr=WordNetLemmatizer()
    tokens=[lmtzr.lemmatize(token) for token in tokens]
    preprocessed_text=' '.join(tokens)
    return preprocessed_text
preprocessing (text)

#读取数据集
import csv
file_path=r'D:\SMSSpamCollectionjs.txt'
sms=open(file_path,'r',encoding='utf-8')
sms_data=[]
sms_label=[]
csv_rreader=csv.reader(sms,delimiter='\t')
for line in csv_reader:
    sms_label.append(line[0])
    sms_data.append(preprocessing(line[1]))
sms.close()

#按0.7,0.3比例分为训练集和测试集
import numpy as np
sms_data=np.array(sms_data)
sms_label=np.array(sms_label)

from sklearn.model_selection import train_text_split
x_train, x_test, y_train, y_test = train_text_split(sms_data, sms_label, test_size=0.3, random_state=0, stratify=sms_label)

#将其向量化
from sklearn.feature_extraction.text import TfidfVectorizer
vectorizer=TfidfVectorizer(min_df=2,ngram_range=(1,2),stop_words='english',strip_accents='unicode',norm='l2')
X_train=vectorizer.fit_transform(x_train)
X_test=vectorizer.transform(x_test)

X_train
a=X_train.toarray()
print(a)

for i in range(1000):
    for j in range(5984):
        if a[i,j]!=0:
            print(i,j,a[i,j])

#朴素贝叶斯分类器
from sklearn.navie_bayes import MultionmialNB
clf= MultionmialNB().fit(X_train,y_train)
y_nb_pred=clf.predict(X_test)

#分类结果显示
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report

print(y_nb_pred.shape, y_nb_pred) #x_test预测结果

#
from sklearn.metrics import classification_report
cr = classification_report(y_nb_pred,y_test)
print(cr)