malware_detaction_ML

使用ML对恶意软件进行识别

import pandas as pd

data = pd.read_csv("MalwareData.csv", sep='|')

print(data.columns)

legit = data[0:41323].drop('legitimate', axis=1)
mal = data[41323::].drop('legitimate', axis=1)

print("合法数据有：%s 个，特征有：%s "%(legit.shape[0], legit.shape[1]))
print("恶意数据有：%s 个，特征有：%s "%(mal.shape[0], mal.shape[1]))

Index(['Name', 'md5', 'Machine', 'SizeOfOptionalHeader', 'Characteristics',
       'MajorLinkerVersion', 'MinorLinkerVersion', 'SizeOfCode',
       'SizeOfInitializedData', 'SizeOfUninitializedData',
       'AddressOfEntryPoint', 'BaseOfCode', 'BaseOfData', 'ImageBase',
       'SectionAlignment', 'FileAlignment', 'MajorOperatingSystemVersion',
       'MinorOperatingSystemVersion', 'MajorImageVersion', 'MinorImageVersion',
       'MajorSubsystemVersion', 'MinorSubsystemVersion', 'SizeOfImage',
       'SizeOfHeaders', 'CheckSum', 'Subsystem', 'DllCharacteristics',
       'SizeOfStackReserve', 'SizeOfStackCommit', 'SizeOfHeapReserve',
       'SizeOfHeapCommit', 'LoaderFlags', 'NumberOfRvaAndSizes', 'SectionsNb',
       'SectionsMeanEntropy', 'SectionsMinEntropy', 'SectionsMaxEntropy',
       'SectionsMeanRawsize', 'SectionsMinRawsize', 'SectionMaxRawsize',
       'SectionsMeanVirtualsize', 'SectionsMinVirtualsize',
       'SectionMaxVirtualsize', 'ImportsNbDLL', 'ImportsNb',
       'ImportsNbOrdinal', 'ExportNb', 'ResourcesNb', 'ResourcesMeanEntropy',
       'ResourcesMinEntropy', 'ResourcesMaxEntropy', 'ResourcesMeanSize',
       'ResourcesMinSize', 'ResourcesMaxSize', 'LoadConfigurationSize',
       'VersionInformationSize', 'legitimate'],
      dtype='object')
合法数据有：41323 个，特征有：56 
恶意数据有：96724 个，特征有：56 

data.head(-5)

mal.head(5)

data['legitimate']

0         1
1         1
2         1
3         1
4         1
         ..
138042    0
138043    0
138044    0
138045    0
138046    0
Name: legitimate, Length: 138047, dtype: int64

from sklearn.ensemble import ExtraTreesClassifier
from sklearn.feature_selection import SelectFromModel
from sklearn.model_selection import train_test_split

data_in = data.drop(['Name', 'md5', 'legitimate'], axis=1).values
labels = data['legitimate'].values
extratree = ExtraTreesClassifier().fit(data_in, labels)
select = SelectFromModel(extratree, prefit=True)
data_in_new = select.transform(data_in)
print(data_in.shape, data_in_new.shape)

(138047, 54) (138047, 13)

import numpy as np

features = data_in_new.shape[1]
importances = extratree.feature_importances_
indices = np.argsort(importances)[::-1]

for f in range(features):
    print('%d'%(f+1),data.columns[2+indices[f]],importances[indices[f]])

1 DllCharacteristics 0.13159378465829966
2 Characteristics 0.12898426863444917
3 Machine 0.10501819146671434
4 VersionInformationSize 0.059924950764671775
5 Subsystem 0.05488002624255342
6 SectionsMaxEntropy 0.05349804545740858
7 MajorSubsystemVersion 0.052116049448295995
8 ResourcesMaxEntropy 0.044890679185710124
9 ImageBase 0.04252264661828136
10 SizeOfOptionalHeader 0.04178342613704668
11 ResourcesMinEntropy 0.039008608631288154
12 SectionsMinEntropy 0.028637455162767493
13 MajorOperatingSystemVersion 0.019217124710739743

from sklearn.ensemble import RandomForestClassifier

legit_train, legit_test, mal_train, mal_test = train_test_split(data_in_new, labels)
classifier = RandomForestClassifier(n_estimators=50)
classifier.fit(legit_train,mal_train)

RandomForestClassifier(n_estimators=50)

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

print("the score of the algorithm:", classifier.score(legit_test, mal_test) * 100)

the score of the algorithm: 99.4233889661567

from sklearn.metrics import accuracy_score, confusion_matrix, precision_score, recall_score, f1_score

def print_stats_metrics(y_test, y_pred):
    print('Accuracy: %f' % accuracy_score(y_test,y_pred) )
    confmat = confusion_matrix(y_true=y_test, y_pred=y_pred)
    print ("confusion matrix")
    print(confmat)
    print (pd.crosstab(y_test, y_pred, rownames=['True'], colnames=['Predicted'], margins=True))
    print('Precision: %f' % precision_score(y_true=y_test, y_pred=y_pred,average='binary'))
    print('Recall: %f' % recall_score(y_true=y_test, y_pred=y_pred))
    print('F1-measure: %f' % f1_score(y_true=y_test, y_pred=y_pred))

predictions = classifier.predict(legit_test)
print_stats_metrics(mal_test,predictions)

Accuracy: 0.994234
confusion matrix
[[24109   111]
 [   88 10204]]
Predicted      0      1    All
True                          
0          24109    111  24220
1             88  10204  10292
All        24197  10315  34512
Precision: 0.989239
Recall: 0.991450
F1-measure: 0.990343

from sklearn.ensemble import GradientBoostingClassifier

grad_boost = GradientBoostingClassifier(n_estimators=50)
grad_boost.fit(legit_train, mal_train)

GradientBoostingClassifier(n_estimators=50)

In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.

predictions = grad_boost.predict(legit_test)
print_stats_metrics(mal_test, predictions)

Accuracy: 0.988497
confusion matrix
[[24039   181]
 [  216 10076]]
Predicted      0      1    All
True                          
0          24039    181  24220
1            216  10076  10292
All        24255  10257  34512
Precision: 0.982354
Recall: 0.979013
F1-measure: 0.980680