机器学习朝花夕拾

keras生成器代码示例

def gen_data(data_dir, label, batch_size=batch_size):
    os.chdir(data_dir)
    y_all = to_categorical(label["type"], nb_class)
    label_length = label.shape[0]
    i = 0
    while i < label_length:
        if label_length - i < batch_size:
            batch_size = label_length - i
        x = np.zeros([batch_size, img_rows, img_cols])
        y = y_all[i: i + batch_size]
        for cc, filename in enumerate(label["md5"][i: i + batch_size]):
            mat = bin2array(filename).astype(np.float32)
            x[cc, :, :] = mat
        x = x.reshape(x.shape[0], img_rows, img_cols, 1)
        x /= 255
        i += batch_size
        yield x, y

save_best_callback = ModelCheckpoint('/root/mnist/weights299_{epoch:02d}.h5', monitor='val_acc', mode='max',
                                     save_best_only=True, save_weights_only=True)
model.fit_generator(gen_data(train_data_dir, label), steps_per_epoch=label.shape[0], epochs=epochs,
                    validation_data=gen_data(test_data_dir, label1), validation_steps=label1.shape[0],
                    callbacks=[save_best_callback])

 

设置可见GPU

import tensorflow as tf

from tensorflow.python.client import device_lib

from keras.backend.tensorflow_backend import set_session



os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = "0"

config = tf.ConfigProto()
config.gpu_options.per_process_gpu_memory_fraction = 0.25
config.gpu_options.allow_growth = True
set_session(tf.Session(config=config))

print device_lib.list_local_devices()

 

已有模型

from keras.applications.xception import Xception
from keras.applications.inception_v3 import InceptionV3
from keras.applications.vgg16 import VGG16
from keras.applications.vgg19 import VGG19

其中Xception最小的图像大小为11*11，即input_tensor = Input(shape=(11, 11, 1)),

    Inception最小的图像大小为75*75

使用Xception，数据集为10000*299*299*1，不同的batch_size训练模型时内存占用情况，1080显卡为8G显存

1 2797 M
2 2797
3 2797
4 2797
5 3309
6 3309
7 3309
8 5357
12 5357
13 7837
16 7837

 

算法：

from sklearn.linear_model import  SGDClassifier

from sklearn.linear_model import SGDRegressor

from sklearn.linear_model import  LinearRegression    # 线性回归

from sklearn.linear_model import  LogisticRegression    # 逻辑回归

from sklearn.svm import LinearSVC    # SVM分类器

from sklearn.naive_bayes import MultinomialNB    # 朴素贝叶斯

from sklearn.neighbors import KNeighborsClassifier    # KNN分类

from sklearn.neighbors import KNeighborsRegressor    # KNN回归

from sklearn.tree import DecisionTreeClassifier    # 决策树分类器

from sklearn.cluster import KMeans

 

 

常用函数：

from sklearn.model_selection import train_test_split    # 分割数据，返回x_train,x_test,y_train,y_test

from sklearn.metrics import classification_report, confusion_matrix

from sklearn.datasets import load_iris

from sklearn.pipeline import Pipeline

from sklearn.model_selection import GridSearchCV

 

数据预处理：

数据预处理函数均需要实例化后使用fit_transform(data)，e.g StandardScaler().fit_transform(data)

from sklearn.preprocessing import StandardScaler    # 标准化数据，均值为0，方差为1 

from sklearn.decomposition import PCA    # PCA

from sklearn.feature_extraction import DictVectorizer