· Use CNN to do Kaggle competition handwritten number recognition accuracy rate of 99%+

· Use CNN to do Kaggle competition handwritten number recognition accuracy rate of 99%+

This blog uses Lenet5 implementation, also contains TensorFlow model parameters save and load reference my blog, the practicability is better. Accuracy is 99.85% in training set and 99%+ in test training set.

• Training and model parameter saving

“Train” file

import os
os.environ["KMP_DUPLICATE_LIB_OK"] ="TRUE"
import pandas as pd
import tensorflow as tf  
from tensorflow.examples.tutorials.mnist import input_data  
  
mnist = input_data.read_data_sets('./1CNN/mnistcnn/data', one_hot=True)  
  
#sess = tf.InteractiveSession()  
  
# Training data
x = tf.placeholder("float", shape=[None.784],name="x")  
# Train tag data
y_ = tf.placeholder("float", shape=[None.10],name="y_")  
The first dimension represents the number of samples, the second and third dimensions represent the length and width of the image, the fourth dimension represents the number of image channels, and 1 represents the gray level
x_image = tf.reshape(x, [-1.28.28.1])  
  
# The first layer: convolution layer
conv1_weights = tf.get_variable("conv1_weights"[5.5.1.32], initializer=tf.truncated_normal_initializer(stddev=0.1)) The current layer depth is 1, and the depth of the filter is 32
conv1_biases = tf.get_variable("conv1_biases"[32], initializer=tf.constant_initializer(0.0))  
conv1 = tf.nn.conv2d(x_image, conv1_weights, strides=[1.1.1.1], padding='SAME') # Move step 1, fill with all zeros
relu1 = tf.nn.relu( tf.nn.bias_add(conv1, conv1_biases) ) Activate function Relu to de-linearize

# layer 2: maximum pooling layer
The size of the pool layer filter is 2*2, the move step is 2, and the filter is filled with all zeros
pool1 = tf.nn.max_pool(relu1, ksize=[1.2.2.1], strides=[1.2.2.1], padding='SAME')  

# Third layer: convolution layer
conv2_weights = tf.get_variable("conv2_weights"[5.5.32.64], initializer=tf.truncated_normal_initializer(stddev=0.1)) The current layer depth is 32. The depth of the filter is 64
conv2_biases = tf.get_variable("conv2_biases"[64], initializer=tf.constant_initializer(0.0))  
conv2 = tf.nn.conv2d(pool1, conv2_weights, strides=[1.1.1.1], padding='SAME') # Move step 1, fill with all zeros
relu2 = tf.nn.relu( tf.nn.bias_add(conv2, conv2_biases) )  

# Layer 4: Maximum pooling layer
The size of the pool layer filter is 2*2, the move step is 2, and the filter is filled with all zeros
pool2 = tf.nn.max_pool(relu2, ksize=[1.2.2.1], strides=[1.2.2.1], padding='SAME')  
  
Layer 5: Full connection layer
fc1_weights = tf.get_variable("fc1_weights"[7 * 7 * 64.1024], initializer=tf.truncated_normal_initializer(stddev=0.1)) #7*7*64=3136 Turn the output of the previous layer into eigenvectors
fc1_baises = tf.get_variable("fc1_baises"[1024], initializer=tf.constant_initializer(0.1))  
pool2_vector = tf.reshape(pool2, [-1.7 * 7 * 64])  
fc1 = tf.nn.relu(tf.matmul(pool2_vector, fc1_weights) + fc1_baises)  
  
# To reduce overfitting, add a Dropout layer
keep_prob = tf.placeholder(tf.float32,name="keep_prob") 
fc1_dropout = tf.nn.dropout(fc1, keep_prob)  

Layer 6: Full connection layer
fc2_weights = tf.get_variable("fc2_weights"[1024.10], initializer=tf.truncated_normal_initializer(stddev=0.1)) # Number of neuron nodes is 1024, and classification nodes are 10
fc2_biases = tf.get_variable("fc2_biases"[10], initializer=tf.constant_initializer(0.1))  
fc2 = tf.matmul(fc1_dropout, fc2_weights) + fc2_biases  
  
Layer 7: Output layer
# softmax  
y_conv = tf.nn.softmax(fc2,name="y_conv")  
y_conv_labels = tf.argmax(y_conv,1,name='y_conv_labels')
# Define cross entropy loss function
y_conv = tf.clip_by_value(y_conv,1e-4.1.99)
cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y_conv), reduction_indices=[1]))  
  
# Select the optimizer and let the optimizer minimize loss function/convergence, back propagation
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy)  
  
# tf.argmax() returns the index of the maximum value of the data on a dimension, which in this case represents the predicted and true values
# Judge whether the index of the largest number in the predicted value y and the true value y_ is the same, y is 1-10 probability
correct_prediction = tf.equal(y_conv_labels, tf.argmax(y_,1))  
  
# Use average to count test accuracy
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32),name="accuracy")  

def convert2onehot(data) :
    # covert data to onehot representation
    return pd.get_dummies(data)

file_path = "./1CNN/mnistcnn/datas/train.csv"
df_data = pd.read_csv(file_path, sep=",", header="infer")
np_data = df_data.values
trainx = np_data[:, 1:]/255
trainy = convert2onehot(np_data[:, 0]).values

with tf.Session() as sess:

    # Start training
    srun = sess.run
    srun(tf.global_variables_initializer())
    saver = tf.train.Saver()
    
    for i in range(6001):  
        start_step = i*100 % 42000
        stop_step = start_step+100

        batch_x, batch_y = trainx[start_step:stop_step], trainy[start_step:stop_step]
        srun(train_step,feed_dict={x: batch_x, y_: batch_y, keep_prob: 0.5}) # Train at 50% Dropout
        if i%100= =0:  
            train_accuracy = srun(accuracy,feed_dict={x:batch_x, y_: batch_y, keep_prob: 1.0}) # Dropout is not used in the evaluation phase
            print("step %d, training accuracy %f" % (i, train_accuracy))  
            saver_path = saver.save(sess, "./1CNN/mnistcnn/ckpt/my_model.ckpt",global_step=i)  # Save the model to the save/model.ckpt file

    
    #print("W1:", sess.run(conv1_weights)) #print("W1:", sess.run(conv1_weights)) #print("W1:", sess.run(conv1_weights)
    #print("W2:", sess.run(conv1_biases))

    print("Model saved in file:", saver_path)
    
    # Test accuracy on test data
    print("test accuracy %g" % srun(accuracy,feed_dict={x: trainx, y_: trainy, keep_prob: 1.0}))  
    print("test accuracy %g" % srun(accuracy,feed_dict={x: mnist.test.images, y_: mnist.test.labels, keep_prob: 1.0}))  
Copy the code

Running results:

. Step 5800, training accuracy 1.000000 step 5900, training accuracy 0.990000 step 6000, Training accuracy 1.000000 Model savedin file: ./1CNN/mnistcnn/ckpt/my_model.ckpt-6000
testAccuracy of 0.998571testAccuracy of 0.9958Copy the code

• Model loading and reuse

The app file:

import os
os.environ["KMP_DUPLICATE_LIB_OK"] ="TRUE"
import tensorflow as tf
import pandas as pd


file_path = "./1CNN/mnistcnn/datas/test.csv"
df_data = pd.read_csv(file_path, sep=",", header="infer")
np_data = df_data.values
testx = np_data[:, :]/255

file_path1 = "./1CNN/mnistcnn/datas/sample_submission.csv"
df_data1 = pd.read_csv(file_path1, sep=",", header="infer")

print(df_data1.head())
df_data1.drop(labels="Label",axis = 1,inplace=True)
print(df_data1.head())

with tf.Session() as sess:    
    # load operation graph
    saver = tf.train.import_meta_graph('./1CNN/mnistcnn/ckpt/my_model.ckpt-6000.meta')
    # load parameters
    saver.restore(sess,tf.train.latest_checkpoint('./1CNN/mnistcnn/ckpt'))
    graph = tf.get_default_graph()
    Import the input interface
    x = graph.get_tensor_by_name("x:0")
    Import the output interface
    y_ = graph.get_tensor_by_name("y_:0")

    keep_prob = graph.get_tensor_by_name("keep_prob:0")
    y_conv_labels = graph.get_tensor_by_name("y_conv_labels:0")

    y_conv_labels_val = sess.run(y_conv_labels,{x:testx[:],keep_prob:1.0})

    # Make a prediction
    print("y: ",y_conv_labels_val[:10])

    df_data1["Label"] = y_conv_labels_val
    print(df_data1.head())
    df_data1.to_csv(file_path1,columns=["ImageId"."Label"],index=False)
    print("Ok")
Copy the code

Running results:

 ImageId  Label
0        1      0
1        2      0
2        3      0
3        4      0
4        5      0
   ImageId
0        1
1        2
2        3
3        4
4        5

y:  [2 0 9 9 3 7 0 3 0 3]
   ImageId  Label
0        1      2
1        2      0
2        3      9
3        4      9
4        5      3
Ok
Copy the code

• Results analysis

Kaagle platform results

Your Best Entry 
You advanced 381 places on the leaderboard!

Your submission scored 0.98928, whichIs an improvement of your previous score of 0.98142. Great job!Copy the code

Through this CNN handwritten number recognition practice, I deepened my understanding and application ability of CNN. Using TensorFlow model parameters to save and load reference my blog post also makes the code application more flexible, efficient and clear, I recommend you to use it.