在tensorflow中訓練深度神經網絡時的納米損失教程

Question

我想訓練一個神經網絡，在NOTMNIST上有超過1個隱藏層。當我有一個隱藏層時，它可以正常工作，但是當我添加多個隱藏層時，我開始獲取損失的nan。下面是我使用

from __future__ import print_function 
import numpy as np 
import tensorflow as tf 
from six.moves import cPickle as pickle 
from six.moves import range 

batch_size = 128 
num_hidden = 1024 
num_hidden2 = 300 
num_hidden3 = 50 
SEED = 1234567 
keep_prob = 0.5 

graph1 = tf.Graph() 
with graph1.as_default(): 

    # Input data. For the training data, we use a placeholder that will be fed 
    # at run time with a training minibatch. 
    tf_train_dataset = tf.placeholder(tf.float32, 
            shape=(batch_size, image_size * image_size)) 
    tf_train_labels = tf.placeholder(tf.float32, shape=(batch_size, num_labels)) 
    tf_valid_dataset = tf.constant(valid_dataset) 
    tf_test_dataset = tf.constant(test_dataset) 

    # Variables. 
    weights1 = tf.Variable(tf.truncated_normal([image_size * image_size, num_hidden])) 
    biases1 = tf.Variable(tf.zeros([num_hidden])) 

    weights2 = tf.Variable(tf.truncated_normal([num_hidden, num_hidden2])) 
    biases2 = tf.Variable(tf.zeros([num_hidden2])) 

    weights3 = tf.Variable(tf.truncated_normal([num_hidden2, num_hidden3])) 
    biases3 = tf.Variable(tf.zeros([num_hidden3])) 

    weights4 = tf.Variable(tf.truncated_normal([num_hidden3, num_labels])) 
    biases4 = tf.Variable(tf.zeros([num_labels])) 

    # Training computation. 
    l1 = tf.matmul(tf_train_dataset, weights1) + biases1 
    h1 = tf.nn.relu(l1) 
    h1 = tf.nn.dropout(h1, 0.5, seed=SEED) 

    l2 = tf.matmul(h1, weights2) + biases2 
    h2 = tf.nn.relu(l2) 
    h2 = tf.nn.dropout(h2, 0.5, seed=SEED) 

    l3 = tf.matmul(h2, weights3) + biases3 
    h3 = tf.nn.relu(l3) 
    h3 = tf.nn.dropout(h3, 0.5, seed=SEED) 

    logits = tf.matmul(h3, weights4) + biases4 


    loss = tf.reduce_mean(
    tf.nn.softmax_cross_entropy_with_logits(logits, tf_train_labels)) 

    # L2 regularization for the fully connected parameters. 
    regularizers = (tf.nn.l2_loss(weights1) + tf.nn.l2_loss(biases1) + 
        tf.nn.l2_loss(weights2) + tf.nn.l2_loss(biases2) + 
        tf.nn.l2_loss(weights3) + tf.nn.l2_loss(biases3) + 
        tf.nn.l2_loss(weights4) + tf.nn.l2_loss(biases4)) 
    # Add the regularization term to the loss. 
    loss += 5e-4 * regularizers 

    # Optimizer. 
    optimizer = tf.train.GradientDescentOptimizer(0.5).minimize(loss) 

    # Predictions for the training, validation, and test data. 
    train_prediction = tf.nn.softmax(logits) 

    v_l1 = tf.matmul(tf_valid_dataset, weights1) + biases1 
    v_h1 = tf.nn.relu(v_l1) 

    v_l2 = tf.matmul(v_h1, weights2) + biases2 
    v_h2 = tf.nn.relu(v_l2) 

    v_l3 = tf.matmul(v_h2, weights3) + biases3 
    v_h3 = tf.nn.relu(v_l3) 

    v_logits = tf.matmul(v_h3, weights4) + biases4 
    valid_prediction = tf.nn.softmax(v_logits) 


    t_l1 = tf.matmul(tf_test_dataset, weights1) + biases1 
    t_h1 = tf.nn.relu(t_l1) 

    t_l2 = tf.matmul(t_h1, weights2) + biases2 
    t_h2 = tf.nn.relu(t_l2) 

    t_l3 = tf.matmul(t_h2, weights3) + biases3 
    t_h3 = tf.nn.relu(t_l3) 

    t_logits = tf.matmul(t_h3, weights4) + biases4 
    test_prediction = tf.nn.softmax(t_logits) 


num_steps = 3001 

with tf.Session(graph=graph1) as session: 
    tf.initialize_all_variables().run() 
    print("Initialized") 
    for step in range(num_steps): 
    # Pick an offset within the training data, which has been randomized. 
    # Note: we could use better randomization across epochs. 
    offset = (step * batch_size) % (train_labels.shape[0] - batch_size) 
    # Generate a minibatch. 
    batch_data = train_dataset[offset:(offset + batch_size), :] 
    batch_labels = train_labels[offset:(offset + batch_size), :] 
    # Prepare a dictionary telling the session where to feed the minibatch. 
    # The key of the dictionary is the placeholder node of the graph to be fed, 
    # and the value is the numpy array to feed to it. 
    feed_dict = {tf_train_dataset : batch_data, tf_train_labels : batch_labels} 
    _, l, predictions = session.run(
     [optimizer, loss, train_prediction], feed_dict=feed_dict) 
    if (step % 500 == 0): 
     print("Minibatch loss at step %d: %f" % (step, l)) 
     print("Minibatch accuracy: %.1f%%" % accuracy(predictions, batch_labels)) 
     print("Validation accuracy: %.1f%%" % accuracy(
     valid_prediction.eval(), valid_labels)) 
    print("Test accuracy: %.1f%%" % accuracy(test_prediction.eval(), test_labels)) 

此代碼的輸出我得到

Initialized 
Minibatch loss at step 0: 48759.078125 
Minibatch accuracy: 10.2% 
Validation accuracy: 10.0% 
Minibatch loss at step 500: nan 
Minibatch accuracy: 9.4% 
Validation accuracy: 10.0% 
Minibatch loss at step 1000: nan 
Minibatch accuracy: 8.6% 
Validation accuracy: 10.0% 
Minibatch loss at step 1500: nan 
Minibatch accuracy: 11.7% 
Validation accuracy: 10.0% 
Minibatch loss at step 2000: nan 
Minibatch accuracy: 6.2% 
Validation accuracy: 10.0% 
Minibatch loss at step 2500: nan 
Minibatch accuracy: 10.2% 
Validation accuracy: 10.0% 
Minibatch loss at step 3000: nan 
Minibatch accuracy: 7.8% 
Validation accuracy: 10.0% 
Test accuracy: 10.0% 

Answer 1

嘗試降低權重的標準偏差。默認設置爲1.它爲我工作。