Come accumulare e appy gradienti per l'aggiornamento DQNetwork Async n-step in Tensorflow?

Question

Sto tentando di implementare Asynchronous Methods for Deep Reinforcement Learning e uno dei passaggi richiede di accumulare il gradiente su diversi passaggi e quindi applicarlo. Qual è il modo migliore per ottenere questo in tensorflow? Sono arrivato così lontano da accumulare il gradiente e non credo sia il modo più veloce per raggiungerlo (molti trasferimenti da tensorflow a python e viceversa). Qualsiasi suggerimento è benvenuto. Questo è il mio codice di un giocattolo NN. Non modella o calcola nulla, semplicemente esercita le operazioni che voglio usare.

import tensorflow as tf 

from model import * 


graph = tf.Graph() 

with graph.as_default(): 

    state = tf.placeholder(tf.float32, shape=[None, 80,80,1]) 

    with tf.variable_scope('layer1'): 
     W = weight_variable([8, 8, 1, 32]) 
     variable_summaries(W, "layer1/W") 
     b = bias_variable([32]) 
     variable_summaries(b, "layer1/b") 
     h = conv2d(state, W, 4) + b 
     activation = tf.nn.relu(h) 
     pool1 = max_pool_2x2(activation) 

    print(pool1.get_shape()) 
    pool1 = tf.reshape(pool1, [-1, 3200]) 

    with tf.variable_scope('readout'): 
     W = weight_variable([3200, 3]) 
     b = bias_variable([3]) 
     logits = tf.matmul(pool1, W) + b 
     variable_summaries(h, "y") 

    action_indexes = tf.placeholder(tf.int32, shape=[None], name="action_indexes") 

    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, action_indexes) 

    starter_learning_rate = 1e-6 

    global_step = tf.Variable(0, trainable=False) 

    # decay every 1000 steps with a base of 0.96: 
    learning_rate = tf.train.exponential_decay(starter_learning_rate, 
     global_step, 
     10000, 0.96, staircase=True) 

    optimizer = tf.train.RMSPropOptimizer(learning_rate) 

    gradients_and_variables = optimizer.compute_gradients(loss, tf.trainable_variables()) 

    discounted_values = tf.placeholder(tf.float32, shape=[None, 1]) 

with tf.Session(graph=graph) as s: 

    for v in tf.trainable_variables(): 
     print(v.name, v.dtype, v.get_shape()) 

    s.run(tf.initialize_all_variables()) 

    feed_dict= { 
     state : np.zeros([1, 80, 80, 1]), 
     action_indexes: [1], 
    } 


    var_to_grad = dict((var.name, grad) for grad, var in gradients_and_variables) 
    keys = sorted(var_to_grad.keys()) 
    print(keys) 

    name_to_var = dict((var.name, var) for _, var in gradients_and_variables) 

    for i in range(10): 

     gradients = s.run([ var_to_grad[k] for k in keys], feed_dict=feed_dict) 

     for k,v in zip(keys, gradients): 
      var_to_grad[k] += v 

    for k in keys: 
     print(var_to_grad[k]) 

    s.run(optimizer.apply_gradients((g, name_to_var[v]) for v,g in var_to_grad.iteritems()), feed_dict=feed_dict) 

---

codice aggiornato dopo suggerimento @yaroslave:

import tensorflow as tf 

from model import * 


graph = tf.Graph() 

with graph.as_default(): 

    minibatch = 32 
    state = tf.placeholder(tf.float32, shape=[minibatch, 80,80,1], name="input") 

    with tf.variable_scope('layer1'): 
     W = weight_variable([8, 8, 1, 32]) 
     variable_summaries(W, "layer1/W") 
     b = bias_variable([32]) 
     variable_summaries(b, "layer1/b") 
     h = conv2d(state, W, 4) + b 
     activation = tf.nn.relu(h) 
     pool1 = max_pool_2x2(activation) 

    print(pool1.get_shape()) 
    pool1 = tf.reshape(pool1, [-1, 3200]) 

    with tf.variable_scope('readout'): 
     W = weight_variable([3200, 3]) 
     b = bias_variable([3]) 
     logits = tf.matmul(pool1, W) + b 
     variable_summaries(h, "y") 

    action_indexes = tf.placeholder(tf.int32, shape=[minibatch], name="action_indexes") 

    loss = tf.nn.sparse_softmax_cross_entropy_with_logits(logits, action_indexes) 

    starter_learning_rate = 1e-6 

    global_step = tf.Variable(0, trainable=False) 

    # decay every 1000 steps with a base of 0.96: 
    learning_rate = tf.train.exponential_decay(starter_learning_rate, 
     global_step, 
     10000, 0.96, staircase=True) 

    optimizer = tf.train.RMSPropOptimizer(learning_rate) 

    trainable_variables = tf.trainable_variables() 
    varname_to_var = dict((v.name, v) for v in trainable_variables) 
    keys = sorted(varname_to_var.keys()) 

    gradients_and_variables = optimizer.compute_gradients(loss, [ varname_to_var[k] for k in keys]) 

    var_to_grad = dict((var.name, grad) for grad, var in gradients_and_variables) 

    name_to_var = dict((var.name, var) for _, var in gradients_and_variables) 

    # save the gradients in memory 
    var_to_ref_grad = {} 
    for k in keys: 
     grad = var_to_grad[k] 
     print(k, grad.get_shape()) 
     ref = tf.Variable(tf.zeros_like(grad)) 
     ref = ref.assign_add(grad) 
     var_to_ref_grad[k] = ref 

    discounted_values = tf.placeholder(tf.float32, shape=[None, 1], name='discounted_values') 

    # control when to apply gradients 
    compute_gradients_flag = tf.placeholder(tf.int32, name="compute_gradients") 
    def fn1(): 
     var_grad_list = [] 
     for k in keys: 
      grad = var_to_ref_grad[k] 
      var = varname_to_var[k] 
      var_grad_list.append((grad,var)) 

     optimizer.apply_gradients(var_grad_list) 
     return tf.no_op() 

    fn2 = lambda : tf.no_op() 

    last_op = tf.cond(tf.equal(compute_gradients_flag, 1), fn1, fn2) 

with tf.Session(graph=graph) as s: 

    feed_dict= { 
     state : np.zeros([minibatch, 80, 80, 1]), 
     action_indexes: [1], 
     compute_gradients_flag: False, 
    } 

    s.run(tf.initialize_all_variables()) 

    for i in range(10): 

     # accumulate gradients 
     s.run(last_op, feed_dict=feed_dict) 

Answer 1

Non avete davvero bisogno di accumulare manualmente gradienti. È possibile fare in modo che Tensorflow li accumuli applicando l'aggiornamento di implementazione come batch.

s_list = list_of_states_visited 
a_list = list_of_actions_taken 
R_list = list_of_value_targets 

sess.run(local_net.update, feed_dict={ 
    local_net.input: s_list, 
    local_net.a: a_list, 
    local_net.R: R_list 
}) 

Answer 2

Qualcosa di simile potrebbe funzionare per creare ops per accumulare gradienti, il ripristino delle pendenze accumulate, e applicando i gradienti accumulati (non testata!):

def build_gradient_accumulators(optimizer, gradients_and_variables): 
    accum_grads_and_vars = [] 
    accumulators = [] 
    resetters = [] 

    for grad, var in gradients_and_variables: 
     accum = tf.Variable(tf.zeros_like(grad)) 
     accum = accum.assign_add(grad) 
     accumulators.append(accum) 
     accum_grads_and_vars.append((accum, var)) 
     resetters.append(tf.assign(accum, tf.zeros_like(accum))) 

    reset_op = tf.group(*resetters) 
    accum_op = tf.group(*accumulators) 
    apply_op = optimizer.apply_gradients(accum_grads_and_vars) 
    return reset_op, accum_op, apply_op