Algoritmo di backpropagation della rete neurale non funzionante in Python

Question

Sto scrivendo una rete neurale in Python, seguendo l'esempio here. Sembra che l'algoritmo di backpropagation non funzioni, dato che la rete neurale non riesce a produrre il giusto valore (entro un margine di errore) dopo essere stata addestrata 10 mila volte. In particolare, mi alleno per calcolare la funzione seno nel seguente esempio:

import numpy as np 

class Neuralnet: 
    def __init__(self, neurons): 
     self.weights = [] 
     self.inputs = [] 
     self.outputs = [] 
     self.errors = [] 
     self.rate = .1 
     for layer in range(len(neurons)): 
      self.inputs.append(np.empty(neurons[layer])) 
      self.outputs.append(np.empty(neurons[layer])) 
      self.errors.append(np.empty(neurons[layer])) 
     for layer in range(len(neurons)-1): 
      self.weights.append(
       np.random.normal(
        scale=1/np.sqrt(neurons[layer]), 
        size=[neurons[layer], neurons[layer + 1]] 
        ) 
       ) 

    def feedforward(self, inputs): 
     self.inputs[0] = inputs 
     for layer in range(len(self.weights)): 
      self.outputs[layer] = np.tanh(self.inputs[layer]) 
      self.inputs[layer + 1] = np.dot(self.weights[layer].T, self.outputs[layer]) 
     self.outputs[-1] = np.tanh(self.inputs[-1]) 

    def backpropagate(self, targets): 
     gradient = 1 - self.outputs[-1] * self.outputs[-1] 
     self.errors[-1] = gradient * (self.outputs[-1] - targets) 
     for layer in reversed(range(len(self.errors) - 1)): 
      gradient = 1 - self.outputs[layer] * self.outputs[layer] 
      self.errors[layer] = gradient * np.dot(self.weights[layer], self.errors[layer + 1]) 
     for layer in range(len(self.weights)): 
      self.weights[layer] -= self.rate * np.outer(self.outputs[layer], self.errors[layer + 1]) 

def xor_example(): 
    net = Neuralnet([2, 2, 1]) 
    for step in range(100000): 
     net.feedforward([0, 0]) 
     net.backpropagate([-1]) 
     net.feedforward([0, 1]) 
     net.backpropagate([1]) 
     net.feedforward([1, 0]) 
     net.backpropagate([1]) 
     net.feedforward([1, 1]) 
     net.backpropagate([-1]) 
    net.feedforward([1, 1]) 
    print(net.outputs[-1]) 

def identity_example(): 
    net = Neuralnet([1, 3, 1]) 
    for step in range(100000): 
     x = np.random.normal() 
     net.feedforward([x]) 
     net.backpropagate([np.tanh(x)]) 
    net.feedforward([-2]) 
    print(net.outputs[-1]) 

def sine_example(): 
    net = Neuralnet([1, 6, 1]) 
    for step in range(100000): 
     x = np.random.normal() 
     net.feedforward([x]) 
     net.backpropagate([np.tanh(np.sin(x))]) 
    net.feedforward([3]) 
    print(net.outputs[-1]) 

sine_example() 

L'uscita non riesce ad essere vicino a tanh(sin(3)) = 0.140190616. Sospettavo un errore che coinvolgesse indici o allineamenti sbagliati, ma Numpy non sta sollevando errori come questi. Qualche consiglio su dove ho sbagliato?

MODIFICA: Ho dimenticato di aggiungere i neuroni di bias. Ecco il codice aggiornato:

import numpy as np 

class Neuralnet: 
    def __init__(self, neurons): 
     self.weights = [] 
     self.outputs = [] 
     self.inputs = [] 
     self.errors = [] 
     self.offsets = [] 
     self.rate = .01 
     for layer in range(len(neurons)-1): 
      self.weights.append(
       np.random.normal(
        scale=1/np.sqrt(neurons[layer]), 
        size=[neurons[layer], neurons[layer + 1]] 
        ) 
       ) 
      self.outputs.append(np.empty(neurons[layer])) 
      self.inputs.append(np.empty(neurons[layer])) 
      self.errors.append(np.empty(neurons[layer])) 
      self.offsets.append(np.random.normal(scale=1/np.sqrt(neurons[layer]), size=neurons[layer + 1])) 
     self.inputs.append(np.empty(neurons[-1])) 
     self.errors.append(np.empty(neurons[-1])) 

    def feedforward(self, inputs): 
     self.inputs[0] = inputs 
     for layer in range(len(self.weights)): 
      self.outputs[layer] = np.tanh(self.inputs[layer]) 
      self.inputs[layer + 1] = self.offsets[layer] + np.dot(self.weights[layer].T, self.outputs[layer]) 

    def backpropagate(self, targets): 
     self.errors[-1] = self.inputs[-1] - targets 
     for layer in reversed(range(len(self.errors) - 1)): 
      gradient = 1 - self.outputs[layer] * self.outputs[layer] 
      self.errors[layer] = gradient * np.dot(self.weights[layer], self.errors[layer + 1]) 
     for layer in range(len(self.weights)): 
      self.weights[layer] -= self.rate * np.outer(self.outputs[layer], self.errors[layer + 1]) 
      self.offsets[layer] -= self.rate * self.errors[layer + 1] 

def sine_example(): 
    net = Neuralnet([1, 5, 1]) 
    for step in range(10000): 
     x = np.random.uniform(-5, 5) 
     net.feedforward([x]) 
     net.backpropagate([np.sin(x)]) 
    net.feedforward([np.pi]) 
    print(net.inputs[-1]) 

def xor_example(): 
    net = Neuralnet([2, 2, 1]) 
    for step in range(10000): 
     net.feedforward([0, 0]) 
     net.backpropagate([-1]) 
     net.feedforward([0, 1]) 
     net.backpropagate([1]) 
     net.feedforward([1, 0]) 
     net.backpropagate([1]) 
     net.feedforward([1, 1]) 
     net.backpropagate([-1]) 
    net.feedforward([1, 1]) 
    print(net.outputs[-1]) 

def identity_example(): 
    net = Neuralnet([1, 3, 1]) 
    for step in range(10000): 
     x = np.random.normal() 
     net.feedforward([x]) 
     net.backpropagate([x]) 
    net.feedforward([-2]) 
    print(net.outputs[-1]) 

identity_example() 

Answer 1

Penso che si alleni il NN nel modo sbagliato. Hai un ciclo di oltre 10000 iterazioni e inserisci un nuovo campione in ogni ciclo. Il NN non verrà mai addestrato in questo caso.

(la dichiarazione è sbagliato! Vedere l'aggiornamento!)

Quello che dovete fare è di generare una vasta gamma di veri campioni Y = sin(X), dare alla vostra rete UNA VOLTA e iterare il training set avanti e indietro, al fine di minimizzare la funzione di costo. Per controllare l'algoritmo potrebbe essere necessario tracciare la funzione di costo in base al numero di iterazione e assicurarsi che il costo diminuisca.

Un altro punto importante è l'inizializzazione dei pesi. I tuoi numeri sono piuttosto grandi e la rete impiegherà molto tempo per convergere, specialmente quando si utilizzano tariffe basse. È buona norma generare i pesi iniziali in un piccolo intervallo [-eps .. eps] in modo uniforme.

Nel mio codice ho implementato due diverse funzioni di attivazione: sigmoid() e tanh(). È necessario ridimensionare gli ingressi in base alla funzione selezionata: [0 .. 1] e [-1 .. 1] rispettivamente.

Ecco alcune immagini che mostrano la funzione di costo e le previsioni che ne derivano per sigmoid() e tanh() funzioni di attivazione:

Come si può vedere l'attivazione sigmoid() dà un po ' risultati leggermente migliori, rispetto allo tanh().

Inoltre ho avuto previsioni molto meglio quando si utilizza una rete [1, 6, 1], a fronte di una rete più grande con 4 strati [1, 6, 4, 1]. Quindi la dimensione del NN non è sempre il fattore cruciale.Ecco la previsione per la rete menzionata con 4 strati:

Ecco il mio codice con alcuni commenti. Ho provato ad usare le tue notazioni dov'era possibile.

import numpy as np 
import math 
import matplotlib.pyplot as plt 

class Neuralnet: 
    def __init__(self, neurons, activation): 
     self.weights = [] 
     self.inputs = [] 
     self.outputs = [] 
     self.errors = [] 
     self.rate = 0.5 
     self.activation = activation #sigmoid or tanh 

     self.neurons = neurons 
     self.L = len(self.neurons)  #number of layers 

     eps = 0.12; # range for uniform distribution -eps..+eps    
     for layer in range(len(neurons)-1): 
      self.weights.append(np.random.uniform(-eps,eps,size=(neurons[layer+1], neurons[layer]+1)))    


    ###################################################################################################  
    def train(self, X, Y, iter_count): 

     m = X.shape[0]; 

     for layer in range(self.L): 
      self.inputs.append(np.empty([m, self.neurons[layer]]))   
      self.errors.append(np.empty([m, self.neurons[layer]])) 

      if (layer < self.L -1): 
       self.outputs.append(np.empty([m, self.neurons[layer]+1])) 
      else: 
       self.outputs.append(np.empty([m, self.neurons[layer]])) 

     #accumulate the cost function 
     J_history = np.zeros([iter_count, 1]) 


     for i in range(iter_count): 

      self.feedforward(X) 

      J = self.cost(Y, self.outputs[self.L-1]) 
      J_history[i, 0] = J 

      self.backpropagate(Y) 


     #plot the cost function to check the descent 
     plt.plot(J_history) 
     plt.show() 


    ###################################################################################################  
    def cost(self, Y, H):  
     J = np.sum(np.sum(np.power((Y - H), 2), axis=0))/(2*m) 
     return J 

    ################################################################################################### 
    def feedforward(self, X): 

     m = X.shape[0]; 

     self.outputs[0] = np.concatenate( (np.ones([m, 1]), X), axis=1) 

     for i in range(1, self.L): 
      self.inputs[i] = np.dot(self.outputs[i-1], self.weights[i-1].T ) 

      if (self.activation == 'sigmoid'): 
       output_temp = self.sigmoid(self.inputs[i]) 
      elif (self.activation == 'tanh'): 
       output_temp = np.tanh(self.inputs[i]) 


      if (i < self.L - 1): 
       self.outputs[i] = np.concatenate( (np.ones([m, 1]), output_temp), axis=1) 
      else: 
       self.outputs[i] = output_temp 

    ################################################################################################### 
    def backpropagate(self, Y): 

     self.errors[self.L-1] = self.outputs[self.L-1] - Y 

     for i in range(self.L - 2, 0, -1): 

      if (self.activation == 'sigmoid'): 
       self.errors[i] = np.dot( self.errors[i+1], self.weights[i][:, 1:] ) * self.sigmoid_prime(self.inputs[i]) 
      elif (self.activation == 'tanh'): 
       self.errors[i] = np.dot( self.errors[i+1], self.weights[i][:, 1:] ) * (1 - self.outputs[i][:, 1:]*self.outputs[i][:, 1:]) 

     for i in range(0, self.L-1): 
      grad = np.dot(self.errors[i+1].T, self.outputs[i])/m 
      self.weights[i] = self.weights[i] - self.rate*grad 

    ################################################################################################### 
    def sigmoid(self, z): 
     s = 1.0/(1.0 + np.exp(-z)) 
     return s 

    ################################################################################################### 
    def sigmoid_prime(self, z): 
     s = self.sigmoid(z)*(1 - self.sigmoid(z)) 
     return s  

    ################################################################################################### 
    def predict(self, X, weights): 

     m = X.shape[0]; 

     self.inputs = [] 
     self.outputs = [] 
     self.weights = weights 

     for layer in range(self.L): 
      self.inputs.append(np.empty([m, self.neurons[layer]]))   

      if (layer < self.L -1): 
       self.outputs.append(np.empty([m, self.neurons[layer]+1])) 
      else: 
       self.outputs.append(np.empty([m, self.neurons[layer]])) 

     self.feedforward(X) 

     return self.outputs[self.L-1] 


################################################################################################### 
#    MAIN PART 

activation1 = 'sigmoid'  # the input should be scaled into [ 0..1] 
activation2 = 'tanh'  # the input should be scaled into [-1..1] 

activation = activation1 

net = Neuralnet([1, 6, 1], activation) # structure of the NN and its activation function 


########################################################################################## 
#    TRAINING 

m = 1000 #size of the training set 
X = np.linspace(0, 4*math.pi, num = m).reshape(m, 1); # input training set 


Y = np.sin(X) # target 

kx = 0.1 # noise parameter 
noise = (2.0*np.random.uniform(0, kx, m) - kx).reshape(m, 1) 
Y = Y + noise # noisy target 

# scaling of the target depending on the activation function 
if (activation == 'sigmoid'): 
    Y_scaled = (Y/(1+kx) + 1)/2.0 
elif (activation == 'tanh'): 
    Y_scaled = Y/(1+kx) 


# number of the iteration for the training stage 
iter_count = 20000 
net.train(X, Y_scaled, iter_count) #training 

# gained weights 
trained_weights = net.weights 

########################################################################################## 
#     PREDICTION 

m_new = 40 #size of the prediction set 
X_new = np.linspace(0, 4*math.pi, num = m_new).reshape(m_new, 1); 

Y_new = net.predict(X_new, trained_weights) # prediction 

#rescaling of the result 
if (activation == 'sigmoid'): 
    Y_new = (2.0*Y_new - 1.0) * (1+kx) 
elif (activation == 'tanh'): 
    Y_new = Y_new * (1+kx) 

# visualization 
plt.plot(X, Y) 
plt.plot(X_new, Y_new, 'ro') 
plt.show() 

raw_input('press any key to exit') 

UPDATE

Vorrei riprendere l'affermazione per quanto riguarda il metodo di allenamento utilizzati nel codice. La rete può essere effettivamente addestrata usando un solo campione per iterazione. Ho ottenuto risultati interessanti in online-formazione utilizzando entrambe le funzioni sigma e attivazione tanh:

Online-formazione utilizzando Sigmoid (funzione di costo e la previsione)

Online-formazione utilizzando Tanh (funzione di costo e di predizione)

Come si può vedere, la scelta di Sigmoid come funzione di attivazione offre prestazioni migliori. La funzione di costo non sembra buona come durante l'allenamento offline, ma almeno tende a scendere.

ho tracciato la funzione di costo nell'implementazione, sembra piuttosto a scatti così:

Forse è una buona idea per cercare il codice con il sigma o anche la funzione di Relu.

Ecco il codice sorgente aggiornato. Per passare tra le modalità di allenamento online e offline basta cambiare la variabile method.

import numpy as np 
import math 
import matplotlib.pyplot as plt 

class Neuralnet: 
    def __init__(self, neurons, activation): 
     self.weights = [] 
     self.inputs = [] 
     self.outputs = [] 
     self.errors = [] 
     self.rate = 0.2 
     self.activation = activation #sigmoid or tanh 

     self.neurons = neurons 
     self.L = len(self.neurons)  #number of layers 

     eps = 0.12; #range for uniform distribution -eps..+eps    
     for layer in range(len(neurons)-1): 
      self.weights.append(np.random.uniform(-eps,eps,size=(neurons[layer+1], neurons[layer]+1)))    


    ###################################################################################################  
    def train(self, X, Y, iter_count): 

     m = X.shape[0]; 

     for layer in range(self.L): 
      self.inputs.append(np.empty([m, self.neurons[layer]]))   
      self.errors.append(np.empty([m, self.neurons[layer]])) 

      if (layer < self.L -1): 
       self.outputs.append(np.empty([m, self.neurons[layer]+1])) 
      else: 
       self.outputs.append(np.empty([m, self.neurons[layer]])) 

     #accumulate the cost function 
     J_history = np.zeros([iter_count, 1]) 


     for i in range(iter_count): 

      self.feedforward(X) 

      J = self.cost(Y, self.outputs[self.L-1]) 
      J_history[i, 0] = J 

      self.backpropagate(Y) 


     #plot the cost function to check the descent 
     #plt.plot(J_history) 
     #plt.show() 


    ###################################################################################################  
    def cost(self, Y, H):  
     J = np.sum(np.sum(np.power((Y - H), 2), axis=0))/(2*m) 
     return J 


    ################################################################################################### 
    def cost_online(self, min_x, max_x, iter_number): 
     h_arr = np.zeros([iter_number, 1]) 
     y_arr = np.zeros([iter_number, 1]) 

     for step in range(iter_number): 
      x = np.random.uniform(min_x, max_x, 1).reshape(1, 1) 

      self.feedforward(x) 
      h_arr[step, 0] = self.outputs[-1] 
      y_arr[step, 0] = np.sin(x) 



     J = np.sum(np.sum(np.power((y_arr - h_arr), 2), axis=0))/(2*iter_number) 
     return J 

    ################################################################################################### 
    def feedforward(self, X): 

     m = X.shape[0]; 

     self.outputs[0] = np.concatenate( (np.ones([m, 1]), X), axis=1) 

     for i in range(1, self.L): 
      self.inputs[i] = np.dot(self.outputs[i-1], self.weights[i-1].T ) 

      if (self.activation == 'sigmoid'): 
       output_temp = self.sigmoid(self.inputs[i]) 
      elif (self.activation == 'tanh'): 
       output_temp = np.tanh(self.inputs[i]) 


      if (i < self.L - 1): 
       self.outputs[i] = np.concatenate( (np.ones([m, 1]), output_temp), axis=1) 
      else: 
       self.outputs[i] = output_temp 

    ################################################################################################### 
    def backpropagate(self, Y): 

     self.errors[self.L-1] = self.outputs[self.L-1] - Y 

     for i in range(self.L - 2, 0, -1): 

      if (self.activation == 'sigmoid'): 
       self.errors[i] = np.dot( self.errors[i+1], self.weights[i][:, 1:] ) * self.sigmoid_prime(self.inputs[i]) 
      elif (self.activation == 'tanh'): 
       self.errors[i] = np.dot( self.errors[i+1], self.weights[i][:, 1:] ) * (1 - self.outputs[i][:, 1:]*self.outputs[i][:, 1:]) 

     for i in range(0, self.L-1): 
      grad = np.dot(self.errors[i+1].T, self.outputs[i])/m 
      self.weights[i] = self.weights[i] - self.rate*grad 


    ################################################################################################### 
    def sigmoid(self, z): 
     s = 1.0/(1.0 + np.exp(-z)) 
     return s 

    ################################################################################################### 
    def sigmoid_prime(self, z): 
     s = self.sigmoid(z)*(1 - self.sigmoid(z)) 
     return s  

    ################################################################################################### 
    def predict(self, X, weights): 

     m = X.shape[0]; 

     self.inputs = [] 
     self.outputs = [] 
     self.weights = weights 

     for layer in range(self.L): 
      self.inputs.append(np.empty([m, self.neurons[layer]]))   

      if (layer < self.L -1): 
       self.outputs.append(np.empty([m, self.neurons[layer]+1])) 
      else: 
       self.outputs.append(np.empty([m, self.neurons[layer]])) 

     self.feedforward(X) 

     return self.outputs[self.L-1] 


################################################################################################### 
#    MAIN PART 

activation1 = 'sigmoid'  #the input should be scaled into [0..1] 
activation2 = 'tanh'  #the input should be scaled into [-1..1] 

activation = activation1 

net = Neuralnet([1, 6, 1], activation) # structure of the NN and its activation function 


method1 = 'online' 
method2 = 'offline' 

method = method1 

kx = 0.1 #noise parameter 

################################################################################################### 
#    TRAINING 

if (method == 'offline'): 

    m = 1000 #size of the training set 
    X = np.linspace(0, 4*math.pi, num = m).reshape(m, 1); #input training set 


    Y = np.sin(X) #target 


    noise = (2.0*np.random.uniform(0, kx, m) - kx).reshape(m, 1) 
    Y = Y + noise #noisy target 

    #scaling of the target depending on the activation function 
    if (activation == 'sigmoid'): 
     Y_scaled = (Y/(1+kx) + 1)/2.0 
    elif (activation == 'tanh'): 
     Y_scaled = Y/(1+kx) 


    #number of the iteration for the training stage 
    iter_count = 20000 
    net.train(X, Y_scaled, iter_count) #training 

elif (method == 'online'): 

    sampling_count = 100000 # number of samplings during the training stage 


    m = 1 #batch size 

    iter_count = sampling_count/m 

    for layer in range(net.L): 
     net.inputs.append(np.empty([m, net.neurons[layer]]))   
     net.errors.append(np.empty([m, net.neurons[layer]])) 

     if (layer < net.L -1): 
      net.outputs.append(np.empty([m, net.neurons[layer]+1])) 
     else: 
      net.outputs.append(np.empty([m, net.neurons[layer]]))  

    J_history = [] 
    step_history = [] 

    for i in range(iter_count): 
     X = np.random.uniform(0, 4*math.pi, m).reshape(m, 1) 

     Y = np.sin(X) #target 
     noise = (2.0*np.random.uniform(0, kx, m) - kx).reshape(m, 1) 
     Y = Y + noise #noisy target 

     #scaling of the target depending on the activation function 
     if (activation == 'sigmoid'): 
      Y_scaled = (Y/(1+kx) + 1)/2.0 
     elif (activation == 'tanh'): 
      Y_scaled = Y/(1+kx) 

     net.feedforward(X) 
     net.backpropagate(Y_scaled) 


     if (np.remainder(i, 1000) == 0): 
      J = net.cost_online(0, 4*math.pi, 1000) 
      J_history.append(J) 
      step_history.append(i) 

    plt.plot(step_history, J_history) 
    plt.title('Batch size ' + str(m) + ', rate ' + str(net.rate) + ', samples ' + str(sampling_count)) 
    #plt.ylim([0, 0.1]) 

    plt.show() 

#gained weights 
trained_weights = net.weights 

########################################################################################## 
#     PREDICTION 

m_new = 40 #size of the prediction set 
X_new = np.linspace(0, 4*math.pi, num = m_new).reshape(m_new, 1); 

Y_new = net.predict(X_new, trained_weights) #prediction 

#rescaling of the result 
if (activation == 'sigmoid'): 
    Y_new = (2.0*Y_new - 1.0) * (1+kx) 
elif (activation == 'tanh'): 
    Y_new = Y_new * (1+kx) 

#visualization 

#fake sine curve to show the ideal signal 
if (method == 'online'): 
    X = np.linspace(0, 4*math.pi, num = 100) 
    Y = np.sin(X) 

plt.plot(X, Y) 

plt.plot(X_new, Y_new, 'ro') 
if (method == 'online'): 
    plt.title('Batch size ' + str(m) + ', rate ' + str(net.rate) + ', samples ' + str(sampling_count)) 
plt.ylim([-1.5, 1.5]) 
plt.show() 

raw_input('press any key to exit') 

ora ho alcune osservazioni al codice corrente:

tua funzione seno assomiglia a questo:

Non so il motivo per cui si utilizza tanh nel vostro ingresso di destinazione . Se vuoi veramente usare tanh di seno come target, devi ridimensionarlo a [-1..1], perché tanh (sin (x)) restituisce valori nell'intervallo [-0.76..0.76].

La prossima cosa è la gamma del set di allenamento. Si utilizza x = np.random.normal() per generare i campioni. Ecco la distribuzione di un tale input:

Dopo che si desidera la vostra rete per predire il seno di 3, ma la rete è quasi mai visto questo numero durante la fase di formazione. Vorrei utilizzare la distribuzione uniforme in una gamma più ampia per la generazione di campioni.