有沒有辦法來減少RMSProp的代碼量

Question

我有一些簡單的循環神經網絡的代碼，並想知道是否有一種方法可以減少更新階段所需的代碼量。我有這樣的代碼：

class RNN(object): 
    def__init___(self, data, hidden_size, eps=0.0001): 
     self.data = data 
     self.hidden_size = hidden_size 
     self.weights_hidden = np.random.rand(hidden_size, hidden_size) * 0.1 # W 
     self.weights_input = np.random.rand(hidden_size, len(data[0])) * 0.1 # U 
     self.weights_output = np.random.rand(len(data[0]), hidden_size) * 0.1 # V 
     self.bias_hidden = np.array([np.random.rand(hidden_size)]).T # b 
     self.bias_output = np.array([np.random.rand(len(data[0]))]).T # c 

     self.cache_w_hid, self.cache_w_in, self.cache_w_out = 0, 0, 0 
     self.cache_b_hid, self.cache_b_out = 0, 0 
     self.eps = eps 

    def train(self, seq_length, epochs, eta, decay_rate=0.9, learning_decay=0.0): 
     # Other stuff 
     self.update(seq, epoch, eta, decay_rate, learning_decay) 
     # Other Stuff 

    def update(self, seq, epoch, eta, decay_rate, learning_decay): 
     """Updates the network's weights and biases by applying gradient 
     descent using backpropagation through time and RMSPROP. 
     """ 
     delta_nabla_c, delta_nabla_b,\ 
     delta_nabla_V, delta_nabla_W, delta_nabla_U = self.backward_pass(seq) 

     eta = eta*np.exp(-epoch*learning_decay) 

     self.cache_w_hid += decay_rate * self.cache_w_hid \ 
          + (1 - decay_rate) * delta_nabla_W**2 
     self.weights_hidden -= eta * delta_nabla_W/(np.sqrt(self.cache_w_hid) + self.eps) 

     self.cache_w_in += decay_rate * self.cache_w_in \ 
          + (1 - decay_rate) * delta_nabla_U**2   
     self.weights_input -= eta * delta_nabla_U/(np.sqrt(self.cache_w_in) + self.eps) 

     self.cache_w_out += decay_rate * self.cache_w_out \ 
          + (1 - decay_rate) * delta_nabla_V**2 
     self.weights_output -= eta * delta_nabla_V/(np.sqrt(self.cache_w_out) + self.eps) 

     self.cache_b_hid += decay_rate * self.cache_b_hid \ 
          + (1 - decay_rate) * delta_nabla_b**2 
     self.bias_hidden -= eta * delta_nabla_b/(np.sqrt(self.cache_b_hid) + self.eps) 

     self.cache_b_out += decay_rate * self.cache_b_out \ 
          + (1 - decay_rate) * delta_nabla_c**2 
     self.bias_output -= eta * delta_nabla_c/(np.sqrt(self.cache_b_out) + self.eps) 

爲#RMSProp下每個變量如下更新規則，即：

cache = decay_rate * cache + (1 - decay_rate) * dx**2 
x += - learning_rate * dx/(np.sqrt(cache) + eps) 

我cache_都宣佈之後self.weight_或self.bias_並希望有這樣寫更緊湊。我在尋找使用zip()，但我不知道如何去做。

Answer 1

從你的問題來看，我猜你正在努力提高可讀性/優雅性，優於任何其他類型的優化。

您可以引入一個函數來實現更新規則，然後爲每個變量調用一次。這裏的訣竅是Python允許您按名稱訪問屬性，因此您可以傳遞緩存和權重屬性的名稱而不是值。這將讓你對未來的通行證更新值：

def update_rule(self, cache_attr, x_attr, decay_rate, learning_rate, dx): 
    cache = getattr(self, cache_attr) 
    cache = decay_rate * cache + (1 - decay_rate) * dx**2 
    setattr(self, cache_attr, cache) 

    x = getattr(self, x_attr) 
    x += - learning_rate * dx/(np.sqrt(cache) + self.eps) 
    setattr(self, x_attr, x) 

def update(self, seq, epoch, eta, decay_rate, learning_decay): 
    """Updates the network's weights and biases by applying gradient 
    descent using backpropagation through time and RMSPROP. 
    """ 
    delta_nabla_c, delta_nabla_b,\ 
    delta_nabla_V, delta_nabla_W, delta_nabla_U = self.backward_pass(seq) 

    eta = eta*np.exp(-epoch*learning_decay) 

    self.update_rule('cache_w_hid', 'weights_hidden', decay_rate, eta, delta_nabla_W) 
    self.update_rule('cache_w_in', 'weights_input', decay_rate, eta, delta_nabla_U) 
    self.update_rule('cache_w_out', 'weights_output', decay_rate, eta, delta_nabla_V) 
    self.update_rule('cache_b_hid', 'bias_hidden', decay_rate, eta, delta_nabla_b) 
    self.update_rule('cache_b_out', 'bias_output', decay_rate, eta, delta_nabla_c) 

事實上，你可以節省額外的參數和避免暴露什麼基本上是通過把update_rule爲update的私有方法。這將暴露的update到update_rule命名空間，當它被調用，所以你不必在decay_rate和learning_rate經過：

def update(self, seq, epoch, eta, decay_rate, learning_decay): 
    """Updates the network's weights and biases by applying gradient 
    descent using backpropagation through time and RMSPROP. 
    """ 

    def update_rule(cache_attr, x_attr, dx): 
     cache = getattr(self, cache_attr) 
     cache = decay_rate * cache + (1 - decay_rate) * dx**2 
     setattr(self, cache_attr, cache) 

     x = getattr(self, x_attr) 
     x += - eta * dx/(np.sqrt(cache) + self.eps) 
     setattr(self, x_attr, x) 

    delta_nabla_c, delta_nabla_b,\ 
    delta_nabla_V, delta_nabla_W, delta_nabla_U = self.backward_pass(seq) 

    eta = eta*np.exp(-epoch*learning_decay) 

    update_rule('cache_w_hid', 'weights_hidden', delta_nabla_W) 
    update_rule('cache_w_in', 'weights_input', delta_nabla_U) 
    update_rule('cache_w_out', 'weights_output', delta_nabla_V) 
    update_rule('cache_b_hid', 'bias_hidden', delta_nabla_b) 
    update_rule('cache_b_out', 'bias_output', delta_nabla_c) 

最後，如果你真的想，你可以使用zip把調用update_rule成一個循環。請注意，對於此版本，調用的順序已更改爲與self.backward_pass返回的值的順序相匹配。我個人不會使用這個最後的版本，除非你真的有很多更新要做，因爲除了它對backward_pass的結果非常敏感之外，它開始看起來模糊不清。

def update(self, seq, epoch, eta, decay_rate, learning_decay): 
    """Updates the network's weights and biases by applying gradient 
    descent using backpropagation through time and RMSPROP. 
    """ 

    def update_rule(cache_attr, x_attr, dx): 
     cache = getattr(self, cache_attr) 
     cache = decay_rate * cache + (1 - decay_rate) * dx**2 
     setattr(self, cache_attr, cache) 

     x = getattr(self, x_attr) 
     x += - eta * dx/(np.sqrt(cache) + self.eps) 
     setattr(self, x_attr, x) 

    dx = self.backward_pass(seq) 

    eta = eta*np.exp(-epoch*learning_decay) 

    cache_attrs = ('cache_b_out', 'cache_b_hid', 'cache_w_out', 'cache_w_hid', 'cache_w_in') 
    x_attrs = ('bias_output', 'bias_hidden', 'weights_output', 'weights_hidden', 'weights_input') 

    for args in zip(cache_attrs, x_attrs, dx): 
     update_rule(*args)