Обратное распространение не заставляет мою сеть изучать простые задачи в большинстве случаев

Question

Я хочу больше узнать о внутренних принципах машинного обучения, поэтому я написал кое-что в Typescript, чтобы разобраться с основами. Это не учиться однако. Я создал небольшой скрипт для клонирования проекта и его визуализации: https://gist.github.com/teh-mICON/7192320a97653a77a95fdc1b7621656c

однако вот соответствующий код:

activate(activation: number, memory: Memory, connection: Connection = null) {
    if (connection !== null)
      this.activationValues[connection.innovation] = activation;
    // if there has been no activations in this forward pass, the activation passed here is the initial input
    if (this.activations == 0) {
      this.netInput = activation;
    } else {
      // otherwise, add the activation to the input
      this.netInput += activation;
    }

    // if all incoming node connections have fired
    if (++this.activations >= this.connectionsBackward.length) {
      // reset the activations counter
      this.activations = 0;

      // calculate the activation value (squash input+ bias)
      // except if input node
      const output = this.getActivation();
      // fire on all outgoing connections
      _.each(this.connectionsForward, (connection: Connection) => {
        if (!memory.allowed(connection.innovation)) return;
        connection.to.activate(output * connection.weight, memory, connection);
        memory.activated(connection.innovation);
      })
    }
  }

    /*
    ∂Eₜₒₜₐₗ / ∂wᵢⱼ  = ( ∂Eₜₒₜₐₗ / ∂outⱼ ) * ( ∂outⱼ / ∂netⱼ ) * (  ∂netⱼ / ∂wᵢⱼ )

    ∂Eₜₒₜₐₗ / ∂outⱼ = - ( target - outⱼ)
    ∂outⱼ / ∂netⱼ = outⱼ (1 - outⱼ) (derivative of activation function, here: sigmoid)
    ∂netⱼ / ∂wᵢⱼ  = outᵢ

    Δw = -η * ∂Eₜₒₜₐₗ / ∂wᵢⱼ = -η * ∂ⱼ * outᵢ
  */
  propagateOutput(ideal, learningRate) {
    // calculate partial derivatives
    const partialDerivativeErrorOut = -(ideal - this.getActivation());
    const partialDerivativeOutNetinput = this.getActivation(true);

    // for all incoming connections
    _.each(this.connectionsBackward, (connection: Connection) => {
      const partialDerivativeNetinputWeight = connection.from.getActivation();

      // calculate partial derivative for error to weight of connection
      const derivativeErrorWeight = partialDerivativeErrorOut * partialDerivativeOutNetinput * partialDerivativeNetinputWeight;
      // assign weight adjustment to connection
      connection.adjustment = connection.delta = -learningRate * derivativeErrorWeight;

      // propagate error backwards
      if (connection.from.type == NODE_TYPE.input) return;
      connection.from.propagateHidden(partialDerivativeErrorOut, learningRate);
    });
  }

  /*
    ∂Eₜₒₜₐₗ / ∂wᵢⱼ  = ( ∂Eₜₒₜₐₗ / ∂outⱼ ) * ( ∂outⱼ / ∂netⱼ ) * (  ∂netⱼ / ∂wᵢⱼ )

    ∂Eₜₒₜₐₗ / ∂outⱼ = Σ [ (∂Eₖ₁ / doutⱼ) + (∂Eₖ₂ / doutⱼ) + ... (∂Eₖₙ / / doutⱼ) ]
                  -> ∂Eₖ / ∂outⱼ   = ∂Eₖ * / ∂netₖ
                                                -> ∂netₖ / ∂outⱼ = wᵢⱼ
    ∂outⱼ / ∂netⱼ = outⱼ (1 - outⱼ)
    ∂netⱼ / ∂wᵢⱼ  = input from this connection

    Δw = -η * ∂Eₜₒₜₐₗ / ∂wᵢⱼ
  */
  propagateHidden(partialDerivativeErrorOutConnected, learningRate) {
    // add up all incoming error derivatives
    this.partialDerivativeErrorOutConnectedSum += partialDerivativeErrorOutConnected;

    // if all incoming connections have backpropagated
    if (++this.propagations == this.connectionsForward.length) {
      this.propagations = 0;

      // calculate partial derivatives
      const partialDerivativeOutNetinput = this.getActivation(true);
      const partialDerivativeNetinputBias = 1;

      // calculate partial derivative for error to bias
      const derivativeErrorBias = this.partialDerivativeErrorOutConnectedSum * partialDerivativeOutNetinput * partialDerivativeNetinputBias;
      // assign bias adjustment to node
      this.adjustment = -learningRate * derivativeErrorBias;

      // for all incoming connections
      _.each(this.connectionsBackward, (connection: Connection) => {
        const partialDerivativeNetinputWeight = this.activationValues[connection.innovation];

        // calculate partial derivative for error to weight of connection
        const derivativeErrorWeight = this.partialDerivativeErrorOutConnectedSum * partialDerivativeOutNetinput * partialDerivativeNetinputWeight;
        // assign weight adjustment to connection
        connection.adjustment = connection.delta = -learningRate * derivativeErrorWeight;

        // propagate errors backwards
        if (connection.from.type == NODE_TYPE.input) return;
        connection.from.propagateHidden(this.partialDerivativeErrorOutConnectedSum, learningRate);
      });
    }
  }

  adjust(memory) {
    // actually adjust bias and connection weights (recursively)
    this.bias += this.adjustment;
    this.adjustment = 0;
    _.each(this.getConnectionsBackward(), (connection: Connection) => {
      if (!memory.allowed(connection.innovation)) return;
      connection.weight += connection.adjustment;
      connection.adjustment = 0;
      memory.activated(connection.innovation);
      connection.from.adjust(memory)
    });
  }

  getActivation(derivative = false) {
    if(this.type == NODE_TYPE.input) {
      return this.netInput;
    }
    return this.squash(this.netInput + this.bias, derivative);
  }

Например, если я хочу, чтобы он отражал ввод / выводЯ получаю одинаковый вывод на каждом узле после нескольких тысяч итераций:

https://i.imgur.com/QzqP5m9.png

То же самое для AND: https://i.imgur.com/fjFIayK.png

То же самое для XOR: https://i.imgur.com/XTAbuXP.png

Буду очень признателен, если кто-то сможет помочьменя с этим.

С уважением, MICON