Fix activation functions support (#163)

- Backpropagation using the neuron activation functions derivative
- instead of hardcoded sigmoid derivative
- Added missing activation functions derivatives
- Sigmoid forced for the output layer
- Updated ThresholdedReLU default threshold to 0 (acts as a ReLU)
- Unit tests for derivatives
- Unit tests for classifiers using different activation functions
- Added missing docs
This commit is contained in:
David Monllaó 2018-01-09 11:09:59 +01:00 committed by Arkadiusz Kondas
parent 9938cf2911
commit e83f7b95d5
18 changed files with 254 additions and 6 deletions

View File

@ -66,4 +66,6 @@ $mlp->predict([[1, 1, 1, 1], [0, 0, 0, 0]]);
* BinaryStep * BinaryStep
* Gaussian * Gaussian
* HyperbolicTangent * HyperbolicTangent
* Parametric Rectified Linear Unit
* Sigmoid (default) * Sigmoid (default)
* Thresholded Rectified Linear Unit

View File

@ -10,4 +10,10 @@ interface ActivationFunction
* @param float|int $value * @param float|int $value
*/ */
public function compute($value): float; public function compute($value): float;
/**
* @param float|int $value
* @param float|int $computedvalue
*/
public function differentiate($value, $computedvalue): float;
} }

View File

@ -15,4 +15,17 @@ class BinaryStep implements ActivationFunction
{ {
return $value >= 0 ? 1.0 : 0.0; return $value >= 0 ? 1.0 : 0.0;
} }
/**
* @param float|int $value
* @param float|int $computedvalue
*/
public function differentiate($value, $computedvalue): float
{
if ($value === 0 || $value === 0.0) {
return 1;
}
return 0;
}
} }

View File

@ -15,4 +15,13 @@ class Gaussian implements ActivationFunction
{ {
return exp(-pow($value, 2)); return exp(-pow($value, 2));
} }
/**
* @param float|int $value
* @param float|int $calculatedvalue
*/
public function differentiate($value, $calculatedvalue): float
{
return -2 * $value * $calculatedvalue;
}
} }

View File

@ -25,4 +25,13 @@ class HyperbolicTangent implements ActivationFunction
{ {
return tanh($this->beta * $value); return tanh($this->beta * $value);
} }
/**
* @param float|int $value
* @param float|int $computedvalue
*/
public function differentiate($value, $computedvalue): float
{
return 1 - pow($computedvalue, 2);
}
} }

View File

@ -25,4 +25,13 @@ class PReLU implements ActivationFunction
{ {
return $value >= 0 ? $value : $this->beta * $value; return $value >= 0 ? $value : $this->beta * $value;
} }
/**
* @param float|int $value
* @param float|int $computedvalue
*/
public function differentiate($value, $computedvalue): float
{
return $computedvalue >= 0 ? 1.0 : $this->beta;
}
} }

View File

@ -25,4 +25,13 @@ class Sigmoid implements ActivationFunction
{ {
return 1 / (1 + exp(-$this->beta * $value)); return 1 / (1 + exp(-$this->beta * $value));
} }
/**
* @param float|int $value
* @param float|int $computedvalue
*/
public function differentiate($value, $computedvalue): float
{
return $computedvalue * (1 - $computedvalue);
}
} }

View File

@ -13,7 +13,7 @@ class ThresholdedReLU implements ActivationFunction
*/ */
private $theta; private $theta;
public function __construct(float $theta = 1.0) public function __construct(float $theta = 0.0)
{ {
$this->theta = $theta; $this->theta = $theta;
} }
@ -25,4 +25,13 @@ class ThresholdedReLU implements ActivationFunction
{ {
return $value > $this->theta ? $value : 0.0; return $value > $this->theta ? $value : 0.0;
} }
/**
* @param float|int $value
* @param float|int $calculatedvalue
*/
public function differentiate($value, $calculatedvalue): float
{
return $calculatedvalue >= $this->theta ? 1.0 : 0.0;
}
} }

View File

@ -9,6 +9,7 @@ use Phpml\Exception\InvalidArgumentException;
use Phpml\Helper\Predictable; use Phpml\Helper\Predictable;
use Phpml\IncrementalEstimator; use Phpml\IncrementalEstimator;
use Phpml\NeuralNetwork\ActivationFunction; use Phpml\NeuralNetwork\ActivationFunction;
use Phpml\NeuralNetwork\ActivationFunction\Sigmoid;
use Phpml\NeuralNetwork\Layer; use Phpml\NeuralNetwork\Layer;
use Phpml\NeuralNetwork\Node\Bias; use Phpml\NeuralNetwork\Node\Bias;
use Phpml\NeuralNetwork\Node\Input; use Phpml\NeuralNetwork\Node\Input;
@ -125,7 +126,10 @@ abstract class MultilayerPerceptron extends LayeredNetwork implements Estimator,
{ {
$this->addInputLayer($this->inputLayerFeatures); $this->addInputLayer($this->inputLayerFeatures);
$this->addNeuronLayers($this->hiddenLayers, $this->activationFunction); $this->addNeuronLayers($this->hiddenLayers, $this->activationFunction);
$this->addNeuronLayers([count($this->classes)], $this->activationFunction);
// Sigmoid function for the output layer as we want a value from 0 to 1.
$sigmoid = new Sigmoid();
$this->addNeuronLayers([count($this->classes)], $sigmoid);
$this->addBiasNodes(); $this->addBiasNodes();
$this->generateSynapses(); $this->generateSynapses();

View File

@ -26,6 +26,11 @@ class Neuron implements Node
*/ */
protected $output = 0.0; protected $output = 0.0;
/**
* @var float
*/
protected $z = 0.0;
public function __construct(?ActivationFunction $activationFunction = null) public function __construct(?ActivationFunction $activationFunction = null)
{ {
$this->activationFunction = $activationFunction ?: new Sigmoid(); $this->activationFunction = $activationFunction ?: new Sigmoid();
@ -47,19 +52,25 @@ class Neuron implements Node
public function getOutput(): float public function getOutput(): float
{ {
if ($this->output === 0.0) { if ($this->output === 0.0) {
$sum = 0.0; $this->z = 0;
foreach ($this->synapses as $synapse) { foreach ($this->synapses as $synapse) {
$sum += $synapse->getOutput(); $this->z += $synapse->getOutput();
} }
$this->output = $this->activationFunction->compute($sum); $this->output = $this->activationFunction->compute($this->z);
} }
return $this->output; return $this->output;
} }
public function getDerivative(): float
{
return $this->activationFunction->differentiate($this->z, $this->output);
}
public function reset(): void public function reset(): void
{ {
$this->output = 0.0; $this->output = 0.0;
$this->z = 0.0;
} }
} }

View File

@ -64,7 +64,7 @@ class Backpropagation
private function getSigma(Neuron $neuron, int $targetClass, int $key, bool $lastLayer): float private function getSigma(Neuron $neuron, int $targetClass, int $key, bool $lastLayer): float
{ {
$neuronOutput = $neuron->getOutput(); $neuronOutput = $neuron->getOutput();
$sigma = $neuronOutput * (1 - $neuronOutput); $sigma = $neuron->getDerivative();
if ($lastLayer) { if ($lastLayer) {
$value = 0; $value = 0;

View File

@ -7,6 +7,11 @@ namespace Phpml\Tests\Classification;
use Phpml\Classification\MLPClassifier; use Phpml\Classification\MLPClassifier;
use Phpml\Exception\InvalidArgumentException; use Phpml\Exception\InvalidArgumentException;
use Phpml\ModelManager; use Phpml\ModelManager;
use Phpml\NeuralNetwork\ActivationFunction;
use Phpml\NeuralNetwork\ActivationFunction\HyperbolicTangent;
use Phpml\NeuralNetwork\ActivationFunction\PReLU;
use Phpml\NeuralNetwork\ActivationFunction\Sigmoid;
use Phpml\NeuralNetwork\ActivationFunction\ThresholdedReLU;
use Phpml\NeuralNetwork\Node\Neuron; use Phpml\NeuralNetwork\Node\Neuron;
use PHPUnit\Framework\TestCase; use PHPUnit\Framework\TestCase;
@ -141,6 +146,33 @@ class MLPClassifierTest extends TestCase
$this->assertEquals(4, $network->predict([0, 0, 0, 0, 0])); $this->assertEquals(4, $network->predict([0, 0, 0, 0, 0]));
} }
/**
* @dataProvider activationFunctionsProvider
*/
public function testBackpropagationActivationFunctions(ActivationFunction $activationFunction): void
{
$network = new MLPClassifier(5, [3], ['a', 'b'], 10000, $activationFunction);
$network->train(
[[1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 1, 0], [1, 1, 1, 1, 1]],
['a', 'b', 'a', 'a']
);
$this->assertEquals('a', $network->predict([1, 0, 0, 0, 0]));
$this->assertEquals('b', $network->predict([0, 1, 0, 0, 0]));
$this->assertEquals('a', $network->predict([0, 0, 1, 1, 0]));
$this->assertEquals('a', $network->predict([1, 1, 1, 1, 1]));
}
public function activationFunctionsProvider(): array
{
return [
[new Sigmoid()],
[new HyperbolicTangent()],
[new PReLU()],
[new ThresholdedReLU()],
];
}
public function testSaveAndRestore(): void public function testSaveAndRestore(): void
{ {
// Instantinate new Percetron trained for OR problem // Instantinate new Percetron trained for OR problem

View File

@ -27,4 +27,23 @@ class BinaryStepTest extends TestCase
[0, -0.1], [0, -0.1],
]; ];
} }
/**
* @dataProvider binaryStepDerivativeProvider
*/
public function testBinaryStepDerivative($expected, $value): void
{
$binaryStep = new BinaryStep();
$activatedValue = $binaryStep->compute($value);
$this->assertEquals($expected, $binaryStep->differentiate($value, $activatedValue));
}
public function binaryStepDerivativeProvider(): array
{
return [
[0, -1],
[1, 0],
[0, 1],
];
}
} }

View File

@ -29,4 +29,27 @@ class GaussianTest extends TestCase
[0, -3], [0, -3],
]; ];
} }
/**
* @dataProvider gaussianDerivativeProvider
*/
public function testGaussianDerivative($expected, $value): void
{
$gaussian = new Gaussian();
$activatedValue = $gaussian->compute($value);
$this->assertEquals($expected, $gaussian->differentiate($value, $activatedValue), '', 0.001);
}
public function gaussianDerivativeProvider(): array
{
return [
[0, -5],
[0.735, -1],
[0.779, -0.5],
[0, 0],
[-0.779, 0.5],
[-0.735, 1],
[0, 5],
];
}
} }

View File

@ -30,4 +30,28 @@ class HyperboliTangentTest extends TestCase
[0.3, 0, 0], [0.3, 0, 0],
]; ];
} }
/**
* @dataProvider tanhDerivativeProvider
*/
public function testHyperbolicTangentDerivative($beta, $expected, $value): void
{
$tanh = new HyperbolicTangent($beta);
$activatedValue = $tanh->compute($value);
$this->assertEquals($expected, $tanh->differentiate($value, $activatedValue), '', 0.001);
}
public function tanhDerivativeProvider(): array
{
return [
[1.0, 0, -6],
[1.0, 0.419, -1],
[1.0, 1, 0],
[1.0, 0.419, 1],
[1.0, 0, 6],
[0.5, 0.786, 1],
[0.5, 0.786, -1],
[0.3, 1, 0],
];
}
} }

View File

@ -29,4 +29,27 @@ class PReLUTest extends TestCase
[0.02, -0.06, -3], [0.02, -0.06, -3],
]; ];
} }
/**
* @dataProvider preluDerivativeProvider
*/
public function testPReLUDerivative($beta, $expected, $value): void
{
$prelu = new PReLU($beta);
$activatedValue = $prelu->compute($value);
$this->assertEquals($expected, $prelu->differentiate($value, $activatedValue));
}
public function preluDerivativeProvider(): array
{
return [
[0.5, 0.5, -3],
[0.5, 1, 0],
[0.5, 1, 1],
[0.01, 1, 1],
[1, 1, 1],
[0.3, 1, 0.1],
[0.1, 0.1, -0.1],
];
}
} }

View File

@ -30,4 +30,28 @@ class SigmoidTest extends TestCase
[2.0, 0, -3.75], [2.0, 0, -3.75],
]; ];
} }
/**
* @dataProvider sigmoidDerivativeProvider
*/
public function testSigmoidDerivative($beta, $expected, $value): void
{
$sigmoid = new Sigmoid($beta);
$activatedValue = $sigmoid->compute($value);
$this->assertEquals($expected, $sigmoid->differentiate($value, $activatedValue), '', 0.001);
}
public function sigmoidDerivativeProvider(): array
{
return [
[1.0, 0, -10],
[1, 0.006, -5],
[1.0, 0.25, 0],
[1, 0.006, 5],
[1.0, 0, 10],
[2.0, 0.25, 0],
[0.5, 0.246, 0.5],
[0.5, 0.241, 0.75],
];
}
} }

View File

@ -28,4 +28,26 @@ class ThresholdedReLUTest extends TestCase
[0.9, 0, 0.1], [0.9, 0, 0.1],
]; ];
} }
/**
* @dataProvider thresholdDerivativeProvider
*/
public function testThresholdedReLUDerivative($theta, $expected, $value): void
{
$thresholdedReLU = new ThresholdedReLU($theta);
$activatedValue = $thresholdedReLU->compute($value);
$this->assertEquals($expected, $thresholdedReLU->differentiate($value, $activatedValue));
}
public function thresholdDerivativeProvider(): array
{
return [
[0, 1, 1],
[0, 1, 0],
[0.5, 1, 1],
[0.5, 1, 1],
[0.5, 0, 0],
[2, 0, -1],
];
}
} }