smallnnde: Neural Network that uses Differential Evolutionary (DE) algorithm for training
by Benjamin Kenwright
 | smallnnde |  |
smallnnde is a flexible neural network implementation with DE training algorithm. Designed to be small, free, and open source. The test case example
uses a 1-3-1 network configuration to emulate a sine wave function.
• Neural network with DE training
• Support multiple layers - i.e., multilayer perceptron (MLP)
• Customizable (easy to configure for different topologies/layers/connections)
• Educational version (sometimes using libraries and packages masks the beauty of just how simple a neural network is at its heart)
• Not dependent on any libraries (vanilla C++)
• Importantly, it's a fun piece of code to play around with and learn about neural networks
The full implementation details are given below:
n>#include <algorithm> //std::max(..) std::min(..)
#include <iostream> // cout
#include <vector>
#include <map>
#include <math.h>
#include <time.h>
using namespace std;
#define TrainSetSize 20
#define PI 3.141592653589793238463
#define epoch 1000000
#define POPULATIONSIZE 50
#define DBG_ASSERT(f) { if(!(f)){ __debugbreak(); }; }
double sigmoidF(double x) { return (1.0f / (1.0f + std::exp(-x))); }
//double random(double min, double max) { return min + rand() / (RAND_MAX / (max - min + 1.0) + 1.0); }
double random(double min, double max) { return min + (max - min) * ((double)rand() / (double)RAND_MAX); }
double clamp(double n, double lower, double upper) { return std::max(lower, std::min(n, upper)); }
struct neuron {
neuron() { id = rand(); bias = random(-1, 1); outputF = -1; }
int id;
std::map<int, neuron*> incomingtargets;
double bias;
std::map<int, double > incomingweights;
double outputF; // f(x) forward
void connect(neuron* n, double weight = random(-1, 1)) {
n->incomingtargets[this->id] = this;
n->incomingweights[this->id] = weight;
}
double activate(const double* input) {
if (input != NULL) {
//this->outputB = 1; // f'(x)
this->outputF = *input; // f(x)
}
else {
double sum = 0; // sum (x * w) + b
map<int, neuron*>::iterator it;
for (it = this->incomingtargets.begin(); it != this->incomingtargets.end(); it++)
{
sum += (this->incomingtargets[it->first]->outputF * this->incomingweights[it->first]);
}
sum += this->bias;
this->outputF = sigmoidF(sum);// f(x)
}
return this->outputF;
}
};
struct network {
vector<neuron*> inputs;// Input Layer w/ 1 neurons
vector<neuron*> hiddens;// Hiddent Layer w/ 3 neurons
vector<neuron*> outputs;// Output Layer w/ 1 neuron
// Connect Input Layer to Hidden Layer
double error;
network()
{
error = 0;
inputs.push_back(new neuron()); // Input Layer w/ 1 neurons
hiddens.push_back(new neuron()); hiddens.push_back(new neuron()); hiddens.push_back(new neuron()); // Hiddent Layer w/ 3 neurons
outputs.push_back(new neuron()); // Output Layer w/ 1 neuron
inputs[0]->connect(hiddens[0]); inputs[0]->connect(hiddens[1]); inputs[0]->connect(hiddens[2]);
// Connect Hidden Layer to Output Layer
hiddens[0]->connect(outputs[0]); hiddens[1]->connect(outputs[0]); hiddens[2]->connect(outputs[0]);
}
~network()
{
delete inputs[0];
delete hiddens[0];
delete hiddens[1];
delete hiddens[2];
delete outputs[0];
}
double activate(const double* input, const double* output) {
for (int i = 0; i < 1; i++) inputs[i]->activate(input);
for (int i = 0; i < 3; i++) hiddens[i]->activate(NULL);
for (int i = 0; i < 1; i++) outputs[i]->activate(NULL);
if (output != NULL) error += abs(*output - outputs[0]->outputF);
return outputs[0]->outputF;
}
double GetFitness() const { return error; }
int getDimension() const
{
return 5 + 6;
} // 5 neurons (5xbias), 6 connections (6xweights)
double& getData(int dimensionIndex)
{
// bias and incomingweights
neuron* allneurons[] = { inputs[0], hiddens[0], hiddens[1], hiddens[2], outputs[0] };
int count = 0;
for (int i = 0; i<5; i++)
{
if (count == dimensionIndex)
{
return allneurons[i]->bias;
}
count++;
for (int k = 0; k<(int)allneurons[i]->incomingweights.size(); k++)
{
if (count == dimensionIndex)
{
return allneurons[i]->incomingweights[k];
}
count++;
}
}
DBG_ASSERT(false);
return allneurons[0]->bias; // never get here!
}
void setData(int dimensionIndex, double val)
{
// bias and incomingweights
neuron* allneurons[] = { inputs[0], hiddens[0], hiddens[1], hiddens[2], outputs[0] };
int count = 0;
for (int i = 0; i<5; i++)
{
if (count == dimensionIndex)
{
allneurons[i]->bias = val;
return;
}
count++;
for (int k = 0; k<(int)allneurons[i]->incomingweights.size(); k++)
{
if (count == dimensionIndex)
{
allneurons[i]->incomingweights[k] = val;
return;
}
count++;
}
}
DBG_ASSERT(false);
//return allneurons[0]->bias; // never get here!
}
};
void main() {
srand((unsigned int)time(NULL));
double F = 0.1; //differential weight [0,2]
double CR = 0.5f; //crossover probability [0,1]
for (int test = 0; test < 10; test++)
{
F = random(0.001, 1.9); //differential weight [0,2]
CR = random(0.001, 0.9); //crossover probability [0,1]
vector<network*> population;// = new network*[POPULATIONSIZE];// = new network[ POPULATIONSIZE ];
for (int i = 0; i < POPULATIONSIZE; i++)
{
population.push_back(new network());
}
vector<pair<double, double>> trainSet; trainSet.resize(TrainSetSize);
for (int i = 0; i < TrainSetSize; i++) { trainSet[i] = make_pair(i / (double)TrainSetSize, sin((i / (double)TrainSetSize) * (2.0 * PI))*0.5 + 0.5); }
//dimensionality of problem, means how many variables problem has. this case 3 (data1,data2,data3)
const int N = population[0]->getDimension(); // 5 neurons and each neuron has connected weights and bias (should be 11 for this case)
for (int j = 0; j < POPULATIONSIZE; j++)
{
for (int ts = 0; ts < TrainSetSize; ts++)
{
const double input = trainSet[ts].first;
const double output = trainSet[ts].second;
population[j]->activate(&input, &output);
}
}
for (int e = 0; e < epoch; e++)
{
//F = random(0.001, 0.5);
//CR = random(0.01, 0.6);
//main loop of evolution.
for (int j = 0; j < POPULATIONSIZE; j++)
{
//calculate new candidate solution
//pick random point from population
int x = ::floor(random(0, POPULATIONSIZE - 1));
int a, b, c; //pick three different random points from population
do { a = ::floor(random(0, POPULATIONSIZE - 1)); } while (a == x);
do { b = ::floor(random(0, POPULATIONSIZE - 1)); } while (b == x || b == a);
do { c = ::floor(random(0, POPULATIONSIZE - 1)); } while (c == x || c == a || c == b);
// Pick a random index [0-Dimensionality]
int R = ::floor(random(0, N));
//Compute the agent's new position
network* original = population[x];
network* candidate = new network();// = original;
network* individual1 = population[a];
network* individual2 = population[b];
network* individual3 = population[c];
//if(i==R | i<CR)
//candidate=a+f*(b-c)
//else
//candidate=x
for (int s = 0; s < N; ++s)
{
//if (s == R || random(0, 1)<CR)
if (s == R || random(0, 1) < CR)
{
//candidate->getData(s) = individual1->getData(s) + F*(individual2->getData(s) - individual3->getData(s));
double val = individual1->getData(s) + F*(individual2->getData(s) - individual3->getData(s));
//val = val * 0.5;
candidate->setData(s, val);
//candidate->getData(s) *= 0.5;
//double val = candidate->getData(s);
//DBG_ASSERT(val > -1000 && val < 1000);
}
else
{
//candidate->getData(s) = original->getData(s);
double val = original->getData(s);
candidate->setData(s, val);
}
}// End for b
// calculate fitness for our new candidate
for (int ts = 0; ts < TrainSetSize; ts++)
{
const double input = trainSet[ts].first;
const double output = trainSet[ts].second;
candidate->activate(&input, &output);
}
double candidateFitness = candidate->GetFitness();
double originalFitness = original->GetFitness();
// if better replace
if (candidateFitness < originalFitness)
{
population[x] = candidate;
delete original;
}
else
{
delete candidate;
}
}// end population
#if 1 // print out the fitness as it evolves
{
int bestFitnessIndex = 0;
double bestFitnessValue = population[bestFitnessIndex]->GetFitness();
for (int i = 1; i < POPULATIONSIZE; ++i)
{
if (population[i]->GetFitness() < bestFitnessValue)
{
bestFitnessValue = population[i]->GetFitness();
bestFitnessIndex = i;
}
}// End for i
cout << "epoch:" << e << " fitness:" << bestFitnessValue << "\r";
}
#endif
}// epoch
int bestFitnessIndex = 0;
double bestFitnessValue = population[bestFitnessIndex]->GetFitness();
for (int i = 1; i < POPULATIONSIZE; ++i)
{
if (population[i]->GetFitness() < bestFitnessValue)
{
bestFitnessValue = population[i]->GetFitness();
bestFitnessIndex = i;
}
}// End for i
DBG_ASSERT(bestFitnessIndex > -1);
#if 0
cout << "input, idea, predicted (y =sin(x))" << "\n";
for (int i = 0; i < 100; i++) { //Print out test results (ideal vs predicted for sin wave)
double input = i / 100.0;
double predicted = population[bestFitnessIndex].activate(&input, NULL);
std::cout << (input) << ", " << sin(input * (2.0 * PI))*0.5 + 0.5 << ", " << predicted << "\n";
} system("pause"); // wait for user to press a key before closing
#endif
#if 1
char filename[256] = { 0 };
sprintf(filename, "data_F=%.5f_CR=%.5f.txt", F, CR);
FILE* fp = fopen(filename, "wt");
fprintf(fp, "epoch %d, populationsize: %d\n", epoch, POPULATIONSIZE);
fprintf(fp, "error %f\n", bestFitnessValue);
fprintf(fp, "input, idea, predicted (y =sin(x))\n");
for (int i = 0; i < 100; i++) { //Print out test results (ideal vs predicted for sin wave)
double input = i / 100.0;
double predicted = population[bestFitnessIndex]->activate(&input, NULL);
double exact = sin(input * (2.0 * PI))*0.5 + 0.5;
fprintf(fp, "%f %f %f \n", input, exact, predicted);
}
fflush(fp);
fclose(fp);
fp = NULL;
#endif
for (int i = 0; i < POPULATIONSIZE; i++)
{
delete population[i];
}
population.resize(0);
}// end for test
system("pause"); // wait for user to press a key before closing
}
| Key Features | |
• Optionally saves the trained data to a file (compare the ideal against predicted)
• Runs through various DE parameters (CR/F) to see which provides better convergence on a result (most optimal)
• Prints out the epoch and the fitness (watch the algorithm improve as it searches for the neural network weights and biases)
• Compare the accuracy/time/computational cost with other training version (e.g., back propagation) - however, for problems that have very non-linear constraints or are based on fitness critera (no-data), these alternative training algorithms may be limited.
| References | |
1. Deep Learning with Javascript: Example-Based Approach (Kenwright) ISBN: 979-8660010767
2. Game C++ Programming: A Practical Introduction (Kenwright) ISBN: 979-8653531095
3. smallnn: Neural Network in 99 lines of C++ (Kenwright)
4. Inverse kinematic solutions for articulated characters using massively parallel architectures and differential evolutionary algorithms (Kenwright). VRIPHYS '17: Proceedings of the 13th Workshop on Virtual Reality Interactions and Physical Simulations, April 2017
5. Neural network in combination with a differential evolutionary training algorithm for addressing ambiguous articulated inverse kinematic problems (Kenwright). SA '18: SIGGRAPH Asia 2018 Technical BriefsDecember 2018
6. smallnnde homepage
|
