Project Adaptive Systems:
Study of a Back-Propagating Network for Binary Operators
Kristof Van Laerhoven
3th year Computer Science
Description.
The network has 2 input nodes, 2 hidden nodes and 1 output node. The hidden nodes and output nodes receive also input from a bias node. This way, the network can perform the binary operations OR, AND, NOR, NAND, and even XOR. The training is done with back-propagation.
Source Code:
/* a simple ANN for binary operators,
// trained with backpropagation
// by Kristof Van Laerhoven, 3th year computer science
*/
#include <stdio.h>
#include <stdlib.h>
#include <math.h>
#include <conio.h>
class BinOp {
public:
BinOp( float newbias=-1, float newlr=0.7, float newmr=0.9,
int newiter=300, char* newfname="xor.dat",
float newlowlim=0, float newupplim=1, float newsteep=0.5 ):
bias(newbias), lr(newlr), mr(newmr), iter(newiter),
fname(newfname), lowlim(newlowlim), upplim(newupplim),
steep(newsteep) {};
void set_bias( int value ) { bias = value; };
void set_lr( float value ) { lr = value; };
void set_mr( float value ) { mr = value; };
void set_steep( float value ) { steep = value; };
void set_iter( int value ) { iter = value; };
void set_fname( char* value ) { fname = value; };
void set_lowlim( float value ) { lowlim = value; };
void set_upplim( float value ) { upplim = value; };
void set_weights(float w1, float w2, float w3, float w4, float w5,
float w6, float w7, float w8, float w9);
float get_bias() { return bias; };
float get_lr() { return lr; };
float get_mr() { return mr; };
int get_iter() { return iter; };
void printschema();
void printinfo();
void printtest();
float train_bp();
float test( float i1, float i2 );
char stoptest();
protected:
void initweights();
void printweights();
char readfile();
void fillnodes(int i1, int i2, int o);
void adjusterrors();
float rand_ab( float a, float b );
float activationf( float weight1, float O1,
float weight2, float O2,
float weight3, float O3 );
float errorf1( float target, float output );
float adjustweight1( float pweight, float delta, float Oi);
float errorf2( float Om, float weight, float err );
float adjustweight2( float mtoi, float pweight, float I );
private:
int iter; // # of iterations
float bias;
float lr; // learning rate
float mr; // momentum rate
float steep; // steepness
float weight[10]; // the weights
float pweight[10];// the previous weights
float IA, IB, MA, MB, O; // the node values
float target; // the actual target
float lowlim, upplim; // lower & upper limit of randomness
float errorO; // output -> middle error
float errorMA, errorMB; // middle -> input errors
FILE *fileptr; // filepointer
int in[4][2], out[4]; // for file-reading
char *fname;
};
/********************************************************************/
void BinOp::printinfo() {
printf("settings for the network:\n learning rate: %f\n", lr);
printf(" momentum rate: %f\n bias: %f\n", mr, bias);
printf(" iterations: %d\n filename: %s\n\n", iter, fname );
}
void BinOp::printschema() {
printf(" ------------------\n");
printf(" ³ Output: %+2.3f ³------ w7: %+2.3f\n",O, weight[7] );
printf(" ------------------\n");
printf(" / \\\n");
printf(" w5: %+2.3f w6: %+2.3f\n",weight[5], weight[6]);
printf(" / \\\n");
printf(" ---------------- ----------------\n");
printf(" w8:%+2.3f --³ Ma: %+2.3f ³ ³ Mb: %+2.3f ³----- w9:%+2.3f\n",weight[8],MA,MB,weight[9]);
printf(" ---------------- ----------------\n");
printf(" ³ \\ / ³\n");
printf(" ³ w3:%+2.3f w2:%+2.3f ³\n", weight[3], weight[2]);
printf(" ³ \\/ ³\n");
printf(" w1: %+2.3f / \\ w4: %+2.3f \n", weight[1], weight[4]);
printf(" ³ / \\ ³\n");
printf(" ---------------- ----------------\n");
printf(" ³ IA: %+2.3f ³ ³ IB: %+2.3f ³\n",IA,IB);
printf(" ---------------- ----------------\n");
}
void BinOp::printtest() {
for (float i=0; i<2; i++)
for (float j=0; j<2; j++) {
printf( " %2g %2g = %+2.4g", i, j, test( i, j ));
if ( test( i, j ) < 0.2 ) printf(" => 0 \n");
else if ( test( i, j ) > 0.8 ) printf(" => 1 \n");
else printf("\n");
}
}
char BinOp::stoptest(){
return ( ( test(0,0) < 0.1 || test(0,0) > 0.9 ) &&
( test(1,1) < 0.1 || test(1,1) > 0.9 ) &&
( test(1,0) < 0.1 || test(1,0) > 0.9 ) &&
( test(0,1) < 0.1 || test(0,1) > 0.9 ) );
}
float BinOp::train_bp() {
if ( readfile() < 0 ) printf("file error\n");
else printf("file loaded. Press a key.\n");
getch();
initweights();
int j=0;
float maxerror;
while (j<iter) {
gotoxy(1,2);printf(" iteration #: %4d\n", j);
j++;
maxerror=0;
for (int i=0; i<4; i++) { // for each input pattern :
fillnodes( in[i][0], in[i][1], out[i] );
gotoxy(1,3);
printschema();
adjusterrors();
printf("error = %+2.6f\n", errorO );
if ( errorO > abs(maxerror) ) maxerror=errorO;
}
printf("%2.6f\n",maxerror);
}
return maxerror;
}
float BinOp::test( float i1, float i2 ) {
IA = i1;
IB = i2;
MB = activationf( weight[2], IA, weight[4], IB, weight[9], bias);
MA = activationf( weight[1], IA, weight[3], IB, weight[8], bias);
return 1/(1+exp(-((MA*weight[5])+(MB*weight[6])+(bias*weight[7]))));
}
void BinOp::initweights() {
for (char i=0; i<10; i++) {
weight[i]=rand_ab(lowlim,upplim);
pweight[i]=0;
}
}
void BinOp::fillnodes(int i1, int i2, int o) {
IA = i1;
IB = i2;
target = o;
MB = activationf( weight[2], IA, weight[4], IB, weight[9], bias);
MA = activationf( weight[1], IA, weight[3], IB, weight[8], bias);
O = activationf( weight[5], MA, weight[6], MB, weight[7], bias);
}
void BinOp::adjusterrors() {
errorO = errorf1(target, O); // error at output
pweight[5] = adjustweight1( pweight[5], errorO, MA);
pweight[6] = adjustweight1( pweight[6], errorO, MB);
pweight[7] = adjustweight1( pweight[7], errorO, bias);
for( char k=5; k<8; k++) weight[k] = weight[k] + pweight[k];
errorMA = errorf2( MA, weight[5], errorO); // error at middle nodes
errorMB = errorf2( MB, weight[6], errorO);
pweight[1] = adjustweight2( errorMA, pweight[1], IA);
pweight[2] = adjustweight2( errorMB, pweight[2], IA);
pweight[3] = adjustweight2( errorMA, pweight[3], IB);
pweight[4] = adjustweight2( errorMB, pweight[4], IB);
pweight[8] = adjustweight2( errorMA, pweight[8], bias);
pweight[9] = adjustweight2( errorMB, pweight[9], bias);
for( k=1; k<5; k++) weight[k] = weight[k] + pweight[k];
for( k=8; k<10; k++) weight[k] = weight[k] + pweight[k];
}
void BinOp::printweights() {
printf("%3.3g %3.3g %1.3g %1.3g %1.3g %1.3g %1.3g %1.3g %1.3g\n",
weight[1], weight[2], weight[3], weight[4], weight[5],
weight[6], weight[7], weight[8], weight[9] );
}
char BinOp::readfile() {
fileptr = fopen(fname, "r");
if (fileptr==0) {
return -1; // The file could not be found
}
for (char i=0; i< 4; i++) {
fscanf(fileptr,"%i",&in[i][0]); // inputs
fscanf(fileptr,"%i",&in[i][1]);
fscanf(fileptr,"%i",&out[i]); // target
}
fclose(fileptr); // close file
return 0;
}
float BinOp::rand_ab(float a, float b) {
return ( a + (b - a) * rand() / RAND_MAX );
}
float BinOp::activationf( float weight1,float O1,
float weight2,float O2,
float weight3, float O3) {
float t[3]; // temporary array
t[0]=(weight1*O1);
t[1]=(weight2*O2);
t[2]=(weight3*O3);
return 1/(1+exp(-2*steep*(t[0]+t[1]+t[2]))); // returns sigmoid
}
float BinOp::errorf1( float target, float output ) {
return ( (target-output)*2*steep*(output)*(1-output) );
}
float BinOp::adjustweight1( float pweight, float delta, float Oi) {
return ( (lr*delta*Oi)+(mr*pweight) );
}
float BinOp::errorf2( float Om, float weight, float err ) {
return ( 2*steep*Om*(1-Om)*(err*weight) );
}
float BinOp::adjustweight2( float mtoi, float pweight, float I) {
return ( (lr*mtoi*I)+(mr*pweight) );
}
void BinOp::set_weights(float w1, float w2, float w3, float w4, float w5,
float w6, float w7, float w8, float w9) {
weight[1]=w1; weight[2]=w2; weight[3]=w3;
weight[4]=w4; weight[5]=w5; weight[6]=w6;
weight[7]=w7; weight[8]=w8; weight[9]=w9;
}
/********************************************************************/
void main(void) {
BinOp binop;
clrscr();
//randomize();
for (int i=0; i<1; i++) {
binop.set_iter(300);
binop.set_mr(0.99);
binop.set_lr(0.3);
binop.set_bias(1);
binop.set_fname("xor.dat");
binop.train_bp();
binop.printtest();
}
}
Example Output:
------------------
│ Output: +0.004 │------ w7: -8.367
------------------
/ \
w5: -12.044 w6: +14.698
/ \
---------------- ----------------
w8:+4.334 --│ Ma: +0.987 │ │ Mb: +1.000 │----- w9:+14.675
---------------- ----------------
│ \ / │
│ w3:-11.748 w2:-6.582 │
│ \/ │
w1: -11.902 / \ w4: -10.411
│ / \ │
---------------- ----------------
│ IA: +0.000 │ │ IB: +0.000 │
---------------- ----------------
error = -0.000015
0.000005
0 0 = +0.003845 => 0
0 1 = +0.9978 => 1
1 0 = +0.9982 => 1
1 1 = +0.0008667 => 0
Evaluation plots:
