SHARE
TWEET

Untitled

a guest Sep 23rd, 2019 69 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. #include <iostream>
  2. #include <sstream>
  3. #include <fstream>
  4. #include <string>
  5. #include <math.h>
  6. #include <stdlib.h>     /* srand, rand */
  7. #include <time.h>
  8. #include <chrono>
  9.  
  10.  
  11. using namespace std;
  12.  
  13. // activation function
  14. double activation(double x){
  15.     return 1.0/(1.0 + exp(-x));
  16. }
  17.  
  18. // derivative of activation function
  19. double d_activation(double x){
  20.     return (1.0 - activation(x))*activation(x);
  21.  }  
  22.  
  23.  
  24. //
  25. void Randomize(double* a, int n, double min, double max){
  26.     //cout<<"RAND_MAX="<<RAND_MAX<<endl;
  27.     srand (time(NULL));
  28.     for ( int i = 0 ; i < n ; i++){
  29.         double f = (double)rand() / RAND_MAX;
  30.         a[i]= min + f * (max - min);
  31.     }
  32. }
  33.  
  34. void PrintVector(double* a, int n){
  35.     for ( int i = 0 ; i < n ; i++){
  36.         cout<<a[i]<<" ";
  37.     }
  38.     cout<<endl;
  39. }
  40.  
  41.  
  42. class NN{
  43. //  friend class MatrixD;
  44. //  friend class VectorD;
  45.    
  46.     private:
  47.       int nInputs; // number of inputs
  48.       int nOutputs; //number of outputs
  49.       int nHiddenNeurons; // number of neurons in hidden layer
  50.      
  51.       // search parameters
  52.       double dw;   // step for gradient estimation
  53.       double learningRate; // hm, learning  rate
  54.       int nStepsMax;
  55.      
  56.       // whole training set
  57.       double* inputsTraining;   // inputs - column, pattern - row
  58.       double* outputsTraining;  // outputs - column, pattern - row
  59.       int nTrainingEntries;  // number of rows in training set
  60.      
  61.       // current NN situation
  62.       double* currentInputs;  // row picked from inputsTraining
  63.       double* currentAnswers; // ditto for training answers
  64.       double* currentOutputs; // guess what?
  65.       double* currentError;   // current differnce between answers and output
  66.       //output error for current dataset row
  67.       double sumOfOutputErrors;
  68.       // sum of errors for all dataset entries
  69.       double totalDatasetError;
  70.      
  71. //   
  72.       // input to hidden layer
  73.       double* weightsHidden;  // hidden layer weights matrix Wh
  74.       double* biasHidden;     // hidden layer bias vector bh
  75.       double* d_weightsHidden;  // hidden layer weights matrix Wh derivative
  76.       double* d_biasHidden;     // hidden layer bias vector bh derivative
  77.       double* deltaHidden;  // for backpropagation
  78.      
  79.       // state of the hidden layer
  80.       double* netHidden;     // yh = Wh*z + bh - renaming
  81.       double* outHidden;   //  zh = activ(Wh*x + bh) - layer output
  82.      
  83.      
  84.       // hidden layer to output layer
  85.       double* weightsOutput;  // Wh
  86.       double* biasOutput;     // bh
  87.       double* d_weightsOutput;  // dWh/dw - gradients
  88.       double* d_biasOutput;     // dbh / dw - gradients
  89.       double* deltaOutput;     // dbh / dw - gradients
  90.      
  91.      
  92. //
  93.  
  94.       // state of the output layer
  95.       double* nnSumOutput;  
  96.       double* netOutput;  // yo = Wo*zh + bo
  97.       double* nnBiaOutput;
  98.       double* nnOutOutput; // zo = activ(yo)
  99.      
  100.     public:
  101.       NN(){ }; // constructor
  102.       int LoadTrainingSet(string file,int nInp,int nHid , int nOut);
  103.       int LoadWorkingSet(string file,int nInp,int nHid , int nOut);
  104.       void DisplayDigit(int nImage); // ASCII art display
  105.       int InitNet(double min, double max);
  106.       void GetTrainingEntry(int iTrainRow);
  107.        void GetWorkingEntry(int iTrainRow);
  108.       void ForwardProp();
  109.       double GetTotalError(){ return sumOfOutputErrors;};
  110.       void PrintErr(){ cout<<" Error: "<<sumOfOutputErrors<<endl;};
  111.       void DirectGradientEstimation();
  112.       void BackProp();
  113.       void StepByGradient();
  114.       double GetOutputError(){ return sumOfOutputErrors;};
  115.       double TotalDatasetError(); // sum of errors for all rows of train data
  116.       void Train1();
  117.       void PrintOutputs();
  118.       void DisplayResults();   
  119. };
  120.  
  121. // loads inputsTraining and outputsTraining from "file"
  122. int NN::LoadTrainingSet(string file,int nInp,int nHid , int nOut){
  123.     std::ifstream data(file);
  124.     std::string line;
  125.     nTrainingEntries = 0;
  126.     nInputs = nInp;
  127.     nOutputs = nOut;
  128.     nHiddenNeurons  = nHid;
  129.     // count number of lines in input file
  130.     while(std::getline(data,line)) { nTrainingEntries++; }
  131.     cout<<" Thera are "<<nTrainingEntries<<" entries in dataset"<<endl;
  132.     // reserve the memory
  133.     inputsTraining = new double[nTrainingEntries*nInputs];
  134.     outputsTraining = new double[nTrainingEntries*nOutputs];
  135.     cout<<" Memory reserved..."<<endl;
  136.     // rewind the file
  137.     data.clear();
  138.     data.seekg(0);
  139.     // read training data file
  140.     for(int iim = 0; iim<nTrainingEntries; iim++) {
  141.         std::getline(data,line);
  142.         //cout<<" iim= "<<iim<<" Input: "<<line<<endl;
  143.         std::stringstream lineStream(line);
  144.         std::string cell;
  145.         int count = 0;
  146.         // break input string into inputs and answers
  147.         while(std::getline(lineStream,cell,' ')) {
  148.             //cout<<"count="<<count<<"cell="<<cell<<" "<<endl;
  149.             if (count<nInputs) {
  150.                 inputsTraining[iim*nInputs+count] = atof(cell.c_str()) ;
  151.                 //cout<<" count="<<count<<" Inp[][]="<<inputsTraining.GetElement(iim,count)<<endl;
  152.             } else {
  153.                 outputsTraining[iim*nOutputs+count-nInputs] = atof(cell.c_str());
  154.                 //cout<<" count-nInputs="<<count-nInputs<<" Out[][]="<<outputsTraining.GetElement(iim,count-nInputs)<<endl;
  155.             }  
  156.           count++;
  157.         } // while
  158.         //cout<<" Input string "<<iim<<" parsed"<<endl;
  159.         //char stop;
  160.         //cin>>stop;
  161.     } // for
  162.     cout<<" Training set loaded. Inputs:"<<endl;
  163.     //char stop;
  164.     //cin>>stop;
  165.     //inputsTraining.PrintMatrix();
  166.     data.close();
  167.     return 0;
  168. }
  169.  
  170. int NN::LoadWorkingSet(string file,int nInp,int nHid , int nOut){
  171.     std::ifstream data(file);
  172.     std::string line;
  173.     nTrainingEntries = 0;
  174.     nInputs = nInp;
  175.     nOutputs = nOut;
  176.     nHiddenNeurons  = nHid;
  177.     // count number of lines in input file
  178.     while(std::getline(data,line)) { nTrainingEntries++; }
  179.     cout<<" Thera are "<<nTrainingEntries<<" entries in dataset"<<endl;
  180.     // reserve the memory
  181.     inputsTraining = new double[nTrainingEntries*nInputs];
  182.     outputsTraining = new double[nTrainingEntries*nOutputs];
  183.     cout<<" Memory reserved..."<<endl;
  184.     // rewind the file
  185.     data.clear();
  186.     data.seekg(0);
  187.     // read training data file
  188.     for(int iim = 0; iim<nTrainingEntries; iim++) {
  189.         std::getline(data,line);
  190.         //cout<<" iim= "<<iim<<" Input: "<<line<<endl;
  191.         std::stringstream lineStream(line);
  192.         std::string cell;
  193.         int count = 0;
  194.         // break input string into inputs and answers
  195.         while(std::getline(lineStream,cell,' ')) {
  196.             //cout<<"count="<<count<<"cell="<<cell<<" "<<endl;
  197.             if (count<nInputs) {
  198.                 inputsTraining[iim*nInputs+count] = atof(cell.c_str()) ;
  199.                 //cout<<" count="<<count<<" Inp[][]="<<inputsTraining.GetElement(iim,count)<<endl;
  200.             } else {
  201.                 outputsTraining[iim*nOutputs+count-nInputs] = atof(cell.c_str());
  202.                 //cout<<" count-nInputs="<<count-nInputs<<" Out[][]="<<outputsTraining.GetElement(iim,count-nInputs)<<endl;
  203.             }  
  204.           count++;
  205.         } // while
  206.         //cout<<" Input string "<<iim<<" parsed"<<endl;
  207.         //char stop;
  208.         //cin>>stop;
  209.     } // for
  210.     //cout<<" Training set loaded. Inputs:"<<endl;
  211.     //char stop;
  212.     //cin>>stop;
  213.     //inputsTraining.PrintMatrix();
  214.     data.close();
  215.     return 0;
  216. }
  217.  
  218.  
  219.  
  220. // reserves the memory and puts
  221. //random values (range min-max) into weights  and biases
  222. int NN::InitNet(double min, double max){
  223.    
  224.     cout<<" InitNet: nInputs="<<nInputs<<" nHiddenNeurons=";
  225.     cout<<nHiddenNeurons<<" nOutputs="<<nOutputs<<endl;
  226.     // reserve the memory for weights and biases
  227.     // hidden layer
  228.     weightsHidden = new double[nHiddenNeurons*nInputs];
  229.     biasHidden = new double[nHiddenNeurons];
  230.     d_weightsHidden = new double[nHiddenNeurons*nInputs];
  231.     d_biasHidden = new double[nHiddenNeurons];
  232.     deltaHidden = new double[nHiddenNeurons];
  233.     // output layer
  234.     weightsOutput = new double[nHiddenNeurons*nOutputs];
  235.     biasOutput = new double[nOutputs];
  236.     d_weightsOutput = new double[nHiddenNeurons*nOutputs];
  237.     d_biasOutput = new double[nOutputs];
  238.     deltaOutput = new double[nOutputs];
  239.    
  240.     // current input and output vector, answers and error  
  241.     currentInputs = new double[nInputs];
  242.     currentOutputs = new double[nOutputs];
  243.     currentAnswers = new double[nOutputs];
  244.     currentError =  new double[nOutputs];
  245.    
  246.     // reserve memory for current net levels
  247.     netHidden = new double[nHiddenNeurons];
  248.     outHidden = new double[nHiddenNeurons];
  249.     netOutput = new double[nOutputs];
  250.    
  251.     // make weights and biases random
  252.     Randomize(weightsHidden,nHiddenNeurons*nInputs,min,max);
  253.     Randomize(biasHidden,nHiddenNeurons,min,max);
  254.     Randomize(weightsOutput,nHiddenNeurons*nOutputs,min,max);
  255.     Randomize(biasOutput,nOutputs,min,max);
  256.    
  257.     return 0;
  258. }
  259.  
  260. // loads row of dataset into the net for estimation
  261. void NN::GetTrainingEntry(int iTrainRow){
  262.     for ( int i = 0 ; i<nInputs;i++)
  263.        currentInputs[i] = inputsTraining[iTrainRow*nInputs+i];
  264.     for (int i = 0 ; i < nOutputs;i++)  
  265.       currentAnswers[i]= outputsTraining[iTrainRow*nOutputs+i];
  266.    
  267. }
  268.  
  269. void NN::GetWorkingEntry(int iTrainRow){
  270.     for ( int i = 0 ; i<nInputs;i++)
  271.        currentInputs[i] = inputsTraining[iTrainRow*nInputs+i];
  272.     for (int i = 0 ; i < nOutputs;i++)  
  273.       currentAnswers[i]= outputsTraining[iTrainRow*nOutputs+i];
  274.    
  275. }
  276.  
  277. // display digit on the screen as ASCII
  278. void NN::DisplayDigit(int iImage){
  279.   int scan = 0;
  280.   for (int i = 0 ; i < 8; i++){
  281.     for ( int j = 0 ; j < 8;j++){
  282.        if (inputsTraining[iImage*nInputs + scan] > 0.0){
  283.           cout<<"0";
  284.        } else {
  285.          cout<<"-";
  286.        }
  287.        scan++;
  288.     }
  289.     cout<<endl;
  290.   }
  291. }
  292.  
  293.  
  294.  
  295. // direct calculation of forward propagation
  296. void NN::ForwardProp(){
  297.     //
  298.     //cout<<" ForwardProp() started"<<endl;
  299.     //  inputs ->  hidden layer
  300.     // for each neuron in hidden layer
  301.     for ( int hid = 0 ;hid < nHiddenNeurons ; hid++){
  302.         // combine inputs and add bias
  303.         netHidden[hid] = biasHidden[hid];
  304.         for (int inp = 0 ; inp < nInputs ; inp++){
  305. //cout<<" hid="<<hid<<" inp="<<inp<<" ind="<<hid*nInputs + inp<<endl;          
  306.            netHidden[hid] = netHidden[hid] +
  307.               currentInputs[inp]* weightsHidden[hid*nInputs + inp];
  308.         }
  309.         outHidden[hid] = activation(netHidden[hid]);
  310.     }  
  311. //int st;
  312. //cin>>st; 
  313.     sumOfOutputErrors = 0.0;
  314.     // for each neuron in output layer
  315.     for ( int out = 0 ; out < nOutputs ; out++){
  316.         // combine hidden and add bias
  317.         netOutput[out] = biasOutput[out];
  318.         for (int hid = 0 ; hid < nHiddenNeurons ; hid++){
  319.             netOutput[out] = netOutput[out] +
  320.              outHidden[hid]* weightsOutput[out*nHiddenNeurons+hid];
  321.         }
  322.         currentOutputs[out] = activation(netOutput[out]);
  323.         currentError[out] = currentOutputs[out] - currentAnswers[out];
  324.         sumOfOutputErrors = sumOfOutputErrors + currentError[out]*currentError[out];
  325.     }
  326.    
  327. }
  328.  
  329. // calculate gradient by direct estimation
  330. // called by Train1
  331. void NN::DirectGradientEstimation(){
  332.    
  333.     //ForwardProp();
  334.     //double e0 = sumOfOutputErrors;
  335.    
  336.     for ( int i = 0 ; i < (nHiddenNeurons*nOutputs) ; i++){
  337.         double e0 = sumOfOutputErrors;//error given by forward prop
  338.         weightsOutput[i] = weightsOutput[i] + dw;
  339.         ForwardProp();
  340.         double e1 = sumOfOutputErrors;
  341.         d_weightsOutput[i] = (e1-e0)/dw;
  342.         weightsOutput[i] = weightsOutput[i] - dw;
  343.        
  344.     }
  345.    
  346.     for ( int i = 0 ; i < nHiddenNeurons ; i++){
  347.         ForwardProp();
  348.         double e0 = sumOfOutputErrors;
  349.         biasOutput[i] = biasOutput[i] + dw;
  350.         ForwardProp();
  351.         double e1 = sumOfOutputErrors;
  352.         d_biasOutput[i] = (e1-e0)/dw;
  353.         biasOutput[i] = biasOutput[i] - dw;
  354.     }
  355.        
  356.     for ( int i = 0 ; i < (nHiddenNeurons*nOutputs) ; i++){
  357.         ForwardProp();
  358.         double e0 = sumOfOutputErrors;
  359.         weightsHidden[i] = weightsHidden[i] + dw;
  360.         ForwardProp();
  361.         double e1 = sumOfOutputErrors;
  362.         d_weightsHidden[i] = (e1-e0)/dw;
  363.         weightsHidden[i] = weightsHidden[i] - dw;
  364.        
  365.     }
  366.    
  367.     for ( int i = 0 ; i < nHiddenNeurons ; i++){
  368.         ForwardProp();
  369.         double e0 = sumOfOutputErrors;
  370.         biasHidden[i] = biasHidden[i] + dw;
  371.         ForwardProp();
  372.         double e1 = sumOfOutputErrors;
  373.         d_biasHidden[i] = (e1-e0)/dw;
  374.         biasHidden[i] = biasHidden[i] - dw;
  375.        
  376.     }
  377. }
  378.  
  379.  
  380. // calculate gradients by back-propagation
  381. // called by Train1
  382. void NN::BackProp(){
  383.     // to make the process faster
  384.    
  385. }
  386.  
  387. // change weights and biases in direction oppposite to gradient,
  388. // scaled by learning rate (which should be negative)
  389. // called by Train1
  390. void NN::StepByGradient(){
  391.     // do this 
  392.    
  393.     for ( int i = 0 ; i < (nHiddenNeurons*nOutputs) ; i++){
  394.         weightsOutput[i] = weightsOutput[i] - learningRate * d_weightsOutput[i];
  395.     }
  396.    
  397.     for ( int i = 0 ; i < nHiddenNeurons ; i++){
  398.         biasOutput[i] = biasOutput[i] - learningRate * d_biasOutput[i];
  399.     }
  400.    
  401.     for ( int i = 0 ; i < (nHiddenNeurons*nOutputs) ; i++){
  402.         weightsHidden[i] = weightsHidden[i] - learningRate * d_weightsHidden[i];
  403.     }
  404.    
  405.     for ( int i = 0 ; i < nHiddenNeurons ; i++){
  406.         biasHidden[i] = biasHidden[i] - learningRate * d_biasHidden[i];
  407.     }
  408. }
  409.  
  410. // estimates search convergence
  411. // calculates error for all entries in the dataset
  412. // for current values of weights and biases
  413. double NN::TotalDatasetError(){ // sum of errors for all rows of train data
  414. //cout<<" There are "<<nTrainingEntries<<" rows in the dataset"<<endl;
  415.     totalDatasetError = 0.0;
  416.     for ( int entry = 0 ; entry < nTrainingEntries; entry++){
  417. //cout<<"entry = "<<entry<<endl;       
  418.         GetTrainingEntry(entry);
  419.         ForwardProp();
  420.         totalDatasetError = totalDatasetError + GetOutputError();
  421.     }
  422. //cout<<" totalDatasetError/nEntries="<<totalDatasetError/nTrainingEntries<<endl;
  423.     return totalDatasetError; //nTrainingEntries;
  424. }
  425.  
  426.  
  427.  
  428. void NN::Train1(){
  429.     // set net search parameters
  430.     dw = -0.001;  // step to estimate gradient
  431.     learningRate = 1;
  432.     //DisplayDigit(iImage);
  433.     InitNet(-0.1,0.1);
  434.     int iImage = 0;
  435.     srand (time(NULL));  // seed random number generator
  436.     int searchStep = 0;
  437.    
  438.     while (( searchStep < 5000) && (TotalDatasetError() > 10.0) ){
  439.       // pick random entry from training dataset
  440.       iImage = nTrainingEntries*(double)rand() / RAND_MAX;
  441.       // copy inputs and outputs from training matrix into neural netg
  442.       GetTrainingEntry(iImage);
  443.       ForwardProp();
  444.       DirectGradientEstimation();
  445.       //BackProp();
  446.       StepByGradient();
  447.       cout<<"step: "<<searchStep;//<<" image: "<<iImage<<" Error for current row:"<<GetOutputError();
  448.       cout<<" Total dataset error: "<< TotalDatasetError()<<endl;
  449.       searchStep++;
  450.     }
  451.      
  452. }
  453.  
  454. void NN::PrintOutputs(){
  455.     cout<<" Net outputs: ";
  456.     int maxIndex = 0;
  457.     double maxNum = 0;
  458.     for (int out = 0 ; out < nOutputs ; out++){
  459.         cout<<currentOutputs[out]<<"  ";
  460.        
  461.        
  462.         if(currentOutputs[out] > maxNum){
  463.             maxNum = currentOutputs[out];
  464.             maxIndex = out;
  465.            
  466.             }
  467.            
  468.            
  469.     }
  470.     cout<<"This Picture Is a: "<<maxIndex<<"";
  471.     cout<<endl;
  472. }
  473.  
  474.  
  475. void NN::DisplayResults(){
  476.     int iImage = -1;
  477.     cout<<" There are "<< nTrainingEntries<<" entries "<<endl;
  478.     cout<<" Enter number of the entry to display"<<endl;
  479.     cin>>iImage;
  480.     while (iImage < nTrainingEntries) {
  481.       // copy inputs and outputs from big matrix
  482.       GetTrainingEntry(iImage);
  483.       ForwardProp();
  484.       DisplayDigit(iImage);
  485.       PrintVector(currentOutputs, nOutputs);
  486.       cin>>iImage;
  487.    }
  488.    
  489. }
  490.  
  491.  
  492.  
  493. int main(){
  494.     NN neuralNet;
  495.     neuralNet.LoadTrainingSet("train.txt",64,128,8);
  496.     neuralNet.DisplayDigit(700);
  497.    
  498.     //return 0;
  499.    
  500.     int imageIndex = 0 ;
  501.     neuralNet.InitNet(-0.1,0.1);
  502.     neuralNet.ForwardProp();
  503.     //auto t7 = std::chrono::system_clock::now();
  504.     //  neuralNet.DirectGradientEstimation();
  505.     //auto t8 = chrono::system_clock::now();
  506.     //cout<< " Timing Gradient:"<< chrono::duration_cast<std::chrono::milliseconds>(t8 - t7).count() << " ms\n";
  507.    
  508.     neuralNet.Train1();
  509.     string filename;
  510.     cout<<" Enter the file you wish to scan"<<endl;
  511.     getline(cin ,filename);
  512.     neuralNet.LoadWorkingSet(filename,64,128,8);
  513.     /*while ( imageIndex < 750){
  514.       cout<<" Enter index of the imag to display"; cin>>imageIndex;
  515.       if (imageIndex<759){
  516.            neuralNet.GetTrainingEntry(imageIndex);
  517.            neuralNet.ForwardProp();
  518.            neuralNet.DisplayDigit(imageIndex);
  519.            neuralNet.PrintOutputs();
  520.       }
  521.     }*/
  522.    
  523.     while ( imageIndex < 750){
  524.       cout<<" Enter index of the imag to display"; cin>>imageIndex;
  525.       if (imageIndex<759){
  526.            neuralNet.GetTrainingEntry(imageIndex);
  527.            neuralNet.ForwardProp();
  528.            neuralNet.DisplayDigit(imageIndex);
  529.            neuralNet.PrintOutputs();
  530.       }
  531.     }
  532.      
  533. }
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top