perceptron.cpp

/****************************************************************************************************************/
/*                                                                                                              */
/*   OpenNN: Open Neural Networks Library                                                                       */
/*   www.artelnics.com/opennn                                                                                   */
/*                                                                                                              */
/*   P E R C E P T R O N   C L A S S                                                                            */
/*                                                                                                              */
/*   Roberto Lopez                                                                                              */
/*   Artelnics - Making intelligent use of data                                                                 */
/*   [email protected]                                                                                 */
/*                                                                                                              */
/****************************************************************************************************************/

// OpenNN includes

#include "perceptron.h"

namespace OpenNN
{

    
Perceptron::Perceptron(void)
{
   set();
}



Perceptron::Perceptron(const size_t& new_inputs_number)
{
   set(new_inputs_number);
}



Perceptron::Perceptron(const size_t& new_inputs_number, const double& new_parameters_value)
{
   set(new_inputs_number, new_parameters_value);
}



Perceptron::Perceptron(const Perceptron& other_perceptron)
{
   set(other_perceptron);
}



Perceptron::~Perceptron(void)
{
}


// ASSIGNMENT OPERATOR


Perceptron& Perceptron::operator=(const Perceptron& other_perceptron)
{
   if(this != &other_perceptron) 
   {
      bias = other_perceptron.bias;
   
      synaptic_weights = other_perceptron.synaptic_weights;

      activation_function = other_perceptron.activation_function;

      display = other_perceptron.display;
   }

   return(*this);
}


// EQUAL TO OPERATOR

// bool operator == (const Perceptron&) const method


bool Perceptron::operator == (const Perceptron& other_perceptron) const
{
   if(bias == other_perceptron.bias
   && synaptic_weights == other_perceptron.synaptic_weights
   && activation_function == other_perceptron.activation_function
   && display == other_perceptron.display)
   {
      return(true);
   }
   else
   {
      return(false);
   }
}


// METHODS

// const ActivationFunction& get_activation_function(void) const method


const Perceptron::ActivationFunction& Perceptron::get_activation_function(void) const 
{
   return(activation_function);                           
}


// std::string write_activation_function(void) const method


std::string Perceptron::write_activation_function(void) const
{
   switch(activation_function)
   {
      case Perceptron::Logistic:   
      {
         return("logistic");
      }
      break;

      case Perceptron::HyperbolicTangent:   
      {
         return("tanh");
      }
      break;

      case Perceptron::Threshold:   
      {
         return("threshold");
      }
      break;

      case Perceptron::SymmetricThreshold:   
      {
         return("symmetric_threshold");
      }
      break;

      case Perceptron::Linear:   
      {
         return("");
      }
      break;

      default:
      {
         std::ostringstream buffer;

         buffer << "OpenNN Exception: Perceptron class.\n" 
                << "std::string get_activation_function(void) const method.\n"
                << "Unknown activation function.\n";

         throw std::logic_error(buffer.str());
      }
      break;
   }
}


// size_t get_inputs_number(void) const method


size_t Perceptron::get_inputs_number(void) const
{
   return(synaptic_weights.size());
}    


// double get_bias(void) const method


const double& Perceptron::get_bias(void) const
{
   return(bias);
}


// Vector<double>& arrange_synaptic_weights(void)


const Vector<double>& Perceptron::arrange_synaptic_weights(void) const
{
   return(synaptic_weights);
}


// double get_synaptic_weight(const size_t&) const method

 
const double& Perceptron::get_synaptic_weight(const size_t& synaptic_weight_index) const
{
   // Control sentence (if debug)

   #ifndef NDEBUG 
      
   const size_t inputs_number = get_inputs_number();

   if(synaptic_weight_index >= inputs_number)
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: Perceptron class.\n"
                << "double get_synaptic_weight(const size_t&) const method.\n"
                << "Index of synaptic weight must be less than number of inputs.\n";

      throw std::logic_error(buffer.str());
   }

   #endif

   // Get single synaptic weights

   return(synaptic_weights[synaptic_weight_index]);   
}


// const bool& get_display(void) const method


const bool& Perceptron::get_display(void) const
{
   return(display);
}


//  void set(void) method


void Perceptron::set(void)
{
   initialize_bias_normal(0.0, 0.2);

   synaptic_weights.set();

   activation_function = HyperbolicTangent;

   display = true;
}


// void set(const size_t&) method


void Perceptron::set(const size_t& new_inputs_number)
{
   // Set synaptic weights

   activation_function = HyperbolicTangent;

   initialize_bias_normal(0.0, 0.2);

   synaptic_weights.set(new_inputs_number);   
   initialize_synaptic_weights_normal(0.0, 0.2);
   
   display = true;
}


// void set(const size_t&, const double&) method


void Perceptron::set(const size_t& new_inputs_number, const double& new_parameters_value)
{
   bias = new_parameters_value;
   synaptic_weights.set(new_inputs_number, new_parameters_value);

   activation_function = HyperbolicTangent;

   display = true;
}


// void set(const Perceptron&)


void Perceptron::set(const Perceptron& other_perceptron)
{
   bias = other_perceptron.bias;
   
   synaptic_weights = other_perceptron.synaptic_weights;

   activation_function = other_perceptron.activation_function;

   display = other_perceptron.display;
}


// void set_activation_function(const ActivationFunction&) method


void Perceptron::set_activation_function(const Perceptron::ActivationFunction& new_activation_function)
{
   activation_function = new_activation_function;
}


// void set_activation_function(const std::string&) method


void Perceptron::set_activation_function(const std::string& new_activation_function_name)
{
   if(new_activation_function_name == "Logistic")
   {
      activation_function = Logistic;
   }
   else if(new_activation_function_name == "HyperbolicTangent")
   {
      activation_function = HyperbolicTangent;
   }
   else if(new_activation_function_name == "Threshold")
   {
      activation_function = Threshold;
   }
   else if(new_activation_function_name == "SymmetricThreshold")
   {
      activation_function = SymmetricThreshold;
   }
   else if(new_activation_function_name == "Linear")
   {
      activation_function = Linear;
   }
   else
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: Perceptron class.\n"
             << "void set_activation_function(const std::string&) method.\n"
             << "Unknown activation function: " << new_activation_function_name << ".\n";

      throw std::logic_error(buffer.str());
   }
}


// void set_bias(const double&) method

 
void Perceptron::set_bias(const double& new_bias)
{
   bias = new_bias;   
}


// void set_synaptic_weights(const Vector<double>&) method

 
void Perceptron::set_synaptic_weights(const Vector<double>& new_synaptic_weights)
{
   // Control sentence (if debug)

   #ifndef NDEBUG 

   const size_t inputs_number = get_inputs_number();

   if(new_synaptic_weights.size() != inputs_number)
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: Perceptron class.\n"
             << "void set_synaptic_weights(const Vector<double>&) method.\n"
             << "Size of synaptic weights vector must be equal to number of inputs.\n";

      throw std::logic_error(buffer.str());
   }

   #endif

   // Set synaptic weights
   
   synaptic_weights = new_synaptic_weights;
}


// void set_synaptic_weight(const size_t&, const double&) method


void Perceptron::set_synaptic_weight(const size_t& synaptic_weight_index, const double& new_synaptic_weight)
{
   // Control sentence (if debug)

   #ifndef NDEBUG 

   const size_t inputs_number = get_inputs_number();

   if(synaptic_weight_index >= inputs_number)
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: Perceptron class.\n"
             << "void set_synaptic_weight(const size_t&, const double&) method.\n"
             << "Index of synaptic weight must be less than number of inputs.\n";

      throw std::logic_error(buffer.str());
   }

   #endif

   // Set single synaptic weight

   synaptic_weights[synaptic_weight_index] = new_synaptic_weight;
}


// void set_display(const bool&) method

 
void Perceptron::set_display(const bool& new_display)
{
   display = new_display;   
}


// void set_inputs_number(size_t) method

 
void Perceptron::set_inputs_number(const size_t& new_inputs_number)
{
   initialize_bias_normal(0.0,1.0);

   synaptic_weights.set(new_inputs_number);
   initialize_synaptic_weights_normal(0.0,1.0);
}


// size_t count_parameters_number(void) const method


size_t Perceptron::count_parameters_number(void) const
{
   const size_t inputs_number = get_inputs_number();
   
   return(1 + inputs_number);
}


// Vector<double> arrange_parameters(void) const method


Vector<double> Perceptron::arrange_parameters(void) const
{
   const size_t parameters_number = count_parameters_number();

   Vector<double> parameters(parameters_number);

   parameters[0] = bias;

   const size_t inputs_number = get_inputs_number();

   for(size_t i = 0; i < inputs_number; i++)
   {
      parameters[(size_t)1+i] = synaptic_weights[i];
   }

   return(parameters); 
}


// void set_parameters(const Vector<double>&) method


void Perceptron::set_parameters(const Vector<double>& new_parameters)
{
   const size_t inputs_number = get_inputs_number();

   // Control sentence (if debug)

   #ifndef NDEBUG 

   const size_t size = new_parameters.size();

   if(size != 1+inputs_number)
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: Perceptron class.\n"
             << "void set_parameters(const Vector<double>&) method.\n"
             << "Size must be equal to one plus number of inputs.\n";

      throw std::logic_error(buffer.str());
   }

   #endif

   bias = new_parameters[0];

   for(size_t i = 0; i < inputs_number; i++)
   {
      synaptic_weights[i] = new_parameters[i+1];
   }
}


// void initialize_bias(const double&) method


void Perceptron::initialize_bias(const double& value)
{
   bias = value;
}


// void initialize_bias_uniform(const double&, const double&) method


void Perceptron::initialize_bias_uniform(const double& minimum, const double& maximum)
{
   // Control sentence (if debug)

   #ifndef NDEBUG 

   if(minimum > maximum)
   {
     std::ostringstream buffer;

      buffer << "OpenNN Exception: Perceptron class.\n" 
             << "initialize_bias_uniform(const double&, const double&) method.\n"
             << "Minimum value must be less than maximum value.\n";

      throw std::logic_error(buffer.str());
   }

   #endif

   bias = calculate_random_uniform(minimum, maximum);
}


// void initialize_synaptic_weights(const double&) method


void Perceptron::initialize_synaptic_weights(const double& value)
{
   synaptic_weights.initialize(value);
}


// void initialize_synaptic_weights_uniform(const double&, const double&) method


void Perceptron::initialize_synaptic_weights_uniform(const double& minimum, const double& maximum)
{
   synaptic_weights.randomize_uniform(minimum, maximum);
}


// void initialize_bias_normal(const double&, const double&) method


void Perceptron::initialize_bias_normal(const double& mean, const double& standard_deviation)
{
   // Control sentence (if debug)

   #ifndef NDEBUG 

   if(standard_deviation < 0.0)
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: Perceptron class.\n" 
             << "initialize_bias_normal(const double&, const double&) method.\n"
             << "Standard deviation must be equal or greater than zero.\n";

      throw std::logic_error(buffer.str());
   }

   #endif

   bias = calculate_random_normal(mean, standard_deviation);
}


// void initialize_synaptic_weights_normal(const double&, const double&) method


void Perceptron::initialize_synaptic_weights_normal(const double& mean, const double& standard_deviation)
{
   synaptic_weights.randomize_normal(mean, standard_deviation);
}


// void initialize_parameters(const double&) method


void Perceptron::initialize_parameters(const double& value)
{
   bias = value;
   synaptic_weights.initialize(value);
}


// double calculate_combination(const Vector<double>&) method


double Perceptron::calculate_combination(const Vector<double>& inputs) const
{
    const size_t inputs_number = get_inputs_number();

   // Control sentence (if debug)

   #ifndef NDEBUG 

   if(inputs_number == 0)
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: Perceptron class.\n" 
             << "calculate_combination(const Vector<double>&) method.\n"
             << "Number of inputs must be greater than zero.\n";

      throw std::logic_error(buffer.str());
   }

   const size_t inputs_size = inputs.size();

   if(inputs_size != inputs_number)
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: Perceptron class.\n"
             << "double calculate_combination(const Vector<double>&) method.\n"
             << "Size of inputs (" << inputs_size << ") must be equal to number of inputs (" << inputs_number << ").\n";

      throw std::logic_error(buffer.str());
   }

   #endif

   // Calculate combination

   double combination = bias;

   for(size_t i = 0; i < inputs_number; i++)
   {
       combination += synaptic_weights[i]*inputs[i];
   }

   return(combination);
}


// double calculate_combination(const Vector<double>&, const Vector<double>&) const method


double Perceptron::calculate_combination(const Vector<double>& inputs, const Vector<double>& parameters) const
{
    const size_t inputs_number = get_inputs_number();

   // Control sentence (if debug)

   #ifndef NDEBUG 

   std::ostringstream buffer;

   const size_t inputs_size = inputs.size();

   if(inputs_size != inputs_number)
   {
      buffer << "OpenNN Exception: Perceptron class.\n" 
             << "double calculate_combination(const Vector<double>&, const Vector<double>&) const method.\n"
             << "Size of inputs must be equal to number of inputs.\n";

      throw std::logic_error(buffer.str());
   }

   const size_t parameters_size = parameters.size();

   const size_t parameters_number = count_parameters_number();

   if(parameters_size != parameters_number)
   {
      buffer << "OpenNN Exception: Perceptron class.\n" 
             << "double calculate_combination(const Vector<double>&, const Vector<double>&) const method.\n"
             << "Size of potential parameters (" << parameters_size << ") must be equal to number of parameters (" << parameters_number << ").\n";

      throw std::logic_error(buffer.str());
   }

   #endif

   // Modified for performance.

   double combination = parameters[0];

   for(size_t i = 0; i < inputs_number; i++)
   {
       combination += parameters[i+1]*inputs[i];
   }

   return(combination);
}


// double calculate_activation(const double&) const  method


double Perceptron::calculate_activation(const double& combination) const
{
   switch(activation_function)   
   {
      case Perceptron::Logistic:
      {
         return(1.0/(1.0 + exp(-combination)));
      }
      break;
                                     
      case Perceptron::HyperbolicTangent:
      {
         return(1.0-2.0/(exp(2.0*combination)+1.0));   
      }
      break;

      case Perceptron::Threshold:
      {
         if(combination < 0)
         {
            return(0.0);
         }
         else
         {
            return(1.0);
         }
      }
      break;

      case Perceptron::SymmetricThreshold:
      {
         if(combination < 0)
         {
            return(-1.0);
         }
         else
         {
            return(1.0);
         }
      }
      break;

      case Perceptron::Linear:
      {
         return(combination);        
      }
      break;
      
      default:
      {
         std::ostringstream buffer;

         buffer << "OpenNN Exception: Perceptron class.\n" 
                << "double calculate_activation(const double&) const  method.\n"
                << "Unknown activation function.\n";

         throw std::logic_error(buffer.str());
      }
      break;
   }
}


// double calculate_activation_derivative(const double&) const  method


double Perceptron::calculate_activation_derivative(const double& combination) const
{
   switch(activation_function)   
   {
      case Perceptron::Logistic:
      {
         const double exponent = exp(-combination);

         return(exponent/((1.0+exponent)*(1.0+exponent)));
      }
      break;
                                     
      case Perceptron::HyperbolicTangent:
      {
         const double tanh_combination = tanh(combination);

         return(1.0 - tanh_combination*tanh_combination);
      }
      break;

      case Perceptron::Threshold:
      {
         if(combination != 0.0)
         {
            return(0.0); 
         }
         else
         {
            std::ostringstream buffer;

            buffer << "OpenNN Exception: Perceptron class.\n" 
                   << "double calculate_activation_derivative(const double&) const  method.\n"
                   << "Threshold activation function is not derivable.\n";

            throw std::logic_error(buffer.str());
         }
      }
      break;

      case Perceptron::SymmetricThreshold:
      {
         if(combination != 0.0)
         {
            return(0.0); 
         }
         else
         {
            std::ostringstream buffer;

            buffer << "OpenNN Exception: Perceptron class.\n" 
                      << "double calculate_activation_derivative(const double&) const  method.\n"
                      << "Symmetric threshold activation function is not derivable.\n";

            throw std::logic_error(buffer.str());
         }
      }
      break;

      case Perceptron::Linear:
      {
         return(1.0);
      }
      break;

      default:
      {
         std::ostringstream buffer;

         buffer << "OpenNN Exception: Perceptron class.\n" 
                << "double calculate_activation_derivative(const double&) const  method.\n"
                << "Unknown activation function.\n";

         throw std::logic_error(buffer.str());
      }
      break;
   }
}


// double calculate_activation_second_derivative(const double&) const  method


double Perceptron::calculate_activation_second_derivative(const double& combination) const
{
   switch(activation_function)   
   {
      case Perceptron::Logistic:
      {
         const double exponent = exp(combination);

         return(-exponent*(exponent-1.0)/((exponent+1.0)*(exponent+1.0)*(exponent+1.0)));
      }
      break;
                                     
      case Perceptron::HyperbolicTangent:
      {
         return(-2.0*tanh(combination)*(1.0 - pow(tanh(combination),2)));
      }
      break;

      case Perceptron::Threshold:
      {
         if(combination != 0.0)
         {
            return(0.0); 
         }
         else
         {
            std::ostringstream buffer;

            buffer << "OpenNN Exception: Perceptron class.\n" 
                   << "double calculate_activation_second_derivative(const double&) const  method.\n"
                   << "Threshold activation function is not derivable.\n";

            throw std::logic_error(buffer.str());
         }
      }
      break;

      case Perceptron::SymmetricThreshold:
      {
         if(combination != 0.0)
         {
            return(0.0); 
         }
         else
         {
            std::ostringstream buffer;

            buffer << "OpenNN Exception: Perceptron class.\n" 
                   << "double calculate_activation_second_derivative(const double&) const  method.\n"
                   << "Symmetric threshold activation function is not derivable.\n";

            throw std::logic_error(buffer.str());
         }
      }
      break;

      case Perceptron::Linear:
      {
         return(0.0);        
      }
      break;

      default:
      {
         std::ostringstream buffer;

         buffer << "OpenNN Exception: Perceptron class.\n" 
                << "double calculate_activation_second_derivative(const double&) const  method.\n"
                << "Unknown activation function.\n";

         throw std::logic_error(buffer.str());
      }
      break;
   }
}


// double calculate_output(const Vector<double>&) const method


double Perceptron::calculate_output(const Vector<double>& inputs) const
{
   // Control sentence (if debug)

   #ifndef NDEBUG 

   const size_t size = inputs.size();
   const size_t inputs_number = get_inputs_number();

   if(size != inputs_number)
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: Perceptron class.\n" 
             << "double calculate_output(const Vector<double>&) const method.\n"
             << "Size must be equal to number of inputs.\n";

      throw std::logic_error(buffer.str());
   }

   #endif

   // Calculate outputs 

   return(calculate_activation(calculate_combination(inputs)));  
}


// double calculate_output(const Vector<double>&, const Vector<double>&) const method


double Perceptron::calculate_output(const Vector<double>& inputs, const Vector<double>& parameters) const
{
   // Control sentence (if debug)

   #ifndef NDEBUG 

   const size_t inputs_size = inputs.size();
   const size_t inputs_number = get_inputs_number();

   if(inputs_size != inputs_number)
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: Perceptron class.\n" 
             << "double calculate_output(const Vector<double>&, const Vector<double>&) const method.\n"
             << "Size of inputs must be equal to number of inputs.\n";

      throw std::logic_error(buffer.str());
   }

   const size_t parameters_size = parameters.size();

   const size_t parameters_number = count_parameters_number();

   if(parameters_size != parameters_number)
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: Perceptron class.\n" 
             << "double calculate_output(const Vector<double>&, const Vector<double>&) const method.\n"
             << "Size of potential parameters (" << parameters_size << ") must be equal to number of parameters (" << parameters_number << ").\n";

      throw std::logic_error(buffer.str());
   }

   #endif

   return(calculate_activation(calculate_combination(inputs, parameters)));
}


// Vector<double> calculate_gradient(const Vector<double>&) const method


Vector<double> Perceptron::calculate_gradient(const Vector<double>& inputs) const
{
   // Control sentence (if debug)

   #ifndef NDEBUG 

   const size_t size = inputs.size();
   const size_t inputs_number = get_inputs_number();

   if(size != inputs_number)
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: Perceptron class.\n" 
             << "Vector<double> calculate_gradient(const Vector<double>&) const method.\n"
             << "Size must be equal to number of inputs.\n";

      throw std::logic_error(buffer.str());
   }

   #endif

   // Calculate gradient

   const double combination = calculate_combination(inputs);

   const double activation_derivative = calculate_activation_derivative(combination);

   return(synaptic_weights*activation_derivative);
}


// Vector<double> calculate_gradient(const Vector<double>&, const Vector<double>&) const method


Vector<double> Perceptron::calculate_gradient(const Vector<double>& inputs, const Vector<double>& parameters) const
{
   const size_t inputs_number = get_inputs_number();

   // Control sentence (if debug)

   #ifndef NDEBUG 

   const size_t size = inputs.size();

   if(size != inputs_number)
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: Perceptron class.\n" 
             << "double calculate_gradient(const Vector<double>&, const Vector<double>&) const method.\n"
             << "Size must be equal to number of inputs.\n";

      throw std::logic_error(buffer.str());
   }

   #endif

   // Calculate parameters gradient

   const double combination = calculate_combination(inputs, parameters);

   const double activation_derivative = calculate_activation_derivative(combination);

   Vector<double> gradient(1+inputs_number);

   // Bias

   gradient[0] = activation_derivative;

   // Synaptic weights

   for(size_t i = 1; i < 1+inputs_number; i++)
   {
      gradient[i] = inputs[i-1]*activation_derivative;
   }

   return(gradient);
}


// Vector<double> calculate_combination_gradient(const Vector<double>&) const method


Vector<double> Perceptron::calculate_combination_gradient(const Vector<double>&) const
{
   return(synaptic_weights);
}


// Vector<double> calculate_combination_gradient(const Vector<double>&, const Vector<double>&) const method


Vector<double> Perceptron::calculate_combination_gradient(const Vector<double>& inputs, const Vector<double>&) const
{   
   const size_t inputs_number = get_inputs_number();

   // Control sentence (if debug)

   #ifndef NDEBUG 

   const size_t size = inputs.size();

   if(size != inputs_number)
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: Perceptron class.\n" 
             << "double calculate_combination_gradient(const Vector<double>&, const Vector<double>&) const method.\n"
             << "Size must be equal to number of inputs.\n";

      throw std::logic_error(buffer.str());
   }

   #endif

   // Calculate combination gradient

   Vector<double> combination_gradient(1+inputs_number);

   // Bias

   combination_gradient[0] = 1.0;

   // Synaptic weights

   for(size_t i = 1; i < 1+inputs_number; i++)
   {
      combination_gradient[i] = inputs[i-1];
   }

   return(combination_gradient);
}


// Matrix<double> calculate_Hessian(const Vector<double>&) const method


Matrix<double> Perceptron::calculate_Hessian(const Vector<double>& inputs) const
{   
   // Control sentence (if debug)

   #ifndef NDEBUG 

   const size_t inputs_number = get_inputs_number();
   const size_t inputs_size = inputs.size();

   if(inputs_size != inputs_number)
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: Perceptron class.\n" 
             << "Matrix<double> calculate_Hessian(const Vector<double>&) const method.\n"
             << "Size of inputs must be equal to number of inputs.\n";

      throw std::logic_error(buffer.str());
   }

   #endif

   const double combination = calculate_combination(inputs);
   const double activation_second_derivative = calculate_activation_second_derivative(combination);

   return(synaptic_weights.direct(synaptic_weights)*activation_second_derivative);
}


// Matrix<double> calculate_Hessian(const Vector<double>&, const Vector<double>&) const method


Matrix<double> Perceptron::calculate_Hessian(const Vector<double>& inputs, const Vector<double>& parameters) const
{
   // Control sentence (if debug)

   #ifndef NDEBUG 

   const size_t inputs_size = inputs.size();
   const size_t inputs_number = get_inputs_number();

   if(inputs_size != inputs_number)
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: Perceptron class.\n" 
             << "Matrix<double> calculate_Hessian(const Vector<double>&, const Vector<double>&) const method.\n"
             << "Size of inputs must be equal to number of inputs.\n";

      throw std::logic_error(buffer.str());
   }

   #endif

   const double combination = calculate_combination(inputs, parameters);
   const double activation_second_derivative = calculate_activation_second_derivative(combination);

   const size_t parameters_number = count_parameters_number();

   Matrix<double> Hessian(parameters_number, parameters_number);

   // Bias - bias derivative

   Hessian(0,0) = activation_second_derivative;

   // Bias - synaptic weight derivative

   for(size_t i = 1; i < parameters_number; i++)
   {
      Hessian(0,i) = activation_second_derivative*inputs[i-1];
   }

   // Synaptic weight -synaptic weight derivative

   for(size_t i = 1; i < parameters_number; i++)
   {
      for(size_t j = 1; j < parameters_number; j++)
      {
         Hessian(i,j) = activation_second_derivative*inputs[i-1]*inputs[j-1];
      }
   }

   // Hessian symmetry

   for(size_t i = 0; i < parameters_number; i++)
   {
      for(size_t j = 0; j < i; j++)
      {
         Hessian(i,j) = Hessian(j,i);
      }
   }
   
   return(Hessian);
}


// Matrix<double> calculate_combination_Hessian(const Vector<double>&) const method


Matrix<double> Perceptron::calculate_combination_Hessian(const Vector<double>&) const
{
   const size_t inputs_number = get_inputs_number();

   const Matrix<double> combination_Hessian(inputs_number, inputs_number, 0.0); 

   return(combination_Hessian);   
}


// Matrix<double> calculate_combination_Hessian(const Vector<double>&, const Vector<double>&) const method


Matrix<double> Perceptron::calculate_combination_Hessian(const Vector<double>&, const Vector<double>&) const
{
   const size_t parameters_number = count_parameters_number();

   const Matrix<double> Hessian(parameters_number, parameters_number, 0.0);

   return(Hessian);   
}


// void grow_input(void) method


void Perceptron::grow_input(void)
{
   synaptic_weights.push_back(0.0);
}


// void prune_input(const size_t&) method


void Perceptron::prune_input(const size_t& index)
{
    // Control sentence (if debug)

    #ifndef NDEBUG

    const size_t inputs_number = get_inputs_number();

    if(index >= inputs_number)
    {
       std::ostringstream buffer;

       buffer << "OpenNN Exception: Perceptron class.\n"
              << "void prune_input(const size_t&) method.\n"
              << "Index of input is equal or greater than number of inputs.\n";

       throw std::logic_error(buffer.str());
    }

    #endif

   synaptic_weights.erase(synaptic_weights.begin()+index);
}


// std::string write_expression(const Vector<std::string>&, const std::string&) const method


std::string Perceptron::write_expression(const Vector<std::string>& inputs_name, const std::string& output_name) const
{
   const size_t inputs_number = get_inputs_number();

   std::string activation_function_name = write_activation_function();

   std::ostringstream buffer;

   buffer << output_name << "=" << activation_function_name << "("
          << bias << "\n";

   for(size_t i = 0; i < inputs_number; i++)
   {
      if(synaptic_weights[i] >= 0)
      {
         buffer << "+";      
      }

      buffer << synaptic_weights[i] << "*" << inputs_name[i];

      if(i != 0 && i%4 == 0 && i != inputs_number-1)
      {
            buffer << "\n";
      }
   }

   buffer << ");\n";

   return(buffer.str());
}

}


// OpenNN: Open Neural Networks Library.
// Copyright (c) 2005-2015 Roberto Lopez.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.

// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA