performance_term.cpp

/****************************************************************************************************************/
/*                                                                                                              */
/*   OpenNN: Open Neural Networks Library                                                                       */
/*   www.artelnics.com/opennn                                                                                   */
/*                                                                                                              */
/*   P E R F O R M A N C E   T E R M   C L A S S                                                                */
/*                                                                                                              */
/*   Roberto Lopez                                                                                              */
/*   Artelnics - Making intelligent use of data                                                                 */
/*   [email protected]                                                                                 */
/*                                                                                                              */
/****************************************************************************************************************/

// OpenNN includes

#include "performance_term.h"

namespace OpenNN
{

// DEFAULT CONSTRUCTOR


PerformanceTerm::PerformanceTerm(void)
 : neural_network_pointer(NULL), 
   data_set_pointer(NULL),
   mathematical_model_pointer(NULL),
   numerical_differentiation_pointer(NULL)
{
   set_default();
}


// NEURAL NETWORK CONSTRUCTOR


PerformanceTerm::PerformanceTerm(NeuralNetwork* new_neural_network_pointer)
 : neural_network_pointer(new_neural_network_pointer), 
   data_set_pointer(NULL),
   mathematical_model_pointer(NULL),
   numerical_differentiation_pointer(NULL)
{
   set_default();
}


// DATA SET CONSTRUCTOR


PerformanceTerm::PerformanceTerm(DataSet* new_data_set_pointer)
 : neural_network_pointer(NULL), 
   data_set_pointer(new_data_set_pointer),
   mathematical_model_pointer(NULL),
   numerical_differentiation_pointer(NULL)
{
   set_default();
}


// MATHEMATICAL MODEL CONSTRUCTOR


PerformanceTerm::PerformanceTerm(MathematicalModel* new_mathematical_model_pointer)
 : neural_network_pointer(NULL), 
   data_set_pointer(NULL),
   mathematical_model_pointer(new_mathematical_model_pointer),
   numerical_differentiation_pointer(NULL)
{
   set_default();
}


// NEURAL NETWORK AND DATA SET CONSTRUCTOR


PerformanceTerm::PerformanceTerm(NeuralNetwork* new_neural_network_pointer, DataSet* new_data_set_pointer)
 : neural_network_pointer(new_neural_network_pointer), 
   data_set_pointer(new_data_set_pointer),
   mathematical_model_pointer(NULL),
   numerical_differentiation_pointer(NULL)
{
   set_default();
}


// NEURAL NETWORK AND MATHEMATICAL MODEL CONSTRUCTOR


PerformanceTerm::PerformanceTerm(NeuralNetwork* new_neural_network_pointer, MathematicalModel* new_mathematical_model_pointer)
 : neural_network_pointer(new_neural_network_pointer), 
   data_set_pointer(NULL),
   mathematical_model_pointer(new_mathematical_model_pointer),
   numerical_differentiation_pointer(NULL)
{
   set_default();
}


// NEURAL NETWORK, MATHEMATICAL MODEL AND DATA SET CONSTRUCTOR


PerformanceTerm::PerformanceTerm(NeuralNetwork* new_neural_network_pointer, MathematicalModel* new_mathematical_model_pointer, DataSet* new_data_set_pointer)
 : neural_network_pointer(new_neural_network_pointer), 
   data_set_pointer(new_data_set_pointer),
   mathematical_model_pointer(new_mathematical_model_pointer),
   numerical_differentiation_pointer(NULL)
{
   set_default();
}


// XML CONSTRUCTOR


PerformanceTerm::PerformanceTerm(const tinyxml2::XMLDocument& performance_term_document)
 : neural_network_pointer(NULL), 
   data_set_pointer(NULL),
   mathematical_model_pointer(NULL),
   numerical_differentiation_pointer(NULL)
{
   set_default();

   from_XML(performance_term_document);
}


// COPY CONSTRUCTOR


PerformanceTerm::PerformanceTerm(const PerformanceTerm& other_performance_term)
 : neural_network_pointer(NULL), 
   data_set_pointer(NULL),
   mathematical_model_pointer(NULL),
   numerical_differentiation_pointer(NULL)
{
   neural_network_pointer = other_performance_term.neural_network_pointer;

   data_set_pointer = other_performance_term.data_set_pointer;
   mathematical_model_pointer = other_performance_term.mathematical_model_pointer;

   if(other_performance_term.numerical_differentiation_pointer)
   {
      numerical_differentiation_pointer = new NumericalDifferentiation(*other_performance_term.numerical_differentiation_pointer);
   }

   display = other_performance_term.display;  
}


// DESTRUCTOR


PerformanceTerm::~PerformanceTerm(void)
{
   delete numerical_differentiation_pointer;
}


// ASSIGNMENT OPERATOR

// PerformanceTerm& operator = (const PerformanceTerm&) method


PerformanceTerm& PerformanceTerm::operator = (const PerformanceTerm& other_performance_term)
{
   if(this != &other_performance_term) 
   {
      neural_network_pointer = other_performance_term.neural_network_pointer;

      data_set_pointer = other_performance_term.data_set_pointer;

      mathematical_model_pointer = other_performance_term.mathematical_model_pointer;

      if(other_performance_term.numerical_differentiation_pointer == NULL)
      {
          delete numerical_differentiation_pointer;

          numerical_differentiation_pointer = NULL;
      }
      else
      {
            numerical_differentiation_pointer = new NumericalDifferentiation(*other_performance_term.numerical_differentiation_pointer);
      }

      display = other_performance_term.display;
   }

   return(*this);
}


// EQUAL TO OPERATOR

// bool operator == (const PerformanceTerm&) const method


bool PerformanceTerm::operator == (const PerformanceTerm& other_performance_term) const
{
   if(neural_network_pointer != other_performance_term.neural_network_pointer
   || data_set_pointer != other_performance_term.data_set_pointer
   || mathematical_model_pointer != other_performance_term.mathematical_model_pointer)
   {
       return(false);
   }
/*
   else if((numerical_differentiation_pointer == NULL && other_performance_term.numerical_differentiation_pointer !=NULL)
        || (numerical_differentiation_pointer != NULL && other_performance_term.numerical_differentiation_pointer ==NULL))
   {
       return(false);
   }
   else if(numerical_differentiation_pointer != NULL)
   {
       if(&numerical_differentiation_pointer != &other_performance_term.numerical_differentiation_pointer)
       {
            return(false);
       }
   }
*/
   else if(display != other_performance_term.display)
   {
      return(false);
   }

   return(true);

}


// METHODS

// const bool& get_display(void) const method


const bool& PerformanceTerm::get_display(void) const
{
   return(display);
}


// bool has_neural_network(void) const method


bool PerformanceTerm::has_neural_network(void) const
{
    if(neural_network_pointer)
    {
        return(true);
    }
    else
    {
        return(false);
    }
}


// bool has_mathematical_model(void) const method


bool PerformanceTerm::has_mathematical_model(void) const
{
    if(mathematical_model_pointer)
    {
        return(true);
    }
    else
    {
        return(false);
    }
}


// bool has_data_set(void) const method


bool PerformanceTerm::has_data_set(void) const
{
    if(data_set_pointer)
    {
        return(true);
    }
    else
    {
        return(false);
    }
}


// bool has_numerical_differentiation(void) const method


bool PerformanceTerm::has_numerical_differentiation(void) const
{
    if(numerical_differentiation_pointer)
    {
        return(true);
    }
    else
    {
        return(false);
    }
}


// void set(void) method


void PerformanceTerm::set(void)
{
   neural_network_pointer = NULL;
   data_set_pointer = NULL;
   mathematical_model_pointer = NULL;
   numerical_differentiation_pointer = NULL;

   set_default();
}


// void set(NeuralNetwork*) method


void PerformanceTerm::set(NeuralNetwork* new_neural_network_pointer)
{
   neural_network_pointer = new_neural_network_pointer;
   data_set_pointer = NULL;
   mathematical_model_pointer = NULL;
   numerical_differentiation_pointer = NULL;

   set_default();
}


// void set(DataSet*) method


void PerformanceTerm::set(DataSet* new_data_set_pointer)
{
   neural_network_pointer = NULL;
   data_set_pointer = new_data_set_pointer;
   mathematical_model_pointer = NULL;
   numerical_differentiation_pointer = NULL;

   set_default();
}


// void set(MathematicalModel*) method


void PerformanceTerm::set(MathematicalModel* new_mathematical_model_pointer)
{
   neural_network_pointer = NULL;
   data_set_pointer = NULL;
   mathematical_model_pointer = new_mathematical_model_pointer;
   numerical_differentiation_pointer = NULL;

   set_default();
}


// void set(NeuralNetwork*, DataSet*) method


void PerformanceTerm::set(NeuralNetwork* new_neural_network_pointer, DataSet* new_data_set_pointer)
{
   neural_network_pointer = new_neural_network_pointer;
   data_set_pointer = new_data_set_pointer;
   mathematical_model_pointer = NULL;
   numerical_differentiation_pointer = NULL;

   set_default();
}


// void set(NeuralNetwork*, MathematicalModel*) method


void PerformanceTerm::set(NeuralNetwork* new_neural_network_pointer, MathematicalModel* new_mathematical_model_pointer)
{
   neural_network_pointer = new_neural_network_pointer;
   data_set_pointer = NULL;
   mathematical_model_pointer = new_mathematical_model_pointer;
   numerical_differentiation_pointer = NULL;

   set_default();
}


// void set(NeuralNetwork*, MathematicalModel*, DataSet*) method


void PerformanceTerm::set(NeuralNetwork* new_neural_network_pointer, MathematicalModel* new_mathematical_model_pointer, DataSet* new_data_set_pointer)
{
   neural_network_pointer = new_neural_network_pointer;
   data_set_pointer = new_data_set_pointer;
   mathematical_model_pointer = new_mathematical_model_pointer;
   numerical_differentiation_pointer = NULL;

   set_default();
}


// void set(const PerformanceTerm&) method


void PerformanceTerm::set(const PerformanceTerm& other_performance_term)
{
   neural_network_pointer = other_performance_term.neural_network_pointer;

   data_set_pointer = other_performance_term.data_set_pointer;

   mathematical_model_pointer = other_performance_term.mathematical_model_pointer;

   if(other_performance_term.numerical_differentiation_pointer)
   {
      numerical_differentiation_pointer = new NumericalDifferentiation(*other_performance_term.numerical_differentiation_pointer);
   }

   display = other_performance_term.display;  
}


// void set_neural_network_pointer(NeuralNetwork*) method


void PerformanceTerm::set_neural_network_pointer(NeuralNetwork* new_neural_network_pointer)
{
   neural_network_pointer = new_neural_network_pointer;
}


// void set_mathematical_model_pointer(MathematicalModel*) method


void PerformanceTerm::set_mathematical_model_pointer(MathematicalModel* new_mathematical_model_pointer)
{
   mathematical_model_pointer = new_mathematical_model_pointer;
}


// void set_data_set_pointer(DataSet*) method


void PerformanceTerm::set_data_set_pointer(DataSet* new_data_set_pointer)
{
   data_set_pointer = new_data_set_pointer;
}


// void set_numerical_differentiation_pointer(NumericalDifferentiation*) method


void PerformanceTerm::set_numerical_differentiation_pointer(NumericalDifferentiation* new_numerical_differentiation_pointer)
{
   numerical_differentiation_pointer = new_numerical_differentiation_pointer;
}


// void set_default(void) method


void PerformanceTerm::set_default(void)
{
   display = true;
}


// void set_display(const bool&) method


void PerformanceTerm::set_display(const bool& new_display)
{
   display = new_display;
}


// void construct_numerical_differentiation(void) method


void PerformanceTerm::construct_numerical_differentiation(void)
{
   if(numerical_differentiation_pointer == NULL)
   {
      numerical_differentiation_pointer = new NumericalDifferentiation();
   }
}


// void delete_numerical_differentiation_pointer(void) method


void PerformanceTerm::delete_numerical_differentiation_pointer(void)
{
   delete numerical_differentiation_pointer;

   numerical_differentiation_pointer = NULL;
}


// void check(void) const method


void PerformanceTerm::check(void) const
{
   std::ostringstream buffer;

   if(!neural_network_pointer)
   {
      buffer << "OpenNN Exception: PerformanceTerm class.\n"
             << "void check(void) const.\n"
             << "Pointer to neural network is NULL.\n";

      throw std::logic_error(buffer.str());   
   }
}


// Vector< Vector<double> > calculate_layers_delta(const Vector< Vector<double> >&, const Vector<double>&) method


Vector< Vector<double> > PerformanceTerm::calculate_layers_delta
(const Vector< Vector<double> >& layers_activation_derivative, 
 const Vector<double>& output_objective_gradient) const
{
   // Neural network stuff

   #ifndef NDEBUG 
   
   check();

   #endif

   const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();

   #ifndef NDEBUG 

   std::ostringstream buffer;

   if(!multilayer_perceptron_pointer)
   {
      buffer << "OpenNN Exception: PerformanceTerm class.\n"
             << "Vector< Vector<double> > calculate_layers_delta(const Vector< Vector<double> >&, const Vector<double>&) const method.\n"
             << "Pointer to multilayer perceptron in neural network is NULL.\n";

      throw std::logic_error(buffer.str());   
   }

   #endif

   const size_t layers_number = multilayer_perceptron_pointer->get_layers_number();
   const Vector<size_t> layers_perceptrons_number = multilayer_perceptron_pointer->arrange_layers_perceptrons_numbers();

   // Control sentence (if debug)

   #ifndef NDEBUG 

   // Forward propagation activation derivative size

   const size_t layers_activation_derivative_size = layers_activation_derivative.size();

   if(layers_activation_derivative_size != layers_number)
   {
      buffer << "OpenNN Exception: PerformanceTerm class.\n"
             << "Vector< Vector<double> > calculate_layers_delta(const Vector< Vector<double> >&, const Vector<double>&) method.\n"
             << "Size of forward propagation activation derivative vector must be equal to number of layers.\n";

      throw std::logic_error(buffer.str());   
   }

   if(layers_number > 0)
   {
      const size_t output_objective_gradient_size = output_objective_gradient.size();

      if(output_objective_gradient_size != layers_perceptrons_number[layers_number-1])
      {
         buffer << "OpenNN Exception: PerformanceTerm class.\n"
                << "Vector<double> calculate_layers_delta(const Vector< Vector<double> >&, const Vector<double>&) method.\n"
                << "Size of outputs objective gradient (" << output_objective_gradient_size << ") must be equal to "               
                << "number of outputs (" << layers_perceptrons_number[layers_number-1] << ").\n";

         throw std::logic_error(buffer.str());       
      }
   }

   #endif

   // Neural network stuff

   Matrix<double> layer_synaptic_weights;

   // Performance functional stuff

   Vector< Vector<double> > layers_delta(layers_number);

   // Output layer

   if(layers_number > 0)
   {
      layers_delta[layers_number-1] = layers_activation_derivative[layers_number-1]*output_objective_gradient;

      // Rest of hidden layers

      for(int i = (int)layers_number-2; i >= 0; i--)
      {   
         layer_synaptic_weights = neural_network_pointer->get_multilayer_perceptron_pointer()->get_layer(i+1).arrange_synaptic_weights();

         layers_delta[i] = layers_activation_derivative[i]*(layers_delta[i+1].dot(layer_synaptic_weights));
      }
   }

   return(layers_delta);
}


// Vector< Vector<double> > calculate_layers_delta(const Vector< Vector<double> >&, const Vector<double>&, const Vector<double>&) method


Vector< Vector<double> > PerformanceTerm::calculate_layers_delta
(const Vector< Vector<double> >& layers_activation_derivative, 
 const Vector<double>& homogeneous_solution,
 const Vector<double>& output_objective_gradient) const
{
   const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();

   const size_t layers_number = multilayer_perceptron_pointer->get_layers_number();
   const Vector<size_t> layers_perceptrons_number = multilayer_perceptron_pointer->arrange_layers_perceptrons_numbers();

   // Control sentence (if debug)

   #ifndef NDEBUG 

   // Forward propagation activation derivative size

   const size_t layers_activation_derivative_size = layers_activation_derivative.size();

   if(layers_activation_derivative_size != layers_number)
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: PerformanceTerm class.\n"
             << "Vector< Vector<double> > calculate_layers_delta(const Vector< Vector<double> >&, const Vector<double>&) const method.\n"
             << "Size of forward propagation activation derivative vector must be equal to number of layers.\n";

      throw std::logic_error(buffer.str());   
   }
      
   const size_t objective_function_output_gradient_size = output_objective_gradient.size();

   if(objective_function_output_gradient_size != layers_perceptrons_number[layers_number-1])
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: PerformanceTerm class.\n"
             << "Vector<double> calculate_layers_delta(const Vector< Vector<double> >&, const Vector<double>&) const method.\n"
             << "Size of objective function outputs derivative (" << objective_function_output_gradient_size << ")must be equal to "               
             << "number of outputs (" << layers_perceptrons_number[layers_number-1] << ").\n";

      throw std::logic_error(buffer.str());   
   }

   #endif

   const Vector<PerceptronLayer>& layers = multilayer_perceptron_pointer->get_layers();

   Matrix<double> synaptic_weights;

   double sum;

   // Set layers delta vector of vectors

   Vector< Vector<double> > layers_delta(layers_number);

   for(size_t i = 0; i < layers_number; i++)
   {
      layers_delta[i].set(layers_perceptrons_number[i]);
   }

   // Output layer

   layers_delta[layers_number-1] = layers_activation_derivative[layers_number-1]*homogeneous_solution*output_objective_gradient;

   // Rest of hidden layers

   for(int h = (int)layers_number-2; h >= 0; h--)
   {   
      for(size_t i = 0; i < layers_perceptrons_number[h]; i++)
      {
         sum = 0.0;         

         for(size_t j = 0; j < layers_perceptrons_number[h+1]; j++)
         {
            synaptic_weights = layers[h+1].arrange_synaptic_weights();

            sum += (synaptic_weights(i,j))*layers_delta[h+1][j];
         }       

         layers_delta[h][i] = layers_activation_derivative[h][i]*sum;
      }
   }

   return(layers_delta);
}


// Vector<double> calculate_point_gradient(const Vector<double>&, const Vector< Vector<double> >&, const Vector<double>&) const method


Vector<double> PerformanceTerm::calculate_point_gradient
(const Vector<double>& inputs, 
 const Vector< Vector<double> >& layers_activation, 
 const Vector< Vector<double> >& layers_delta) const
{
   // Control sentence (if debug)

   #ifndef NDEBUG 

   check();

   #endif

   const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();

   #ifndef NDEBUG 

   std::ostringstream buffer;

   if(!multilayer_perceptron_pointer)
   {
      buffer << "OpenNN Exception: PerformanceTerm class.\n"
             << "Vector<double> calculate_point_gradient(const Vector<double>&, const Vector< Vector<double> >&, const Vector<double>&) const method.\n"
             << "Pointer to multilayer perceptron is NULL.\n";

      throw std::logic_error(buffer.str());   
   }

   #endif

   const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
   const size_t layers_number = multilayer_perceptron_pointer->get_layers_number();
   const Vector<size_t> layers_perceptrons_number = multilayer_perceptron_pointer->arrange_layers_perceptrons_numbers();

   // Control sentence (if debug)

   #ifndef NDEBUG 
 
   // Input size

   const size_t inputs_size = inputs.size();

   if(inputs_size != inputs_number)
   {
      buffer << "OpenNN Exception: PerformanceTerm class.\n"
             << "Vector< Vector<double> > calculate_layers_objective_gradient(const Vector< Vector<double> >&, const Vector<double>&, const Vector<double>&) method.\n"
             << "Size of inputs (" << inputs_size << ") must be equal to inputs number (" << inputs_number << ").\n";

      throw std::logic_error(buffer.str());  
   }

   // Forward propagation activation size

   const size_t layers_activation_size = layers_activation.size();

   if(layers_activation_size != layers_number)
   {
      buffer << "OpenNN Exception: PerformanceTerm class.\n"
             << "Vector< Vector<double> > calculate_layers_objective_gradient(const Vector< Vector<double> >&, const Vector<double>&, const Vector<double>&) method.\n"
             << "Size of forward propagation activation (" << layers_activation_size << ") must be equal to number of layers (" << layers_number << ").\n";

      throw std::logic_error(buffer.str());   
   }

   // Hidden errors size

   const size_t layers_delta_size = layers_delta.size();
      
   if(layers_delta_size != layers_number)
   {
      buffer << "OpenNN Exception: PerformanceTerm class.\n"
             << "Vector< Vector<double> > calculate_layers_objective_gradient(const Vector< Vector<double> >&, const Vector<double>&) method.\n"
             << "Size of layers delta ("<< layers_delta_size << ") must be equal to number of layers (" << layers_number << ").\n";

      throw std::logic_error(buffer.str());   
   }

   #endif

   const size_t parameters_number = neural_network_pointer->count_parameters_number();

   Vector<double> point_gradient(parameters_number);

   size_t index = 0;

   const Vector< Vector<double> > layers_inputs = multilayer_perceptron_pointer->arrange_layers_input(inputs, layers_activation);

   const Vector< Matrix<double> > layers_combination_parameters_Jacobian = multilayer_perceptron_pointer->calculate_layers_combination_parameters_Jacobian(layers_inputs);

   for(size_t i = 0; i < layers_number; i++)
   {
      point_gradient.tuck_in(index, layers_delta[i].dot(layers_combination_parameters_Jacobian[i]));

      index += multilayer_perceptron_pointer->get_layer(i).count_parameters_number();
   }

   if(layers_number != 0)
   {
      Vector<double> synaptic_weights;

      size_t index = 0;

      // First layer

      for(size_t i = 0; i < layers_perceptrons_number[0]; i++)
      {
         // Bias

         point_gradient[index] = layers_delta[0][i];
         index++;

         // Synaptic weights

         synaptic_weights = multilayer_perceptron_pointer->get_layer(0).get_perceptron(i).arrange_synaptic_weights();

         for(size_t j = 0; j < inputs_number; j++)
         {
            point_gradient[index] = layers_delta[0][i]*inputs[j];
            index++;   
         }
      }

      // Rest of layers 
    
      for(size_t h = 1; h < layers_number; h++)
      {      
         for(size_t i = 0; i < layers_perceptrons_number[h]; i++)
         {
            // Bias

            point_gradient[index] = layers_delta[h][i];
            index++;

            // Synaptic weights

            synaptic_weights = multilayer_perceptron_pointer->get_layer(h).get_perceptron(i).arrange_synaptic_weights();

            for(size_t j = 0; j < layers_perceptrons_number[h-1]; j++)
            {
               point_gradient[index] = layers_delta[h][i]*layers_activation[h-1][j];
               index++;   
            }
         }
      }

   }

   return(point_gradient);
}


//  Vector<double> calculate_point_gradient(const Vector< Matrix<double> >&, const Vector< Vector<double> >&) const method


Vector<double> PerformanceTerm::calculate_point_gradient
(const Vector< Matrix<double> >& layers_combination_parameters_Jacobian, 
 const Vector< Vector<double> >& layers_delta) const
{
   // Control sentence (if debug)

   #ifndef NDEBUG 

   check();

   #endif

   const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();

   // Control sentence (if debug)

   #ifndef NDEBUG 

   std::ostringstream buffer;

   if(!multilayer_perceptron_pointer)
   {
      buffer << "OpenNN Exception: PerformanceTerm class.\n"
             << "Vector<double> calculate_point_gradient(const Vector<double>&, const Vector< Vector<double> >&, const Vector<double>&) const method.\n"
             << "Pointer to network is NULL.\n";

      throw std::logic_error(buffer.str());   
   }

   #endif

   const size_t layers_number = multilayer_perceptron_pointer->get_layers_number();
   const Vector<size_t> layers_size = multilayer_perceptron_pointer->arrange_layers_perceptrons_numbers();

   // Control sentence (if debug)

   #ifndef NDEBUG 
 
   // Input size

   const size_t layers_combination_parameters_Jacobian_size = layers_combination_parameters_Jacobian.size();

   if(layers_combination_parameters_Jacobian_size != layers_number)
   {
      buffer << "OpenNN Exception: PerformanceTerm class.\n"
             << "Vector< Vector<double> > calculate_layers_objective_gradient(const Vector< Vector<double> >&, const Vector<double>&, const Vector<double>&) method.\n"
             << "Size of forward propagation activation (" << layers_combination_parameters_Jacobian_size << ") must be equal to number of layers (" << layers_number << ").\n";

      throw std::logic_error(buffer.str());   
   }

   // Hidden errors size

   const size_t layers_delta_size = layers_delta.size();
      
   if(layers_delta_size != layers_number)
   {
      buffer << "OpenNN Exception: PerformanceTerm class.\n"
             << "Vector< Vector<double> > calculate_layers_objective_gradient(const Vector< Vector<double> >&, const Vector<double>&) method.\n"
             << "Size of layers delta ("<< layers_delta_size << ") must be equal to number of layers (" << layers_number << ").\n";

      throw std::logic_error(buffer.str());   
   }

   #endif

   const size_t parameters_number = multilayer_perceptron_pointer->count_parameters_number();

   Vector<double> point_gradient(parameters_number);

   size_t index = 0;

   for(size_t i = 0; i < layers_number; i++)
   {
      point_gradient.tuck_in(index, layers_delta[i].dot(layers_combination_parameters_Jacobian[i]));

      index += neural_network_pointer->get_multilayer_perceptron_pointer()->get_layer(i).count_parameters_number();
   }

   return(point_gradient);
}


// Matrix< Matrix<double> > calculate_interlayers_Delta(const Vector< Vector<double> >&, const Vector< Vector<double> >&, const Vector<double>&, const Matrix<double>&, const Vector< Vector<double> >&) method


Matrix< Matrix<double> > PerformanceTerm::calculate_interlayers_Delta
(const Vector< Vector<double> >& layers_activation_derivative,
 const Vector< Vector<double> >& layers_activation_second_derivative,
 const Matrix< Matrix<double> >& interlayers_combination_combination_Jacobian_form, 
 const Vector<double>& output_objective_gradient,
 const Matrix<double>& output_objective_Hessian,
 const Vector< Vector<double> >& layers_delta) const
{
   // Neural network stuff

   #ifndef NDEBUG 

   check();

   #endif

   const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();

   #ifndef NDEBUG 

   std::ostringstream buffer;

   if(!multilayer_perceptron_pointer)
   {
      buffer << "OpenNN Exception: PerformanceTerm class.\n"
             << "Matrix< Matrix<double> > calculate_interlayers_Delta() method.\n"
             << "Multilayer perceptron pointer is NULL.\n";

      throw std::logic_error(buffer.str());   
   }

   #endif

   const size_t layers_number = multilayer_perceptron_pointer->get_layers_number();
   const Vector<size_t> layers_size = multilayer_perceptron_pointer->arrange_layers_perceptrons_numbers();

   // Control sentence (if debug)

   #ifndef NDEBUG 

   if(layers_number != 0)
   {
      const size_t output_objective_gradient_size = output_objective_gradient.size();

      if(output_objective_gradient_size != layers_size[layers_number-1])
      {
         buffer << "OpenNN Exception: PerformanceTerm class.\n"
                << "Vector<double> calculate_interlayers_Delta() method.\n"
                << "Size of layer " << layers_number-1 << " must be equal to size of output objective gradient (" << output_objective_gradient_size << ")."
                << std::endl;

         throw std::logic_error(buffer.str());
      }

      const size_t output_objective_Hessian_rows_number = output_objective_Hessian.get_rows_number();
      const size_t output_objective_Hessian_columns_number = output_objective_Hessian.get_columns_number();

      if(output_objective_Hessian_rows_number != layers_size[layers_number-1])
      {
         buffer << "OpenNN Exception: PerformanceTerm class.\n"
                << "Vector<double> calculate_interlayers_Delta() method.\n"
                << "Size of layer " << layers_number-1 << " must be equal to number of rows in output objective Hessian (" << output_objective_Hessian_rows_number << ")."
                << std::endl;

         throw std::logic_error(buffer.str());
      }

      if(output_objective_Hessian_columns_number != layers_size[layers_number-1])
      {
         buffer << "OpenNN Exception: PerformanceTerm class.\n"
                << "Vector<double> calculate_interlayers_Delta() method.\n"
                << "Size of layer " << layers_number-1 << ") must be equal to number of columns in output objective Hessian (" << output_objective_Hessian_columns_number << ")."
                << std::endl;

         throw std::logic_error(buffer.str());
      }
   }

   #endif

   const Vector< Matrix<double> > layers_synaptic_weights = multilayer_perceptron_pointer->arrange_layers_synaptic_weights();

   // Objective functional stuff

   Matrix< Matrix<double> > interlayers_Delta(layers_number, layers_number);

   for(size_t i = 0; i < layers_number; i++)
   {
      for(size_t j = 0; j < layers_number; j++)
      {
         interlayers_Delta(i,j).set(layers_size[i], layers_size[j]);
      }
   }

   // @todo

   if(layers_number > 0)
   {
      // Output-outputs layer 

      interlayers_Delta(layers_number-1,layers_number-1) = (output_objective_Hessian*(layers_activation_derivative[layers_number-1]*layers_activation_derivative[layers_number-1])).sum_diagonal(output_objective_gradient*layers_activation_second_derivative[layers_number-1]);

      // Rest of hidden layers

      for(int i = (int)layers_number-1; i >= 0; i--)
      {   
         for(int j = (int)layers_number-1; j >= 0; j--)
         {   
            if((int)i != (int)layers_number-1 && (int)j != (int)layers_number-1)
            {
               interlayers_Delta(i,j)
               = layers_activation_second_derivative[j]
                *interlayers_combination_combination_Jacobian_form(i,j)
                *(layers_delta[j+1].dot(layers_synaptic_weights[j+1]))
               + layers_activation_second_derivative[j]
                *(interlayers_Delta(i,j+1).dot(layers_synaptic_weights[j+1]));
            }
         }
      }
   }

   return(interlayers_Delta);
}


// Matrix<double> calculate_point_Hessian(const Vector<double>&, const Matrix< Matrix<double> >&, const Vector< Vector<double> >&, const Matrix< Matrix<double> >&) const method


Matrix<double> PerformanceTerm::calculate_point_Hessian
(const Vector< Vector<double> >& layers_activation_derivative,
 const Vector< Vector< Vector<double> > >& perceptrons_combination_parameters_gradient, 
 const Matrix< Matrix<double> >& interlayers_combination_combination_Jacobian, 
 const Vector< Vector<double> >& layers_delta,
 const Matrix< Matrix<double> >& interlayers_Delta) const
{
   // Neural network stuff

   #ifndef NDEBUG 

   check();

   #endif

   const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();

   #ifndef NDEBUG 

   std::ostringstream buffer;

   if(!multilayer_perceptron_pointer)
   {
      buffer << "OpenNN Exception: PerformanceTerm class.\n"
             << "Matrix<double> calculate_point_Hessian(const Vector<double>&, const Matrix< Matrix<double> >&, const Vector< Vector<double> >&, const Matrix< Matrix<double> >&) const method.\n"
             << "Multilayer perceptron pointer is NULL.\n";

      throw std::logic_error(buffer.str());   
   }

   #endif

   const size_t layers_number = multilayer_perceptron_pointer->get_layers_number();

   const size_t parameters_number = multilayer_perceptron_pointer->count_parameters_number();

   #ifndef NDEBUG 

   const size_t layers_activation_derivative_size = layers_activation_derivative.size();

   if(layers_activation_derivative_size != layers_number)
   {
      buffer << "OpenNN Exception: PerformanceTerm class.\n"
             << "Matrix<double> calculate_point_Hessian(const Vector<double>&, const Matrix< Matrix<double> >&, const Vector< Vector<double> >&, const Matrix< Matrix<double> >&) const method.\n"
             << "Size of layers activation derivative must be equal to number of layers in multilayer perceptron.\n";

      throw std::logic_error(buffer.str());   
   }

//   const size_t perceptrons_combination_parameters_gradient_size = perceptrons_combination_parameters_gradient.size();

//   const size_terlayers_combination_combination_Jacobian_rows_number = interlayers_combination_combination_Jacobian.get_rows_number();
//   const size_terlayers_combination_combination_Jacobian_columns_number = interlayers_combination_combination_Jacobian.get_columns_number();

//   const size_t layers_delta_size = layers_delta.size();

//   const size_terlayers_Delta_rows_number = interlayers_Delta.get_rows_number();
//   const size_terlayers_Delta_columns_number = interlayers_Delta.get_columns_number();

   #endif

   // Objective functional

   Matrix<double> point_Hessian(parameters_number, parameters_number, 0.0);

   Vector<size_t> parameter_indices(3);

   size_t layer_index_i;
   size_t neuron_index_i;
   size_t parameter_index_i;

   size_t layer_index_j;
   size_t neuron_index_j;
   size_t parameter_index_j;

    // @todo


   if(layers_number > 0)
   {
      for(size_t i = 0; i < parameters_number; i++)
      {
         parameter_indices = multilayer_perceptron_pointer->arrange_parameter_indices(i);
         layer_index_i = parameter_indices[0];
         neuron_index_i = parameter_indices[1];
         parameter_index_i = parameter_indices[2];

         for(size_t j = 0; j < parameters_number; j++)
         {
            parameter_indices = multilayer_perceptron_pointer->arrange_parameter_indices(j);
            layer_index_j = parameter_indices[0];
            neuron_index_j = parameter_indices[1];
            parameter_index_j = parameter_indices[2];
           
            point_Hessian(i,j) 
            = perceptrons_combination_parameters_gradient[layer_index_i][neuron_index_i][parameter_index_i]
             *perceptrons_combination_parameters_gradient[layer_index_j][neuron_index_j][parameter_index_j]
             *interlayers_Delta(layer_index_j,layer_index_i)(neuron_index_j,neuron_index_i)
            + perceptrons_combination_parameters_gradient[layer_index_i][neuron_index_i][parameter_index_i]
             *layers_delta[layer_index_j][neuron_index_j]
             *layers_activation_derivative[layer_index_j][neuron_index_j]
             *interlayers_combination_combination_Jacobian(layer_index_j,layer_index_i)(neuron_index_j,neuron_index_i);
         }
      }

      for(size_t i = 0; i < parameters_number; i++)
      {
         for(size_t j = 0; j < i; j++)
         {
            point_Hessian(i,j) = point_Hessian(j,i);
         }
      }
   } 

   return(point_Hessian);
}


// Vector<double> calculate_gradient(void) const method


Vector<double> PerformanceTerm::calculate_gradient(void) const
{
   // Neural network stuff

   #ifndef NDEBUG 

   check();

   #endif

   // Performance functional stuff

   #ifndef NDEBUG 

   std::ostringstream buffer;

   if(!numerical_differentiation_pointer)
   {
      buffer << "OpenNN Exception: PerformanceTerm class.\n"
             << "Vector<double> calculate_gradient(void) const method.\n"
             << "Numerical differentiation pointer is NULL.\n";

      throw std::logic_error(buffer.str());   
   }

   #endif

   const Vector<double> parameters = neural_network_pointer->arrange_parameters();

   return(numerical_differentiation_pointer->calculate_gradient(*this, &PerformanceTerm::calculate_performance, parameters));
}


// Vector<double> calculate_gradient(const Vector<double>&) const method


Vector<double> PerformanceTerm::calculate_gradient(const Vector<double>& parameters) const
{
   // Neural network stuff

   #ifndef NDEBUG

   check();

   #endif

   // Performance functional stuff

   #ifndef NDEBUG

   std::ostringstream buffer;

   if(!numerical_differentiation_pointer)
   {
      buffer << "OpenNN Exception: PerformanceTerm class.\n"
             << "Vector<double> calculate_gradient(const Vector<double>&) const method.\n"
             << "Numerical differentiation pointer is NULL.\n";

      throw std::logic_error(buffer.str());
   }

   #endif

   return(numerical_differentiation_pointer->calculate_gradient(*this, &PerformanceTerm::calculate_performance, parameters));
}


// Matrix<double> calculate_Hessian(void) const method


Matrix<double> PerformanceTerm::calculate_Hessian(void) const
{
   // Neural network stuff

   #ifndef NDEBUG 

   check();

   #endif

   const Vector<double> parameters = neural_network_pointer->arrange_parameters();

   return(numerical_differentiation_pointer->calculate_Hessian(*this, &PerformanceTerm::calculate_performance, parameters));
}


// Matrix<double> calculate_Hessian(const Vector<double>&) const method


Matrix<double> PerformanceTerm::calculate_Hessian(const Vector<double>& parameters) const
{
   // Neural network stuff

   #ifndef NDEBUG

   check();

   #endif

   return(numerical_differentiation_pointer->calculate_Hessian(*this, &PerformanceTerm::calculate_performance, parameters));
}


// Vector<double> calculate_terms(void) const method 


Vector<double> PerformanceTerm::calculate_terms(void) const
{
    std::ostringstream buffer;

    buffer << "OpenNN Exception: PerformanceTerm class.\n"
           << "Vector<double> calculate_terms(void) const method.\n"
           << "The terms function is not defined for this performance term.\n";

    throw std::logic_error(buffer.str());
}


// Vector<double> calculate_terms(const Vector<double>&) const method 


Vector<double> PerformanceTerm::calculate_terms(const Vector<double>&) const
{
    std::ostringstream buffer;

    buffer << "OpenNN Exception: PerformanceTerm class.\n"
           << "Vector<double> calculate_terms(const Vector<double>&) const method.\n"
           << "The terms function is not defined for this performance term.\n";

    throw std::logic_error(buffer.str());
}


// Matrix<double> calculate_terms_Jacobian(void) const method 


Matrix<double> PerformanceTerm::calculate_terms_Jacobian(void) const
{
    std::ostringstream buffer;

    buffer << "OpenNN Exception: PerformanceTerm class.\n"
           << "Matrix<double> calculate_terms_Jacobian(void) const method.\n"
           << "The terms function is not defined for this performance term.\n";

    throw std::logic_error(buffer.str());
}


// PerformanceTerm::FirstOrderTerms calculate_first_order_terms(void) const


PerformanceTerm::FirstOrderTerms PerformanceTerm::calculate_first_order_terms(void) const
{
    std::ostringstream buffer;

    buffer << "OpenNN Exception: PerformanceTerm class.\n"
           << "Vector<double> calculate_terms(void) const method.\n"
           << "The terms function is not defined for this performance term.\n";

    throw std::logic_error(buffer.str());
}


// std::string write_performance_term_type(void) const method


std::string PerformanceTerm::write_performance_term_type(void) const
{
   return("USER_PERFORMANCE_TERM");
}


// std::string write_information(void) const method


std::string PerformanceTerm::write_information(void) const
{
   return("");
}


// std::string to_string(void) const method


std::string PerformanceTerm::to_string(void) const
{
   std::ostringstream buffer;

   buffer << "Performance term\n"
          << "Display: " << display << "\n";

   return(buffer.str());
}


// tinyxml2::XMLDocument* to_XML(void) const method 


tinyxml2::XMLDocument* PerformanceTerm::to_XML(void) const
{
   std::ostringstream buffer;

   tinyxml2::XMLDocument* document = new tinyxml2::XMLDocument;

   // Performance term

   tinyxml2::XMLElement* root_element = document->NewElement("PerformanceTerm");

   document->InsertFirstChild(root_element);

   return(document);
}


// void from_XML(const tinyxml2::XMLDocument&) method


void PerformanceTerm::from_XML(const tinyxml2::XMLDocument& document)
{
   // Display warnings

   const tinyxml2::XMLElement* display_element = document.FirstChildElement("Display");

   if(display_element)
   {
      std::string new_display_string = display_element->GetText();           

      try
      {
         set_display(new_display_string != "0");
      }
      catch(const std::logic_error& e)
      {
         std::cout << e.what() << std::endl;         
      }
   }
}


// size_t calculate_Kronecker_delta(const size_t&, const size_t&) const method


size_t PerformanceTerm::calculate_Kronecker_delta(const size_t& a, const size_t& b) const
{
   if(a == b)
   {
      return(1);
   }
   else
   {
      return(0);
   }
}

}


// OpenNN: Open Neural Networks Library.
// Copyright (c) 2005-2015 Roberto Lopez.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.

// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA