cross_entropy_error.cpp

/****************************************************************************************************************/
/*                                                                                                              */
/*   OpenNN: Open Neural Networks Library                                                                       */
/*   www.artelnics.com/opennn                                                                                   */
/*                                                                                                              */
/*   C R O S S   E N T R O P Y   E R R O R   C L A S S                                                          */
/*                                                                                                              */
/*   Roberto Lopez                                                                                              */
/*   Artelnics - Making intelligent use of data                                                                 */
/*   [email protected]                                                                                 */
/*                                                                                                              */
/****************************************************************************************************************/

// OpenNN includes

#include "cross_entropy_error.h"

namespace OpenNN
{

// DEFAULT CONSTRUCTOR


CrossEntropyError::CrossEntropyError(void) : PerformanceTerm()
{
}


// NEURAL NETWORK CONSTRUCTOR


CrossEntropyError::CrossEntropyError(NeuralNetwork* new_neural_network_pointer)
 : PerformanceTerm(new_neural_network_pointer)
{
}


// DATA SET CONSTRUCTOR


CrossEntropyError::CrossEntropyError(DataSet* new_data_set_pointer) 
: PerformanceTerm(new_data_set_pointer)
{
}


// NEURAL NETWORK AND DATA SET CONSTRUCTOR


CrossEntropyError::CrossEntropyError(NeuralNetwork* new_neural_network_pointer, DataSet* new_data_set_pointer)
: PerformanceTerm(new_neural_network_pointer, new_data_set_pointer)
{
}


// XML CONSTRUCTOR


CrossEntropyError::CrossEntropyError(const tinyxml2::XMLDocument& sum_squared_error_document)
 : PerformanceTerm(sum_squared_error_document)
{
}


// COPY CONSTRUCTOR


CrossEntropyError::CrossEntropyError(const CrossEntropyError& new_cross_entropy_error)
 : PerformanceTerm(new_cross_entropy_error)
{

}


// DESTRUCTOR


CrossEntropyError::~CrossEntropyError(void) 
{
}


// ASSIGNMENT OPERATOR


CrossEntropyError& CrossEntropyError::operator = (const CrossEntropyError& other_cross_entropy_error)
{
   if(this != &other_cross_entropy_error) 
   {
      *neural_network_pointer = *other_cross_entropy_error.neural_network_pointer;
      *data_set_pointer = *other_cross_entropy_error.data_set_pointer;
      display = other_cross_entropy_error.display;
   }

   return(*this);

}

// EQUAL TO OPERATOR


bool CrossEntropyError::operator == (const CrossEntropyError& other_cross_entropy_error) const
{
   if(*neural_network_pointer == *other_cross_entropy_error.neural_network_pointer
   && *mathematical_model_pointer == *other_cross_entropy_error.mathematical_model_pointer
   && display == other_cross_entropy_error.display)    
   {
      return(true);
   }
   else
   {
      return(false);  
   }

}


// METHODS


// void check(void) const method


void CrossEntropyError::check(void) const
{
   std::ostringstream buffer;

   // Neural network stuff

   if(!neural_network_pointer)
   {
      buffer << "OpenNN Exception: CrossEntropyError class.\n"
             << "void check(void) const method.\n"
             << "Pointer to neural network is NULL.\n";

      throw std::logic_error(buffer.str());   
   }

   const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();

   if(!multilayer_perceptron_pointer)
   {
      buffer << "OpenNN Exception: CrossEntropyError class.\n"
             << "void check(void) const method.\n"
             << "Pointer to multilayer perceptron is NULL.\n";

      throw std::logic_error(buffer.str());   
   }

   const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
   const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();

   if(inputs_number == 0)
   {
      buffer << "OpenNN Exception: CrossEntropyError class.\n"
             << "void check(void) const method.\n"
             << "Number of inputs in multilayer perceptron object is zero.\n";

      throw std::logic_error(buffer.str());   
   }

   if(outputs_number == 0)
   {
      buffer << "OpenNN Exception: CrossEntropyError class.\n"
             << "void check(void) const method.\n"
             << "Number of outputs in multilayer perceptron object is zero.\n";

      throw std::logic_error(buffer.str());   
   }

   const ProbabilisticLayer* probabilistic_layer_pointer = neural_network_pointer->get_probabilistic_layer_pointer();

   if(!probabilistic_layer_pointer)
   {
      buffer << "OpenNN Exception: CrossEntropyError class.\n"
             << "void check(void) const method.\n"
             << "Pointer to probabilistic layer is NULL.\n";

      throw std::logic_error(buffer.str());   
   }

   const ProbabilisticLayer::ProbabilisticMethod& outputs_probabilizing_method = probabilistic_layer_pointer->get_probabilistic_method();

   if(outputs_probabilizing_method != ProbabilisticLayer::Softmax)
   {
      buffer << "OpenNN Exception: CrossEntropyError class.\n"
             << "void check(void) const method.\n"
             << "Probabilistic method is not Softmax.\n";

      throw std::logic_error(buffer.str());
   }

   // Data set stuff

   if(!data_set_pointer)
   {
      buffer << "OpenNN Exception: CrossEntropyError class.\n"
             << "void check(void) const method.\n"
             << "Pointer to data set is NULL.\n";

      throw std::logic_error(buffer.str());   
   }

   // Sum squared error stuff

   const Variables& variables = data_set_pointer->get_variables();

   const size_t data_set_inputs_number = variables.count_inputs_number();
   const size_t targets_number = variables.count_targets_number();

   if(inputs_number != data_set_inputs_number)
   {
      buffer << "OpenNN Exception: CrossEntropyError class.\n"
             << "void check(void) const method.\n"
             << "Number of inputs in neural network must be equal to number of inputs in data set.\n";

      throw std::logic_error(buffer.str());   
   }

   if(outputs_number != targets_number)
   {
      buffer << "OpenNN Exception: CrossEntropyError class.\n"
             << "void check(void) const method.\n"
             << "Number of outputs in neural network must be equal to number of targets in data set.\n";

      throw std::logic_error(buffer.str());
   }
}


// double calculate_performance(void) method


double CrossEntropyError::calculate_performance(void) const
{
   std::ostringstream buffer;

   buffer << "OpenNN Exception: CrossEntropyError class.\n"
          << "double calculate_performance(void) method.\n"
          << "This method is under development.\n";

   throw std::logic_error(buffer.str());
/*
   #ifndef NDEBUG 

   check();

   #endif

   // Neural network stuff

   const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();

   const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
   const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();

   // Data set stuff 

   const Instances& instances = data_set_pointer->get_instances();

   const size_t training_instances_number = instances.count_training_instances_number();

   const Vector<size_t> training_indices = instances.arrange_training_indices();

   size_t training_index;

   const MissingValues& missing_values = data_set_pointer->get_missing_values();

   // Cross entropy error

   Vector<double> inputs(inputs_number);
   Vector<double> outputs(outputs_number);
   Vector<double> targets(outputs_number);

   double objective = 0.0;

   #pragma omp parallel for private(i, training_index, inputs, outputs, targets) reduction(+ : sum_squared_error)

   for(size_t i = 0; i < training_instances_number; i++)
   {
       if(missing_values.has_missing_values(i))
       {
           continue;
       }

      // Input vector

      inputs = data_set_pointer->get_training_input_instance(i);

      // Output vector

      outputs = multilayer_perceptron_pointer->calculate_outputs(inputs);

      // Target vector

      targets = data_set_pointer->get_training_target_instance(i);

      // Cross entropy error

      for(size_t j = 0; j < outputs_number; j++)
      {
         objective -= targets[j]*log(outputs[j]) + (1.0 - targets[j])*log(1.0 - outputs[j]);
      }
   }

   return(objective);
*/
}



// double calculate_performance(const Vector<double>&) const method

double CrossEntropyError::calculate_performance(const Vector<double>&) const
{
    return(0.0);
}


// double calculate_minimum_performance(void) method


double CrossEntropyError::calculate_minimum_performance(void)
{
   return(0.0);
}


// double calculate_generalization_performance(void) const method


double CrossEntropyError::calculate_generalization_performance(void) const
{
   // Control sentence

   #ifndef NDEBUG 

   check();

   #endif

   // Neural network stuff

   const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();

   const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
   const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();

   // Data set stuff

   const Instances& instances = data_set_pointer->get_instances();

   const size_t generalization_instances_number = instances.count_generalization_instances_number();

   const Vector<size_t> generalization_indices = instances.arrange_generalization_indices();

   size_t generalization_index;

   const Variables& variables = data_set_pointer->get_variables();

   const Vector<size_t> inputs_indices = variables.arrange_inputs_indices();
   const Vector<size_t> targets_indices = variables.arrange_targets_indices();

   const MissingValues& missing_values = data_set_pointer->get_missing_values();

   // Performance functional

   Vector<double> inputs(inputs_number);
   Vector<double> outputs(outputs_number);
   Vector<double> targets(outputs_number);

   double generalization_performance = 0.0;

   int i = 0;

   #pragma omp parallel for private(i, generalization_index, inputs, outputs, targets) reduction(- : generalization_performance)

   for(i = 0; i < (int)generalization_instances_number; i++)
   {
       generalization_index = generalization_indices[i];

       if(missing_values.has_missing_values(generalization_index))
       {
           continue;
       }

      // Input vector

      inputs = data_set_pointer->get_instance(generalization_index, inputs_indices);

      // Output vector

      outputs = multilayer_perceptron_pointer->calculate_outputs(inputs);

      // Target vector

      targets = data_set_pointer->get_instance(generalization_index, targets_indices);

      // Cross entropy error

      for(size_t j = 0; j < outputs_number; j++)
      {
         generalization_performance -= targets[j]*log(outputs[j]) + (1.0 - targets[j])*log(1.0 - outputs[j]);
      }
   }

   return(generalization_performance);
}


// double calculate_minimum_generalization_performance(void) method


double CrossEntropyError::calculate_minimum_generalization_performance(void)
{
   return(0.0);
}


// Vector<double> calculate_gradient(void) const


Vector<double> CrossEntropyError::calculate_gradient(void) const
{
   #ifndef NDEBUG 

   check();

   #endif

   // Neural network stuff

   const MultilayerPerceptron* multilayer_perceptron_pointer = neural_network_pointer->get_multilayer_perceptron_pointer();

   const size_t inputs_number = multilayer_perceptron_pointer->get_inputs_number();
   const size_t outputs_number = multilayer_perceptron_pointer->get_outputs_number();

   const size_t layers_number = multilayer_perceptron_pointer->get_layers_number();

   const size_t neural_parameters_number = multilayer_perceptron_pointer->count_parameters_number();

   const bool has_conditions_layer = neural_network_pointer->has_conditions_layer();

   const ConditionsLayer* conditions_layer_pointer = has_conditions_layer ? neural_network_pointer->get_conditions_layer_pointer() : NULL;


   #ifndef NDEBUG 

   std::ostringstream buffer;

   const Matrix<double> target_data = data_set_pointer->arrange_target_data();

   if(target_data < 0.0)
   {
      buffer << "OpenNN Exception: CrossEntropyError class.\n"
             << "Vector<double> calculate_gradient(void) const method.\n"
             << "Target data must be equal or greater than zero.\n";

      throw std::logic_error(buffer.str());
   }

   if(target_data > 1.0)
   {
      buffer << "OpenNN Exception: CrossEntropyError class.\n"
             << "Vector<double> calculate_gradient(void) const method.\n"
             << "Target data must be less or equal or than one.\n";

      throw std::logic_error(buffer.str());
   }

   #endif

   // Neural network stuff    

   Vector< Vector< Vector<double> > > first_order_forward_propagation(2); 

   Vector<double> particular_solution;
   Vector<double> homogeneous_solution;

   // Data set stuff

   const Instances& instances = data_set_pointer->get_instances();

   const size_t training_instances_number = instances.count_training_instances_number();

   const Vector<size_t> training_indices = instances.arrange_training_indices();

   size_t training_index;

   const Variables& variables = data_set_pointer->get_variables();

   const Vector<size_t> inputs_indices = variables.arrange_inputs_indices();
   const Vector<size_t> targets_indices = variables.arrange_targets_indices();

   const MissingValues& missing_values = data_set_pointer->get_missing_values();

   Vector<double> inputs(inputs_number);
   Vector<double> targets(outputs_number);

   // Sum squared error stuff

   Vector<double> output_objective_gradient(outputs_number);

   Vector< Vector<double> > layers_delta; 

   Vector<double> point_gradient(neural_parameters_number, 0.0);

   Vector<double> gradient(neural_parameters_number, 0.0);

   for(size_t i = 0; i < training_instances_number; i++)
   {
       training_index = training_indices[i];

       if(missing_values.has_missing_values(training_index))
       {
           continue;
       }

      inputs = data_set_pointer->get_instance(training_index, inputs_indices);

      targets = data_set_pointer->get_instance(training_index, targets_indices);

      first_order_forward_propagation = multilayer_perceptron_pointer->calculate_first_order_forward_propagation(inputs);

      const Vector< Vector<double> >& layers_activation = first_order_forward_propagation[0];
      const Vector< Vector<double> >& layers_activation_derivative = first_order_forward_propagation[1];

      if(!has_conditions_layer)
      {
         const Vector<double>& outputs = layers_activation[layers_number-1];

         for(size_t j = 0; j < outputs_number; j++)
         {
            //output_objective_gradient[j] = -targets[j]/outputs[j]
            //+ (1.0 - targets[j])*(1.0 - outputs[j]);

            output_objective_gradient[j]
            = -targets[j]/outputs[j] + (1.0 - targets[j])/(1.0 - outputs[j]);

         }

         layers_delta = calculate_layers_delta(layers_activation_derivative, output_objective_gradient);
      }
      else
      {
         particular_solution = conditions_layer_pointer->calculate_particular_solution(inputs);
         homogeneous_solution = conditions_layer_pointer->calculate_homogeneous_solution(inputs);

         const Vector<double>& outputs = particular_solution + homogeneous_solution*layers_activation[layers_number-1];

         for(size_t j = 0; j < outputs_number; j++)
         {
//            output_objective_gradient[j]
//            = -targets[j]/outputs[j] + (1.0 - targets[j])*(1.0 - outputs[j]);

            output_objective_gradient[j]
            = -targets[j]/outputs[j] + (1.0 - targets[j])*(1.0 - outputs[j]);

         }

         layers_delta = calculate_layers_delta(layers_activation_derivative, homogeneous_solution, output_objective_gradient);
      }

      point_gradient = calculate_point_gradient(inputs, layers_activation, layers_delta);

      gradient += point_gradient;
   }

   return(gradient);
}


// Matrix<double> calculate_Hessian(void) const


Matrix<double> CrossEntropyError::calculate_Hessian(void) const
{
   Matrix<double> objective_Hessian;

   return(objective_Hessian);
}


// std::string write_performance_term_type(void) const method


std::string CrossEntropyError::write_performance_term_type(void) const
{
   return("CROSS_ENTROPY_ERROR");
}


// tinyxml2::XMLDocument* to_XML(void) const method 


tinyxml2::XMLDocument* CrossEntropyError::to_XML(void) const
{
   std::ostringstream buffer;

   tinyxml2::XMLDocument* document = new tinyxml2::XMLDocument;

   // Cross entropy error 

   tinyxml2::XMLElement* cross_entropy_error_element = document->NewElement("CrossEntropyError");

   document->InsertFirstChild(cross_entropy_error_element);

   // Display

   {
      tinyxml2::XMLElement* display_element = document->NewElement("Display");
      cross_entropy_error_element->LinkEndChild(display_element);

      buffer.str("");
      buffer << display;

      tinyxml2::XMLText* display_text = document->NewText(buffer.str().c_str());
      display_element->LinkEndChild(display_text);
   }

   return(document);
}


// void from_XML(const tinyxml2::XMLDocument&) method


void CrossEntropyError::from_XML(const tinyxml2::XMLDocument& document)
{
    const tinyxml2::XMLElement* root_element = document.FirstChildElement("CrossEntropyError");

    if(!root_element)
    {
        std::ostringstream buffer;

        buffer << "OpenNN Exception: CrossEntropyError class.\n"
               << "void from_XML(const tinyxml2::XMLDocument&) method.\n"
               << "Cross entropy error element is NULL.\n";

        throw std::logic_error(buffer.str());
    }

  // Display
  {
     const tinyxml2::XMLElement* display_element = root_element->FirstChildElement("Display");

     if(display_element)
     {
        const std::string new_display_string = display_element->GetText();

        try
        {
           set_display(new_display_string != "0");
        }
        catch(const std::logic_error& e)
        {
           std::cout << e.what() << std::endl;
        }
     }
  }
}

}


// OpenNN: Open Neural Networks Library.
// Copyright (c) 2005-2015 Roberto Lopez.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.

// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA