quasi_newton_method.h

/****************************************************************************************************************/
/*                                                                                                              */
/*   OpenNN: Open Neural Networks Library                                                                       */
/*   www.artelnics.com/opennn                                                                                   */
/*                                                                                                              */
/*   Q U A S I - N E W T O N   M E T H O D    C L A S S   H E A D E R                                           */
/*                                                                                                              */
/*   Roberto Lopez                                                                                              */
/*   Artelnics - Making intelligent use of data                                                                 */
/*   [email protected]                                                                                 */
/*                                                                                                              */
/****************************************************************************************************************/

#ifndef __QUASINEWTONMETHOD_H__
#define __QUASINEWTONMETHOD_H__

// System includes

#include <string>
#include <sstream>
#include <iostream>
#include <fstream>
#include <algorithm>
#include <functional>
#include <limits>
#include <cmath>
#include <ctime>

// OpenNN includes

#include "performance_functional.h"

#include "training_algorithm.h"
#include "training_rate_algorithm.h"

// TinyXml includes

#include "../tinyxml2/tinyxml2.h"

namespace OpenNN
{


class QuasiNewtonMethod : public TrainingAlgorithm
{

public:

   // ENUMERATIONS


   enum InverseHessianApproximationMethod{DFP, BFGS};


   // DEFAULT CONSTRUCTOR

   explicit QuasiNewtonMethod(void);

   // PERFORMANCE FUNCTIONAL CONSTRUCTOR

   explicit QuasiNewtonMethod(PerformanceFunctional*);

   // XML CONSTRUCTOR

   explicit QuasiNewtonMethod(const tinyxml2::XMLDocument&);


   // DESTRUCTOR

   virtual ~QuasiNewtonMethod(void);

   // STRUCTURES


   struct QuasiNewtonMethodResults : public TrainingAlgorithm::TrainingAlgorithmResults
   {

       explicit QuasiNewtonMethodResults(void) : TrainingAlgorithm::TrainingAlgorithmResults()
       {
           quasi_Newton_method_pointer = NULL;
       }


       explicit QuasiNewtonMethodResults(QuasiNewtonMethod* new_quasi_Newton_method_pointer) : TrainingAlgorithm::TrainingAlgorithmResults()
       {
           quasi_Newton_method_pointer = new_quasi_Newton_method_pointer;
       }


       virtual ~QuasiNewtonMethodResults(void)
       {
           quasi_Newton_method_pointer = NULL;
       }

       // Members


      QuasiNewtonMethod* quasi_Newton_method_pointer;

      // Training history


      Vector< Vector<double> > parameters_history;


      Vector<double> parameters_norm_history;


      Vector<double> performance_history;


      Vector<double> generalization_performance_history;


      Vector< Vector<double> > gradient_history;


      Vector<double> gradient_norm_history;


      Vector< Matrix<double> > inverse_Hessian_history;


      Vector< Vector<double> > training_direction_history;


      Vector<double> training_rate_history;


      Vector<double> elapsed_time_history;

      // Final values


      Vector<double> final_parameters;


      double final_parameters_norm;


      double final_performance;


      double final_generalization_performance;


      Vector<double> final_gradient;


      double final_gradient_norm;


      Matrix<double> final_inverse_Hessian_approximation;


      Vector<double> final_training_direction;


      double final_training_rate;


      double elapsed_time;


      size_t iterations_number;

      // Methods

      QuasiNewtonMethod* get_quasi_Newton_method_pointer(void) const;

      void set_quasi_Newton_method_pointer(QuasiNewtonMethod*);

      void resize_training_history(const size_t&);

      std::string to_string(void) const;

      Matrix<std::string> write_final_results(const size_t& precision = 3) const;
   };

   // METHODS

   // Get methods

   const TrainingRateAlgorithm& get_training_rate_algorithm(void) const;
   TrainingRateAlgorithm* get_training_rate_algorithm_pointer(void);

   const InverseHessianApproximationMethod& get_inverse_Hessian_approximation_method(void) const;
   std::string write_inverse_Hessian_approximation_method(void) const;

   // Training parameters

   const double& get_warning_parameters_norm(void) const;
   const double& get_warning_gradient_norm(void) const;
   const double& get_warning_training_rate(void) const;

   const double& get_error_parameters_norm(void) const;
   const double& get_error_gradient_norm(void) const;
   const double& get_error_training_rate(void) const;

   // Stopping criteria

   const double& get_minimum_parameters_increment_norm(void) const;

   const double& get_minimum_performance_increase(void) const;
   const double& get_performance_goal(void) const;
   const double& get_gradient_norm_goal(void) const;
   const size_t& get_maximum_generalization_performance_decreases(void) const;

   const size_t& get_maximum_iterations_number(void) const;
   const double& get_maximum_time(void) const;

   // Reserve training history

   const bool& get_reserve_parameters_history(void) const;
   const bool& get_reserve_parameters_norm_history(void) const;

   const bool& get_reserve_performance_history(void) const;
   const bool& get_reserve_gradient_history(void) const;
   const bool& get_reserve_gradient_norm_history(void) const;
   const bool& get_reserve_inverse_Hessian_history(void) const;
   const bool& get_reserve_generalization_performance_history(void) const;

   const bool& get_reserve_training_direction_history(void) const;
   const bool& get_reserve_training_rate_history(void) const;
   const bool& get_reserve_elapsed_time_history(void) const;

   // Set methods

   void set_performance_functional_pointer(PerformanceFunctional*);

   void set_inverse_Hessian_approximation_method(const InverseHessianApproximationMethod&);
   void set_inverse_Hessian_approximation_method(const std::string&);

   void set_default(void);

   // Training parameters

   void set_warning_parameters_norm(const double&);
   void set_warning_gradient_norm(const double&);
   void set_warning_training_rate(const double&);

   void set_error_parameters_norm(const double&);
   void set_error_gradient_norm(const double&);
   void set_error_training_rate(const double&);

   // Stopping criteria

   void set_minimum_parameters_increment_norm(const double&);

   void set_minimum_performance_increase(const double&);
   void set_performance_goal(const double&);
   void set_gradient_norm_goal(const double&);
   void set_maximum_generalization_performance_decreases(const size_t&);

   void set_maximum_iterations_number(const size_t&);
   void set_maximum_time(const double&);

   // Reserve training history

   void set_reserve_parameters_history(const bool&);
   void set_reserve_parameters_norm_history(const bool&);

   void set_reserve_performance_history(const bool&);
   void set_reserve_gradient_history(const bool&);
   void set_reserve_gradient_norm_history(const bool&);
   void set_reserve_inverse_Hessian_history(const bool&);
   void set_reserve_generalization_performance_history(const bool&);

   void set_reserve_training_direction_history(const bool&);
   void set_reserve_training_rate_history(const bool&);
   void set_reserve_elapsed_time_history(const bool&);

   // Utilities

   void set_display_period(const size_t&);

   // Training methods

   Vector<double> calculate_gradient_descent_training_direction(const Vector<double>&) const;

   Matrix<double> calculate_DFP_inverse_Hessian
   (const Vector<double>&, const Vector<double>&, const Vector<double>&, const Vector<double>&, const Matrix<double>&) const;

   Matrix<double> calculate_BFGS_inverse_Hessian
   (const Vector<double>&, const Vector<double>&, const Vector<double>&, const Vector<double>&, const Matrix<double>&) const;

   Matrix<double> calculate_inverse_Hessian_approximation(const Vector<double>&, const Vector<double>&, const Vector<double>&, const Vector<double>&, const Matrix<double>&) const;

   Vector<double> calculate_training_direction(const Vector<double>&, const Matrix<double>&) const;

   QuasiNewtonMethodResults* perform_training(void);

   // Training history methods

   void set_reserve_all_training_history(const bool&);

   std::string write_training_algorithm_type(void) const;

   // Serialization methods

   tinyxml2::XMLDocument* to_XML(void) const;

   std::string to_string(void) const;
   Matrix<std::string> to_string_matrix(void) const;

   void from_XML(const tinyxml2::XMLDocument&);

  
private: 


   TrainingRateAlgorithm training_rate_algorithm;


   InverseHessianApproximationMethod inverse_Hessian_approximation_method;



   double warning_parameters_norm;


   double warning_gradient_norm;   


   double warning_training_rate;

   
   double error_parameters_norm;


   double error_gradient_norm;


   double error_training_rate;


   // STOPPING CRITERIA


   double minimum_parameters_increment_norm;


   double minimum_performance_increase;


   double performance_goal;


   double gradient_norm_goal;


   size_t maximum_generalization_performance_decreases;


   size_t maximum_iterations_number;


   double maximum_time;

   // TRAINING HISTORY


   bool reserve_parameters_history;


   bool reserve_parameters_norm_history;


   bool reserve_performance_history;


   bool reserve_gradient_history;


   bool reserve_gradient_norm_history;


   bool reserve_inverse_Hessian_history;

   
   bool reserve_training_direction_history;


   bool reserve_training_rate_history;


   bool reserve_elapsed_time_history;


   bool reserve_generalization_performance_history;
};

}

#endif


// OpenNN: Open Neural Networks Library.
// Copyright (c) 2005-2015 Roberto Lopez.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.

// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA