training_rate_algorithm.cpp

/****************************************************************************************************************/
/*                                                                                                              */
/*   OpenNN: Open Neural Networks Library                                                                       */
/*   www.artelnics.com/opennn                                                                                   */
/*                                                                                                              */
/*   T R A I N I N G   R A T E   A L G O R I T H M   C L A S S                                                  */
/*                                                                                                              */
/*   Roberto Lopez                                                                                              */
/*   Artelnics - Making intelligent use of data                                                                 */
/*   [email protected]                                                                                 */
/*                                                                                                              */
/****************************************************************************************************************/

// OpenNN includes

#include "training_rate_algorithm.h"

namespace OpenNN
{

// DEFAULT CONSTRUCTOR


TrainingRateAlgorithm::TrainingRateAlgorithm(void)
 : performance_functional_pointer(NULL)
{ 
   set_default();
}


// GENERAL CONSTRUCTOR


TrainingRateAlgorithm::TrainingRateAlgorithm(PerformanceFunctional* new_performance_functional_pointer)
 : performance_functional_pointer(new_performance_functional_pointer)
{
   set_default();
}


// XML CONSTRUCTOR


TrainingRateAlgorithm::TrainingRateAlgorithm(const tinyxml2::XMLDocument& document)
 : performance_functional_pointer(NULL)
{ 
   from_XML(document);
}


// DESTRUCTOR 


TrainingRateAlgorithm::~TrainingRateAlgorithm(void)
{ 
}


// METHODS

// PerformanceFunctional* get_performance_functional_pointer(void) const method


PerformanceFunctional* TrainingRateAlgorithm::get_performance_functional_pointer(void) const
{
    #ifndef NDEBUG

    if(!performance_functional_pointer)
    {
        std::ostringstream buffer;

        buffer << "OpenNN Exception: TrainingRateAlgorithm class.\n"
               << "PerformanceFunctional* get_performance_functional_pointer(void) const method.\n"
               << "Performance functional pointer is NULL.\n";

        throw std::logic_error(buffer.str());
    }

    #endif

   return(performance_functional_pointer);
}


// bool has_performance_functional(void) const method


bool TrainingRateAlgorithm::has_performance_functional(void) const
{
    if(performance_functional_pointer)
    {
        return(true);
    }
    else
    {
        return(false);
    }
}


// const TrainingRateMethod& get_training_rate_method(void) const method


const TrainingRateAlgorithm::TrainingRateMethod& TrainingRateAlgorithm::get_training_rate_method(void) const
{
   return(training_rate_method);
}


// std::string write_training_rate_method(void) const method


std::string TrainingRateAlgorithm::write_training_rate_method(void) const
{
   switch(training_rate_method)
   {
      case Fixed:
      {
         return("Fixed");
       }
      break;

      case GoldenSection:
      {
         return("GoldenSection");
       }
      break;

      case BrentMethod:
      {
         return("BrentMethod");
       }
      break;

      default:
      {
         std::ostringstream buffer;

         buffer << "OpenNN Exception: TrainingRateAlgorithm class.\n"
                << "std::string get_training_rate_method(void) const method.\n"
                << "Unknown training rate method.\n";
 
         throw std::logic_error(buffer.str());
       }
      break;
   }
}


// const double& get_bracketing_factor(void) const method


const double& TrainingRateAlgorithm::get_bracketing_factor(void) const
{
   return(bracketing_factor);       
}


// const double& get_training_rate_tolerance(void) const method


const double& TrainingRateAlgorithm::get_training_rate_tolerance(void) const
{
   return(training_rate_tolerance);
}


// const double& get_warning_training_rate(void) const method


const double& TrainingRateAlgorithm::get_warning_training_rate(void) const
{
   return(warning_training_rate);
}


// const double& get_error_training_rate(void) const method


const double& TrainingRateAlgorithm::get_error_training_rate(void) const
{
   return(error_training_rate);
}


// const bool& get_display(void) const method


const bool& TrainingRateAlgorithm::get_display(void) const
{
   return(display);
}


// void set(void) method


void TrainingRateAlgorithm::set(void)
{
   performance_functional_pointer = NULL;
   set_default();
}


// void set(PerformanceFunctional*) method


void TrainingRateAlgorithm::set(PerformanceFunctional* new_performance_functional_pointer)
{
   performance_functional_pointer = new_performance_functional_pointer;
   set_default();
}


// void set_default(void) method 


void TrainingRateAlgorithm::set_default(void)
{
   // TRAINING OPERATORS

   training_rate_method = BrentMethod;

   // TRAINING PARAMETERS

   bracketing_factor = 1.5;

   training_rate_tolerance = 1.0e-6;

   warning_training_rate = 1.0e3;

   error_training_rate = 1.0e6;

   // UTILITIES

   display = true;
}


// void set_performance_functional_pointer(PerformanceFunctional*) method


void TrainingRateAlgorithm::set_performance_functional_pointer(PerformanceFunctional* new_performance_functional_pointer)
{
   performance_functional_pointer = new_performance_functional_pointer;
}


// void set_training_rate_method(const TrainingRateMethod&) method


void TrainingRateAlgorithm::set_training_rate_method(const TrainingRateAlgorithm::TrainingRateMethod& new_training_rate_method)
{
   training_rate_method = new_training_rate_method;
}


// void set_training_rate_method(const std::string&) method


void TrainingRateAlgorithm::set_training_rate_method(const std::string& new_training_rate_method)
{
   if(new_training_rate_method == "Fixed")
   {
      training_rate_method = Fixed;
   }
   else if(new_training_rate_method == "GoldenSection")
   {
      training_rate_method = GoldenSection;
   }
   else if(new_training_rate_method == "BrentMethod")
   {
      training_rate_method = BrentMethod;
   }
   else
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: TrainingRateAlgorithm class.\n"
             << "void set_method(const std::string&) method.\n"
             << "Unknown training rate method: " << new_training_rate_method << ".\n";
   
      throw std::logic_error(buffer.str());
   }
}


// void set_bracketing_factor(const double&) method


void TrainingRateAlgorithm::set_bracketing_factor(const double& new_bracketing_factor)
{ 
   // Control sentence (if debug)

   #ifndef NDEBUG 

   if(new_bracketing_factor < 0.0) 
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: TrainingRateAlgorithm class.\n"
             << "void set_bracketing_factor(const double&) method.\n"
             << "Bracketing factor must be equal or greater than 0.\n";

      throw std::logic_error(buffer.str());
   }

   #endif

   bracketing_factor = new_bracketing_factor;
}


// void set_training_rate_tolerance(const double&) method


void TrainingRateAlgorithm::set_training_rate_tolerance(const double& new_training_rate_tolerance)
{
   // Control sentence (if debug)

   #ifndef NDEBUG 
                                      
   if(new_training_rate_tolerance < 0.0)
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: TrainingRateAlgorithm class.\n"
             << "void set_training_rate_tolerance(const double&) method.\n"
             << "Tolerance must be equal or greater than 0.\n";

      throw std::logic_error(buffer.str());
   }

   #endif

   // Set training rate tolerance

   training_rate_tolerance = new_training_rate_tolerance;
}


// void set_warning_training_rate(const double&) method


void TrainingRateAlgorithm::set_warning_training_rate(const double& new_warning_training_rate)
{
   // Control sentence (if debug)

   #ifndef NDEBUG 

   if(new_warning_training_rate < 0.0)
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: TrainingRateAlgorithm class.\n" 
             << "void set_warning_training_rate(const double&) method.\n"
             << "Warning training rate must be equal or greater than 0.\n";

      throw std::logic_error(buffer.str());
   }

   #endif

   warning_training_rate = new_warning_training_rate;
}


// void set_error_training_rate(const double&) method


void TrainingRateAlgorithm::set_error_training_rate(const double& new_error_training_rate)
{
   // Control sentence (if debug)

   #ifndef NDEBUG 

   if(new_error_training_rate < 0.0)
   {
      std::ostringstream buffer;

      buffer << "OpenNN Exception: TrainingRateAlgorithm class.\n"
             << "void set_error_training_rate(const double&) method.\n"
             << "Error training rate must be equal or greater than 0.\n";

      throw std::logic_error(buffer.str());
   }

   #endif

   // Set error training rate

   error_training_rate = new_error_training_rate;
}


// void set_display(const bool&) method


void TrainingRateAlgorithm::set_display(const bool& new_display)
{
   display = new_display;
}


// Vector<double> calculate_directional_point(const double&, const Vector<double>&, const double&) const method


Vector<double> TrainingRateAlgorithm::calculate_directional_point(const double& performance, const Vector<double>& training_direction, const double& initial_training_rate) const 
{
   #ifndef NDEBUG 

   if(performance_functional_pointer == NULL)
   {
      std::ostringstream buffer;

      buffer << "OpenNN Error: TrainingRateAlgorithm class.\n"
             << "Vector<double> calculate_directional_point(const double&, const Vector<double>&, const double&) const method.\n"
             << "Pointer to performance functional is NULL.\n";

      throw std::logic_error(buffer.str());
   }

   #endif

   #ifndef NDEBUG 

   const NeuralNetwork* neural_network_pointer = performance_functional_pointer->get_neural_network_pointer();

   if(neural_network_pointer == NULL)
   {
      std::ostringstream buffer;

      buffer << "OpenNN Error: TrainingRateAlgorithm class.\n"
             << "Vector<double> calculate_directional_point(const double&, const Vector<double>&, const double&) const method.\n"
             << "Pointer to neural network is NULL.\n";

      throw std::logic_error(buffer.str());
   }

   #endif
   
   switch(training_rate_method)
   {
      case TrainingRateAlgorithm::Fixed:
      {
         return(calculate_fixed_directional_point(performance, training_direction, initial_training_rate));
      }
      break;

      case TrainingRateAlgorithm::GoldenSection:
      {
         return(calculate_golden_section_directional_point(performance, training_direction, initial_training_rate));
      }
      break;

      case TrainingRateAlgorithm::BrentMethod:
      {
         return(calculate_Brent_method_directional_point(performance, training_direction, initial_training_rate));
      }
      break;

      default:
      {
         std::ostringstream buffer;

         buffer << "OpenNN Exception: TrainingRateAlgorithm class\n"
                << "Vector<double> calculate_directional_point(const double&, const Vector<double>&, const double&) const method.\n"
                << "Unknown training rate method.\n";

         throw std::logic_error(buffer.str());
       }
   }
}


// Triplet calculate_bracketing_triplet(const double&, const Vector<double>&, const double&) const method


TrainingRateAlgorithm::Triplet TrainingRateAlgorithm::calculate_bracketing_triplet(
        const double& performance,
        const Vector<double>& training_direction,
        const double& initial_training_rate) const
{    
    Triplet triplet;

   if(training_direction == 0.0)
   {
       triplet.A[0] = 0.0;
       triplet.A[1] = performance;

       triplet.U = triplet.A;

       triplet.B = triplet.A;

       return(triplet);
   }

   if(initial_training_rate == 0.0)
   {
       triplet.A[0] = 0.0;
       triplet.A[1] = performance;

       triplet.U = triplet.A;

       triplet.B = triplet.A;

       return(triplet);
   }

   // Left point

   triplet.A[0] = 0.0;
   triplet.A[1] = performance;

   // Right point

   triplet.B[0] = initial_training_rate;
   triplet.B[1] = performance_functional_pointer->calculate_performance(training_direction, triplet.B[0]);

   while(triplet.A[1] > triplet.B[1])
   {
      triplet.A = triplet.B;

      triplet.B[0] *= bracketing_factor;
      triplet.B[1] = performance_functional_pointer->calculate_performance(training_direction, triplet.B[0]);

      if(triplet.B[0] > error_training_rate)
      {
          std::ostringstream buffer;

          buffer << "OpenNN Warning: TrainingRateAlgorithm class.\n"
                 << "Vector<double> calculate_bracketing_triplet(double, const Vector<double>&, double) const method\n."
                 << "Right point is " << triplet.B[0] << "." << std::endl;

          buffer << "Performance: " << performance << "\n"
                 << "Training direction: " << training_direction << "\n"
                 << "Initial training rate: " << initial_training_rate << std::endl;

          throw std::logic_error(buffer.str());

          return(triplet);
      }
   }

    // Interior point

   triplet.U[0] = triplet.A[0] + (triplet.B[0] - triplet.A[0])/2.0;
   triplet.U[1] = performance_functional_pointer->calculate_performance(training_direction, triplet.U[0]);

   while(triplet.A[1] < triplet.U[1])
   {
      triplet.U[0] = triplet.A[0] + (triplet.U[0]-triplet.A[0])/bracketing_factor;
      triplet.U[1] = performance_functional_pointer->calculate_performance(training_direction, triplet.U[0]);

      if(triplet.U[0] - triplet.A[0] <= training_rate_tolerance)
      {
          triplet.U = triplet.A;
          triplet.B = triplet.A;

          triplet.check();

         return(triplet);
      }
   }

   triplet.check();

   return(triplet);
}


// Vector<double> calculate_fixed_directional_point(const double&, const Vector<double>&, const double&) const method

Vector<double> TrainingRateAlgorithm::calculate_fixed_directional_point(const double&, const Vector<double>& training_direction, const double& initial_training_rate) const
{
   Vector<double> directional_point(2);

   directional_point[0] = initial_training_rate;
   directional_point[1] = performance_functional_pointer->calculate_performance(training_direction, initial_training_rate);

   return(directional_point);
}


// Vector<double> calculate_golden_section_directional_point(double, Vector<double>, double) const method


Vector<double> TrainingRateAlgorithm::calculate_golden_section_directional_point
(const double& performance, const Vector<double>& training_direction, const double& initial_training_rate) const
{
   std::ostringstream buffer;

   // Bracket minimum

   try
   {
      Triplet triplet = calculate_bracketing_triplet(performance, training_direction, initial_training_rate);

      if(triplet.has_length_zero())
      {
         return(triplet.A);
      }

      Vector<double> V(2);

      // Reduce the interval

      do
      {
         V[0] = calculate_golden_section_training_rate(triplet);
         V[1] = performance_functional_pointer->calculate_performance(training_direction, V[0]);

         // Update points
 
         if(V[0] < triplet.U[0] && V[1] >= triplet.U[1])
          {
            triplet.A = V;
            //U = U;
            //B = B;
          }
         else if(V[0] < triplet.U[0] && V[1] <= triplet.U[1])
         {
            //A = A;
            triplet.U = V;
            triplet.B = triplet.U;
          }
          else if(V[0] > triplet.U[0] && V[1] >= triplet.U[1])
          {
            //A = A;
            triplet.B = V;
            //U = U;
         }
         else if(V[0] > triplet.U[0] && V[1] <= triplet.U[1])
         {
            triplet.A = triplet.U;
            triplet.U = V;
            //B = B;
         }
         else if(V[0] == triplet.U[0])
         {
            buffer << "OpenNN Exception: TrainingRateAlgorithm class.\n"
                   << "Vector<double> calculate_golden_section_directional_point(double, const Vector<double>, double) const method.\n"
                   << "Both interior points have the same ordinate.\n";

            std::cout << buffer.str() << std::endl;

            break;
         }
         else
         {
            buffer << "OpenNN Exception: TrainingRateAlgorithm class.\n" 
                   << "Vector<double> calculate_golden_section_directional_point(double, const Vector<double>, double) const method.\n" 
                   << "Unknown set:\n" 
                     << "A = (" << triplet.A[0] << "," << triplet.A[1] << ")\n"
                     << "B = (" << triplet.B[0] << "," << triplet.B[1] << ")\n"
                     << "U = (" << triplet.U[0] << "," << triplet.U[1] << ")\n"
                     << "V = (" << V[0] << "," << V[1] << ")\n";
         
            throw std::logic_error(buffer.str());
         }

         // Check triplet

         triplet.check();

      }while(triplet.B[0] - triplet.A[0] > training_rate_tolerance);

      return(triplet.U);
   }
   catch(const std::logic_error& e)
   {  
      std::cerr << e.what() << std::endl;

      Vector<double> X(2);
      X[0] = initial_training_rate;
      X[1] = performance_functional_pointer->calculate_performance(training_direction, X[0]);

       if(X[1] > performance)
       {
          X[0] = 0.0;
          X[1] = 0.0;
       }

      return(X);
   }
}


// Vector<double> calculate_Brent_method_directional_point(const double&, const Vector<double>, const double&) const method


Vector<double> TrainingRateAlgorithm::calculate_Brent_method_directional_point
(const double& performance, const Vector<double>& training_direction, const double& initial_training_rate) const 
{        
   std::ostringstream buffer;

   // Bracket minimum

   try
   {
      Triplet triplet = calculate_bracketing_triplet(performance, training_direction, initial_training_rate);

      if(triplet.A == triplet.B)
      {
         return(triplet.A);
      }

      Vector<double> V(2);

      // Reduce the interval

      while(triplet.B[0] - triplet.A[0] > training_rate_tolerance)
      {
          try
          {
            V[0] = calculate_Brent_method_training_rate(triplet);
          }
         catch(const std::logic_error&)
          {
            V[0] = calculate_golden_section_training_rate(triplet);
          }

         // Calculate performance for V

         V[1] = performance_functional_pointer->calculate_performance(training_direction, V[0]);

         // Update points
 
         if(V[0] < triplet.U[0] && V[1] >= triplet.U[1])
          {
            triplet.A = V;
            //B = B;
            //U = U;
          }
         else if(V[0] < triplet.U[0] && V[1] <= triplet.U[1])
         {
            //A = A;
            triplet.B = triplet.U;
            triplet.U = V;
         }
         else if(V[0] > triplet.U[0] && V[1] >= triplet.U[1])
         {
            //A = A;
            triplet.B = V;
            //U = U;
         }
         else if(V[0] > triplet.U[0] && V[1] <= triplet.U[1])
         {
            triplet.A = triplet.U;
            //B = B;
            triplet.U = V;
         }
           else if(V[0] == triplet.U[0])
           {
            buffer << "OpenNN Exception: TrainingRateAlgorithm class.\n"
                   << "Vector<double> calculate_Brent_method_directional_point(double, const Vector<double>, double) const method.\n"
                   << "Both interior points have the same ordinate.\n";

            break;
          }
         else
         {
            buffer << "OpenNN Exception: TrainingRateAlgorithm class.\n" 
                   << "Vector<double> calculate_Brent_method_directional_point(double, const Vector<double>, double) const method.\n" 
                   << "Unknown set:\n" 
                     << "A = (" << triplet.A[0] << "," << triplet.A[1] << ")\n"
                     << "B = (" << triplet.B[0] << "," << triplet.B[1] << ")\n"
                     << "U = (" << triplet.U[0] << "," << triplet.U[1] << ")\n"
                     << "V = (" << V[0] << "," << V[1] << ")\n";
         
            throw std::logic_error(buffer.str());
         }

         // Check triplet

         triplet.check();
      }

      return(triplet.U);
   }
   catch(std::range_error& e) // Interval is of length 0
   {  
      std::cerr << e.what() << std::endl;

      Vector<double> A(2);
      A[0] = 0.0;
      A[1] = performance;

      return(A);
   }
   catch(const std::logic_error& e)
   {  
      std::cerr << e.what() << std::endl;

      Vector<double> X(2);
      X[0] = initial_training_rate;
      X[1] = performance_functional_pointer->calculate_performance(training_direction, X[0]);

      if(X[1] > performance)
      {
         X[0] = 0.0;
         X[1] = 0.0;
      }

      return(X);
   }
}


// double calculate_golden_section_training_rate(const Triplet&) const method


double TrainingRateAlgorithm::calculate_golden_section_training_rate(const Triplet& triplet) const
{
//   const double tau = 0.382; // (3.0-sqrt(5.0))/2.0   

    double training_rate;

   if(triplet.U[0] < triplet.A[0] + 0.5*(triplet.B[0] - triplet.A[0]))
   {
      training_rate = triplet.A[0] + 0.618*(triplet.B[0] - triplet.A[0]);
   }
   else
   {
      training_rate = triplet.A[0] + 0.382*(triplet.B[0] - triplet.A[0]);
   }

    #ifndef NDEBUG

    if(training_rate < triplet.A[0])
    {
       std::ostringstream buffer;

       buffer << "OpenNN Error: TrainingRateAlgorithm class.\n"
              << "double calculate_golden_section_training_rate(const Triplet&) const method.\n"
              << "Training rate (" << training_rate << ") is less than triplet left point (" << triplet.A[0] << ").\n";

       throw std::logic_error(buffer.str());
    }

    if(training_rate > triplet.B[0])
    {
       std::ostringstream buffer;

       buffer << "OpenNN Error: TrainingRateAlgorithm class.\n"
              << "double calculate_golden_section_training_rate(const Triplet&) const method.\n"
              << "Training rate (" << training_rate << ") is greater than triplet right point (" << triplet.B[0] << ").\n";

       throw std::logic_error(buffer.str());
    }

    #endif

    return(training_rate);
}


// double calculate_Brent_method_training_rate(const Triplet&) const method


double TrainingRateAlgorithm::calculate_Brent_method_training_rate(const Triplet& triplet) const
{
  const double c = -(triplet.A[1]*(triplet.U[0]-triplet.B[0])
  + triplet.U[1]*(triplet.B[0]-triplet.A[0])
  + triplet.B[1]*(triplet.A[0]-triplet.U[0]))/((triplet.A[0]-triplet.U[0])*(triplet.U[0]-triplet.B[0])*(triplet.B[0]-triplet.A[0]));

   if(c == 0) 
   {
       std::ostringstream buffer;

      buffer << "OpenNN Exception: TrainingRateAlgorithm class.\n"
             << "double calculate_Brent_method_training_rate(Vector<double>&, Vector<double>&, Vector<double>&) const method.\n"
             << "Parabola cannot be constructed.\n";

      throw std::logic_error(buffer.str());
   }
   else if(c < 0) 
   {
       std::ostringstream buffer;

      buffer << "OpenNN Exception: TrainingRateAlgorithm class.\n"
             << "double calculate_Brent_method_training_rate(Vector<double>&, Vector<double>&, Vector<double>&) const method.\n"
             << "Parabola does not have a minimum but a maximum.\n";

      throw std::logic_error(buffer.str());
   }

   const double b = (triplet.A[1]*(triplet.U[0]*triplet.U[0]-triplet.B[0]*triplet.B[0])
   + triplet.U[1]*(triplet.B[0]*triplet.B[0]-triplet.A[0]*triplet.A[0])
   + triplet.B[1]*(triplet.A[0]*triplet.A[0]-triplet.U[0]*triplet.U[0]))/((triplet.A[0]-triplet.U[0])*(triplet.U[0]-triplet.B[0])*(triplet.B[0]-triplet.A[0]));

   const double Brent_method_training_rate = -b/(2.0*c);

   if(Brent_method_training_rate <= triplet.A[0] || Brent_method_training_rate >= triplet.B[0])
   {
       std::ostringstream buffer;

      buffer << "OpenNN Exception: TrainingRateAlgorithm class.\n"
             << "double calculate_parabola_minimal_training_rate(Vector<double>&, Vector<double>&, Vector<double>&) const method.\n"
             << "Brent method training rate is not inside interval.\n"
             << "Interval: (" << triplet.A[0] << "," << triplet.B[0] << ")\n"
             << "Brent method training rate: " << Brent_method_training_rate << std::endl;

      throw std::logic_error(buffer.str());
   }

   return(Brent_method_training_rate);
}


// tinyxml2::XMLDocument* to_XML(void) const method


tinyxml2::XMLDocument* TrainingRateAlgorithm::to_XML(void) const
{
   std::ostringstream buffer;

   tinyxml2::XMLDocument* document = new tinyxml2::XMLDocument;

   // Training algorithm

   tinyxml2::XMLElement* root_element = document->NewElement("TrainingRateAlgorithm");

   document->InsertFirstChild(root_element);

   tinyxml2::XMLElement* element = NULL;
   tinyxml2::XMLText* text = NULL;

   // Training rate method
   {
   element = document->NewElement("TrainingRateMethod");
   root_element->LinkEndChild(element);

   text = document->NewText(write_training_rate_method().c_str());
   element->LinkEndChild(text);
   }

   // Bracketing factor
   {
   element = document->NewElement("BracketingFactor");
   root_element->LinkEndChild(element);

   buffer.str("");
   buffer << bracketing_factor;

   text = document->NewText(buffer.str().c_str());
   element->LinkEndChild(text);
   }

   // Training rate tolerance 
   {
   element = document->NewElement("TrainingRateTolerance");
   root_element->LinkEndChild(element);

   buffer.str("");
   buffer << training_rate_tolerance;

   text = document->NewText(buffer.str().c_str());
   element->LinkEndChild(text);
   }

   // Warning training rate 
   {
   element = document->NewElement("WarningTrainingRate");
   root_element->LinkEndChild(element);

   buffer.str("");
   buffer << warning_training_rate;

   text = document->NewText(buffer.str().c_str());
   element->LinkEndChild(text);
   }

   // Error training rate
   {
   element = document->NewElement("ErrorTrainingRate");
   root_element->LinkEndChild(element);

   buffer.str("");
   buffer << error_training_rate;

   text = document->NewText(buffer.str().c_str());
   element->LinkEndChild(text);
   }

   // Display warnings
   {
   element = document->NewElement("Display");
   root_element->LinkEndChild(element);

   buffer.str("");
   buffer << display;

   text = document->NewText(buffer.str().c_str());
   element->LinkEndChild(text);
   }

   return(document);
}


// virtual void from_XML(const tinyxml2::XMLDocument&) method


void TrainingRateAlgorithm::from_XML(const tinyxml2::XMLDocument& document)
{
   const tinyxml2::XMLElement* root_element = document.FirstChildElement("TrainingRateAlgorithm");

   if(!root_element)
   {
       std::ostringstream buffer;

       buffer << "OpenNN Exception: TrainingRateAlgorithm class.\n"
              << "void from_XML(const tinyxml2::XMLDocument&) method.\n"
              << "Training rate algorithm element is NULL.\n";

       throw std::logic_error(buffer.str());
   }

   // Training rate method
   {
       const tinyxml2::XMLElement* element = root_element->FirstChildElement("TrainingRateMethod");

       if(element)
       {
          std::string new_training_rate_method = element->GetText();

          try
          {
             set_training_rate_method(new_training_rate_method);
          }
          catch(const std::logic_error& e)
          {
             std::cout << e.what() << std::endl;
          }
       }
   }

   // Bracketing factor
   {
       const tinyxml2::XMLElement* element = root_element->FirstChildElement("BracketingFactor");

       if(element)
       {
          const double new_bracketing_factor = atof(element->GetText());

          try
          {
             set_bracketing_factor(new_bracketing_factor);
          }
          catch(const std::logic_error& e)
          {
             std::cout << e.what() << std::endl;
          }
       }
   }
/*
   // First training rate
   {
       const tinyxml2::XMLElement* element = root_element->FirstChildElement("FirstTrainingRate");

       if(element)
       {
          const double new_first_training_rate = atof(element->GetText());

          try
          {
             set_first_training_rate(new_first_training_rate);
          }
          catch(const std::logic_error& e)
          {
             std::cout << e.what() << std::endl;
          }
       }
   }
*/
   // Training rate tolerance 
   {
       const tinyxml2::XMLElement* element = root_element->FirstChildElement("TrainingRateTolerance");

       if(element)
       {
          const double new_training_rate_tolerance = atof(element->GetText());

          try
          {
             set_training_rate_tolerance(new_training_rate_tolerance);
          }
          catch(const std::logic_error& e)
          {
             std::cout << e.what() << std::endl;
          }
       }
   }

   // Warning training rate 
   {
       const tinyxml2::XMLElement* element = root_element->FirstChildElement("WarningTrainingRate");

       if(element)
       {
          const double new_warning_training_rate = atof(element->GetText());

          try
          {
             set_warning_training_rate(new_warning_training_rate);
          }
          catch(const std::logic_error& e)
          {
             std::cout << e.what() << std::endl;
          }
       }
   }

   // Error training rate
   {
       const tinyxml2::XMLElement* element = root_element->FirstChildElement("ErrorTrainingRate");

       if(element)
       {
          const double new_error_training_rate = atof(element->GetText());

          try
          {
             set_error_training_rate(new_error_training_rate);
          }
          catch(const std::logic_error& e)
          {
             std::cout << e.what() << std::endl;
          }
       }
   }

   // Display warnings
   {
       const tinyxml2::XMLElement* element = root_element->FirstChildElement("Display");

       if(element)
       {
          const std::string new_display = element->GetText();

          try
          {
             set_display(new_display != "0");
          }
          catch(const std::logic_error& e)
          {
             std::cout << e.what() << std::endl;
          }
       }
   }
}

}


// OpenNN: Open Neural Networks Library.
// Copyright (c) 2005-2015 Roberto Lopez.
//
// This library is free software; you can redistribute it and/or
// modify it under the terms of the GNU Lesser General Public
// License as published by the Free Software Foundation; either
// version 2.1 of the License, or any later version.
//
// This library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
// Lesser General Public License for more details.

// You should have received a copy of the GNU Lesser General Public
// License along with this library; if not, write to the Free Software
// Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA