OpenNN  2.2
Open Neural Networks Library
quasi_newton_method.h
1 /****************************************************************************************************************/
2 /* */
3 /* OpenNN: Open Neural Networks Library */
4 /* www.artelnics.com/opennn */
5 /* */
6 /* Q U A S I - N E W T O N M E T H O D C L A S S H E A D E R */
7 /* */
8 /* Roberto Lopez */
9 /* Artelnics - Making intelligent use of data */
11 /* */
12 /****************************************************************************************************************/
13 
14 #ifndef __QUASINEWTONMETHOD_H__
15 #define __QUASINEWTONMETHOD_H__
16 
17 // System includes
18 
19 #include <string>
20 #include <sstream>
21 #include <iostream>
22 #include <fstream>
23 #include <algorithm>
24 #include <functional>
25 #include <limits>
26 #include <cmath>
27 #include <ctime>
28 
29 // OpenNN includes
30 
31 #include "performance_functional.h"
32 
33 #include "training_algorithm.h"
34 #include "training_rate_algorithm.h"
35 
36 // TinyXml includes
37 
38 #include "../tinyxml2/tinyxml2.h"
39 
40 namespace OpenNN
41 {
42 
46 
48 {
49 
50 public:
51 
52  // ENUMERATIONS
53 
55 
57 
58 
59  // DEFAULT CONSTRUCTOR
60 
61  explicit QuasiNewtonMethod(void);
62 
63  // PERFORMANCE FUNCTIONAL CONSTRUCTOR
64 
66 
67  // XML CONSTRUCTOR
68 
69  explicit QuasiNewtonMethod(const tinyxml2::XMLDocument&);
70 
71 
72  // DESTRUCTOR
73 
74  virtual ~QuasiNewtonMethod(void);
75 
76  // STRUCTURES
77 
81 
83  {
85 
87  {
89  }
90 
92 
94  {
95  quasi_Newton_method_pointer = new_quasi_Newton_method_pointer;
96  }
97 
99 
101  {
103  }
104 
105  // Members
106 
108 
110 
111  // Training history
112 
114 
116 
118 
120 
122 
124 
126 
128 
130 
132 
134 
136 
138 
140 
142 
144 
146 
148 
150 
152 
153  // Final values
154 
156 
158 
160 
162 
164 
166 
168 
170 
172 
174 
176 
178 
180 
182 
184 
186 
188 
190 
192 
193  double elapsed_time;
194 
196 
198 
199  // Methods
200 
202 
204 
205  void resize_training_history(const size_t&);
206 
207  std::string to_string(void) const;
208 
209  Matrix<std::string> write_final_results(const size_t& precision = 3) const;
210  };
211 
212  // METHODS
213 
214  // Get methods
215 
218 
220  std::string write_inverse_Hessian_approximation_method(void) const;
221 
222  // Training parameters
223 
224  const double& get_warning_parameters_norm(void) const;
225  const double& get_warning_gradient_norm(void) const;
226  const double& get_warning_training_rate(void) const;
227 
228  const double& get_error_parameters_norm(void) const;
229  const double& get_error_gradient_norm(void) const;
230  const double& get_error_training_rate(void) const;
231 
232  // Stopping criteria
233 
234  const double& get_minimum_parameters_increment_norm(void) const;
235 
236  const double& get_minimum_performance_increase(void) const;
237  const double& get_performance_goal(void) const;
238  const double& get_gradient_norm_goal(void) const;
239  const size_t& get_maximum_generalization_performance_decreases(void) const;
240 
241  const size_t& get_maximum_iterations_number(void) const;
242  const double& get_maximum_time(void) const;
243 
244  // Reserve training history
245 
246  const bool& get_reserve_parameters_history(void) const;
247  const bool& get_reserve_parameters_norm_history(void) const;
248 
249  const bool& get_reserve_performance_history(void) const;
250  const bool& get_reserve_gradient_history(void) const;
251  const bool& get_reserve_gradient_norm_history(void) const;
252  const bool& get_reserve_inverse_Hessian_history(void) const;
253  const bool& get_reserve_generalization_performance_history(void) const;
254 
255  const bool& get_reserve_training_direction_history(void) const;
256  const bool& get_reserve_training_rate_history(void) const;
257  const bool& get_reserve_elapsed_time_history(void) const;
258 
259  // Set methods
260 
262 
264  void set_inverse_Hessian_approximation_method(const std::string&);
265 
266  void set_default(void);
267 
268  // Training parameters
269 
270  void set_warning_parameters_norm(const double&);
271  void set_warning_gradient_norm(const double&);
272  void set_warning_training_rate(const double&);
273 
274  void set_error_parameters_norm(const double&);
275  void set_error_gradient_norm(const double&);
276  void set_error_training_rate(const double&);
277 
278  // Stopping criteria
279 
280  void set_minimum_parameters_increment_norm(const double&);
281 
282  void set_minimum_performance_increase(const double&);
283  void set_performance_goal(const double&);
284  void set_gradient_norm_goal(const double&);
286 
287  void set_maximum_iterations_number(const size_t&);
288  void set_maximum_time(const double&);
289 
290  // Reserve training history
291 
292  void set_reserve_parameters_history(const bool&);
293  void set_reserve_parameters_norm_history(const bool&);
294 
295  void set_reserve_performance_history(const bool&);
296  void set_reserve_gradient_history(const bool&);
297  void set_reserve_gradient_norm_history(const bool&);
298  void set_reserve_inverse_Hessian_history(const bool&);
300 
301  void set_reserve_training_direction_history(const bool&);
302  void set_reserve_training_rate_history(const bool&);
303  void set_reserve_elapsed_time_history(const bool&);
304 
305  // Utilities
306 
307  void set_display_period(const size_t&);
308 
309  // Training methods
310 
312 
314  (const Vector<double>&, const Vector<double>&, const Vector<double>&, const Vector<double>&, const Matrix<double>&) const;
315 
317  (const Vector<double>&, const Vector<double>&, const Vector<double>&, const Vector<double>&, const Matrix<double>&) const;
318 
320 
322 
324 
325  // Training history methods
326 
327  void set_reserve_all_training_history(const bool&);
328 
329  std::string write_training_algorithm_type(void) const;
330 
331  // Serialization methods
332 
333  tinyxml2::XMLDocument* to_XML(void) const;
334 
335  std::string to_string(void) const;
337 
338  void from_XML(const tinyxml2::XMLDocument&);
339 
340 
341 private:
342 
345 
347 
349 
351 
352 
354 
356 
358 
360 
362 
364 
366 
368 
370 
372 
374 
376 
377 
378  // STOPPING CRITERIA
379 
381 
383 
385 
387 
389 
391 
393 
395 
398 
400 
402 
404 
406 
407  double maximum_time;
408 
409  // TRAINING HISTORY
410 
412 
414 
416 
418 
420 
422 
424 
426 
428 
430 
432 
434 
436 
438 
440 
442 
444 
446 
448 
450 };
451 
452 }
453 
454 #endif
455 
456 
457 // OpenNN: Open Neural Networks Library.
458 // Copyright (c) 2005-2015 Roberto Lopez.
459 //
460 // This library is free software; you can redistribute it and/or
461 // modify it under the terms of the GNU Lesser General Public
462 // License as published by the Free Software Foundation; either
463 // version 2.1 of the License, or any later version.
464 //
465 // This library is distributed in the hope that it will be useful,
466 // but WITHOUT ANY WARRANTY; without even the implied warranty of
467 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
468 // Lesser General Public License for more details.
469 
470 // You should have received a copy of the GNU Lesser General Public
471 // License along with this library; if not, write to the Free Software
472 // Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
Vector< Matrix< double > > inverse_Hessian_history
History of the inverse Hessian approximation over the training iterations.
double maximum_time
Maximum training time. It is used as a stopping criterion.
const double & get_error_training_rate(void) const
const bool & get_reserve_parameters_history(void) const
Returns true if the parameters history matrix is to be reserved, and false otherwise.
Vector< double > calculate_gradient_descent_training_direction(const Vector< double > &) const
Vector< double > elapsed_time_history
History of the elapsed time over the training iterations.
const double & get_warning_parameters_norm(void) const
Returns the minimum value for the norm of the parameters vector at wich a warning message is written ...
Matrix< double > final_inverse_Hessian_approximation
Final approximation of the inverse Hessian.
double error_training_rate
Training rate at wich the line minimization algorithm is assumed to be unable to bracket a minimum...
std::string write_inverse_Hessian_approximation_method(void) const
Returns the name of the method for the approximation of the inverse Hessian.
const bool & get_reserve_training_rate_history(void) const
Returns true if the training rate history vector is to be reserved, and false otherwise.
const bool & get_reserve_gradient_history(void) const
Returns true if the gradient history vector of vectors is to be reserved, and false otherwise...
InverseHessianApproximationMethod
Enumeration of the available training operators for obtaining the approximation to the inverse Hessia...
void set_reserve_gradient_history(const bool &)
const bool & get_reserve_elapsed_time_history(void) const
Returns true if the elapsed time history vector is to be reserved, and false otherwise.
void set_error_parameters_norm(const double &)
double warning_gradient_norm
Value for the gradient norm at which a warning message is written to the screen.
const double & get_performance_goal(void) const
void set_reserve_all_training_history(const bool &)
void set_performance_goal(const double &)
void set_warning_parameters_norm(const double &)
const double & get_warning_training_rate(void) const
Returns the training rate value at wich a warning message is written to the screen during line minimi...
bool reserve_inverse_Hessian_history
True if the inverse Hessian history vector of matrices is to be reserved, false otherwise.
void set_error_gradient_norm(const double &)
void set_error_training_rate(const double &)
bool reserve_gradient_norm_history
True if the gradient norm history vector is to be reserved, false otherwise.
const bool & get_reserve_inverse_Hessian_history(void) const
Returns true if the inverse Hessian history is to be reserved, and false otherwise.
Vector< double > parameters_norm_history
History of the parameters norm over the training iterations.
const bool & get_reserve_gradient_norm_history(void) const
Returns true if the gradient norm history vector is to be reserved, and false otherwise.
const bool & get_reserve_generalization_performance_history(void) const
Returns true if the Generalization performance history vector is to be reserved, and false otherwise...
void set_display_period(const size_t &)
void set_default(void)
Sets the members of the training algorithm object to their default values.
const double & get_error_parameters_norm(void) const
Returns the value for the norm of the parameters vector at wich an error message is written to the sc...
void set_reserve_parameters_history(const bool &)
Vector< double > final_parameters
Final neural network parameters vector.
void set_reserve_parameters_norm_history(const bool &)
double error_parameters_norm
Value for the parameters norm at which the training process is assumed to fail.
Matrix< std::string > write_final_results(const size_t &precision=3) const
Returns a default (empty) string matrix with the final results from training.
size_t maximum_iterations_number
Maximum number of iterations to perform_training. It is used as a stopping criterion.
double minimum_parameters_increment_norm
Norm of the parameters increment vector at which training stops.
void set_warning_gradient_norm(const double &)
const double & get_warning_gradient_norm(void) const
Returns the minimum value for the norm of the gradient vector at wich a warning message is written to...
void set_maximum_iterations_number(const size_t &)
const double & get_gradient_norm_goal(void) const
void from_XML(const tinyxml2::XMLDocument &)
const TrainingRateAlgorithm & get_training_rate_algorithm(void) const
Returns a constant reference to the training rate algorithm object inside the quasi-Newton method obj...
double final_parameters_norm
Final neural network parameters norm.
double final_performance
Final performance function evaluation.
std::string write_training_algorithm_type(void) const
This method writes a string with the type of training algoritm.
Matrix< double > calculate_inverse_Hessian_approximation(const Vector< double > &, const Vector< double > &, const Vector< double > &, const Vector< double > &, const Matrix< double > &) const
Vector< double > final_training_direction
Final quasi-Newton method training direction.
Vector< double > final_gradient
Final performance function gradient.
QuasiNewtonMethod * get_quasi_Newton_method_pointer(void) const
Returns the pointer to the quasi-Newton method object required by the corresponding results structure...
double gradient_norm_goal
Goal value for the norm of the objective function gradient. It is used as a stopping criterion...
void set_warning_training_rate(const double &)
double warning_parameters_norm
Value for the parameters norm at which a warning message is written to the screen.
InverseHessianApproximationMethod inverse_Hessian_approximation_method
Variable containing the actual method used to obtain a suitable training rate.
const bool & get_reserve_parameters_norm_history(void) const
Returns true if the parameters norm history vector is to be reserved, and false otherwise.
void set_minimum_parameters_increment_norm(const double &)
QuasiNewtonMethodResults * perform_training(void)
double final_generalization_performance
Final generalization performance.
bool reserve_training_rate_history
True if the training rate history vector is to be reserved, false otherwise.
Vector< Vector< double > > gradient_history
History of the performance function gradient over the training iterations.
void set_inverse_Hessian_approximation_method(const InverseHessianApproximationMethod &)
void set_reserve_training_direction_history(const bool &)
Vector< double > generalization_performance_history
History of the generalization performance over the training iterations.
Vector< double > gradient_norm_history
History of the gradient norm over the training iterations.
const double & get_minimum_parameters_increment_norm(void) const
Returns the minimum norm of the parameter increment vector used as a stopping criteria when training...
size_t iterations_number
Maximum number of training iterations.
void set_maximum_generalization_performance_decreases(const size_t &)
void set_reserve_training_rate_history(const bool &)
bool reserve_parameters_norm_history
True if the parameters norm history vector is to be reserved, false otherwise.
bool reserve_performance_history
True if the performance history vector is to be reserved, false otherwise.
void set_reserve_inverse_Hessian_history(const bool &)
void set_reserve_elapsed_time_history(const bool &)
TrainingRateAlgorithm * get_training_rate_algorithm_pointer(void)
Returns a pointer to the training rate algorithm object inside the quasi-Newton method object...
std::string to_string(void) const
Returns a default string representation of a training algorithm.
Matrix< double > calculate_DFP_inverse_Hessian(const Vector< double > &, const Vector< double > &, const Vector< double > &, const Vector< double > &, const Matrix< double > &) const
void set_reserve_gradient_norm_history(const bool &)
QuasiNewtonMethod * quasi_Newton_method_pointer
Pointer to the quasi-Newton method object for which the training results are to be stored...
Vector< Vector< double > > training_direction_history
History of the random search training direction over the training iterations.
std::string to_string(void) const
Returns a string representation of the current quasi-Newton method results structure.
void set_minimum_performance_increase(const double &)
const double & get_error_gradient_norm(void) const
Vector< double > calculate_training_direction(const Vector< double > &, const Matrix< double > &) const
QuasiNewtonMethodResults(QuasiNewtonMethod *new_quasi_Newton_method_pointer)
Quasi-Newton method constructor.
bool reserve_elapsed_time_history
True if the elapsed time history vector is to be reserved, false otherwise.
Matrix< double > calculate_BFGS_inverse_Hessian(const Vector< double > &, const Vector< double > &, const Vector< double > &, const Vector< double > &, const Matrix< double > &) const
void set_reserve_generalization_performance_history(const bool &)
bool reserve_parameters_history
True if the parameters history matrix is to be reserved, false otherwise.
const size_t & get_maximum_generalization_performance_decreases(void) const
Returns the maximum number of generalization failures during the training process.
TrainingRateAlgorithm training_rate_algorithm
void set_performance_functional_pointer(PerformanceFunctional *)
bool reserve_training_direction_history
True if the training direction history matrix is to be reserved, false otherwise. ...
double error_gradient_norm
Value for the gradient norm at which the training process is assumed to fail.
Vector< double > performance_history
History of the performance function performance over the training iterations.
void set_quasi_Newton_method_pointer(QuasiNewtonMethod *)
Returns the pointer to the quasi-Newton method object required by the corresponding results structure...
Vector< Vector< double > > parameters_history
History of the neural network parameters over the training iterations.
double warning_training_rate
Training rate value at wich a warning message is written to the screen.
Vector< double > training_rate_history
History of the random search training rate over the training iterations.
void set_maximum_time(const double &)
double final_training_rate
Final quasi-Newton method training rate.
double elapsed_time
Elapsed time of the training process.
void set_gradient_norm_goal(const double &)
bool reserve_gradient_history
True if the gradient history matrix is to be reserved, false otherwise.
const bool & get_reserve_performance_history(void) const
Returns true if the performance history vector is to be reserved, and false otherwise.
double performance_goal
Goal value for the performance. It is used as a stopping criterion.
const bool & get_reserve_training_direction_history(void) const
Returns true if the training direction history matrix is to be reserved, and false otherwise...
double minimum_performance_increase
Minimum performance improvement between two successive iterations. It is used as a stopping criterion...
const InverseHessianApproximationMethod & get_inverse_Hessian_approximation_method(void) const
Returns the method for approximating the inverse Hessian matrix to be used when training.
tinyxml2::XMLDocument * to_XML(void) const
const double & get_minimum_performance_increase(void) const
Returns the minimum performance improvement during training.
const double & get_maximum_time(void) const
Returns the maximum training time.
void set_reserve_performance_history(const bool &)
Matrix< std::string > to_string_matrix(void) const
const size_t & get_maximum_iterations_number(void) const
Returns the maximum number of iterations for training.
bool reserve_generalization_performance_history
True if the Generalization performance history vector is to be reserved, false otherwise.