mlpack  2.0.1
lbfgs.hpp
Go to the documentation of this file.
1 
15 #ifndef __MLPACK_CORE_OPTIMIZERS_LBFGS_LBFGS_HPP
16 #define __MLPACK_CORE_OPTIMIZERS_LBFGS_LBFGS_HPP
17 
18 #include <mlpack/core.hpp>
19 
20 namespace mlpack {
21 namespace optimization {
22 
35 template<typename FunctionType>
36 class L_BFGS
37 {
38  public:
59  L_BFGS(FunctionType& function,
60  const size_t numBasis = 10, /* same default as scipy */
61  const size_t maxIterations = 0, /* run forever */
62  const double armijoConstant = 1e-4,
63  const double wolfe = 0.9,
64  const double minGradientNorm = 1e-6,
65  const double factr = 1e-15,
66  const size_t maxLineSearchTrials = 50,
67  const double minStep = 1e-20,
68  const double maxStep = 1e20);
69 
76  const std::pair<arma::mat, double>& MinPointIterate() const;
77 
89  double Optimize(arma::mat& iterate);
90 
103  double Optimize(arma::mat& iterate, const size_t maxIterations);
104 
106  const FunctionType& Function() const { return function; }
108  FunctionType& Function() { return function; }
109 
111  size_t NumBasis() const { return numBasis; }
113  size_t& NumBasis() { return numBasis; }
114 
116  size_t MaxIterations() const { return maxIterations; }
118  size_t& MaxIterations() { return maxIterations; }
119 
121  double ArmijoConstant() const { return armijoConstant; }
123  double& ArmijoConstant() { return armijoConstant; }
124 
126  double Wolfe() const { return wolfe; }
128  double& Wolfe() { return wolfe; }
129 
131  double MinGradientNorm() const { return minGradientNorm; }
133  double& MinGradientNorm() { return minGradientNorm; }
134 
136  double Factr() const { return factr; }
138  double& Factr() { return factr; }
139 
141  size_t MaxLineSearchTrials() const { return maxLineSearchTrials; }
144 
146  double MinStep() const { return minStep; }
148  double& MinStep() { return minStep; }
149 
151  double MaxStep() const { return maxStep; }
153  double& MaxStep() { return maxStep; }
154 
155  private:
157  FunctionType& function;
158 
160  arma::mat newIterateTmp;
162  arma::cube s;
164  arma::cube y;
165 
167  size_t numBasis;
173  double wolfe;
177  double factr;
181  double minStep;
183  double maxStep;
184 
186  std::pair<arma::mat, double> minPointIterate;
187 
194  double Evaluate(const arma::mat& iterate);
195 
203  double ChooseScalingFactor(const size_t iterationNum,
204  const arma::mat& gradient);
205 
212  bool GradientNormTooSmall(const arma::mat& gradient);
213 
227  bool LineSearch(double& functionValue,
228  arma::mat& iterate,
229  arma::mat& gradient,
230  const arma::mat& searchDirection);
231 
240  void SearchDirection(const arma::mat& gradient,
241  const size_t iterationNum,
242  const double scalingFactor,
243  arma::mat& searchDirection);
244 
256  void UpdateBasisSet(const size_t iterationNum,
257  const arma::mat& iterate,
258  const arma::mat& oldIterate,
259  const arma::mat& gradient,
260  const arma::mat& oldGradient);
261 };
262 
263 } // namespace optimization
264 } // namespace mlpack
265 
266 #include "lbfgs_impl.hpp"
267 
268 #endif // __MLPACK_CORE_OPTIMIZERS_LBFGS_LBFGS_HPP
269 
double factr
Minimum relative function value decrease to continue the optimization.
Definition: lbfgs.hpp:177
double ArmijoConstant() const
Get the Armijo condition constant.
Definition: lbfgs.hpp:121
void SearchDirection(const arma::mat &gradient, const size_t iterationNum, const double scalingFactor, arma::mat &searchDirection)
Find the L-BFGS search direction.
Linear algebra utility functions, generally performed on matrices or vectors.
double Factr() const
Get the factr value.
Definition: lbfgs.hpp:136
size_t & MaxIterations()
Modify the maximum number of iterations.
Definition: lbfgs.hpp:118
double Wolfe() const
Get the Wolfe parameter.
Definition: lbfgs.hpp:126
size_t maxLineSearchTrials
Maximum number of trials for the line search.
Definition: lbfgs.hpp:179
arma::cube y
Stores all the y matrices in memory.
Definition: lbfgs.hpp:164
size_t & NumBasis()
Modify the memory size.
Definition: lbfgs.hpp:113
L_BFGS(FunctionType &function, const size_t numBasis=10, const size_t maxIterations=0, const double armijoConstant=1e-4, const double wolfe=0.9, const double minGradientNorm=1e-6, const double factr=1e-15, const size_t maxLineSearchTrials=50, const double minStep=1e-20, const double maxStep=1e20)
Initialize the L-BFGS object.
size_t MaxIterations() const
Get the maximum number of iterations.
Definition: lbfgs.hpp:116
bool GradientNormTooSmall(const arma::mat &gradient)
Check to make sure that the norm of the gradient is not smaller than 1e-5.
double & ArmijoConstant()
Modify the Armijo condition constant.
Definition: lbfgs.hpp:123
double ChooseScalingFactor(const size_t iterationNum, const arma::mat &gradient)
Calculate the scaling factor, gamma, which is used to scale the Hessian approximation matrix...
double minGradientNorm
Minimum gradient norm required to continue the optimization.
Definition: lbfgs.hpp:175
double maxStep
Maximum step of the line search.
Definition: lbfgs.hpp:183
double minStep
Minimum step of the line search.
Definition: lbfgs.hpp:181
double & MinGradientNorm()
Modify the minimum gradient norm.
Definition: lbfgs.hpp:133
double armijoConstant
Parameter for determining the Armijo condition.
Definition: lbfgs.hpp:171
double & MaxStep()
Modify the maximum line search step size.
Definition: lbfgs.hpp:153
size_t & MaxLineSearchTrials()
Modify the maximum number of line search trials.
Definition: lbfgs.hpp:143
bool LineSearch(double &functionValue, arma::mat &iterate, arma::mat &gradient, const arma::mat &searchDirection)
Perform a back-tracking line search along the search direction to calculate a step size satisfying th...
FunctionType & Function()
Modify the function that is being optimized.
Definition: lbfgs.hpp:108
double & Wolfe()
Modify the Wolfe parameter.
Definition: lbfgs.hpp:128
const FunctionType & Function() const
Return the function that is being optimized.
Definition: lbfgs.hpp:106
Include all of the base components required to write MLPACK methods, and the main MLPACK Doxygen docu...
arma::mat newIterateTmp
Position of the new iterate.
Definition: lbfgs.hpp:160
arma::cube s
Stores all the s matrices in memory.
Definition: lbfgs.hpp:162
double & MinStep()
Modify the minimum line search step size.
Definition: lbfgs.hpp:148
size_t maxIterations
Maximum number of iterations.
Definition: lbfgs.hpp:169
size_t NumBasis() const
Get the memory size.
Definition: lbfgs.hpp:111
size_t MaxLineSearchTrials() const
Get the maximum number of line search trials.
Definition: lbfgs.hpp:141
const std::pair< arma::mat, double > & MinPointIterate() const
Return the point where the lowest function value has been found.
double Evaluate(const arma::mat &iterate)
Evaluate the function at the given iterate point and store the result if it is a new minimum...
std::pair< arma::mat, double > minPointIterate
Best point found so far.
Definition: lbfgs.hpp:186
double Optimize(arma::mat &iterate)
Use L-BFGS to optimize the given function, starting at the given iterate point and finding the minimu...
The generic L-BFGS optimizer, which uses a back-tracking line search algorithm to minimize a function...
Definition: lbfgs.hpp:36
double wolfe
Parameter for detecting the Wolfe condition.
Definition: lbfgs.hpp:173
double MinStep() const
Return the minimum line search step size.
Definition: lbfgs.hpp:146
double & Factr()
Modify the factr value.
Definition: lbfgs.hpp:138
double MaxStep() const
Return the maximum line search step size.
Definition: lbfgs.hpp:151
double MinGradientNorm() const
Get the minimum gradient norm.
Definition: lbfgs.hpp:131
void UpdateBasisSet(const size_t iterationNum, const arma::mat &iterate, const arma::mat &oldIterate, const arma::mat &gradient, const arma::mat &oldGradient)
Update the y and s matrices, which store the differences between the iterate and old iterate and the ...
size_t numBasis
Size of memory for this L-BFGS optimizer.
Definition: lbfgs.hpp:167