Gradient Descent Methods¶
Steepest Descent¶
Using Packages¶
Unconstrained Minimization¶
Example problem is
\[f(x) = \sum_{i = 1}^{N-1} \:100(x_i - x_{i-1}^{2})\]
Nelder-Mead Method¶
Function calls only
Needs a starting vector that is non-degenerate
import numpy as np
from scipy.optimize import minimize
def rosen(x):
"""The Rosenbrock function"""
return sum(100.0*(x[1:]-x[:-1]**2.0)**2.0 + (1-x[:-1])**2.0)
x0 = np.array([1.3, 0.7, 0.8, 1.9, 1.2])
res = minimize(rosen, x0, method='nelder-mead',options={'disp': True})
print(res.x)
Optimization terminated successfully.
Current function value: 0.000066
Iterations: 141
Function evaluations: 243
[0.99910115 0.99820923 0.99646346 0.99297555 0.98600385]
Powell’s Direction Set Method¶
Function calls only
Needs a starting vector that is non-degenerate
import numpy as np
from scipy.optimize import minimize
def rosen(x):
"""The Rosenbrock function"""
return sum(100.0*(x[1:]-x[:-1]**2.0)**2.0 + (1-x[:-1])**2.0)
x0 = np.array([1.3, 0.7, 0.8, 1.9, 1.2])
res = minimize(rosen, x0, method='powell',options={'disp': True})
print(res.x)
Optimization terminated successfully.
Current function value: 0.000000
Iterations: 18
Function evaluations: 1084
[1. 1. 1. 1. 1.]
def rosen(x):
nminus1 = len(x)-1
acc = 0.0
for i in range(nminus1):
acc = acc + 100*(x[i+1]-x[i]**2)
return acc
import numpy as np
from scipy.optimize import minimize
#Rosenbrock Function
def fun_rosenbrock(x):
return np.array([10 * (x[1] - x[0]**2), (1 - x[0])])
from scipy.optimize import least_squares
input = np.array([2, 2])
res = least_squares(fun_rosenbrock, input)
print(res)
active_mask: array([0., 0.])
cost: 2.2211364862629114e-29
fun: array([-6.66133815e-15, -2.22044605e-16])
grad: array([ 1.33448809e-13, -6.66133815e-14])
jac: array([[-20.00000015, 10. ],
[ -1. , 0. ]])
message: '`gtol` termination condition is satisfied.'
nfev: 3
njev: 3
optimality: 1.3344880855256055e-13
status: 1
success: True
x: array([1., 1.])
def rosen_der(x):
xm = x[1:-1]
xm_m1 = x[:-2]
xm_p1 = x[2:]
der = np.zeros_like(x)
der[1:-1] = 200*(xm-xm_m1**2) - 400*(xm_p1 - xm**2)*xm - 2*(1-xm)
der[0] = -400*x[0]*(x[1]-x[0]**2) - 2*(1-x[0])
der[-1] = 200*(x[-1]-x[-2]**2)
return der
from scipy.optimize import Bounds
bounds = Bounds([0, -0.5], [1.0, 2.0])
ineq_cons = {'type': 'ineq',
'fun' : lambda x: np.array([1 - x[0] - 2*x[1],
1 - x[0]**2 - x[1],
1 - x[0]**2 + x[1]]),
# 'jac' : lambda x: np.array([[-1.0, -2.0],
# [-2*x[0], -1.0],
# [-2*x[0], 1.0]])
}
eq_cons = {'type': 'eq',
'fun' : lambda x: np.array([2*x[0] + x[1] - 1]),
# 'jac' : lambda x: np.array([2.0, 1.0])
}
x0 = np.array([0.5, 0])
res = minimize(rosen, x0, method='SLSQP', jac="2-point",
constraints=[eq_cons, ineq_cons], options={'ftol': 1e-9, 'disp': True},
bounds=bounds)
Optimization terminated successfully (Exit mode 0)
Current function value: -100.0000009896211
Iterations: 10
Function evaluations: 38
Gradient evaluations: 6
res
fun: -100.0000009896211
jac: array([-146.41016388, 100. ])
message: 'Optimization terminated successfully'
nfev: 38
nit: 10
njev: 6
status: 0
success: True
x: array([ 0.73205081, -0.46410162])