test function list¶

In this part, we will show gradient descent optimization on various 2-dimensioanl test function from this wiki page wiki/test-function-for-optimization.

We will plot the function landscape around the optimal point and also the optimization path.

We choose the initial point for optimization uniformly from the range $[-1, 1]$, so it could be biased to the local optimal point around the origin.

In [1]:

import numpy as np
import matplotlib.pyplot as plt
import torch

try:
    import numqi
except ImportError:
    %pip install numqi
    import numqi

np_rng = np.random.default_rng()
hf_uniform_para = lambda *x: torch.nn.Parameter(torch.tensor(np_rng.uniform(-1, 1, size=x), dtype=torch.float64))

import numpy as np import matplotlib.pyplot as plt import torch try: import numqi except ImportError: %pip install numqi import numqi np_rng = np.random.default_rng() hf_uniform_para = lambda *x: torch.nn.Parameter(torch.tensor(np_rng.uniform(-1, 1, size=x), dtype=torch.float64))

In [2]:

def hf_demo(model, num_repeat=3, xlim=None, ylim=None, x_optim=None, tag_logscale=False):
    callback = numqi.optimize.MinimizeCallback(extra_key='path', print_freq=0)
    theta_optim = numqi.optimize.minimize(model, tol=1e-10, num_repeat=num_repeat, callback=callback)
    path = np.stack(callback.state['path'])
    print(f'optimal theta: {theta_optim.x}')
    print(f'optimal loss: {theta_optim.fun}')

    hf_model = numqi.optimize.hf_model_wrapper(model)
    xdata = np.linspace(*xlim, 100)
    ydata = np.linspace(*ylim, 100)
    if len(x_optim)==2:
        zdata = np.array([[hf_model(np.array([x, y]), tag_grad=False) for x in xdata] for y in ydata])
    else:
        zdata = np.array([[hf_model(np.concatenate([np.array([x, y]),x_optim[2:]]), tag_grad=False) for x in xdata] for y in ydata])
    if tag_logscale:
        zdata = np.log(np.maximum(1e-4,zdata))
    fig,ax = plt.subplots()
    contour_set = ax.contourf(xdata, ydata, zdata, levels=15, cmap='winter')
    fig.colorbar(contour_set)
    ax.plot([x_optim[0]], [x_optim[1]], 'rx', label='optimal')
    ax.plot(path[:,0], path[:,1], '-', color='orange', label='path')
    ax.set_xlim(*xlim)
    ax.set_ylim(*ylim)
    ax.legend()
    ax.set_xlabel('x')
    ax.set_ylabel('y')
    if tag_logscale:
        ax.set_title('log scale plot of loss function')
    fig.tight_layout()

def hf_demo(model, num_repeat=3, xlim=None, ylim=None, x_optim=None, tag_logscale=False): callback = numqi.optimize.MinimizeCallback(extra_key='path', print_freq=0) theta_optim = numqi.optimize.minimize(model, tol=1e-10, num_repeat=num_repeat, callback=callback) path = np.stack(callback.state['path']) print(f'optimal theta: {theta_optim.x}') print(f'optimal loss: {theta_optim.fun}') hf_model = numqi.optimize.hf_model_wrapper(model) xdata = np.linspace(*xlim, 100) ydata = np.linspace(*ylim, 100) if len(x_optim)==2: zdata = np.array([[hf_model(np.array([x, y]), tag_grad=False) for x in xdata] for y in ydata]) else: zdata = np.array([[hf_model(np.concatenate([np.array([x, y]),x_optim[2:]]), tag_grad=False) for x in xdata] for y in ydata]) if tag_logscale: zdata = np.log(np.maximum(1e-4,zdata)) fig,ax = plt.subplots() contour_set = ax.contourf(xdata, ydata, zdata, levels=15, cmap='winter') fig.colorbar(contour_set) ax.plot([x_optim[0]], [x_optim[1]], 'rx', label='optimal') ax.plot(path[:,0], path[:,1], '-', color='orange', label='path') ax.set_xlim(*xlim) ax.set_ylim(*ylim) ax.legend() ax.set_xlabel('x') ax.set_ylabel('y') if tag_logscale: ax.set_title('log scale plot of loss function') fig.tight_layout()

Rastrigin function¶

$$ f(x)=An+\sum_{i=1}^n [x_i^2-A\cos(2\pi x_i)], A=10$$

$$ f(0,\cdots,0)=0 $$

In [3]:

class Rastrigin(torch.nn.Module):
    def __init__(self, n):
        super().__init__()
        self.x = hf_uniform_para(n)
        self.A = 10
        # solution [0,0,...,0], 0

    def forward(self):
        x = self.x
        loss = self.A*x.shape[0] + (x*x - self.A*torch.cos(2*np.pi*x)).sum()
        return loss

n = 2
model = Rastrigin(n)
x_optim = np.zeros(n)
hf_demo(model, num_repeat=10, xlim=(-5.12, 5.12), ylim=(-5.12, 5.12), x_optim=x_optim)
# when n is large, it's almost impossible to find the global minimum

class Rastrigin(torch.nn.Module): def __init__(self, n): super().__init__() self.x = hf_uniform_para(n) self.A = 10 # solution [0,0,...,0], 0 def forward(self): x = self.x loss = self.A*x.shape[0] + (x*x - self.A*torch.cos(2*np.pi*x)).sum() return loss n = 2 model = Rastrigin(n) x_optim = np.zeros(n) hf_demo(model, num_repeat=10, xlim=(-5.12, 5.12), ylim=(-5.12, 5.12), x_optim=x_optim) # when n is large, it's almost impossible to find the global minimum

[round=0] min(f)=3.979831190554144, current(f)=3.979831190554144
[round=1] min(f)=0.9949590570932969, current(f)=0.9949590570932969
[round=2] min(f)=0.9949590570932898, current(f)=0.9949590570932898
[round=3] min(f)=0.0, current(f)=0.0
[round=4] min(f)=0.0, current(f)=1.9899181141865832
[round=5] min(f)=0.0, current(f)=0.9949590570932969
[round=6] min(f)=0.0, current(f)=1.9899181141865832
[round=7] min(f)=0.0, current(f)=1.989918114186601
[round=8] min(f)=0.0, current(f)=0.9949590570932898
[round=9] min(f)=0.0, current(f)=7.959662381108174
optimal theta: [-3.84273733e-10  2.66948780e-10]
optimal loss: 0.0

Ackley function¶

$$ f(x,y)=-20\mathrm{exp}[-0.2\sqrt{0.5(x^2+y^2)}] - \mathrm{exp}[0.5(\cos(2\pi x)+\cos(2\pi y))] + e + 20 $$

$$ f(0,0)=0 $$

In [4]:

class Ackley(torch.nn.Module):
    def __init__(self):
        super().__init__()
        self.theta = hf_uniform_para(2)
        # solution [0,0] 0

    def forward(self):
        x,y = self.theta
        tmp0 = -20*torch.exp(-0.2*torch.sqrt(0.5*(x*x + y*y)))
        tmp1 = -torch.exp(0.5*(torch.cos(2*np.pi*x)+torch.cos(2*np.pi*y))) + np.e + 20
        ret = tmp0 + tmp1
        return ret

model = Ackley()
x_optim = np.zeros(2)
hf_demo(model, num_repeat=10, xlim=(-5, 5), ylim=(-5, 5), x_optim=x_optim)

class Ackley(torch.nn.Module): def __init__(self): super().__init__() self.theta = hf_uniform_para(2) # solution [0,0] 0 def forward(self): x,y = self.theta tmp0 = -20*torch.exp(-0.2*torch.sqrt(0.5*(x*x + y*y))) tmp1 = -torch.exp(0.5*(torch.cos(2*np.pi*x)+torch.cos(2*np.pi*y))) + np.e + 20 ret = tmp0 + tmp1 return ret model = Ackley() x_optim = np.zeros(2) hf_demo(model, num_repeat=10, xlim=(-5, 5), ylim=(-5, 5), x_optim=x_optim)

[round=0] min(f)=9.019274216370832e-11, current(f)=9.019274216370832e-11
[round=1] min(f)=9.019274216370832e-11, current(f)=3.5744518772576797
[round=2] min(f)=5.0754067615343956e-11, current(f)=5.0754067615343956e-11
[round=3] min(f)=5.0754067615343956e-11, current(f)=8.823519692668924e-11
[round=4] min(f)=5.0754067615343956e-11, current(f)=5.0818016461562365e-11
[round=5] min(f)=5.0754067615343956e-11, current(f)=6.419043074856745e-11
[round=6] min(f)=5.0754067615343956e-11, current(f)=3.574451877257683
[round=7] min(f)=5.0754067615343956e-11, current(f)=2.5799275570298725
[round=8] min(f)=5.0754067615343956e-11, current(f)=6.552269837811764e-11
[round=9] min(f)=5.0754067615343956e-11, current(f)=2.579927557029876
optimal theta: [-3.61427855e-12  1.75769978e-11]
optimal loss: 5.0754067615343956e-11

Rosenbrock function¶

$$ f(x)=\sum_{i=1}^{n-1}[100(x_{i+1}-x_i^2)^2 + (1-x_i)^2] $$

$$ f(1,\cdots,1)=0 $$

In [5]:

class Rosenbrock(torch.nn.Module):
    def __init__(self, n):
        super().__init__()
        self.theta = hf_uniform_para(n)
        # solution [1,1,...,1] 0

    def forward(self):
        tmp0 = self.theta[1:] - self.theta[:-1]**2
        tmp1 = 1-self.theta[:-1]
        ret = 100*torch.dot(tmp0, tmp0) + torch.dot(tmp1,tmp1)
        return ret

n = 2
model = Rosenbrock(n)
x_optim = np.ones(n)
hf_demo(model, num_repeat=10, xlim=(-2, 2), ylim=(-1, 3), x_optim=x_optim, tag_logscale=True)

class Rosenbrock(torch.nn.Module): def __init__(self, n): super().__init__() self.theta = hf_uniform_para(n) # solution [1,1,...,1] 0 def forward(self): tmp0 = self.theta[1:] - self.theta[:-1]**2 tmp1 = 1-self.theta[:-1] ret = 100*torch.dot(tmp0, tmp0) + torch.dot(tmp1,tmp1) return ret n = 2 model = Rosenbrock(n) x_optim = np.ones(n) hf_demo(model, num_repeat=10, xlim=(-2, 2), ylim=(-1, 3), x_optim=x_optim, tag_logscale=True)

[round=0] min(f)=1.7971290582349114e-18, current(f)=1.7971290582349114e-18
[round=1] min(f)=2.773232190186765e-20, current(f)=2.773232190186765e-20
[round=2] min(f)=2.773232190186765e-20, current(f)=2.6410879016286126e-16
[round=3] min(f)=2.773232190186765e-20, current(f)=1.9782756615671537e-19
[round=4] min(f)=2.773232190186765e-20, current(f)=1.1283200280291766e-14
[round=5] min(f)=2.773232190186765e-20, current(f)=2.6785466048288873e-19
[round=6] min(f)=2.773232190186765e-20, current(f)=2.5675095947541786e-16
[round=7] min(f)=2.773232190186765e-20, current(f)=6.871924863159172e-15
[round=8] min(f)=2.773232190186765e-20, current(f)=1.922609756083945e-17
[round=9] min(f)=2.773232190186765e-20, current(f)=1.8247711806972154e-19
optimal theta: [1. 1.]
optimal loss: 2.773232190186765e-20

Beale function¶

$$ f(x,y)=(1.5-x+xy)^2 + (2.25-x+xy^2)^2 + (2.625-x+xy^3)^2 $$

$$ f(3,0.5)=0 $$

In [6]:

class Beale(torch.nn.Module):
    def __init__(self) -> None:
        super().__init__()
        self.theta = hf_uniform_para(2)
        # [3,0.5] 0

    def forward(self):
        x,y = self.theta
        ret = (1.5 - x + x*y)**2 + (2.25 - x + x*y**2)**2 + (2.625 - x + x*y**3)**2
        return ret

model = Beale()
x_optim = np.array([3, 0.5])
hf_demo(model, num_repeat=10, xlim=(-4.5, 4.5), ylim=(-4.5, 4.5), x_optim=x_optim, tag_logscale=True)

class Beale(torch.nn.Module): def __init__(self) -> None: super().__init__() self.theta = hf_uniform_para(2) # [3,0.5] 0 def forward(self): x,y = self.theta ret = (1.5 - x + x*y)**2 + (2.25 - x + x*y**2)**2 + (2.625 - x + x*y**3)**2 return ret model = Beale() x_optim = np.array([3, 0.5]) hf_demo(model, num_repeat=10, xlim=(-4.5, 4.5), ylim=(-4.5, 4.5), x_optim=x_optim, tag_logscale=True)

[round=0] min(f)=0.45340812414290266, current(f)=0.45340812414290266
[round=1] min(f)=8.850474420258804e-19, current(f)=8.850474420258804e-19
[round=2] min(f)=8.850474420258804e-19, current(f)=2.1196636791003204e-16
[round=3] min(f)=9.250682044357888e-21, current(f)=9.250682044357888e-21
[round=4] min(f)=9.250682044357888e-21, current(f)=1.1741450056601588e-20
[round=5] min(f)=9.250682044357888e-21, current(f)=2.873580616207381e-16
[round=6] min(f)=9.250682044357888e-21, current(f)=9.106850793062285e-18
[round=7] min(f)=9.250682044357888e-21, current(f)=1.3203596835271353e-18
[round=8] min(f)=9.250682044357888e-21, current(f)=2.5546702642102734e-15
[round=9] min(f)=9.250682044357888e-21, current(f)=8.424591766475669e-17
optimal theta: [3.  0.5]
optimal loss: 9.250682044357888e-21

Goldstein-Price function¶

$$ f(x,y)=[1+(x+y+1)^2(19-14x+3x^2-14y+6xy+3y^2)][30+(2x-3y)^2(18-32x+12x^2+48y-36xy+27y^2)]-3 $$

$$ f(0,-1)=0 $$

comment: the function value is shifted by a constant $3$ for good plotting.

In [7]:

class GoldsteinPrice(torch.nn.Module):
    def __init__(self) -> None:
        super().__init__()
        self.theta = hf_uniform_para(2)
        # [0,-1] 0

    def forward(self):
        x,y = self.theta
        tmp0 = 1 + (x+y+1)**2 * (19-14*x+3*x*x-14*y+6*x*y+3*y*y)
        tmp1 = 30 + (2*x-3*y)**2 * (18-32*x+12*x*x+48*y-36*x*y+27*y*y)
        ret = tmp0*tmp1 - 3 #shift 3 for good plotting
        return ret

model = GoldsteinPrice()
x_optim = np.array([0, -1])
hf_demo(model, num_repeat=10, xlim=(-2, 2), ylim=(-3, 1), x_optim=x_optim, tag_logscale=True)

class GoldsteinPrice(torch.nn.Module): def __init__(self) -> None: super().__init__() self.theta = hf_uniform_para(2) # [0,-1] 0 def forward(self): x,y = self.theta tmp0 = 1 + (x+y+1)**2 * (19-14*x+3*x*x-14*y+6*x*y+3*y*y) tmp1 = 30 + (2*x-3*y)**2 * (18-32*x+12*x*x+48*y-36*x*y+27*y*y) ret = tmp0*tmp1 - 3 #shift 3 for good plotting return ret model = GoldsteinPrice() x_optim = np.array([0, -1]) hf_demo(model, num_repeat=10, xlim=(-2, 2), ylim=(-3, 1), x_optim=x_optim, tag_logscale=True)

[round=0] min(f)=80.99999999999946, current(f)=80.99999999999946
[round=1] min(f)=27.0, current(f)=27.0
[round=2] min(f)=7.105427357601002e-15, current(f)=7.105427357601002e-15
[round=3] min(f)=7.105427357601002e-15, current(f)=7.105427357601002e-15
[round=4] min(f)=7.105427357601002e-15, current(f)=27.0
[round=5] min(f)=7.105427357601002e-15, current(f)=836.9999999999997
[round=6] min(f)=0.0, current(f)=0.0
[round=7] min(f)=0.0, current(f)=2.1316282072803006e-14
[round=8] min(f)=0.0, current(f)=6.394884621840902e-14
[round=9] min(f)=0.0, current(f)=1.7763568394002505e-14
optimal theta: [-2.61030413e-10 -1.00000000e+00]
optimal loss: 0.0

Booth function¶

$$ f(x,y)=(x+2y-7)^2 + (2x+y-5)^2 $$

$$ f(1,3)=0 $$

In [8]:

class Booth(torch.nn.Module):
    def __init__(self) -> None:
        super().__init__()
        self.theta = hf_uniform_para(2)
        # [1,3] 0

    def forward(self):
        x,y = self.theta
        tmp0 = x+2*y-7
        tmp1 = 2*x+y-5
        ret = tmp0*tmp0 + tmp1*tmp1
        return ret

model = Booth()
x_optim = np.array([1, 3])
hf_demo(model, num_repeat=10, xlim=(-10, 10), ylim=(-10, 10), x_optim=x_optim, tag_logscale=True)

class Booth(torch.nn.Module): def __init__(self) -> None: super().__init__() self.theta = hf_uniform_para(2) # [1,3] 0 def forward(self): x,y = self.theta tmp0 = x+2*y-7 tmp1 = 2*x+y-5 ret = tmp0*tmp0 + tmp1*tmp1 return ret model = Booth() x_optim = np.array([1, 3]) hf_demo(model, num_repeat=10, xlim=(-10, 10), ylim=(-10, 10), x_optim=x_optim, tag_logscale=True)

[round=0] min(f)=1.8406491501521868e-26, current(f)=1.8406491501521868e-26
[round=1] min(f)=1.8406491501521868e-26, current(f)=2.2157302813369593e-16
[round=2] min(f)=9.947536014836959e-27, current(f)=9.947536014836959e-27
[round=3] min(f)=9.947536014836959e-27, current(f)=3.12325809595103e-26
[round=4] min(f)=3.1562324817892682e-27, current(f)=3.1562324817892682e-27
[round=5] min(f)=3.1562324817892682e-27, current(f)=3.719551355200786e-21
[round=6] min(f)=3.1562324817892682e-27, current(f)=6.089225390269416e-19
[round=7] min(f)=3.1562324817892682e-27, current(f)=3.878225466146488e-17
[round=8] min(f)=3.1562324817892682e-27, current(f)=1.6599562587807312e-18
[round=9] min(f)=6.310887241768095e-30, current(f)=6.310887241768095e-30
optimal theta: [1. 3.]
optimal loss: 6.310887241768095e-30

Bukin Function N.6¶

$$ f(x,y)=100\sqrt{|y-0.01x^2|}+0.01|x+10| $$

$$ f(-10,1)=0 $$

In [9]:

class BukinFunctionN6(torch.nn.Module):
    def __init__(self) -> None:
        super().__init__()
        self.theta = hf_uniform_para(2)
        # [-10,1] 0

    def forward(self):
        x,y = self.theta
        # possible numerical unstable here, also hard for Adam
        ret = 100*torch.sqrt(torch.abs(y-0.01*x*x)) + 0.01*torch.abs(x+10)
        return ret

model = BukinFunctionN6()
x_optim = np.array([-10, 1])
hf_demo(model, num_repeat=10, xlim=(-15, -5), ylim=(-4, 6), x_optim=x_optim, tag_logscale=True)

class BukinFunctionN6(torch.nn.Module): def __init__(self) -> None: super().__init__() self.theta = hf_uniform_para(2) # [-10,1] 0 def forward(self): x,y = self.theta # possible numerical unstable here, also hard for Adam ret = 100*torch.sqrt(torch.abs(y-0.01*x*x)) + 0.01*torch.abs(x+10) return ret model = BukinFunctionN6() x_optim = np.array([-10, 1]) hf_demo(model, num_repeat=10, xlim=(-15, -5), ylim=(-4, 6), x_optim=x_optim, tag_logscale=True)

[round=0] min(f)=0.4732169099356066, current(f)=0.4732169099356066
[round=1] min(f)=0.4293510676920937, current(f)=0.4293510676920937
[round=2] min(f)=0.4293510676920937, current(f)=4.68048440199357
[round=3] min(f)=0.4293510676920937, current(f)=454.65341950800143
[round=4] min(f)=0.4293510676920937, current(f)=61.93726162885764
[round=5] min(f)=0.22523087613117052, current(f)=0.22523087613117052
[round=6] min(f)=0.22523087613117052, current(f)=1460.7500579128332
[round=7] min(f)=0.22523087613117052, current(f)=23.483118580121804

[round=8] min(f)=0.22523087613117052, current(f)=82.35946329669395
[round=9] min(f)=0.22523087613117052, current(f)=12.491087541121109
optimal theta: [0.94342342 0.00903503]
optimal loss: 0.22523087613117052

Matyas function¶

$$ f(x,y)=0.26(x^2+y^2)-0.48xy $$

$$ f(0,0)=0 $$

In [10]:

class Matyas(torch.nn.Module):
    def __init__(self) -> None:
        super().__init__()
        self.theta = hf_uniform_para(2)
        # [-0,0] 0

    def forward(self):
        x,y = self.theta
        ret = 0.26*(x*x + y*y) - 0.48*x*y
        return ret

model = Matyas()
x_optim = np.array([0, 0])
hf_demo(model, num_repeat=10, xlim=(-10, 10), ylim=(-10, 10), x_optim=x_optim, tag_logscale=True)

class Matyas(torch.nn.Module): def __init__(self) -> None: super().__init__() self.theta = hf_uniform_para(2) # [-0,0] 0 def forward(self): x,y = self.theta ret = 0.26*(x*x + y*y) - 0.48*x*y return ret model = Matyas() x_optim = np.array([0, 0]) hf_demo(model, num_repeat=10, xlim=(-10, 10), ylim=(-10, 10), x_optim=x_optim, tag_logscale=True)

[round=0] min(f)=3.276854244578216e-31, current(f)=3.276854244578216e-31
[round=1] min(f)=3.276854244578216e-31, current(f)=2.488334850544609e-18
[round=2] min(f)=3.276854244578216e-31, current(f)=7.002273850429576e-28
[round=3] min(f)=3.3971786738764116e-38, current(f)=3.3971786738764116e-38
[round=4] min(f)=3.3971786738764116e-38, current(f)=1.1308538279455482e-16
[round=5] min(f)=3.3971786738764116e-38, current(f)=2.7345123722387732e-31
[round=6] min(f)=3.3971786738764116e-38, current(f)=2.041485741050471e-34
[round=7] min(f)=3.3971786738764116e-38, current(f)=1.4496591359086796e-18
[round=8] min(f)=3.3971786738764116e-38, current(f)=8.091987254337407e-32
[round=9] min(f)=3.3971786738764116e-38, current(f)=2.9070756952558668e-31
optimal theta: [9.21571847e-19 9.21571847e-19]
optimal loss: 3.3971786738764116e-38

Levi function N.13¶

$$ f(x,y)=\sin^2(3\pi x)+ (x-1)^2(1+\sin^2(3\pi y)) + (y-1)^2(1+\sin^2(2\pi y)) $$

$$ f(1,1)=0 $$

In [11]:

class LeviFunction(torch.nn.Module):
    def __init__(self) -> None:
        super().__init__()
        self.theta = hf_uniform_para(2)
        # [0,0] 0

    def forward(self):
        x,y = self.theta
        ret = torch.sin(3*np.pi*x)**2 + (x-1)**2*(1+torch.sin(3*np.pi*y)**2) + (y-1)**2*(1+torch.sin(2*np.pi*y)**2)
        return ret

model = LeviFunction()
x_optim = np.array([1, 1])
hf_demo(model, num_repeat=10, xlim=(-10, 10), ylim=(-10, 10), x_optim=x_optim)

class LeviFunction(torch.nn.Module): def __init__(self) -> None: super().__init__() self.theta = hf_uniform_para(2) # [0,0] 0 def forward(self): x,y = self.theta ret = torch.sin(3*np.pi*x)**2 + (x-1)**2*(1+torch.sin(3*np.pi*y)**2) + (y-1)**2*(1+torch.sin(2*np.pi*y)**2) return ret model = LeviFunction() x_optim = np.array([1, 1]) hf_demo(model, num_repeat=10, xlim=(-10, 10), ylim=(-10, 10), x_optim=x_optim)

[round=0] min(f)=1.2550683095287517e-15, current(f)=1.2550683095287517e-15
[round=1] min(f)=1.2550683095287517e-15, current(f)=0.988826485986504
[round=2] min(f)=1.2550683095287517e-15, current(f)=1.0890993446202546
[round=3] min(f)=1.2550683095287517e-15, current(f)=2.7465302928244624
[round=4] min(f)=1.2550683095287517e-15, current(f)=0.6316425103184422
[round=5] min(f)=1.2550683095287517e-15, current(f)=2.4383888177766733
[round=6] min(f)=1.2550683095287517e-15, current(f)=1.073324641073469
[round=7] min(f)=1.2550683095287517e-15, current(f)=4.4190344336411025
[round=8] min(f)=1.2550683095287517e-15, current(f)=1.9809214686955976
[round=9] min(f)=1.2550683095287517e-15, current(f)=3.9585513262103103
optimal theta: [1.         1.00000003]
optimal loss: 1.2550683095287517e-15

Himmelblau's function¶

$$ f(x,y)=(x^2+y-11)^2 + (x+y^2-7)^2 $$

$$ f(3,2)=f(-2.805118,3.131312)=f(-3.779310,-3.283186)=f(3.584428,-1.848126)=0 $$

In [12]:

class Himmelblau(torch.nn.Module):
    def __init__(self) -> None:
        super().__init__()
        self.theta = hf_uniform_para(2)
        # [3,2],[-2.805118,3.131312],[-3.779310,-3.283186],[3.584428,-1.848126] 0

    def forward(self):
        x,y = self.theta
        tmp0 = x*x + y - 11
        tmp1 = x + y*y - 7
        ret = tmp0*tmp0 + tmp1*tmp1
        return ret

model = Himmelblau()
x_optim = np.array([[3, 2],[-2.805118,3.131312],[-3.779310,-3.283186],[3.584428,-1.848126]])
hf_demo(model, num_repeat=10, xlim=(-5, 5), ylim=(-5, 5), x_optim=x_optim[0], tag_logscale=True)

class Himmelblau(torch.nn.Module): def __init__(self) -> None: super().__init__() self.theta = hf_uniform_para(2) # [3,2],[-2.805118,3.131312],[-3.779310,-3.283186],[3.584428,-1.848126] 0 def forward(self): x,y = self.theta tmp0 = x*x + y - 11 tmp1 = x + y*y - 7 ret = tmp0*tmp0 + tmp1*tmp1 return ret model = Himmelblau() x_optim = np.array([[3, 2],[-2.805118,3.131312],[-3.779310,-3.283186],[3.584428,-1.848126]]) hf_demo(model, num_repeat=10, xlim=(-5, 5), ylim=(-5, 5), x_optim=x_optim[0], tag_logscale=True)

[round=0] min(f)=7.025728504878649e-19, current(f)=7.025728504878649e-19
[round=1] min(f)=7.025728504878649e-19, current(f)=1.411981621409988e-17
[round=2] min(f)=2.510588642822407e-22, current(f)=2.510588642822407e-22
[round=3] min(f)=2.510588642822407e-22, current(f)=5.847626564376177e-17
[round=4] min(f)=2.510588642822407e-22, current(f)=7.246908517804936e-15
[round=5] min(f)=2.510588642822407e-22, current(f)=1.1331638897341891e-17
[round=6] min(f)=2.510588642822407e-22, current(f)=3.823768523540559e-21
[round=7] min(f)=2.510588642822407e-22, current(f)=2.3604223574980464e-12
[round=8] min(f)=2.510588642822407e-22, current(f)=4.593760546732333e-15
[round=9] min(f)=2.510588642822407e-22, current(f)=2.4439003941229784e-18
optimal theta: [-2.80511809  3.13131252]
optimal loss: 2.510588642822407e-22

Three-hump camel function¶

$$ f(x,y)=2x^2-1.05x^4+\frac{x^6}{6}+xy+y^2 $$

$$ f(0,0)=0 $$

In [13]:

class ThreeHumpCamel(torch.nn.Module):
    def __init__(self) -> None:
        super().__init__()
        self.theta = hf_uniform_para(2)
        # [0,0] 0

    def forward(self):
        x,y = self.theta
        ret = 2*x*x - 1.05*x*x*x*x + x*x*x*x*x*x/6 + x*y + y*y
        return ret

model = ThreeHumpCamel()
x_optim = np.array([0, 0])
hf_demo(model, num_repeat=10, xlim=(-5, 5), ylim=(-5, 5), x_optim=x_optim, tag_logscale=True)

class ThreeHumpCamel(torch.nn.Module): def __init__(self) -> None: super().__init__() self.theta = hf_uniform_para(2) # [0,0] 0 def forward(self): x,y = self.theta ret = 2*x*x - 1.05*x*x*x*x + x*x*x*x*x*x/6 + x*y + y*y return ret model = ThreeHumpCamel() x_optim = np.array([0, 0]) hf_demo(model, num_repeat=10, xlim=(-5, 5), ylim=(-5, 5), x_optim=x_optim, tag_logscale=True)

[round=0] min(f)=1.918820813341499e-14, current(f)=1.918820813341499e-14
[round=1] min(f)=5.453726835885964e-20, current(f)=5.453726835885964e-20
[round=2] min(f)=5.453726835885964e-20, current(f)=4.560636686973354e-17
[round=3] min(f)=1.8358380319335708e-24, current(f)=1.8358380319335708e-24
[round=4] min(f)=1.8358380319335708e-24, current(f)=5.25251671997231e-20
[round=5] min(f)=1.8358380319335708e-24, current(f)=3.813070433132284e-17
[round=6] min(f)=1.8358380319335708e-24, current(f)=2.5913332299477873e-15
[round=7] min(f)=1.8358380319335708e-24, current(f)=2.4087468780385718e-14
[round=8] min(f)=1.8358380319335708e-24, current(f)=3.443653864905338e-18
[round=9] min(f)=1.8358380319335708e-24, current(f)=4.5444414777935776e-20
optimal theta: [7.32024353e-13 5.81660573e-13]
optimal loss: 1.8358380319335708e-24

Easom function¶

$$ f(x,y)=-\cos(x)\cos(y)\exp[-(x-\pi)^2-(y-\pi)^2] $$

$$ f(\pi,\pi)=-1 $$

In [14]:

class Easom(torch.nn.Module):
    def __init__(self) -> None:
        super().__init__()
        self.theta = hf_uniform_para(2)
        # [pi,pi] -1

    def forward(self):
        x,y = self.theta
        ret = -torch.cos(x)*torch.cos(y)*torch.exp(-(x-np.pi)**2-(y-np.pi)**2)
        return ret

model = Easom()
x_optim = np.array([np.pi, np.pi])
hf_demo(model, num_repeat=10, xlim=(-100, 100), ylim=(-100, 100), x_optim=x_optim)

class Easom(torch.nn.Module): def __init__(self) -> None: super().__init__() self.theta = hf_uniform_para(2) # [pi,pi] -1 def forward(self): x,y = self.theta ret = -torch.cos(x)*torch.cos(y)*torch.exp(-(x-np.pi)**2-(y-np.pi)**2) return ret model = Easom() x_optim = np.array([np.pi, np.pi]) hf_demo(model, num_repeat=10, xlim=(-100, 100), ylim=(-100, 100), x_optim=x_optim)

[round=0] min(f)=-1.0, current(f)=-1.0
[round=1] min(f)=-1.0, current(f)=-0.9999999999999999
[round=2] min(f)=-1.0, current(f)=-1.0
[round=3] min(f)=-1.0, current(f)=-1.0
[round=4] min(f)=-1.0, current(f)=-1.0
[round=5] min(f)=-1.0, current(f)=-1.0
[round=6] min(f)=-1.0, current(f)=-1.0
[round=7] min(f)=-1.0, current(f)=-8.907024498671357e-16
[round=8] min(f)=-1.0, current(f)=-1.0
[round=9] min(f)=-1.0, current(f)=-1.0
optimal theta: [3.14159265 3.14159265]
optimal loss: -1.0

Cross-in-tray function¶

$$ f(x,y)=-0.0001\left[\left|\sin(x)\sin(y)\exp\left(\left|100-\frac{\sqrt{x^2+y^2}}{\pi}\right|\right)\right|+1\right]^{0.1} $$

$$ f(1.34941,-1.34941)=f(1.34941,1.34941)=f(-1.34941,1.34941)=f(-1.34941,-1.34941)=-2.06261 $$

In [15]:

class CrossInTray(torch.nn.Module):
    def __init__(self) -> None:
        super().__init__()
        self.theta = hf_uniform_para(2)
        # [1.34941,-1.34941],[1.34941,1.34941],[-1.34941,1.34941],[-1.34941,-1.34941] -2.06261

    def forward(self):
        x,y = self.theta
        tmp0 = torch.abs(100 - torch.sqrt(x*x + y*y)/np.pi)
        tmp1 = torch.abs(torch.sin(x)*torch.sin(y)*torch.exp(tmp0))
        ret = -0.0001*tmp1**0.1
        return ret

model = CrossInTray()
x_optim = np.array([[1.34941,-1.34941],[1.34941,1.34941],[-1.34941,1.34941],[-1.34941,-1.34941]])
hf_demo(model, num_repeat=10, xlim=(-10, 10), ylim=(-10, 10), x_optim=x_optim[0])

class CrossInTray(torch.nn.Module): def __init__(self) -> None: super().__init__() self.theta = hf_uniform_para(2) # [1.34941,-1.34941],[1.34941,1.34941],[-1.34941,1.34941],[-1.34941,-1.34941] -2.06261 def forward(self): x,y = self.theta tmp0 = torch.abs(100 - torch.sqrt(x*x + y*y)/np.pi) tmp1 = torch.abs(torch.sin(x)*torch.sin(y)*torch.exp(tmp0)) ret = -0.0001*tmp1**0.1 return ret model = CrossInTray() x_optim = np.array([[1.34941,-1.34941],[1.34941,1.34941],[-1.34941,1.34941],[-1.34941,-1.34941]]) hf_demo(model, num_repeat=10, xlim=(-10, 10), ylim=(-10, 10), x_optim=x_optim[0])

[round=0] min(f)=-2.0626118708227374, current(f)=-2.0626118708227374
[round=1] min(f)=-2.0626118708227374, current(f)=-2.0626118708225607
[round=2] min(f)=-2.0626118708227374, current(f)=-2.0626118708227144
[round=3] min(f)=-2.0626118708227374, current(f)=-2.0626118708223866
[round=4] min(f)=-2.0626118708227374, current(f)=-2.062611870822678
[round=5] min(f)=-2.0626118708227397, current(f)=-2.0626118708227397
[round=6] min(f)=-2.0626118708227397, current(f)=-2.062611870822491
[round=7] min(f)=-2.0626118708227397, current(f)=-2.0626118708227374
[round=8] min(f)=-2.0626118708227397, current(f)=-2.0626118708227255
[round=9] min(f)=-2.0626118708227397, current(f)=-2.062611870822715
optimal theta: [ 1.34940661 -1.34940662]
optimal loss: -2.0626118708227397

Eggholder function¶

$$ f(x,y)=-(y+47)\sin\sqrt{|x/2+(y+47)|}-x\sin\sqrt{|x-(y+47)|} $$

$$ f(512,404.2319)=-959.6407 $$

In [16]:

class Eggholder(torch.nn.Module):
    def __init__(self) -> None:
        super().__init__()
        self.theta = hf_uniform_para(2)
        # [512,404.2319] -959.6407

    def forward(self):
        x,y = self.theta
        tmp0 = -(y+47)*torch.sin(torch.sqrt(torch.abs(x/2 + (y+47))))
        tmp1 = -x*torch.sin(torch.sqrt(torch.abs(x-(y+47))))
        ret = tmp0 + tmp1
        return ret

model = Eggholder()
x_optim = np.array([512, 404.2319])
hf_demo(model, num_repeat=10, xlim=(-1000, 1000), ylim=(-1000, 1000), x_optim=x_optim)

class Eggholder(torch.nn.Module): def __init__(self) -> None: super().__init__() self.theta = hf_uniform_para(2) # [512,404.2319] -959.6407 def forward(self): x,y = self.theta tmp0 = -(y+47)*torch.sin(torch.sqrt(torch.abs(x/2 + (y+47)))) tmp1 = -x*torch.sin(torch.sqrt(torch.abs(x-(y+47)))) ret = tmp0 + tmp1 return ret model = Eggholder() x_optim = np.array([512, 404.2319]) hf_demo(model, num_repeat=10, xlim=(-1000, 1000), ylim=(-1000, 1000), x_optim=x_optim)

[round=0] min(f)=-66.84371732946398, current(f)=-66.84371732946398
[round=1] min(f)=-66.84371732946398, current(f)=-66.84371732946397
[round=2] min(f)=-66.84371732946398, current(f)=-66.84371732946398
[round=3] min(f)=-66.84371732946398, current(f)=-66.8437173294622
[round=4] min(f)=-66.84371732946398, current(f)=-66.84371732946394
[round=5] min(f)=-66.84371732946398, current(f)=-66.84371732946377
[round=6] min(f)=-66.84371732946401, current(f)=-66.84371732946401
[round=7] min(f)=-66.84371732946401, current(f)=-66.84371732946359
[round=8] min(f)=-66.84371732946401, current(f)=-66.84371732945586
[round=9] min(f)=-66.84371732946401, current(f)=-66.84371732946398
optimal theta: [ 8.45693449 15.65091816]
optimal loss: -66.84371732946401

Holder table function¶

$$ f(x,y)=-\left|\sin(x)\cos(y)\exp\left(\left|1-\frac{\sqrt{x^2+y^2}}{\pi}\right|\right)\right| $$

$$ f(8.05502,9.66459)=f(-8.05502, 9.66459)=f(8.05502, -9.66459)=f(-8.05502, -9.66459)=-19.2085 $$

In [17]:

class HolderTable(torch.nn.Module):
    def __init__(self) -> None:
        super().__init__()
        self.theta = hf_uniform_para(2)
        # [8.05502,9.66459] -19.2085

    def forward(self):
        x,y = self.theta
        tmp0 = torch.sin(x)*torch.cos(y)
        tmp1 = torch.exp(torch.abs(1 - torch.sqrt(x*x + y*y)/np.pi))
        ret = -torch.abs(tmp0*tmp1)
        return ret

model = HolderTable()
x_optim = np.array([[8.05502, 9.66459], [-8.05502, 9.66459], [8.05502, -9.66459], [-8.05502, -9.66459]])
hf_demo(model, num_repeat=10, xlim=(-10, 10), ylim=(-10, 10), x_optim=x_optim[1])

class HolderTable(torch.nn.Module): def __init__(self) -> None: super().__init__() self.theta = hf_uniform_para(2) # [8.05502,9.66459] -19.2085 def forward(self): x,y = self.theta tmp0 = torch.sin(x)*torch.cos(y) tmp1 = torch.exp(torch.abs(1 - torch.sqrt(x*x + y*y)/np.pi)) ret = -torch.abs(tmp0*tmp1) return ret model = HolderTable() x_optim = np.array([[8.05502, 9.66459], [-8.05502, 9.66459], [8.05502, -9.66459], [-8.05502, -9.66459]]) hf_demo(model, num_repeat=10, xlim=(-10, 10), ylim=(-10, 10), x_optim=x_optim[1])

[round=0] min(f)=-1.7329722438956618, current(f)=-1.7329722438956618
[round=1] min(f)=-1.7329722438960455, current(f)=-1.7329722438960455
[round=2] min(f)=-1.7329722438960669, current(f)=-1.7329722438960669
[round=3] min(f)=-1.7329722438960702, current(f)=-1.7329722438960702
[round=4] min(f)=-1.7329722438960702, current(f)=-1.73297224389607
[round=5] min(f)=-1.7329722438960702, current(f)=-1.73297224389607
[round=6] min(f)=-1.7329722438960702, current(f)=-1.7329722438960702
[round=7] min(f)=-1.7329722438960702, current(f)=-1.732972243896063
[round=8] min(f)=-1.7329722438960704, current(f)=-1.7329722438960704
[round=9] min(f)=-1.7329722438960704, current(f)=-1.7329722438960702
optimal theta: [-1.26262726e+00  8.86713466e-13]
optimal loss: -1.7329722438960704

McCormick function¶

$$ f(x,y)=\sin(x+y)+(x-y)^2-1.5x+2.5y+1 $$

$$ f(-0.54719,-1.54719)=-1.9133 $$

In [18]:

class McCormick(torch.nn.Module):
    def __init__(self) -> None:
        super().__init__()
        self.theta = hf_uniform_para(2)
        # [-0.54719,-1.54719] -1.9133

    def forward(self):
        x,y = self.theta
        tmp0 = torch.sin(x+y)
        tmp1 = (x-y)**2
        tmp2 = -1.5*x + 2.5*y + 1
        ret = tmp0 + tmp1 + tmp2
        return ret

# out of domain
model = McCormick()
x_optim = np.array([-0.54719,-1.54719])
hf_demo(model, num_repeat=10, xlim=(-2.5, 4), ylim=(-3, 4), x_optim=x_optim)

class McCormick(torch.nn.Module): def __init__(self) -> None: super().__init__() self.theta = hf_uniform_para(2) # [-0.54719,-1.54719] -1.9133 def forward(self): x,y = self.theta tmp0 = torch.sin(x+y) tmp1 = (x-y)**2 tmp2 = -1.5*x + 2.5*y + 1 ret = tmp0 + tmp1 + tmp2 return ret # out of domain model = McCormick() x_optim = np.array([-0.54719,-1.54719]) hf_demo(model, num_repeat=10, xlim=(-2.5, 4), ylim=(-3, 4), x_optim=x_optim)

[round=0] min(f)=-595.6742344834522, current(f)=-595.6742344834522
[round=1] min(f)=-595.6742344834522, current(f)=-1.9132229549810362
[round=2] min(f)=-595.6742344834522, current(f)=-1.9132229549810358
[round=3] min(f)=-595.6742344834522, current(f)=-1.9132229549810362
[round=4] min(f)=-595.6742344834522, current(f)=-1.9132229549810362
[round=5] min(f)=-595.6742344834522, current(f)=-1.9132229549810367
[round=6] min(f)=-595.6742344834522, current(f)=-1.9132229549810367
[round=7] min(f)=-595.6742344834522, current(f)=-1.9132229549810058
[round=8] min(f)=-595.6742344834522, current(f)=-1.9132229549810362
[round=9] min(f)=-595.6742344834522, current(f)=-1.913222954981018
optimal theta: [-594.30820908 -595.30820908]
optimal loss: -595.6742344834522

Schaffer function N.2¶

$$ f(x,y)=0.5+\frac{\sin^2(x^2-y^2)-0.5}{[1+0.001(x^2+y^2)]^2} $$

$$ f(0,0)=0 $$

In [19]:

class SchafferFunctionN2(torch.nn.Module):
    def __init__(self) -> None:
        super().__init__()
        self.theta = hf_uniform_para(2)
        # [0,0] 0

    def forward(self):
        x,y = self.theta
        tmp0 = torch.sin(x*x-y*y)
        tmp1 = 1 + 0.001*(x*x + y*y)
        ret = 0.5 + (tmp0*tmp0 - 0.5)/(tmp1*tmp1)
        return ret

model = SchafferFunctionN2()
x_optim = np.array([0, 0])
hf_demo(model, num_repeat=10, xlim=(-50, 50), ylim=(-50, 50), x_optim=x_optim)

class SchafferFunctionN2(torch.nn.Module): def __init__(self) -> None: super().__init__() self.theta = hf_uniform_para(2) # [0,0] 0 def forward(self): x,y = self.theta tmp0 = torch.sin(x*x-y*y) tmp1 = 1 + 0.001*(x*x + y*y) ret = 0.5 + (tmp0*tmp0 - 0.5)/(tmp1*tmp1) return ret model = SchafferFunctionN2() x_optim = np.array([0, 0]) hf_demo(model, num_repeat=10, xlim=(-50, 50), ylim=(-50, 50), x_optim=x_optim)

[round=0] min(f)=2.4424906541753444e-15, current(f)=2.4424906541753444e-15
[round=1] min(f)=2.4424906541753444e-15, current(f)=2.1245227799226996e-12
[round=2] min(f)=2.4424906541753444e-15, current(f)=4.351952131997905e-11
[round=3] min(f)=2.4424906541753444e-15, current(f)=1.2101430968414206e-13
[round=4] min(f)=2.4424906541753444e-15, current(f)=1.9544366125501256e-12
[round=5] min(f)=2.4424906541753444e-15, current(f)=1.318944953254686e-13
[round=6] min(f)=0.0, current(f)=0.0
[round=7] min(f)=0.0, current(f)=2.220446049250313e-16
[round=8] min(f)=0.0, current(f)=0.0
[round=9] min(f)=0.0, current(f)=0.0
optimal theta: [ 8.48431843e-08 -1.33979508e-07]
optimal loss: 0.0

Schaffer function N.4¶

$$ f(x,y)=0.5+\frac{\cos^2(\sin|x^2-y^2|)-0.5}{[1+0.001(x^2+y^2)]^2} $$

$$ f(0,1.25313)=f(0,-1.25313)=f(1.25313,0)=f(-1.25313,0)=0.292579 $$

In [20]:

class SchafferFunctionN4(torch.nn.Module):
    def __init__(self) -> None:
        super().__init__()
        self.theta = hf_uniform_para(2)
        # [0,1.25313] 0.292579

    def forward(self):
        x,y = self.theta
        tmp0 = torch.cos(torch.sin(torch.abs(x*x-y*y)))
        tmp1 = 1 + 0.001*(x*x + y*y)
        ret = 0.5 + (tmp0*tmp0 - 0.5)/(tmp1*tmp1)
        return ret

model = SchafferFunctionN4()
x_optim = np.array([[0, 1.25313], [0, -1.25313], [1.25313,0], [-1.25313,0]])
hf_demo(model, num_repeat=10, xlim=(-50, 50), ylim=(-50, 50), x_optim=x_optim[0])

class SchafferFunctionN4(torch.nn.Module): def __init__(self) -> None: super().__init__() self.theta = hf_uniform_para(2) # [0,1.25313] 0.292579 def forward(self): x,y = self.theta tmp0 = torch.cos(torch.sin(torch.abs(x*x-y*y))) tmp1 = 1 + 0.001*(x*x + y*y) ret = 0.5 + (tmp0*tmp0 - 0.5)/(tmp1*tmp1) return ret model = SchafferFunctionN4() x_optim = np.array([[0, 1.25313], [0, -1.25313], [1.25313,0], [-1.25313,0]]) hf_demo(model, num_repeat=10, xlim=(-50, 50), ylim=(-50, 50), x_optim=x_optim[0])

[round=0] min(f)=0.2925786320360111, current(f)=0.2925786320360111
[round=1] min(f)=0.2925786320359848, current(f)=0.2925786320359848
[round=2] min(f)=0.2925786320359848, current(f)=0.29257863203606227
[round=3] min(f)=0.2925786320359848, current(f)=0.2925786320372108
[round=4] min(f)=0.2925786320359808, current(f)=0.2925786320359808
[round=5] min(f)=0.2925786320359808, current(f)=0.2925786320359814
[round=6] min(f)=0.2925786320359808, current(f)=0.29257863203602985
[round=7] min(f)=0.2925786320359808, current(f)=0.2925786320360161
[round=8] min(f)=0.2925786320359808, current(f)=0.2925786320365067
[round=9] min(f)=0.2925786320359808, current(f)=0.29257863203601453
optimal theta: [1.25313183e+00 4.91465170e-07]
optimal loss: 0.2925786320359808

Styblinski-Tang function¶

$$ f(x)=\frac{1}{2}\sum_{i=1}^n[x_i^4-16x_i^2+5x_i] $$

$$ f(-2.903534,-2.903534,\cdots,-2.903534)=-39.16599n $$

In [21]:

class StyblinskiTangFunction(torch.nn.Module):
    def __init__(self, n) -> None:
        super().__init__()
        self.theta = hf_uniform_para(n)
        # [-2.903534]*n -39.16599*n

    def forward(self):
        x = self.theta
        ret = 0.5*(x*x*x*x - 16*x*x + 5*x).sum()
        return ret

n = 2
model = StyblinskiTangFunction(n)
x_optim = np.array([-2.903534]*n)
hf_demo(model, num_repeat=10, xlim=(-5, 5), ylim=(-5, 5), x_optim=x_optim)

class StyblinskiTangFunction(torch.nn.Module): def __init__(self, n) -> None: super().__init__() self.theta = hf_uniform_para(n) # [-2.903534]*n -39.16599*n def forward(self): x = self.theta ret = 0.5*(x*x*x*x - 16*x*x + 5*x).sum() return ret n = 2 model = StyblinskiTangFunction(n) x_optim = np.array([-2.903534]*n) hf_demo(model, num_repeat=10, xlim=(-5, 5), ylim=(-5, 5), x_optim=x_optim)

[round=0] min(f)=-64.19561235905354, current(f)=-64.19561235905354
[round=1] min(f)=-78.33233140754282, current(f)=-78.33233140754282
[round=2] min(f)=-78.33233140754282, current(f)=-78.33233140754243
[round=3] min(f)=-78.33233140754282, current(f)=-50.05889331056789
[round=4] min(f)=-78.33233140754282, current(f)=-50.05889331056788
[round=5] min(f)=-78.33233140754282, current(f)=-50.05889331056788
[round=6] min(f)=-78.33233140754282, current(f)=-78.33233140754282
[round=7] min(f)=-78.33233140754282, current(f)=-50.05889331056783
[round=8] min(f)=-78.33233140754282, current(f)=-78.33233140754282
[round=9] min(f)=-78.33233140754282, current(f)=-64.19561235905478
optimal theta: [-2.90353401 -2.90353404]
optimal loss: -78.33233140754282