From 2060829b67ee50812376d8c5499d7d1002a537a3 Mon Sep 17 00:00:00 2001 From: Marcin Date: Fri, 7 Feb 2025 21:15:38 +0100 Subject: [PATCH 01/10] Add files for Bounds handling unit tests --- tests/data_classes/bounds_handle/conftest.py | 92 +++++++++++++++++++ .../bounds_handle/test_bounds_project.py | 0 .../bounds_handle/test_bounds_reflect.py | 0 .../bounds_handle/test_bounds_wrap.py | 0 tests/data_classes/test_bounds.py | 3 +- 5 files changed, 94 insertions(+), 1 deletion(-) create mode 100644 tests/data_classes/bounds_handle/conftest.py create mode 100644 tests/data_classes/bounds_handle/test_bounds_project.py create mode 100644 tests/data_classes/bounds_handle/test_bounds_reflect.py create mode 100644 tests/data_classes/bounds_handle/test_bounds_wrap.py diff --git a/tests/data_classes/bounds_handle/conftest.py b/tests/data_classes/bounds_handle/conftest.py new file mode 100644 index 0000000..148bdcc --- /dev/null +++ b/tests/data_classes/bounds_handle/conftest.py @@ -0,0 +1,92 @@ +""" +Pytest fixtures used in Bounds handling unit tests. +""" + +import pytest + +from optilab.data_classes import Bounds, Point + + +@pytest.fixture() +def example_bounds() -> Bounds: + """ + Example bounds used in bounds handling unit tests. + """ + return Bounds(10, 20) + + +@pytest.fixture() +def point_in_bounds() -> Point: + """ + Point that lies within the example_bounds fixture. + Expected values for bounds handlers is the same as the point. + """ + return Point(14) + + +@pytest.fixture() +def point_equal_lower_bound() -> Point: + """ + Point that lies on the lower bound of example_bounds fixture. + Expected values for all bounds handlers are equal to the point. + """ + return Point(10) + + +@pytest.fixture() +def point_equal_upper_bound() -> Point: + """ + Point that lies on the upper bound of example_bounds fixture. + Expected values for all bounds handlers are equal to the point. + """ + return Point(20) + + +@pytest.fixture() +def point_below_bounds() -> Point: + """ + Point that lies below the example_bounds fixture. + Expected values for bounds handlers are: + - reflect: 18 + - wrap: 2 + - project: 10 + """ + return Point(2) + + +@pytest.fixture() +def point_above_bounds() -> Point: + """ + Point that lies above the example_bounds fixture. + Expected values for bounds handlers are: + - reflect: 17 + - wrap: 13 + - project: 20 + """ + return Point(23) + + +@pytest.fixture +def point_twice_below_bounds() -> Point: + """ + Point that lies below the lower bound of the example_bounds, and the difference + in distance from the lower bound is bigger than the length of the bounds. + Expected values for bounds handlers are: + - reflect: 16 + - wrap: 16 + - project: 10 + """ + return Point(-4) + + +@pytest.fixture +def point_twice_above_bounds() -> Point: + """ + Point that lies below the lower bound of the example_bounds, and the difference + in distance from the lower bound is bigger than the length of the bounds. + Expected values for bounds handlers are: + - reflect: 12 + - wrap: 12 + - project: 20 + """ + return Point(32) diff --git a/tests/data_classes/bounds_handle/test_bounds_project.py b/tests/data_classes/bounds_handle/test_bounds_project.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/data_classes/bounds_handle/test_bounds_reflect.py b/tests/data_classes/bounds_handle/test_bounds_reflect.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/data_classes/bounds_handle/test_bounds_wrap.py b/tests/data_classes/bounds_handle/test_bounds_wrap.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/data_classes/test_bounds.py b/tests/data_classes/test_bounds.py index 75c30cc..7eefc69 100644 --- a/tests/data_classes/test_bounds.py +++ b/tests/data_classes/test_bounds.py @@ -1,5 +1,5 @@ """ -Bounds dataclass unit tests. +Bounds dataclass unit tests. Unit tests for bound handling methods are in separate scripts. """ import numpy as np @@ -10,6 +10,7 @@ class TestBounds: """ Bounds dataclass unit tests class. + Unit tests for bound handling methods are in separate scripts. """ def test_valid_bounds(self): From 47c35b60f271eff12f9f5e8b5fa1c7ea7e1bf14d Mon Sep 17 00:00:00 2001 From: Marcin Date: Fri, 7 Feb 2025 21:16:49 +0100 Subject: [PATCH 02/10] Bump version to 18 --- docs/conf.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index c8582e4..d55fa77 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,7 +13,7 @@ project = 'optilab' copyright = '2025, Marcin Łojek' author = 'Marcin Łojek' -release = '17' +release = '18' # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration diff --git a/pyproject.toml b/pyproject.toml index e3bc391..a93fa9b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "optilab" -version = "17" +version = "18" authors = [ { name="mlojek", email="marcin.lojek@pw.edu.pl" }, ] From c7c0963dda2af5bf88ebc7c253eb748fb36a7119 Mon Sep 17 00:00:00 2001 From: Marcin Date: Fri, 7 Feb 2025 21:54:23 +0100 Subject: [PATCH 03/10] Implement Bounds.project unit tests --- src/optilab/data_classes/bounds.py | 69 +++++++++++++++++++ tests/data_classes/bounds_handle/conftest.py | 26 +++++-- .../bounds_handle/test_bounds_project.py | 65 +++++++++++++++++ 3 files changed, 153 insertions(+), 7 deletions(-) diff --git a/src/optilab/data_classes/bounds.py b/src/optilab/data_classes/bounds.py index a088e4a..445e770 100644 --- a/src/optilab/data_classes/bounds.py +++ b/src/optilab/data_classes/bounds.py @@ -2,6 +2,7 @@ Class representing bounds of the search space. """ +from copy import deepcopy from dataclasses import dataclass from typing import List @@ -83,3 +84,71 @@ def random_point_list(self, num_points: int, dim: int) -> PointList: Point: List of randomly sampled points from the search space. """ return PointList([self.random_point(dim) for _ in range(num_points)]) + + # search space bounds handling methods + def reflect(self, point: Point) -> Point: + """ + Handle bounds by reflecting the point back into the + search area. + + Args: + point (Point): The point to handle. + + Returns: + Point: Reflected point. + """ + raise NotImplementedError + + def wrap(self, point: Point) -> Point: + """ + Handle bounds by wrapping the point around the + search area. + + Args: + point (Point): The point to handle. + + Returns: + Point: Wrapped point. + """ + raise NotImplementedError + + def project(self, point: Point) -> Point: + """ + Handle bounds by projecting the point onto the bounds + of the search area. + + Args: + point (Point): The point to handle. + + Returns: + Point: Projected point. + """ + new_x = [] + + for val in point.x: + if val < self.lower: + new_x.append(deepcopy(self.lower)) + elif val > self.upper: + new_x.append(deepcopy(self.upper)) + else: + new_x.append(val) + + point.x = new_x + return point + + def handle_bounds(self, point: Point, mode: str) -> Point: + """ + Function to choose the bound handling method by name of the method. + + Args: + point (Point): The point to handle. + mode (str): Bound handling mode to use, choose from reflect, wrap or project. + + Returns: + Point: Handled point. + """ + methods = {"reflect": self.reflect, "wrap": self.wrap, "project": self.project} + try: + return methods[mode](point) + except KeyError as err: + raise ValueError(f"Invalid mode {mode} in Bounds.handle_bounds!") from err diff --git a/tests/data_classes/bounds_handle/conftest.py b/tests/data_classes/bounds_handle/conftest.py index 148bdcc..1ae07bd 100644 --- a/tests/data_classes/bounds_handle/conftest.py +++ b/tests/data_classes/bounds_handle/conftest.py @@ -21,7 +21,7 @@ def point_in_bounds() -> Point: Point that lies within the example_bounds fixture. Expected values for bounds handlers is the same as the point. """ - return Point(14) + return Point([14]) @pytest.fixture() @@ -30,7 +30,7 @@ def point_equal_lower_bound() -> Point: Point that lies on the lower bound of example_bounds fixture. Expected values for all bounds handlers are equal to the point. """ - return Point(10) + return Point([10]) @pytest.fixture() @@ -39,7 +39,7 @@ def point_equal_upper_bound() -> Point: Point that lies on the upper bound of example_bounds fixture. Expected values for all bounds handlers are equal to the point. """ - return Point(20) + return Point([20]) @pytest.fixture() @@ -51,7 +51,7 @@ def point_below_bounds() -> Point: - wrap: 2 - project: 10 """ - return Point(2) + return Point([2]) @pytest.fixture() @@ -63,7 +63,7 @@ def point_above_bounds() -> Point: - wrap: 13 - project: 20 """ - return Point(23) + return Point([23]) @pytest.fixture @@ -76,7 +76,7 @@ def point_twice_below_bounds() -> Point: - wrap: 16 - project: 10 """ - return Point(-4) + return Point([-4]) @pytest.fixture @@ -89,4 +89,16 @@ def point_twice_above_bounds() -> Point: - wrap: 12 - project: 20 """ - return Point(32) + return Point([32]) + + +@pytest.fixture +def point_multidimensional() -> Point: + """ + A multidimensional point that has all the values of the previous fixtures. + Expected values for bounds handlers are: + - reflect: [14, 10, 20, 18, 17, 16, 12] + - wrap: [14, 10, 20, 2, 14, 16, 12] + - project: [14, 10, 20, 10, 20, 10, 20] + """ + return Point([14, 10, 20, 2, 23, -4, 23]) diff --git a/tests/data_classes/bounds_handle/test_bounds_project.py b/tests/data_classes/bounds_handle/test_bounds_project.py index e69de29..ebf077e 100644 --- a/tests/data_classes/bounds_handle/test_bounds_project.py +++ b/tests/data_classes/bounds_handle/test_bounds_project.py @@ -0,0 +1,65 @@ +""" +Unit tests for Bounds.project method. +""" + + +class TestBoundsProject: + """ + Unit tests for Bounds.project method. + """ + + def test_point_in_bounds(self, example_bounds, point_in_bounds): + """ + Test if projection works as expected when the point lies within bounds. + """ + handled_point = example_bounds.project(point_in_bounds) + assert handled_point.x == [14] + + def test_point_equal_lower_bound(self, example_bounds, point_equal_lower_bound): + """ + Test if projection works as expected when the point lies on the lower bound. + """ + handled_point = example_bounds.project(point_equal_lower_bound) + assert handled_point.x == [10] + + def test_point_equal_upper_bound(self, example_bounds, point_equal_upper_bound): + """ + Test if projection works as expected when the point lies on the upper bound. + """ + handled_point = example_bounds.project(point_equal_upper_bound) + assert handled_point.x == [20] + + def test_point_below_bounds(self, example_bounds, point_below_bounds): + """ + Test if projection works as expected when the point lies below the lower bound. + """ + handled_point = example_bounds.project(point_below_bounds) + assert handled_point.x == [10] + + def test_point_above_bounds(self, example_bounds, point_above_bounds): + """ + Test if projection works as expected when the point lies above the upper bound. + """ + handled_point = example_bounds.project(point_above_bounds) + assert handled_point.x == [20] + + def test_point_twice_below_bounds(self, example_bounds, point_twice_below_bounds): + """ + Test if projection works as expected when the point lies far below the lower bound. + """ + handled_point = example_bounds.project(point_twice_below_bounds) + assert handled_point.x == [10] + + def test_point_twice_above_bounds(self, example_bounds, point_twice_above_bounds): + """ + Test if projection works as expected when the point lies far above the upper bound. + """ + handled_point = example_bounds.project(point_twice_above_bounds) + assert handled_point.x == [20] + + def test_multidimensional(self, example_bounds, point_multidimensional): + """ + Test if projection works as expected for a multidimensional point. + """ + handled_point = example_bounds.project(point_multidimensional) + assert handled_point.x == [14, 10, 20, 10, 20, 10, 20] From 307ad87766d94084377c413ea6b70f70e6bab244 Mon Sep 17 00:00:00 2001 From: Marcin Date: Fri, 7 Feb 2025 22:57:08 +0100 Subject: [PATCH 04/10] Add xfailing tests for remaining bounds handlers --- .../bounds_handle/test_bounds_reflect.py | 68 +++++++++++++++++++ .../bounds_handle/test_bounds_wrap.py | 68 +++++++++++++++++++ .../bounds_handle/test_handle_bounds.py | 29 ++++++++ 3 files changed, 165 insertions(+) create mode 100644 tests/data_classes/bounds_handle/test_handle_bounds.py diff --git a/tests/data_classes/bounds_handle/test_bounds_reflect.py b/tests/data_classes/bounds_handle/test_bounds_reflect.py index e69de29..07c8e3a 100644 --- a/tests/data_classes/bounds_handle/test_bounds_reflect.py +++ b/tests/data_classes/bounds_handle/test_bounds_reflect.py @@ -0,0 +1,68 @@ +""" +Unit tests for Bounds.reflect method. +""" + +import pytest + + +@pytest.mark.xfail +class TestBoundsReflect: + """ + Unit tests for Bounds.reflect method. + """ + + def test_point_in_bounds(self, example_bounds, point_in_bounds): + """ + Test if reflection works as expected when the point lies within bounds. + """ + handled_point = example_bounds.reflect(point_in_bounds) + assert handled_point.x == [14] + + def test_point_equal_lower_bound(self, example_bounds, point_equal_lower_bound): + """ + Test if reflection works as expected when the point lies on the lower bound. + """ + handled_point = example_bounds.reflect(point_equal_lower_bound) + assert handled_point.x == [10] + + def test_point_equal_upper_bound(self, example_bounds, point_equal_upper_bound): + """ + Test if reflection works as expected when the point lies on the upper bound. + """ + handled_point = example_bounds.reflect(point_equal_upper_bound) + assert handled_point.x == [20] + + def test_point_below_bounds(self, example_bounds, point_below_bounds): + """ + Test if reflection works as expected when the point lies below the lower bound. + """ + handled_point = example_bounds.reflect(point_below_bounds) + assert handled_point.x == [18] + + def test_point_above_bounds(self, example_bounds, point_above_bounds): + """ + Test if reflection works as expected when the point lies above the upper bound. + """ + handled_point = example_bounds.reflect(point_above_bounds) + assert handled_point.x == [17] + + def test_point_twice_below_bounds(self, example_bounds, point_twice_below_bounds): + """ + Test if reflection works as expected when the point lies far below the lower bound. + """ + handled_point = example_bounds.reflect(point_twice_below_bounds) + assert handled_point.x == [16] + + def test_point_twice_above_bounds(self, example_bounds, point_twice_above_bounds): + """ + Test if reflection works as expected when the point lies far above the upper bound. + """ + handled_point = example_bounds.reflect(point_twice_above_bounds) + assert handled_point.x == [12] + + def test_multidimensional(self, example_bounds, point_multidimensional): + """ + Test if reflection works as expected for a multidimensional point. + """ + handled_point = example_bounds.reflect(point_multidimensional) + assert handled_point.x == [14, 10, 20, 18, 17, 16, 12] diff --git a/tests/data_classes/bounds_handle/test_bounds_wrap.py b/tests/data_classes/bounds_handle/test_bounds_wrap.py index e69de29..125cc45 100644 --- a/tests/data_classes/bounds_handle/test_bounds_wrap.py +++ b/tests/data_classes/bounds_handle/test_bounds_wrap.py @@ -0,0 +1,68 @@ +""" +Unit tests for Bounds.wrap method. +""" + +import pytest + + +@pytest.mark.xfail +class TestBoundswrap: + """ + Unit tests for Bounds.wrap method. + """ + + def test_point_in_bounds(self, example_bounds, point_in_bounds): + """ + Test if wrapping works as expected when the point lies within bounds. + """ + handled_point = example_bounds.wrap(point_in_bounds) + assert handled_point.x == [14] + + def test_point_equal_lower_bound(self, example_bounds, point_equal_lower_bound): + """ + Test if wrapping works as expected when the point lies on the lower bound. + """ + handled_point = example_bounds.wrap(point_equal_lower_bound) + assert handled_point.x == [10] + + def test_point_equal_upper_bound(self, example_bounds, point_equal_upper_bound): + """ + Test if wrapping works as expected when the point lies on the upper bound. + """ + handled_point = example_bounds.wrap(point_equal_upper_bound) + assert handled_point.x == [20] + + def test_point_below_bounds(self, example_bounds, point_below_bounds): + """ + Test if wrapping works as expected when the point lies below the lower bound. + """ + handled_point = example_bounds.wrap(point_below_bounds) + assert handled_point.x == [2] + + def test_point_above_bounds(self, example_bounds, point_above_bounds): + """ + Test if wrapping works as expected when the point lies above the upper bound. + """ + handled_point = example_bounds.wrap(point_above_bounds) + assert handled_point.x == [14] + + def test_point_twice_below_bounds(self, example_bounds, point_twice_below_bounds): + """ + Test if wrapping works as expected when the point lies far below the lower bound. + """ + handled_point = example_bounds.wrap(point_twice_below_bounds) + assert handled_point.x == [16] + + def test_point_twice_above_bounds(self, example_bounds, point_twice_above_bounds): + """ + Test if wrapping works as expected when the point lies far above the upper bound. + """ + handled_point = example_bounds.wrap(point_twice_above_bounds) + assert handled_point.x == [12] + + def test_multidimensional(self, example_bounds, point_multidimensional): + """ + Test if wrapping works as expected for a multidimensional point. + """ + handled_point = example_bounds.wrap(point_multidimensional) + assert handled_point.x == [14, 10, 20, 2, 14, 16, 12] diff --git a/tests/data_classes/bounds_handle/test_handle_bounds.py b/tests/data_classes/bounds_handle/test_handle_bounds.py new file mode 100644 index 0000000..85a71ea --- /dev/null +++ b/tests/data_classes/bounds_handle/test_handle_bounds.py @@ -0,0 +1,29 @@ +""" +Unit tests for Bounds.handle_bounds method. +""" + +import pytest + + +class TestBoundsHandleBounds: + """ + Unit tests for Bounds.handle_bounds method. + """ + + def test_project(self, example_bounds, point_multidimensional): + handled_point = example_bounds.handle_bounds(point_multidimensional, "project") + assert handled_point.x == [14, 10, 20, 10, 20, 10, 20] + + @pytest.mark.xfail() + def test_reflect(self, example_bounds, point_multidimensional): + handled_point = example_bounds.handle_bounds(point_multidimensional, "reflect") + assert handled_point.x == [14, 10, 20, 18, 17, 16, 12] + + @pytest.mark.xfail() + def test_wrap(self, example_bounds, point_multidimensional): + handled_point = example_bounds.handle_bounds(point_multidimensional, "wrap") + assert handled_point.x == [14, 10, 20, 2, 14, 16, 12] + + def test_invalid_mode(self, example_bounds, point_multidimensional): + with pytest.raises(ValueError, match="Invalid mode"): + example_bounds.handle_bounds(point_multidimensional, "invalid") From 57414d5dffa31ec7201ff25414293b669899f469 Mon Sep 17 00:00:00 2001 From: Marcin Date: Fri, 7 Feb 2025 23:09:33 +0100 Subject: [PATCH 05/10] Implemented wrap bound handling --- src/optilab/data_classes/bounds.py | 14 +++++++++++++- tests/data_classes/bounds_handle/conftest.py | 6 +++--- .../data_classes/bounds_handle/test_bounds_wrap.py | 9 +++------ .../bounds_handle/test_handle_bounds.py | 3 +-- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/optilab/data_classes/bounds.py b/src/optilab/data_classes/bounds.py index 445e770..e44bd29 100644 --- a/src/optilab/data_classes/bounds.py +++ b/src/optilab/data_classes/bounds.py @@ -110,7 +110,19 @@ def wrap(self, point: Point) -> Point: Returns: Point: Wrapped point. """ - raise NotImplementedError + new_x = [] + + for val in point.x: + if val < self.lower or val > self.upper: + val -= self.lower + val %= self.upper - self.lower + val += self.lower + new_x.append(val) + else: + new_x.append(val) + + point.x = new_x + return point def project(self, point: Point) -> Point: """ diff --git a/tests/data_classes/bounds_handle/conftest.py b/tests/data_classes/bounds_handle/conftest.py index 1ae07bd..eafae7e 100644 --- a/tests/data_classes/bounds_handle/conftest.py +++ b/tests/data_classes/bounds_handle/conftest.py @@ -48,7 +48,7 @@ def point_below_bounds() -> Point: Point that lies below the example_bounds fixture. Expected values for bounds handlers are: - reflect: 18 - - wrap: 2 + - wrap: 12 - project: 10 """ return Point([2]) @@ -98,7 +98,7 @@ def point_multidimensional() -> Point: A multidimensional point that has all the values of the previous fixtures. Expected values for bounds handlers are: - reflect: [14, 10, 20, 18, 17, 16, 12] - - wrap: [14, 10, 20, 2, 14, 16, 12] + - wrap: [14, 10, 20, 12, 13, 16, 12] - project: [14, 10, 20, 10, 20, 10, 20] """ - return Point([14, 10, 20, 2, 23, -4, 23]) + return Point([14, 10, 20, 2, 23, -4, 32]) diff --git a/tests/data_classes/bounds_handle/test_bounds_wrap.py b/tests/data_classes/bounds_handle/test_bounds_wrap.py index 125cc45..91d5b85 100644 --- a/tests/data_classes/bounds_handle/test_bounds_wrap.py +++ b/tests/data_classes/bounds_handle/test_bounds_wrap.py @@ -2,10 +2,7 @@ Unit tests for Bounds.wrap method. """ -import pytest - -@pytest.mark.xfail class TestBoundswrap: """ Unit tests for Bounds.wrap method. @@ -37,14 +34,14 @@ def test_point_below_bounds(self, example_bounds, point_below_bounds): Test if wrapping works as expected when the point lies below the lower bound. """ handled_point = example_bounds.wrap(point_below_bounds) - assert handled_point.x == [2] + assert handled_point.x == [12] def test_point_above_bounds(self, example_bounds, point_above_bounds): """ Test if wrapping works as expected when the point lies above the upper bound. """ handled_point = example_bounds.wrap(point_above_bounds) - assert handled_point.x == [14] + assert handled_point.x == [13] def test_point_twice_below_bounds(self, example_bounds, point_twice_below_bounds): """ @@ -65,4 +62,4 @@ def test_multidimensional(self, example_bounds, point_multidimensional): Test if wrapping works as expected for a multidimensional point. """ handled_point = example_bounds.wrap(point_multidimensional) - assert handled_point.x == [14, 10, 20, 2, 14, 16, 12] + assert handled_point.x == [14, 10, 20, 12, 13, 16, 12] diff --git a/tests/data_classes/bounds_handle/test_handle_bounds.py b/tests/data_classes/bounds_handle/test_handle_bounds.py index 85a71ea..e10f004 100644 --- a/tests/data_classes/bounds_handle/test_handle_bounds.py +++ b/tests/data_classes/bounds_handle/test_handle_bounds.py @@ -19,10 +19,9 @@ def test_reflect(self, example_bounds, point_multidimensional): handled_point = example_bounds.handle_bounds(point_multidimensional, "reflect") assert handled_point.x == [14, 10, 20, 18, 17, 16, 12] - @pytest.mark.xfail() def test_wrap(self, example_bounds, point_multidimensional): handled_point = example_bounds.handle_bounds(point_multidimensional, "wrap") - assert handled_point.x == [14, 10, 20, 2, 14, 16, 12] + assert handled_point.x == [14, 10, 20, 12, 13, 16, 12] def test_invalid_mode(self, example_bounds, point_multidimensional): with pytest.raises(ValueError, match="Invalid mode"): From f98db62c417828471c10c1d549cda8ed4ec45596 Mon Sep 17 00:00:00 2001 From: Marcin Date: Sat, 8 Feb 2025 08:47:07 +0100 Subject: [PATCH 06/10] Finish implementing bound handling and unit tests for it --- src/optilab/data_classes/bounds.py | 18 +++++++++++++++++- tests/data_classes/bounds_handle/conftest.py | 9 +++++++++ .../bounds_handle/test_bounds_project.py | 8 ++++++++ .../bounds_handle/test_bounds_reflect.py | 9 ++++++++- .../bounds_handle/test_bounds_wrap.py | 8 ++++++++ .../bounds_handle/test_handle_bounds.py | 1 - 6 files changed, 50 insertions(+), 3 deletions(-) diff --git a/src/optilab/data_classes/bounds.py b/src/optilab/data_classes/bounds.py index e44bd29..b5b1848 100644 --- a/src/optilab/data_classes/bounds.py +++ b/src/optilab/data_classes/bounds.py @@ -97,7 +97,23 @@ def reflect(self, point: Point) -> Point: Returns: Point: Reflected point. """ - raise NotImplementedError + new_x = [] + + for val in point.x: + if val < self.lower or val > self.upper: + val -= self.lower + remainder = val % (self.upper - self.lower) + relative_distance = val // (self.upper - self.lower) + + if relative_distance % 2 == 0: + new_x.append(self.lower + remainder) + else: + new_x.append(self.upper - remainder) + else: + new_x.append(val) + + point.x = new_x + return point def wrap(self, point: Point) -> Point: """ diff --git a/tests/data_classes/bounds_handle/conftest.py b/tests/data_classes/bounds_handle/conftest.py index eafae7e..225ba3e 100644 --- a/tests/data_classes/bounds_handle/conftest.py +++ b/tests/data_classes/bounds_handle/conftest.py @@ -102,3 +102,12 @@ def point_multidimensional() -> Point: - project: [14, 10, 20, 10, 20, 10, 20] """ return Point([14, 10, 20, 2, 23, -4, 32]) + + +@pytest.fixture +def evaluated_point() -> Point: + """ + An evaluated point, with y value and is_evaluated set to True. + Used to check if the handled point has the same y and is_evaluated values. + """ + return Point(x=[14, 10, 20, -4], y=10.1, is_evaluated=True) diff --git a/tests/data_classes/bounds_handle/test_bounds_project.py b/tests/data_classes/bounds_handle/test_bounds_project.py index ebf077e..3d3cc64 100644 --- a/tests/data_classes/bounds_handle/test_bounds_project.py +++ b/tests/data_classes/bounds_handle/test_bounds_project.py @@ -63,3 +63,11 @@ def test_multidimensional(self, example_bounds, point_multidimensional): """ handled_point = example_bounds.project(point_multidimensional) assert handled_point.x == [14, 10, 20, 10, 20, 10, 20] + + def test_evaluated_point(self, example_bounds, evaluated_point): + """ + Test if projecting a point leaves y and is_evaluated members unchanged. + """ + handled_point = example_bounds.project(evaluated_point) + assert handled_point.y == 10.1 + assert handled_point.is_evaluated diff --git a/tests/data_classes/bounds_handle/test_bounds_reflect.py b/tests/data_classes/bounds_handle/test_bounds_reflect.py index 07c8e3a..db62aef 100644 --- a/tests/data_classes/bounds_handle/test_bounds_reflect.py +++ b/tests/data_classes/bounds_handle/test_bounds_reflect.py @@ -5,7 +5,6 @@ import pytest -@pytest.mark.xfail class TestBoundsReflect: """ Unit tests for Bounds.reflect method. @@ -66,3 +65,11 @@ def test_multidimensional(self, example_bounds, point_multidimensional): """ handled_point = example_bounds.reflect(point_multidimensional) assert handled_point.x == [14, 10, 20, 18, 17, 16, 12] + + def test_evaluated_point(self, example_bounds, evaluated_point): + """ + Test if reflecting a point leaves y and is_evaluated members unchanged. + """ + handled_point = example_bounds.reflect(evaluated_point) + assert handled_point.y == 10.1 + assert handled_point.is_evaluated diff --git a/tests/data_classes/bounds_handle/test_bounds_wrap.py b/tests/data_classes/bounds_handle/test_bounds_wrap.py index 91d5b85..91f35f8 100644 --- a/tests/data_classes/bounds_handle/test_bounds_wrap.py +++ b/tests/data_classes/bounds_handle/test_bounds_wrap.py @@ -63,3 +63,11 @@ def test_multidimensional(self, example_bounds, point_multidimensional): """ handled_point = example_bounds.wrap(point_multidimensional) assert handled_point.x == [14, 10, 20, 12, 13, 16, 12] + + def test_evaluated_point(self, example_bounds, evaluated_point): + """ + Test if wrapping a point leaves y and is_evaluated members unchanged. + """ + handled_point = example_bounds.wrap(evaluated_point) + assert handled_point.y == 10.1 + assert handled_point.is_evaluated diff --git a/tests/data_classes/bounds_handle/test_handle_bounds.py b/tests/data_classes/bounds_handle/test_handle_bounds.py index e10f004..06ec4d7 100644 --- a/tests/data_classes/bounds_handle/test_handle_bounds.py +++ b/tests/data_classes/bounds_handle/test_handle_bounds.py @@ -14,7 +14,6 @@ def test_project(self, example_bounds, point_multidimensional): handled_point = example_bounds.handle_bounds(point_multidimensional, "project") assert handled_point.x == [14, 10, 20, 10, 20, 10, 20] - @pytest.mark.xfail() def test_reflect(self, example_bounds, point_multidimensional): handled_point = example_bounds.handle_bounds(point_multidimensional, "reflect") assert handled_point.x == [14, 10, 20, 18, 17, 16, 12] From 3332b6ce80892d84c16a699708e72bb9358096f2 Mon Sep 17 00:00:00 2001 From: Marcin Date: Sat, 8 Feb 2025 10:47:56 +0100 Subject: [PATCH 07/10] Minor changes to documentation --- docs/index.rst | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/docs/index.rst b/docs/index.rst index fdfd20d..eadd064 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,15 +1,10 @@ -.. optilab documentation master file, created by - sphinx-quickstart on Sat Dec 7 23:13:57 2024. - You can adapt this file completely to your liking, but it should at least - contain the root `toctree` directive. - optilab documentation ===================== -Optilab is a python framework for black-box optimization. +Optilab is a python framework for black-box optimization. To learn more visit the official repository of this project at `github `_. .. toctree:: :maxdepth: 2 :caption: Contents: - modules + optilab From 2badc26851e5f4ee138ac399e9fe1dbc722bc099 Mon Sep 17 00:00:00 2001 From: Marcin Date: Sat, 8 Feb 2025 11:22:12 +0100 Subject: [PATCH 08/10] Start adding first tutorial jupyter notebook --- demo/perform_optimization.ipynb | 198 ++++++++++++++++++++++++++++++++ pyproject.toml | 3 +- 2 files changed, 200 insertions(+), 1 deletion(-) create mode 100644 demo/perform_optimization.ipynb diff --git a/demo/perform_optimization.ipynb b/demo/perform_optimization.ipynb new file mode 100644 index 0000000..b239753 --- /dev/null +++ b/demo/perform_optimization.ipynb @@ -0,0 +1,198 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a933bf15-cd99-41d9-9833-2b6657dfb763", + "metadata": {}, + "source": [ + "# Optilab tutorial: performing optimization\n", + "This tutorial notebook aims to explain how to perform optimization using optilab API. In this notebook you will learn how to:\n", + "- create an instance of optimizer and objective function,\n", + "- run optimization on the function,\n", + "- visualize the results of the optimization,\n", + "- save optimization result to a pickle file,\n", + "- read and visualize the results using optilab's CLI tool." + ] + }, + { + "cell_type": "markdown", + "id": "37633541-1aee-460f-adad-f60cab7dd0d7", + "metadata": {}, + "source": [ + "## Creating an objective function\n", + "Optilab comes with a lot of common black-box optimization functions. Here, lets create instances of a sphere function. Let's say we want to perform optimization in 10 dimensions:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b4d70253-c4db-4179-9c05-99eb15995729", + "metadata": {}, + "outputs": [], + "source": [ + "# define a constant to hold the dimensionality of the problem\n", + "DIM = 10\n", + "\n", + "# import the sphere function from optilab\n", + "from optilab.functions.unimodal import SphereFunction\n", + "\n", + "# create a sphere function instance with given dimensionality\n", + "objective_function = SphereFunction(DIM)" + ] + }, + { + "cell_type": "markdown", + "id": "0db11933-97c0-4b78-93c6-aebfbd1b5715", + "metadata": {}, + "source": [ + "Let's also create an instance of a harder, multimodal objective function:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "289046df-4ef8-4f18-824d-1665bae347f8", + "metadata": {}, + "outputs": [], + "source": [ + "from optilab.functions.multimodal import RastriginFunction\n", + "\n", + "multimodal_objective = RastriginFunction(DIM)" + ] + }, + { + "cell_type": "markdown", + "id": "6ee2577f-be0b-429f-9908-b2b0573c9d03", + "metadata": {}, + "source": [ + "We can see some information about a function in it's metadata:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41bac97b-94b5-4855-8f25-577b8723373f", + "metadata": {}, + "outputs": [], + "source": [ + "print(objective_function.get_metadata())\n", + "print(multimodal_objective.get_metadata())" + ] + }, + { + "cell_type": "markdown", + "id": "850fbcf3-f6b5-47d7-b5a9-5c915f08264a", + "metadata": {}, + "source": [ + "## Creating an optimizer\n", + "Now let's create an instance of CMA-ES optimizer. Since optilab comes with implementations of some optimizers it's as easy as importing it from the library. Let's also create an instance of LMM-CMA-ES, which we will use to compare the two later on:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "345e17bb-25dd-493e-8e1f-70ff3093de8a", + "metadata": {}, + "outputs": [], + "source": [ + "# define constants for the population size and sigma0 for cmaes optimizers\n", + "POPSIZE = DIM * 2\n", + "SIGMA0 = 1\n", + "\n", + "# import optimizers from optilab\n", + "from optilab.optimizers import CmaEs, LmmCmaEs\n", + "\n", + "# create instances of the optimizers\n", + "cmaes_optimizer = CmaEs(POPSIZE, SIGMA0)" + ] + }, + { + "cell_type": "markdown", + "id": "f799c8e1-791c-4a34-9adb-d0016343af8f", + "metadata": {}, + "source": [ + "- [ ] print metadata\n", + "### optimize\n", + "- [ ] use optimize\n", + "- [ ] use .run_optimization\n", + "- [ ] show the optimizationRun structure\n", + "### plotting\n", + "- [ ] convergence\n", + "- [ ] ecdf\n", + "- [ ] box\n", + "### save and run\n", + "- [ ] save to pickle\n", + "- [ ] !python -m optilab pickle_name.pkl" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "05ac0707-01de-49c7-b11e-5ee058136443", + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "from optilab.data_classes import Bounds\n", + "from optilab.utils import dump_to_pickle\n", + "\n", + "# hyperparams:\n", + "DIM = args.dim\n", + "POPSIZE = DIM * 2\n", + "NUM_NEIGHBORS = DIM + 2\n", + "BUFFER_SIZES = [m * POPSIZE for m in [2, 5, 10, 20, 30, 50]]\n", + "NUM_RUNS = 51\n", + "CALL_BUDGET = 1e4 * DIM\n", + "TOL = 1e-8\n", + "SIGMA0 = 1\n", + "NUM_PROCESSES = 16\n", + "\n", + "# optimized problem\n", + "BOUNDS = Bounds(-100, 100)\n", + "TARGET = 0.0\n", + "\n", + "for func in FUNCS[args.year]:\n", + " print(func.name)\n", + " results = []\n", + "\n", + " cmaes_optimizer = CmaEs(POPSIZE, SIGMA0)\n", + " cmaes_results = cmaes_optimizer.run_optimization(\n", + " NUM_RUNS, func, BOUNDS, CALL_BUDGET, TOL, num_processes=NUM_PROCESSES\n", + " )\n", + " results.append(cmaes_results)\n", + "\n", + " for buffer_size in BUFFER_SIZES:\n", + " knn_optimizer = KnnCmaEs(POPSIZE, SIGMA0, NUM_NEIGHBORS, buffer_size)\n", + " knn_results = knn_optimizer.run_optimization(\n", + " NUM_RUNS, func, BOUNDS, CALL_BUDGET, TOL, num_processes=NUM_PROCESSES\n", + " )\n", + " results.append(knn_results)\n", + "\n", + " dump_to_pickle(\n", + " results, f\"003_knn_benchmark_{func.name}_{DIM}.pkl\"\n", + " )" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/pyproject.toml b/pyproject.toml index a93fa9b..b8e35c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,7 +31,8 @@ dependencies = [ "cma==4.0.0", "pandas==2.2.3", "scikit-learn==1.5.2", - "shapely==2.0.6" + "shapely==2.0.6", + "jupyter" ] [project.urls] From f573e4d96dec6746916c2115e83d7185cda10fad Mon Sep 17 00:00:00 2001 From: Marcin Date: Sat, 8 Feb 2025 18:07:33 +0100 Subject: [PATCH 09/10] Finish the first tutorial notebook --- demo/perform_optimization.ipynb | 271 ++++++++++++++++++++++++++------ 1 file changed, 222 insertions(+), 49 deletions(-) diff --git a/demo/perform_optimization.ipynb b/demo/perform_optimization.ipynb index b239753..335fe76 100644 --- a/demo/perform_optimization.ipynb +++ b/demo/perform_optimization.ipynb @@ -20,7 +20,7 @@ "metadata": {}, "source": [ "## Creating an objective function\n", - "Optilab comes with a lot of common black-box optimization functions. Here, lets create instances of a sphere function. Let's say we want to perform optimization in 10 dimensions:" + "Optilab comes with a lot of common black-box optimization functions. Here, lets create instances of a sphere function. Let's say we want to perform optimization in 2 dimensions:" ] }, { @@ -31,7 +31,7 @@ "outputs": [], "source": [ "# define a constant to hold the dimensionality of the problem\n", - "DIM = 10\n", + "DIM = 2\n", "\n", "# import the sphere function from optilab\n", "from optilab.functions.unimodal import SphereFunction\n", @@ -103,74 +103,247 @@ "from optilab.optimizers import CmaEs, LmmCmaEs\n", "\n", "# create instances of the optimizers\n", - "cmaes_optimizer = CmaEs(POPSIZE, SIGMA0)" + "cmaes_optimizer = CmaEs(POPSIZE, SIGMA0)\n", + "lmm_cmaes_optimizer = LmmCmaEs(POPSIZE, SIGMA0, 2)" ] }, { "cell_type": "markdown", - "id": "f799c8e1-791c-4a34-9adb-d0016343af8f", + "id": "95bccbe6-28a7-4159-9800-a3c634df08fc", "metadata": {}, "source": [ - "- [ ] print metadata\n", - "### optimize\n", - "- [ ] use optimize\n", - "- [ ] use .run_optimization\n", - "- [ ] show the optimizationRun structure\n", - "### plotting\n", - "- [ ] convergence\n", - "- [ ] ecdf\n", - "- [ ] box\n", - "### save and run\n", - "- [ ] save to pickle\n", - "- [ ] !python -m optilab pickle_name.pkl" + "Let's see some information about the optimizers." ] }, { "cell_type": "code", "execution_count": null, - "id": "05ac0707-01de-49c7-b11e-5ee058136443", + "id": "a3d5062b-3bfa-4e32-b417-ab4e89caff75", "metadata": {}, "outputs": [], "source": [ - "\n", + "print(cmaes_optimizer.metadata)\n", + "print(lmm_cmaes_optimizer.metadata)" + ] + }, + { + "cell_type": "markdown", + "id": "764f5e92-0c44-4e54-88e9-2b2ce33ace7d", + "metadata": {}, + "source": [ + "## Perform optimization\n", + "Let's now put the two together and optimize the objective functions using the cmaes and lmm_cmaes optimizers. Let's start by defining the bounds of the problem, the number of allowed calls to the function, and the tolerance of value: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a476b4c5-aa38-4229-8d9e-a7e663c7027a", + "metadata": {}, + "outputs": [], + "source": [ + "# import Bounds class from optilab\n", "from optilab.data_classes import Bounds\n", - "from optilab.utils import dump_to_pickle\n", "\n", - "# hyperparams:\n", - "DIM = args.dim\n", - "POPSIZE = DIM * 2\n", - "NUM_NEIGHBORS = DIM + 2\n", - "BUFFER_SIZES = [m * POPSIZE for m in [2, 5, 10, 20, 30, 50]]\n", - "NUM_RUNS = 51\n", - "CALL_BUDGET = 1e4 * DIM\n", - "TOL = 1e-8\n", - "SIGMA0 = 1\n", - "NUM_PROCESSES = 16\n", - "\n", - "# optimized problem\n", + "# define the constants\n", "BOUNDS = Bounds(-100, 100)\n", - "TARGET = 0.0\n", + "CALL_BUDGET = DIM * 10e4\n", + "TOLERANCE = 1e-8" + ] + }, + { + "cell_type": "markdown", + "id": "94e8f068-79c2-4c24-a2a1-84868f754f86", + "metadata": {}, + "source": [ + "Optimization can be done using `optimize()` method. It returns a `PointList` object, which is a log of all points evaluated using the objective function." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "244e9a8f-6eeb-45b2-8671-750b4b2d9e8a", + "metadata": {}, + "outputs": [], + "source": [ + "# perform optimization\n", + "cmaes_log = cmaes_optimizer.optimize(objective_function, BOUNDS, CALL_BUDGET, TOLERANCE)\n", + "lmm_cmaes_log = lmm_cmaes_optimizer.optimize(objective_function, BOUNDS, CALL_BUDGET, TOLERANCE)\n", "\n", - "for func in FUNCS[args.year]:\n", - " print(func.name)\n", - " results = []\n", + "# see the results\n", + "print(f'CMAES: num_evals: {len(cmaes_log)}, best value: {cmaes_log.best_y()}')\n", + "print(f'LMM CMAES: num_evals: {len(lmm_cmaes_log)}, best value: {lmm_cmaes_log.best_y()}')" + ] + }, + { + "cell_type": "markdown", + "id": "6c8758fe-62bb-4821-a26b-c1ba60d42f82", + "metadata": {}, + "source": [ + "However the recommended way is to use `run_optimization()` method, which performs the desired number of runs, displays a progessbar, allows for multiprocessing, and returns the result in the format used in optilab CLI tool:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d471e7c0-cf14-4059-b82c-e361b3e67495", + "metadata": {}, + "outputs": [], + "source": [ + "# define the number of runs\n", + "RUNS = 51\n", "\n", - " cmaes_optimizer = CmaEs(POPSIZE, SIGMA0)\n", - " cmaes_results = cmaes_optimizer.run_optimization(\n", - " NUM_RUNS, func, BOUNDS, CALL_BUDGET, TOL, num_processes=NUM_PROCESSES\n", - " )\n", - " results.append(cmaes_results)\n", + "# perform optimization\n", + "cmaes_runs = cmaes_optimizer.run_optimization(RUNS, objective_function, BOUNDS, CALL_BUDGET, TOLERANCE)\n", + "lmm_cmaes_runs = lmm_cmaes_optimizer.run_optimization(RUNS, objective_function, BOUNDS, CALL_BUDGET, TOLERANCE)" + ] + }, + { + "cell_type": "markdown", + "id": "e7544cb4-eecb-4a90-b428-9c853fdd004b", + "metadata": {}, + "source": [ + "To speed up your experiments you can use multiprocessing. Bear in mind however, that it does not work for all optimizers and hyperparameter configurations." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2c8484dc-b1b9-48e8-b5a5-99b9dcae9845", + "metadata": {}, + "outputs": [], + "source": [ + "# number of processes to use\n", + "PROCESSES = 4\n", "\n", - " for buffer_size in BUFFER_SIZES:\n", - " knn_optimizer = KnnCmaEs(POPSIZE, SIGMA0, NUM_NEIGHBORS, buffer_size)\n", - " knn_results = knn_optimizer.run_optimization(\n", - " NUM_RUNS, func, BOUNDS, CALL_BUDGET, TOL, num_processes=NUM_PROCESSES\n", - " )\n", - " results.append(knn_results)\n", + "# perform multiprocessed optimization\n", + "_ = cmaes_optimizer.run_optimization(RUNS, objective_function, BOUNDS, CALL_BUDGET, TOLERANCE, num_processes=PROCESSES)" + ] + }, + { + "cell_type": "markdown", + "id": "91e20989-fbe1-4c51-8cf2-9c89e19387b2", + "metadata": {}, + "source": [ + "Optimization results are returned in a data structure called `OptimizationRun`. It stores the metadata of the optimizer, objective function, other optimization hyperparameters and the log of objective values." + ] + }, + { + "cell_type": "markdown", + "id": "f3c37e58-2fb8-4c20-93a5-9b1e713decf0", + "metadata": {}, + "source": [ + "## Plotting optimization results\n", + "Optilab provides user with functions for plotting the convergence curves, ecdf curves and box plots of optimization runs. Let's first plot the convergence curves for CMA-ES and LMM-CMA-ES:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1d14b734-e86b-4bcc-b5ec-5b68ddad759d", + "metadata": {}, + "outputs": [], + "source": [ + "from optilab.plotting import plot_box_plot, plot_convergence_curve, plot_ecdf_curves\n", + "\n", + "# plot convergence curve for the two optimizers\n", + "plot_convergence_curve({run.model_metadata.name: run.logs for run in [cmaes_runs, lmm_cmaes_runs]})" + ] + }, + { + "cell_type": "markdown", + "id": "f8c7346b-bb57-42ac-b0d8-65a5c816bea3", + "metadata": {}, + "source": [ + "As you can see the LMM-CMA-ES converges much quicker than regular CMA-ES. Now let's plot the ECDF curve:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a792d285-351b-49bd-a327-252805d8eebf", + "metadata": {}, + "outputs": [], + "source": [ + "plot_ecdf_curves({run.model_metadata.name: run.logs for run in [cmaes_runs, lmm_cmaes_runs]}, DIM, TOLERANCE)" + ] + }, + { + "cell_type": "markdown", + "id": "ab0ff345-f254-4ced-a010-f595f83876bf", + "metadata": {}, + "source": [ + "Lastly, let's plot the box plot of optimization results:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eca95650-aabb-4526-a62b-e5822781d107", + "metadata": {}, + "outputs": [], + "source": [ + "plot_box_plot(data={run.model_metadata.name: run.bests_y() for run in [cmaes_runs, lmm_cmaes_runs]})" + ] + }, + { + "cell_type": "markdown", + "id": "9b71e00a-3f5f-4bb4-8e5a-fa2269656358", + "metadata": {}, + "source": [ + "Plotting functions also alow for saving the plots to an image file. Use keyword argument `savepath=` to specify where to save the image." + ] + }, + { + "cell_type": "markdown", + "id": "feb6e473-3fd6-45c4-b5dc-9dbf7bbf35e1", + "metadata": {}, + "source": [ + "## Save results to a pickle file and analyze it using optilab's CLI tool\n", + "To save the results of an experiment you can dump optimization runs to a pickle file and then read it and plot is using optilab's CLI functionality. Firstly, pack all OptimizationRun objects into a list. Then use a utility function to save it to a pickle file." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a23ac042-18a0-40da-94da-f7704be2b4b3", + "metadata": {}, + "outputs": [], + "source": [ + "from optilab.utils import dump_to_pickle\n", + "\n", + "runs = [cmaes_runs, lmm_cmaes_runs]\n", "\n", - " dump_to_pickle(\n", - " results, f\"003_knn_benchmark_{func.name}_{DIM}.pkl\"\n", - " )" + "SAVEFILE_NAME = 'tutorial.pkl'\n", + "\n", + "dump_to_pickle(runs, SAVEFILE_NAME)" + ] + }, + { + "cell_type": "markdown", + "id": "8fc57ab4-c8b2-4527-83c6-2afc5e3235f8", + "metadata": {}, + "source": [ + "Now that you saved the results to a pickle file, you can read it into the CLI tool to get various information about the results. The CLI tool also allows to perform statistical testing on the results to determine if the difference in results is statistically significant." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1266ec91-4670-4883-94cd-751e5f549d11", + "metadata": {}, + "outputs": [], + "source": [ + "!python -m optilab $SAVEFILE_NAME --test_y --test_eval" + ] + }, + { + "cell_type": "markdown", + "id": "66a083f2-a583-440b-8d87-aea1cc433004", + "metadata": {}, + "source": [ + "## Closing remarks\n", + "Thank you for choosing optilab for your project. If you wish to learn more checkout the projects repo on [github](https://github.com/mlojek/optilab) and the project's documentation on [readthedocs](https://optilab.readthedocs.io). Feel free to use optilab in your research and work. If you wish to contribute to the project, feel free to do so yourself or leave an issue in the repo. Best of luck, Marcin." ] } ], From ee5788816bfc32f375d93e7ee27f73df189dc981 Mon Sep 17 00:00:00 2001 From: Marcin Date: Mon, 10 Feb 2025 16:36:30 +0100 Subject: [PATCH 10/10] Improve readme, rename tutorial notebook --- README.md | 39 ++- demo/perform_optimization.ipynb | 371 ------------------------ demo/tutorial.ipynb | 488 ++++++++++++++++++++++++++++++++ 3 files changed, 519 insertions(+), 379 deletions(-) delete mode 100644 demo/perform_optimization.ipynb create mode 100644 demo/tutorial.ipynb diff --git a/README.md b/README.md index 37b44e7..3655506 100644 --- a/README.md +++ b/README.md @@ -6,17 +6,40 @@ Optilab is a lightweight and flexible python framework for testing black-box optimization. ## Features -- Intuitive interface to quickly prototype and run optimizers and metamodels. -- High quality documentation. -- Objective functions, optimizers, plotting and data handling. -- CLI functionality to easily summarize results of previous experiments. -- Multiprocessing for faster computation. +- ✅ Intuitive interface to quickly prototype and run optimizers and metamodels. +- 📚 High quality documentation. +- 📈 Objective functions, optimizers, plotting and data handling. +- ⋙ CLI functionality to easily summarize results of previous experiments. +- 🚀 Multiprocessing for faster computation. -## How to run -Optilab has been tested to work on the latest python versions. To install it, just run `make install`. +## How to install +Optilab has been tested to work on python versions 3.11 and above. To install it from PyPI, run: +``` +pip install optilab +``` +You can also install from source by cloning this repo and running: +``` +make install +``` ## Try the demos -If you're not sure how to start using optilab, see some examples in `demo` directory. +Learn how to use optilab by using our demo notebook. See `demo/tutorial.ipynb`. + +## CLI tool +Optilab comes with a powerful CLI tool to easily summarize your experiments. It allows for plotting the results and performing statistical testing to check for statistical significance in optimization results. +``` +Optilab CLI utility. +usage: python -m optilab [-h] [--hide_plots] [--test_y] [--test_evals] pickle_path + +positional arguments: + pickle_path Path to pickle file or directory with optimization runs. + +options: + -h, --help show this help message and exit + --hide_plots Hide plots when running the script. + --test_y Perform Mann-Whitney U test on y values. + --test_evals Perform Mann-Whitney U test on eval values. +``` ## Docker This project comes with a docker container. You can pull it from dockerhub: diff --git a/demo/perform_optimization.ipynb b/demo/perform_optimization.ipynb deleted file mode 100644 index 335fe76..0000000 --- a/demo/perform_optimization.ipynb +++ /dev/null @@ -1,371 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "a933bf15-cd99-41d9-9833-2b6657dfb763", - "metadata": {}, - "source": [ - "# Optilab tutorial: performing optimization\n", - "This tutorial notebook aims to explain how to perform optimization using optilab API. In this notebook you will learn how to:\n", - "- create an instance of optimizer and objective function,\n", - "- run optimization on the function,\n", - "- visualize the results of the optimization,\n", - "- save optimization result to a pickle file,\n", - "- read and visualize the results using optilab's CLI tool." - ] - }, - { - "cell_type": "markdown", - "id": "37633541-1aee-460f-adad-f60cab7dd0d7", - "metadata": {}, - "source": [ - "## Creating an objective function\n", - "Optilab comes with a lot of common black-box optimization functions. Here, lets create instances of a sphere function. Let's say we want to perform optimization in 2 dimensions:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b4d70253-c4db-4179-9c05-99eb15995729", - "metadata": {}, - "outputs": [], - "source": [ - "# define a constant to hold the dimensionality of the problem\n", - "DIM = 2\n", - "\n", - "# import the sphere function from optilab\n", - "from optilab.functions.unimodal import SphereFunction\n", - "\n", - "# create a sphere function instance with given dimensionality\n", - "objective_function = SphereFunction(DIM)" - ] - }, - { - "cell_type": "markdown", - "id": "0db11933-97c0-4b78-93c6-aebfbd1b5715", - "metadata": {}, - "source": [ - "Let's also create an instance of a harder, multimodal objective function:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "289046df-4ef8-4f18-824d-1665bae347f8", - "metadata": {}, - "outputs": [], - "source": [ - "from optilab.functions.multimodal import RastriginFunction\n", - "\n", - "multimodal_objective = RastriginFunction(DIM)" - ] - }, - { - "cell_type": "markdown", - "id": "6ee2577f-be0b-429f-9908-b2b0573c9d03", - "metadata": {}, - "source": [ - "We can see some information about a function in it's metadata:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "41bac97b-94b5-4855-8f25-577b8723373f", - "metadata": {}, - "outputs": [], - "source": [ - "print(objective_function.get_metadata())\n", - "print(multimodal_objective.get_metadata())" - ] - }, - { - "cell_type": "markdown", - "id": "850fbcf3-f6b5-47d7-b5a9-5c915f08264a", - "metadata": {}, - "source": [ - "## Creating an optimizer\n", - "Now let's create an instance of CMA-ES optimizer. Since optilab comes with implementations of some optimizers it's as easy as importing it from the library. Let's also create an instance of LMM-CMA-ES, which we will use to compare the two later on:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "345e17bb-25dd-493e-8e1f-70ff3093de8a", - "metadata": {}, - "outputs": [], - "source": [ - "# define constants for the population size and sigma0 for cmaes optimizers\n", - "POPSIZE = DIM * 2\n", - "SIGMA0 = 1\n", - "\n", - "# import optimizers from optilab\n", - "from optilab.optimizers import CmaEs, LmmCmaEs\n", - "\n", - "# create instances of the optimizers\n", - "cmaes_optimizer = CmaEs(POPSIZE, SIGMA0)\n", - "lmm_cmaes_optimizer = LmmCmaEs(POPSIZE, SIGMA0, 2)" - ] - }, - { - "cell_type": "markdown", - "id": "95bccbe6-28a7-4159-9800-a3c634df08fc", - "metadata": {}, - "source": [ - "Let's see some information about the optimizers." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a3d5062b-3bfa-4e32-b417-ab4e89caff75", - "metadata": {}, - "outputs": [], - "source": [ - "print(cmaes_optimizer.metadata)\n", - "print(lmm_cmaes_optimizer.metadata)" - ] - }, - { - "cell_type": "markdown", - "id": "764f5e92-0c44-4e54-88e9-2b2ce33ace7d", - "metadata": {}, - "source": [ - "## Perform optimization\n", - "Let's now put the two together and optimize the objective functions using the cmaes and lmm_cmaes optimizers. Let's start by defining the bounds of the problem, the number of allowed calls to the function, and the tolerance of value: " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a476b4c5-aa38-4229-8d9e-a7e663c7027a", - "metadata": {}, - "outputs": [], - "source": [ - "# import Bounds class from optilab\n", - "from optilab.data_classes import Bounds\n", - "\n", - "# define the constants\n", - "BOUNDS = Bounds(-100, 100)\n", - "CALL_BUDGET = DIM * 10e4\n", - "TOLERANCE = 1e-8" - ] - }, - { - "cell_type": "markdown", - "id": "94e8f068-79c2-4c24-a2a1-84868f754f86", - "metadata": {}, - "source": [ - "Optimization can be done using `optimize()` method. It returns a `PointList` object, which is a log of all points evaluated using the objective function." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "244e9a8f-6eeb-45b2-8671-750b4b2d9e8a", - "metadata": {}, - "outputs": [], - "source": [ - "# perform optimization\n", - "cmaes_log = cmaes_optimizer.optimize(objective_function, BOUNDS, CALL_BUDGET, TOLERANCE)\n", - "lmm_cmaes_log = lmm_cmaes_optimizer.optimize(objective_function, BOUNDS, CALL_BUDGET, TOLERANCE)\n", - "\n", - "# see the results\n", - "print(f'CMAES: num_evals: {len(cmaes_log)}, best value: {cmaes_log.best_y()}')\n", - "print(f'LMM CMAES: num_evals: {len(lmm_cmaes_log)}, best value: {lmm_cmaes_log.best_y()}')" - ] - }, - { - "cell_type": "markdown", - "id": "6c8758fe-62bb-4821-a26b-c1ba60d42f82", - "metadata": {}, - "source": [ - "However the recommended way is to use `run_optimization()` method, which performs the desired number of runs, displays a progessbar, allows for multiprocessing, and returns the result in the format used in optilab CLI tool:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d471e7c0-cf14-4059-b82c-e361b3e67495", - "metadata": {}, - "outputs": [], - "source": [ - "# define the number of runs\n", - "RUNS = 51\n", - "\n", - "# perform optimization\n", - "cmaes_runs = cmaes_optimizer.run_optimization(RUNS, objective_function, BOUNDS, CALL_BUDGET, TOLERANCE)\n", - "lmm_cmaes_runs = lmm_cmaes_optimizer.run_optimization(RUNS, objective_function, BOUNDS, CALL_BUDGET, TOLERANCE)" - ] - }, - { - "cell_type": "markdown", - "id": "e7544cb4-eecb-4a90-b428-9c853fdd004b", - "metadata": {}, - "source": [ - "To speed up your experiments you can use multiprocessing. Bear in mind however, that it does not work for all optimizers and hyperparameter configurations." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2c8484dc-b1b9-48e8-b5a5-99b9dcae9845", - "metadata": {}, - "outputs": [], - "source": [ - "# number of processes to use\n", - "PROCESSES = 4\n", - "\n", - "# perform multiprocessed optimization\n", - "_ = cmaes_optimizer.run_optimization(RUNS, objective_function, BOUNDS, CALL_BUDGET, TOLERANCE, num_processes=PROCESSES)" - ] - }, - { - "cell_type": "markdown", - "id": "91e20989-fbe1-4c51-8cf2-9c89e19387b2", - "metadata": {}, - "source": [ - "Optimization results are returned in a data structure called `OptimizationRun`. It stores the metadata of the optimizer, objective function, other optimization hyperparameters and the log of objective values." - ] - }, - { - "cell_type": "markdown", - "id": "f3c37e58-2fb8-4c20-93a5-9b1e713decf0", - "metadata": {}, - "source": [ - "## Plotting optimization results\n", - "Optilab provides user with functions for plotting the convergence curves, ecdf curves and box plots of optimization runs. Let's first plot the convergence curves for CMA-ES and LMM-CMA-ES:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1d14b734-e86b-4bcc-b5ec-5b68ddad759d", - "metadata": {}, - "outputs": [], - "source": [ - "from optilab.plotting import plot_box_plot, plot_convergence_curve, plot_ecdf_curves\n", - "\n", - "# plot convergence curve for the two optimizers\n", - "plot_convergence_curve({run.model_metadata.name: run.logs for run in [cmaes_runs, lmm_cmaes_runs]})" - ] - }, - { - "cell_type": "markdown", - "id": "f8c7346b-bb57-42ac-b0d8-65a5c816bea3", - "metadata": {}, - "source": [ - "As you can see the LMM-CMA-ES converges much quicker than regular CMA-ES. Now let's plot the ECDF curve:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a792d285-351b-49bd-a327-252805d8eebf", - "metadata": {}, - "outputs": [], - "source": [ - "plot_ecdf_curves({run.model_metadata.name: run.logs for run in [cmaes_runs, lmm_cmaes_runs]}, DIM, TOLERANCE)" - ] - }, - { - "cell_type": "markdown", - "id": "ab0ff345-f254-4ced-a010-f595f83876bf", - "metadata": {}, - "source": [ - "Lastly, let's plot the box plot of optimization results:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "eca95650-aabb-4526-a62b-e5822781d107", - "metadata": {}, - "outputs": [], - "source": [ - "plot_box_plot(data={run.model_metadata.name: run.bests_y() for run in [cmaes_runs, lmm_cmaes_runs]})" - ] - }, - { - "cell_type": "markdown", - "id": "9b71e00a-3f5f-4bb4-8e5a-fa2269656358", - "metadata": {}, - "source": [ - "Plotting functions also alow for saving the plots to an image file. Use keyword argument `savepath=` to specify where to save the image." - ] - }, - { - "cell_type": "markdown", - "id": "feb6e473-3fd6-45c4-b5dc-9dbf7bbf35e1", - "metadata": {}, - "source": [ - "## Save results to a pickle file and analyze it using optilab's CLI tool\n", - "To save the results of an experiment you can dump optimization runs to a pickle file and then read it and plot is using optilab's CLI functionality. Firstly, pack all OptimizationRun objects into a list. Then use a utility function to save it to a pickle file." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "a23ac042-18a0-40da-94da-f7704be2b4b3", - "metadata": {}, - "outputs": [], - "source": [ - "from optilab.utils import dump_to_pickle\n", - "\n", - "runs = [cmaes_runs, lmm_cmaes_runs]\n", - "\n", - "SAVEFILE_NAME = 'tutorial.pkl'\n", - "\n", - "dump_to_pickle(runs, SAVEFILE_NAME)" - ] - }, - { - "cell_type": "markdown", - "id": "8fc57ab4-c8b2-4527-83c6-2afc5e3235f8", - "metadata": {}, - "source": [ - "Now that you saved the results to a pickle file, you can read it into the CLI tool to get various information about the results. The CLI tool also allows to perform statistical testing on the results to determine if the difference in results is statistically significant." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "1266ec91-4670-4883-94cd-751e5f549d11", - "metadata": {}, - "outputs": [], - "source": [ - "!python -m optilab $SAVEFILE_NAME --test_y --test_eval" - ] - }, - { - "cell_type": "markdown", - "id": "66a083f2-a583-440b-8d87-aea1cc433004", - "metadata": {}, - "source": [ - "## Closing remarks\n", - "Thank you for choosing optilab for your project. If you wish to learn more checkout the projects repo on [github](https://github.com/mlojek/optilab) and the project's documentation on [readthedocs](https://optilab.readthedocs.io). Feel free to use optilab in your research and work. If you wish to contribute to the project, feel free to do so yourself or leave an issue in the repo. Best of luck, Marcin." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.13.1" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/demo/tutorial.ipynb b/demo/tutorial.ipynb new file mode 100644 index 0000000..d280d8d --- /dev/null +++ b/demo/tutorial.ipynb @@ -0,0 +1,488 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a933bf15-cd99-41d9-9833-2b6657dfb763", + "metadata": {}, + "source": [ + "# Optilab tutorial: performing optimization\n", + "This tutorial notebook aims to explain how to perform optimization using optilab API. In this notebook you will learn how to:\n", + "- create an instance of optimizer and objective function,\n", + "- run optimization on the function,\n", + "- visualize the results of the optimization,\n", + "- save optimization result to a pickle file,\n", + "- read and visualize the results using optilab's CLI tool." + ] + }, + { + "cell_type": "markdown", + "id": "37633541-1aee-460f-adad-f60cab7dd0d7", + "metadata": {}, + "source": [ + "## Creating an objective function\n", + "Optilab comes with a lot of common black-box optimization functions. Here, lets create instances of a sphere function. Let's say we want to perform optimization in 2 dimensions:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "b4d70253-c4db-4179-9c05-99eb15995729", + "metadata": {}, + "outputs": [], + "source": [ + "# define a constant to hold the dimensionality of the problem\n", + "DIM = 2\n", + "\n", + "# import the sphere function from optilab\n", + "from optilab.functions.unimodal import SphereFunction\n", + "\n", + "# create a sphere function instance with given dimensionality\n", + "objective_function = SphereFunction(DIM)" + ] + }, + { + "cell_type": "markdown", + "id": "0db11933-97c0-4b78-93c6-aebfbd1b5715", + "metadata": {}, + "source": [ + "Let's also create an instance of a harder, multimodal objective function:" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "289046df-4ef8-4f18-824d-1665bae347f8", + "metadata": {}, + "outputs": [], + "source": [ + "from optilab.functions.multimodal import RastriginFunction\n", + "\n", + "multimodal_objective = RastriginFunction(DIM)" + ] + }, + { + "cell_type": "markdown", + "id": "6ee2577f-be0b-429f-9908-b2b0573c9d03", + "metadata": {}, + "source": [ + "We can see some information about a function in it's metadata:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "41bac97b-94b5-4855-8f25-577b8723373f", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "FunctionMetadata(name='sphere', dim=2, hyperparameters={})\n", + "FunctionMetadata(name='rastrigin', dim=2, hyperparameters={})\n" + ] + } + ], + "source": [ + "print(objective_function.get_metadata())\n", + "print(multimodal_objective.get_metadata())" + ] + }, + { + "cell_type": "markdown", + "id": "850fbcf3-f6b5-47d7-b5a9-5c915f08264a", + "metadata": {}, + "source": [ + "## Creating an optimizer\n", + "Now let's create an instance of CMA-ES optimizer. Since optilab comes with implementations of some optimizers it's as easy as importing it from the library. Let's also create an instance of LMM-CMA-ES, which we will use to compare the two later on:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "345e17bb-25dd-493e-8e1f-70ff3093de8a", + "metadata": {}, + "outputs": [], + "source": [ + "# define constants for the population size and sigma0 for cmaes optimizers\n", + "POPSIZE = DIM * 2\n", + "SIGMA0 = 1\n", + "\n", + "# import optimizers from optilab\n", + "from optilab.optimizers import CmaEs, LmmCmaEs\n", + "\n", + "# create instances of the optimizers\n", + "cmaes_optimizer = CmaEs(POPSIZE, SIGMA0)\n", + "lmm_cmaes_optimizer = LmmCmaEs(POPSIZE, SIGMA0, 2)" + ] + }, + { + "cell_type": "markdown", + "id": "95bccbe6-28a7-4159-9800-a3c634df08fc", + "metadata": {}, + "source": [ + "Let's see some information about the optimizers." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "a3d5062b-3bfa-4e32-b417-ab4e89caff75", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "OptimizerMetadata(name='cma-es', population_size=4, hyperparameters={'sigma0': 1})\n", + "OptimizerMetadata(name='lmm-cma-es', population_size=4, hyperparameters={'sigma0': 1, 'polynomial_dim': 2})\n" + ] + } + ], + "source": [ + "print(cmaes_optimizer.metadata)\n", + "print(lmm_cmaes_optimizer.metadata)" + ] + }, + { + "cell_type": "markdown", + "id": "764f5e92-0c44-4e54-88e9-2b2ce33ace7d", + "metadata": {}, + "source": [ + "## Perform optimization\n", + "Let's now put the two together and optimize the objective functions using the cmaes and lmm_cmaes optimizers. Let's start by defining the bounds of the problem, the number of allowed calls to the function, and the tolerance of value: " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "a476b4c5-aa38-4229-8d9e-a7e663c7027a", + "metadata": {}, + "outputs": [], + "source": [ + "# import Bounds class from optilab\n", + "from optilab.data_classes import Bounds\n", + "\n", + "# define the constants\n", + "BOUNDS = Bounds(-100, 100)\n", + "CALL_BUDGET = DIM * 10e4\n", + "TOLERANCE = 1e-8" + ] + }, + { + "cell_type": "markdown", + "id": "94e8f068-79c2-4c24-a2a1-84868f754f86", + "metadata": {}, + "source": [ + "Optimization can be done using `optimize()` method. It returns a `PointList` object, which is a log of all points evaluated using the objective function." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "244e9a8f-6eeb-45b2-8671-750b4b2d9e8a", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CMAES: num_evals: 284, best value: 4.0930336381123275e-09\n", + "LMM CMAES: num_evals: 79, best value: 1.6974317411842597e-09\n" + ] + } + ], + "source": [ + "# perform optimization\n", + "cmaes_log = cmaes_optimizer.optimize(objective_function, BOUNDS, CALL_BUDGET, TOLERANCE)\n", + "lmm_cmaes_log = lmm_cmaes_optimizer.optimize(objective_function, BOUNDS, CALL_BUDGET, TOLERANCE)\n", + "\n", + "# see the results\n", + "print(f'CMAES: num_evals: {len(cmaes_log)}, best value: {cmaes_log.best_y()}')\n", + "print(f'LMM CMAES: num_evals: {len(lmm_cmaes_log)}, best value: {lmm_cmaes_log.best_y()}')" + ] + }, + { + "cell_type": "markdown", + "id": "6c8758fe-62bb-4821-a26b-c1ba60d42f82", + "metadata": {}, + "source": [ + "However the recommended way is to use `run_optimization()` method, which performs the desired number of runs, displays a progessbar, allows for multiprocessing, and returns the result in the format used in optilab CLI tool:" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "d471e7c0-cf14-4059-b82c-e361b3e67495", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimizing...: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 51/51 [00:01<00:00, 27.68run/s]\n", + "Optimizing...: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 51/51 [01:20<00:00, 1.59s/run]\n" + ] + } + ], + "source": [ + "# define the number of runs\n", + "RUNS = 51\n", + "\n", + "# perform optimization\n", + "cmaes_runs = cmaes_optimizer.run_optimization(RUNS, objective_function, BOUNDS, CALL_BUDGET, TOLERANCE)\n", + "lmm_cmaes_runs = lmm_cmaes_optimizer.run_optimization(RUNS, objective_function, BOUNDS, CALL_BUDGET, TOLERANCE)" + ] + }, + { + "cell_type": "markdown", + "id": "e7544cb4-eecb-4a90-b428-9c853fdd004b", + "metadata": {}, + "source": [ + "To speed up your experiments you can use multiprocessing. Bear in mind however, that it does not work for all optimizers and hyperparameter configurations." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "2c8484dc-b1b9-48e8-b5a5-99b9dcae9845", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Optimizing...: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 51/51 [00:00<00:00, 101.40run/s]\n" + ] + } + ], + "source": [ + "# number of processes to use\n", + "PROCESSES = 4\n", + "\n", + "# perform multiprocessed optimization\n", + "_ = cmaes_optimizer.run_optimization(RUNS, objective_function, BOUNDS, CALL_BUDGET, TOLERANCE, num_processes=PROCESSES)" + ] + }, + { + "cell_type": "markdown", + "id": "91e20989-fbe1-4c51-8cf2-9c89e19387b2", + "metadata": {}, + "source": [ + "Optimization results are returned in a data structure called `OptimizationRun`. It stores the metadata of the optimizer, objective function, other optimization hyperparameters and the log of objective values." + ] + }, + { + "cell_type": "markdown", + "id": "f3c37e58-2fb8-4c20-93a5-9b1e713decf0", + "metadata": {}, + "source": [ + "## Plotting optimization results\n", + "Optilab provides user with functions for plotting the convergence curves, ecdf curves and box plots of optimization runs. Let's first plot the convergence curves for CMA-ES and LMM-CMA-ES:" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "1d14b734-e86b-4bcc-b5ec-5b68ddad759d", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkIAAAHHCAYAAABTMjf2AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAB1sUlEQVR4nO3dd3wUdf7H8ddukk2vhCSEhARCJ5DQRaQHARVFLJwNBPVs2LCc3Cme5Secepz1RL1T7GDXsyACIqJID70TCC0JLZ3Und8fa1YjLYEks5t9Px+PfczszOzM+5uN5ON3vjNjMQzDQERERMQDWc0OICIiImIWFUIiIiLisVQIiYiIiMdSISQiIiIeS4WQiIiIeCwVQiIiIuKxVAiJiIiIx1IhJCIiIh5LhZCIiIh4LBVCIiIi4rFUCIm4qR07dnDzzTfTqlUr/Pz8CAkJoW/fvjz33HMcO3bM7HgiIm7B2+wAIlJ7X331FVdccQW+vr6MHTuW5ORkysrKWLx4Mffffz8bNmzg1VdfNTumiIjLs+ihqyLuJSMjgy5duhAXF8eCBQto1qxZtfXbt2/nq6++4q677jIp4dkrKSnBZrNhtarT+mxUVFRgt9ux2WxmRxFxWfpXRsTNPPXUUxQWFvLf//73uCIIoHXr1tWKoIqKCh5//HGSkpLw9fUlMTGRv/71r5SWllb7XGJiIhdddBGLFy+mV69e+Pn50apVK9566y3nNitWrMBisfDmm28ed9xvv/0Wi8XCl19+6Vy2b98+JkyYQHR0NL6+vnTq1InXX3+92ucWLlyIxWJh1qxZPPTQQzRv3pyAgADy8/MB+PDDD+nYsSN+fn4kJyfz6aefcv3115OYmFhtP3a7nWeffZZOnTrh5+dHdHQ0N998M0ePHq11O6vk5uZyzz33kJiYiK+vL3FxcYwdO5ZDhw45tyktLeWRRx6hdevW+Pr6Eh8fzwMPPHDcz/dkli5dygUXXEB4eDiBgYF06dKF5557zrl+4MCBDBw48LjP/fFnsGvXLiwWC8888wzPPvus8/tevXo13t7ePProo8ftY8uWLVgsFl588cVqbb777ruJj4/H19eX1q1b849//AO73V7ts7NmzaJ79+4EBwcTEhJC586dq+UWcRuGiLiV5s2bG61atarx9uPGjTMA4/LLLzdeeuklY+zYsQZgjBo1qtp2CQkJRrt27Yzo6Gjjr3/9q/Hiiy8a3bp1MywWi7F+/Xrndq1atTIuuOCC444zfvx4Izw83CgrKzMMwzCysrKMuLg4Iz4+3njssceMl19+2bj44osNwPjXv/7l/Nz3339vAEbHjh2N1NRUY/r06cbUqVONoqIi48svvzQsFovRpUsXY/r06cbDDz9shIeHG8nJyUZCQkK14994442Gt7e3cdNNNxkzZsww/vKXvxiBgYFGz549nZlq086CggIjOTnZ8PLyMm666Sbj5ZdfNh5//HGjZ8+exurVqw3DMIzKykrj/PPPNwICAoy7777beOWVV4yJEyca3t7exiWXXHLa72bu3LmGzWYzEhISjEceecR4+eWXjTvvvNNIS0tzbjNgwABjwIABx3123Lhx1X4GGRkZzp9jq1atjGnTphn/+te/jN27dxuDBw82OnbseNw+Hn30UcPLy8vIysoyDMMwioqKjC5duhhNmjQx/vrXvxozZswwxo4da1gsFuOuu+6qlhswhgwZYrz00kvGSy+9ZEycONG44oorTttmEVejQkjEjeTl5RlAjf7IGoZhpKenG4Bx4403Vlt+3333GYCxYMEC57KEhAQDMBYtWuRclpOTY/j6+hr33nuvc9nkyZMNHx8f48iRI85lpaWlRlhYmDFhwgTnshtuuMFo1qyZcejQoWrH/tOf/mSEhoYaxcXFhmH8Vgi1atXKuaxK586djbi4OKOgoMC5bOHChQZQrQj48ccfDcB49913q31+zpw5xy2vaTunTJliAMYnn3xi/JHdbjcMwzDefvttw2q1Gj/++GO19TNmzDAA46effjrus1UqKiqMli1bGgkJCcbRo0dPuH/DqH0hFBISYuTk5FTb9pVXXjEAY926ddWWd+zY0Rg8eLDz/eOPP24EBgYaW7durbbdgw8+aHh5eRmZmZmGYRjGXXfdZYSEhBgVFRUnbZ+Iu9CpMRE3UnW6KDg4uEbbf/311wBMmjSp2vJ7770XcAy6/r2OHTvSr18/5/umTZvSrl07du7c6Vw2ZswYysvL+eSTT5zL5s6dS25uLmPGjAHAMAw+/vhjRo4ciWEYHDp0yPkaNmwYeXl5rFq1qtqxx40bh7+/v/P9/v37WbduHWPHjiUoKMi5fMCAAXTu3LnaZz/88ENCQ0MZOnRotWN1796doKAgvv/++1q38+OPPyYlJYVLL730uJ+rxWJxHrdDhw60b9++2nEHDx4McNxxf2/16tVkZGRw9913ExYWdsL9n4nLLruMpk2bVls2evRovL29mT17tnPZ+vXr2bhxo/M7q2pPv379CA8Pr9aetLQ0KisrWbRoEQBhYWEUFRXx3XffnXFOEVehq8ZE3EhISAgABQUFNdp+9+7dWK1WWrduXW15TEwMYWFh7N69u9ryFi1aHLeP8PDwauNsUlJSaN++PbNnz+aGG24AYPbs2URGRjoLgIMHD5Kbm8urr7560qvXcnJyqr1v2bLlcdmB47JXLft9IbVt2zby8vKIioqq0bFq0s4dO3Zw2WWXnXB/vz/upk2bjis8Tnbc39uxYwcAycnJpzxGbf3x5wgQGRnJkCFD+OCDD3j88ccBx3fm7e3N6NGjndtt27aNtWvXnrY9t912Gx988AEjRoygefPmnH/++Vx55ZUMHz68Ttsi0hBUCIm4kZCQEGJjY1m/fn2tPlfTHgYvL68TLjf+cHHpmDFj+L//+z8OHTpEcHAwX3zxBVdddRXe3o5/UqoG1l577bWMGzfuhPvs0qVLtfe/7w2qLbvdTlRUFO++++4J1//xD3tN21mT43bu3Jnp06efcH18fHyt9nciFovlhLkqKytPuP3Jfo5/+tOfGD9+POnp6aSmpvLBBx8wZMgQIiMjndvY7XaGDh3KAw88cMJ9tG3bFoCoqCjS09P59ttv+eabb/jmm2944403GDt27AkH0ou4MhVCIm7moosu4tVXX2XJkiX06dPnlNsmJCRgt9vZtm0bHTp0cC7Pzs4mNzeXhISEM8owZswYHn30UT7++GOio6PJz8/nT3/6k3N906ZNCQ4OprKykrS0tDM6RlW27du3H7fuj8uSkpKYN28effv2PauC6o/7PF3BmZSUxJo1axgyZEitT2clJSUBjlNUp/oZhYeHVztlV+WPvXmnM2rUKG6++Wbn6bGtW7cyefLk4zIVFhbW6Duz2WyMHDmSkSNHYrfbue2223jllVd4+OGHT9iLJ+KqNEZIxM088MADBAYGcuONN5KdnX3c+h07djgvY77gggsAePbZZ6ttU9WDceGFF55Rhg4dOtC5c2dmz57N7NmzadasGf3793eu9/Ly4rLLLuPjjz8+YTFx8ODB0x4jNjaW5ORk3nrrLQoLC53Lf/jhB9atW1dt2yuvvJLKykrnaZ/fq6ioIDc3txatc7jssstYs2YNn3766XHrqnporrzySvbt28drr7123DbHjh2jqKjopPvv1q0bLVu25Nlnnz0u3+97gJKSkti8eXO1n9maNWv46aefatWesLAwhg0bxgcffMCsWbOw2WyMGjWq2jZXXnklS5Ys4dtvvz3u87m5uVRUVABw+PDhauusVquzh6+mtw0QcRXqERJxM0lJSbz33nuMGTOGDh06VLuz9M8//8yHH37I9ddfDzjG84wbN45XX32V3NxcBgwYwLJly3jzzTcZNWoUgwYNOuMcY8aMYcqUKfj5+XHDDTccd/PDadOm8f3339O7d29uuukmOnbsyJEjR1i1ahXz5s3jyJEjpz3Gk08+ySWXXELfvn0ZP348R48e5cUXXyQ5OblacTRgwABuvvlmpk6dSnp6Oueffz4+Pj5s27aNDz/8kOeee47LL7+8Vu27//77+eijj7jiiiuYMGEC3bt358iRI3zxxRfMmDGDlJQUrrvuOj744ANuueUWvv/+e/r27UtlZSWbN2/mgw8+4Ntvv6VHjx4n3L/VauXll19m5MiRpKamMn78eJo1a8bmzZvZsGGDsxiZMGEC06dPZ9iwYdxwww3k5OQwY8YMOnXq5Bw8X1Njxozh2muv5d///jfDhg07bpD2/fffzxdffMFFF13E9ddfT/fu3SkqKmLdunV89NFH7Nq1i8jISG688UaOHDnC4MGDiYuLY/fu3bzwwgukpqZW63kUcQsmXrEmImdh69atxk033WQkJiYaNpvNCA4ONvr27Wu88MILRklJiXO78vJy49FHHzVatmxp+Pj4GPHx8cbkyZOrbWMYjsvKL7zwwuOOc7LLt7dt22YABmAsXrz4hBmzs7ON22+/3YiPjzd8fHyMmJgYY8iQIcarr77q3Kbq8vkPP/zwhPuYNWuW0b59e8PX19dITk42vvjiC+Oyyy4z2rdvf9y2r776qtG9e3fD39/fCA4ONjp37mw88MADxv79+8+onYcPHzYmTpxoNG/e3LDZbEZcXJwxbty4arcEKCsrM/7xj38YnTp1Mnx9fY3w8HCje/fuxqOPPmrk5eWdsE2/t3jxYmPo0KFGcHCwERgYaHTp0sV44YUXqm3zzjvvGK1atTJsNpuRmppqfPvttye9fP7pp58+6bHy8/MNf39/AzDeeeedE25TUFBgTJ482WjdurVhs9mMyMhI49xzzzWeeeYZ5/2YPvroI+P88883oqKiDJvNZrRo0cK4+eabjQMHDpy2vSKuRo/YEBG3k5qaStOmTXX5toicNY0REhGXVV5e7hyXUmXhwoWsWbPmhI+dEBGpLfUIiYjL2rVrF2lpaVx77bXExsayefNmZsyYQWhoKOvXr6dJkyZmRxQRN6fB0iLissLDw+nevTv/+c9/OHjwIIGBgVx44YVMmzZNRZCI1An1CImIiIjH0hghERER8VgqhERERMRjaYzQadjtdvbv309wcPBZPRFaREREGo5hGBQUFBAbG3vcDV9/T4XQaezfv79OHpwoIiIiDW/Pnj3ExcWddL0KodMIDg4GHD/IkJAQk9OIiIhITeTn5xMfH+/8O34yKoROo+p0WEhIiAohERERN3O6YS0aLC0iIiIeS4WQiIiIeCwVQiIiIuKxNEZIRERcnt1up6yszOwY4kJ8fHzw8vI66/2oEBIREZdWVlZGRkYGdrvd7CjiYsLCwoiJiTmr+/ypEBIREZdlGAYHDhzAy8uL+Pj4U94YTzyHYRgUFxeTk5MDQLNmzc54XyqERETEZVVUVFBcXExsbCwBAQFmxxEX4u/vD0BOTg5RUVFnfJpMpbWIiLisyspKAGw2m8lJxBVVFcfl5eVnvA8VQiIi4vL0rEc5kbr4vVAhJCIiIh5LhZCIiIh4LBVCIiIi4rFUCJkk83Ax2fkllFZUmh1FRETEY6kQMsl//vMCf5n2DCMefp0BU2YxeNo3jHxhMWNfX8at76zkntnpPPq/Dby+OIMPV+xhyY7DrN2bS1FphdnRRUSkBux2O0899RStW7fG19eXFi1a8H//93/s2rULi8XCBx98QL9+/fD396dnz55s3bqV5cuX06NHD4KCghgxYgQHDx507m/58uUMHTqUyMhIQkNDGTBgAKtWrTptjj179nDllVcSFhZGREQEl1xyCbt27XKuX7hwIb169SIwMJCwsDD69u3L7t276+NH4pJ0HyGT3Fj6Ji1s+35bUAKbiluQlRPuXJRjhLPYnkwpPpThw1J7e8qs/sSE+NEkyEaTQBsJTQIZ1D6KLs1DCQ/U5aUi0rgZhsGxcnN60v19vGp1ldLkyZN57bXX+Ne//sV5553HgQMH2Lx5s3P9I488wrPPPkuLFi2YMGECV199NcHBwTz33HMEBARw5ZVXMmXKFF5++WUACgoKGDduHC+88AKGYfDPf/6TCy64gG3bthEcHHzCDOXl5QwbNow+ffrw448/4u3tzRNPPMHw4cNZu3YtVquVUaNGcdNNN/H+++9TVlbGsmXLPOoqPYthGIbZIVxZfn4+oaGh5OXlERISUjc7NQz45CaM7A2QmwllRVg4/ddQQAD77E341t6TFypGUfGHOrZpsC8hft4M7RjD6G7NiQn1I9jX26N+oUWkcSkpKSEjI4OWLVvi5+dHcVkFHad8a0qWjY8NI8BWs/6DgoICmjZtyosvvsiNN95Ybd2uXbto2bIl//nPf7jhhhsAmDVrFldddRXz589n8ODBAEybNo2ZM2dWK55+z263ExYWxnvvvcdFF110wm3eeecdnnjiCTZt2uT8W1BWVkZYWBifffYZPXr0oEmTJixcuJABAwbUqG2u5I+/H79X07/f6hEyg8UCl/0HZ3liGFB0EHb/DGVFVQth/2rIWudYX3CA4Lw9tLcW0966h+ua7WNB9xdYtreEJTsPs+fIMQ4WlHKwoJQdP+xgxg87AAiwedGhWQh/H9mJznGhZrRWRMTjbNq0idLSUoYMGXLSbbp06eKcj46OBqBz587VllU9QgIgOzubhx56iIULF5KTk0NlZSXFxcVkZmYCcMstt/DOO+84ty8sLGTNmjVs3779uB6jkpISduzYwfnnn8/111/PsGHDGDp0KGlpaVx55ZVn9cgKd6NCyBVYLBAUBZ1GVV/e9drf5u2VsD8dcjbCnMlEHFzK5Tsf4fI/vQdWKwUl5WQcKiLzSDFv/LSL7TmF5B0rp7iskpW7jzLyxcXEhPjRvlkwqfFhXNEjnuZh/g3ZShGRs+bv48XGx4aZduwab+t/+n9ffXx8nPNVvTV/XPb7B82OGzeOw4cP89xzz5GQkICvry99+vShrKwMgMcee4z77ruv2jEKCwvp3r0777777nHHb9q0KQBvvPEGd955J3PmzGH27Nk89NBDfPfdd5xzzjk1bq87UyHkLqxeENfd8WqSBG9fClu/gZ+fh/PuJtjPhy5xYXSJC+OiLrEAHCurZF9uMf+at42v1h4gK7+ErPwSFm45yMyfd/HIyI70a9OUyCBfkxsnIlIzFoulxqenzNSmTRv8/f2ZP3/+cafGztRPP/3Ev//9by644ALAMQj60KFDzvVRUVFERUVV+0y3bt2YPXs2UVFRpzw91LVrV7p27crkyZPp06cP7733nscUQrpqzB0lnAsjnnLML3gcDu844Wb+Ni9aRwXz0tXdWPf38/n41j48dkknOsWGkFtczj2z19DjiXmkTf+BqV9v4liZLuUXEakLfn5+/OUvf+GBBx7grbfeYseOHfzyyy/897//PeN9tmnThrfffptNmzaxdOlSrrnmmtP2PF1zzTVERkZyySWX8OOPP5KRkcHChQu588472bt3LxkZGUyePJklS5awe/du5s6dy7Zt2+jQocMZ53Q3KoTcVbexkDQE7BWOXqHTCPbzoXtCBGP7JPLRLedy28Ak2sc4zhlvzynklUU7+fPbK8jKK6nv5CIiHuHhhx/m3nvvZcqUKXTo0IExY8ZUG/NTW//97385evQo3bp147rrruPOO+88rgfojwICAli0aBEtWrRg9OjRdOjQgRtuuIGSkhJCQkIICAhg8+bNXHbZZbRt25Y///nP3H777dx8881nnNPd6Kqx06iXq8bqyu4l8MZw8LLB3esgOKbWuzhaVMYPWw8y+ZN1HCuvxGqBlPgw+rWO5OreCcSE+p1+JyIi9eRUVwWJ1MVVY+oRcmcJfSD+HKgsgyUvntEuwgNtjOranHdu7E2vxAjsBqzOzOX5BdsZ/e+fOFRYWsehRUREXIcKIXfX717HdPnrUHzkjHfTPSGcD27pw5LJg3nq8i4kNAlgf14Jg55ZyI1vruDrdQfIztdpMxERaVxUCLm7NkMhpguUF8Hqt896d81C/bmyRzyvX9+T6BBfCkoqmLcpm9veXcWAp79n+a4zL7ZERERcjQohd2exQPdxjvlN/6uz3SY1DWLRA4P48o7zuOG8lgTYvCgpt/PwZ+upqLSffgciIiJuQIVQY9D+IsACe5dD/v46262vtxfJzUN5+KKOLP7LYEL8vNmcVcC50xbw9pJd2O0aZy8iIu5NhVBjEBwDcT0d85u/qpdDRATaeOryFEL9fcgpKOXhzzcw9F8/8OO2g6f/sIiIiItSIdRYdBjpmNbh6bE/Gp4cw/K/pfH3kR0J9vNmx8Eibnl7Jftzj9XbMUVEROqTCqHGosOvTx7etfisrh47HZu3lev7tuTnBwfTPSGcorJK7pq1mn0qhkRExA2pEGosIlpBVCcwKmHrt/V+uGA/H/5xWWf8fKws33WUgU9/z6TZ6azfl1fvxxYREakrjb4Qys3NpUePHqSmppKcnMxrr71mdqT6U9UrtKV+xgn9UeuoYD6//TzOaRVBeaXBJ6v3MfLFxXy7IatBji8i4qoGDhzI3XffbXYMqYFGXwgFBwezaNEi0tPTWbp0KU8++SSHDx82O1b9aDPMMd25CCorGuSQ7WKCmfXnPnx+e1+GtI/CMGDyJ+vYfbioQY4vIiJyNhp9IeTl5UVAQAAApaWlGIZBo328Wmwq+IVBaR7sX9Wgh06JD+Pf13ajfUwwR4rKGDp9ER+t3NugGURERGrL5QuhRYsWMXLkSGJjY7FYLHz22WfHbfPSSy+RmJiIn58fvXv3ZtmyZdXW5+bmkpKSQlxcHPfffz+RkZENlL6BWb2g1UDH/I4FDX54X28vXhvbg76tm1BWaecvH6/lwxV7dL8hEfFoiYmJPPHEE4wdO5agoCASEhL44osvOHjwIJdccglBQUF06dKFFStWOD8zc+ZMwsLC+PLLL2nXrh0BAQFcfvnlFBcX8+abb5KYmEh4eDh33nknlZWVpzx+bm4uN998M9HR0fj5+ZGcnMyXX355Vsd5++236dGjB8HBwcTExHD11VeTk5Nz2p/F4sWL6devH/7+/sTHx3PnnXdSVPTbGYR///vftGnTBj8/P6Kjo7n88str++OuNZcvhIqKikhJSeGll1464frZs2czadIkHnnkEVatWkVKSgrDhg2r9oWEhYWxZs0aMjIyeO+998jOzm6o+A0vaZBjakIhBBAfEcA7N/RmdLfmVNoN7v9oLRPfb9jeKRFpxAwDyorMeZ3F2YR//etf9O3bl9WrV3PhhRdy3XXXMXbsWK699lpWrVpFUlISY8eOrXbGori4mOeff55Zs2YxZ84cFi5cyKWXXsrXX3/N119/zdtvv80rr7zCRx99dNLj2u12RowYwU8//cQ777zDxo0bmTZtGl5eXmd1nPLych5//HHWrFnDZ599xq5du7j++utP+TPYsWMHw4cP57LLLmPt2rXMnj2bxYsXM3HiRABWrFjBnXfeyWOPPcaWLVuYM2cO/fv3P8OfeM1ZDDc6T2SxWPj0008ZNWqUc1nv3r3p2bMnL77oePq63W4nPj6eO+64gwcffPC4fdx2220MHjz4pFVmaWkppaW/PXE9Pz+f+Ph48vLyCAkJqdsG1Yeju+G5LmD1hr/sAt9gU2KUV9qZsXAHzy/YRnmlwTs39Oa8No20J05E6k1JSQkZGRm0bNkSPz8/R0HyZKw5Yf66H2yBNdp04MCBpKam8uyzz5KYmEi/fv14+23H8yCzsrJo1qwZDz/8MI899hgAv/zyC3369OHAgQPExMQwc+ZMxo8fz/bt20lKSgLglltu4e233yY7O5ugoCAAhg8fTmJiIjNmzDhhjrlz5zJixAg2bdpE27Ztj1tfV8dZsWIFPXv2pKCgwPmZP7rxxhvx8vLilVdecS5bvHgxAwYMoKioiK+//prx48ezd+9egoNr9rfruN+P38nPzyc0NPS0f79dvkfoVMrKyli5ciVpaWnOZVarlbS0NJYsWQJAdnY2BQUFAOTl5bFo0SLatWt30n1OnTqV0NBQ5ys+Pr5+G1HXwhMgLAHsFZD5i2kxfLys3DGkDdf0TgDg/o/W8ML8bRwqLD3NJ0VEGp8uXbo456OjowHo3Lnzcct+fzYjICDAWZxUbZOYmFit0IiOjnZ+5sknnyQoKMj5yszMJD09nbi4uBMWQWd6HICVK1cycuRIWrRoQXBwMAMGDAAgMzMTgE6dOjlzjBgxAoA1a9Ywc+bMahmHDRuG3W4nIyODoUOHkpCQQKtWrbjuuut49913KS4uPu3P9mx51/sR6tGhQ4eorKx0/gJViY6OZvPmzQDs3r2bP//5z85B0nfccUe1X74/mjx5MpMmTXK+r+oRcist+zueRJ/xg+Pp9Ca6Y3Brvll/gAN5Jfzzu628+P12xvSM528XdsDX2+v0OxAR+T2fAEfPjFnHPtOP+vg45y0Wy0mX2e32E36mapsTLav6zC233MKVV17pXBcbG4u/v3+tstXkOEVFRQwbNoxhw4bx7rvv0rRpUzIzMxk2bBhlZWUAfP3115SXlwM4MxQWFnLzzTdz5513HpehRYsW2Gw2Vq1axcKFC5k7dy5Tpkzh73//O8uXLycsLOy07ThTbl0I1USvXr1IT0+v8fa+vr74+vrWX6CG0HLAr4XQIrOT0CTIl7l3D2Duxize+WU3a/bm8daS3cSG+XPLgKTT70BE5PcslhqfnvI0ERERREREVFvWpUsX9u7dy9atW0/ZK1Qbmzdv5vDhw0ybNs3ZUfD7gd4ACQkJx32uW7dubNy4kdatW590397e3qSlpZGWlsYjjzxCWFgYCxYsYPTo0XWS/UTc+tRYZGQkXl5exw1+zs7OJiYmxqRULqBlP8f0wFo4dtTcLEBogA9X9Ijns9v78tCFHQCYvXxP472NgYiIixgwYAD9+/fnsssu47vvviMjI4NvvvmGOXPmnPE+q3pvXnjhBXbu3MkXX3zB448/ftrP/eUvf+Hnn39m4sSJpKens23bNj7//HPnYOkvv/yS559/nvT0dHbv3s1bb72F3W4/5XCWuuDWhZDNZqN79+7Mnz/fucxutzN//nz69OljYjKTBcdAZFvAgF0/mZ3GyWKxcFWvFgTavMg4VMTi7YfMjiQi0uh9/PHH9OzZk6uuuoqOHTvywAMPnPaS+1Np2rQpM2fO5MMPP6Rjx45MmzaNZ5555rSf69KlCz/88ANbt26lX79+dO3alSlTphAb6xj8HhYWxieffMLgwYPp0KEDM2bM4P3336dTp05nnLUmXP6qscLCQrZv3w5A165dmT59OoMGDSIiIoIWLVowe/Zsxo0bxyuvvEKvXr149tln+eCDD9i8efNxY4fORE1HnbucLyfBiv9C71tgxD/MTlPN3z5dx7tLMwn28+bf13SjX5umZkcSERd1qquCRDziqrEVK1bQtWtXunbtCsCkSZOcVSTAmDFjeOaZZ5gyZQqpqamkp6czZ86cOimC3FrV6bGMH83NcQIPDG9Pz8RwCkoquO6/y3h+/jazI4mIiIdy+R4hs7ltj1DRIXj618HI92yE0Obm5vmDkvJKnvhqI+/84rjU8pkrUri8e5zJqUTE1ahHSE7FI3qE5AwFRkJCX8f82tnmZjkBPx8vnhjVmTsGO64eePSLDRSWNsyDYkVERKqoEGrMUq92TNPfO6tbw9ene9La0qppIAWlFXy4Yo/ZcURExMOoEGrMOl7iuAHY4W1wYI3ZaU7IarUwvm9LAJ6fv43n52+jvNJ+mk+JiKfRKA45kbr4vVAh1Jj5BjturgiOu0y7qMu6NadV00COFpcz/butTJi5nJLyM7+0U0Qaj6qHg1bdsVjk96oewfHHO2HXRqO/s7THa9kPtn7juMt037vMTnNCATZv/jfxPL5ae4C//28DP247xAsLtnH/sPZmRxMRk3l7exMQEMDBgwfx8fHBatX/v4ujJ6i4uJicnBzCwsKcBfOZUCHU2LXs75juXgKV5eB15lVzfQr09ebKnvGE+PtwyzsreXXRTi7vHk/LSN1KX8STWSwWmjVrRkZGBrt37zY7jriYsLCws36ShAqhxi6qE/hHwLEjsG8VtOhtdqJTGtYpmvNaR7J4+yG+SN/PXWltzI4kIiaz2Wy0adNGp8ekGh8fn7PqCaqiQqixs1qh1QDY8Cls/p/LF0IWi4W0DlEs3n6I9D3mPydNRFyD1WrVfYSkXuhkqyfo9OtTe9d/CnbXvyIrtUU4AOl7cnWliIiI1CsVQp6gzflgC4b8vbB3mdlpTqtjsxBs3laOFpez+3Cx2XFERKQRUyHkCXz8oP2FjvnNX5mbpQZs3lY6xTpuh75yt06PiYhI/VEh5CmSBjumu1zvIawn0isxAoBHvtjAj9sOmpxGREQaKxVCnqLqafQH1kBJnrlZauDmAUn0SAinsLSC+z5cQ5GeQyYiIvVAhZCnCImFJq3BsMPun81Oc1oRgTbeubE38RH+ZOeX8uy8rWZHEhGRRkiFkCdJ/LVXaKfrPm7j9/x8vHjowo4AvPZjBs/P32ZyIhERaWxUCHmSVgMd050LzUxRK8M6xfDA8HYA/GveVtbvc/3TeiIi4j5UCHmSlv3BYoWDmyB/v9lpauy2ga0ZmRKLYcCUz9dzpEh3lxURkbqhQsiTBERAbDfH/I7vzc1SSw+OaI+fj5VVmbn0+8cC7v9wDcsyjuiGiyIiclZUCHmapEGO6Y755uaopeZh/rx30zl0bBZCUVklH67cy5WvLOHvX2zAblcxJCIiZ0aFkKdpPdQx3TbP8TR6N9KtRThf3XkeH97Shyt7xGGxwJtLdvPyDzvMjiYiIm5KhZCniesBgU2hNA92LTY7Ta1ZLBZ6Jkbw1OUpPHZxJwDeWrKLikrXf4aaiIi4HhVCnsbqBW2HO+a3fG1ulrN0Zc94IgJtZOeX8sNW3X1aRERqT4WQJ2p/kWO6+h3I/MXcLGfB19uL0V2bA3Dvh2uY9s1mMg4VmZxKRETciQohT9RmqOPZY+XFMPtaqHTfx1f8uX8r2kYHkVtczowfdnDxC4vZe1RPrBcRkZpRIeSJrF4w5l3wC4Oig7B/ldmJzlhUiB/f3NWfl67uRodmIRSUVnDP7HTyit1rILiIiJhDhZCnsgU4brAIbvPIjZPxslq4sEszZlzbjQCbF8t3HaXfUwu49j9LmfzJOrLySsyOKCIiLkqFkCdrNcAxzXDvQqhKQpNA3r2xN60iA8kvqWDx9kO8vyyTx77cYHY0ERFxUSqEPFnLgY7pnqVQ1jjG1XRtEc639/Tn09vO5Y7BrQH4el0Wj3y+Xj1DIiJyHBVCnqxJEviFQmUZ5GaanabO+HhZ6doinHvPb8fFKbGA48aLF72wmPQ9uboTtYiIOKkQ8mQWCwRFO+aLGud9eB66sANX9ogjPMCHQ4WljHrpJy54/key89U7JCIiKoQksKljWpRjbo56EhXix1OXpzBv0gAGt4/Cz8fK5qwCrnrtF47qKfYiIh5PhZCncxZCh8zNUc+aBPny+vU9+e6eAcSG+rHzYBG3vruSQ4WlZkcTERETqRDydM5CqHGeGvuj+IgA3hjfi0CbF7/sPEKPJ+bR/uFv+N+a/WZHExERE6gQ8nRBUY5pYeM8NXYi7WKCeeuGXqTEhwFQUm7nb5+uY/dhPZ5DRMTTqBDydIGRjmkjPzX2R90TIvjstnNZ/rc02kYHkV9SwYCnF/LElxvNjiYiIg1IhZCnC/y1R6iRDpY+FYvFQtNgX2Zc2502UUEAvPXLbj2eQ0TEg6gQ8nQeNkboRFo1DWLuPf1pGx1EWYWdr9YdMDuSiIg0EBVCni7o10Ko0HMLIXD0Dl3WLQ6Av366jkmz06nUjRdFRBo9FUKerqpHqLwIyjx7sPClXZsT5OsNwCer9/Hmz7vMDSQiIvVOhZCnswWBt79j3oNPj4Hj5ouLHhjEhL4tAZj6zSZufWclR3TjRRGRRkuFkKezWH7rFfKgS+hPJiLQxkMXdiCtQxTllQbfrM/iprdWsOtQkZ5RJiLSCHmbHaAhXHrppSxcuJAhQ4bw0UcfmR3H9US0hLxMyNkI8b3MTmM6q9XCa2N7sCzjCNe9voyVu48y8JmFRAb50i4miOgQP5qF+tEjMYJB7aLMjisiImfBIwqhu+66iwkTJvDmm2+aHcU1xfWAjB9g73Lofr3ZaVyCxWKhd6smPHNFCk/N2czBglIOFZZyaPvvH8mxg6t6xTO4fTSRQTYiAh2vYD8f03KLiEjteEQhNHDgQBYuXGh2DNcV19Mx3bvC3Bwu6OKUWC5OiaWswk76nlz2Hi0mK7+E7TmFfLJqH+8v28P7y/ZU+8ygdk15cnRnmoX6m5RaRERqyuXHCC1atIiRI0cSGxuLxWLhs88+O26bl156icTERPz8/OjduzfLli1r+KDurHkPx/TgFijJMzeLi7J5W+nVMoLR3eK4bWBrpl+ZyhvX9+SiLs3o3DyU5mH+BNi8APh+y0HOn76ID5bvwTA0rkhExJW5fI9QUVERKSkpTJgwgdGjRx+3fvbs2UyaNIkZM2bQu3dvnn32WYYNG8aWLVuIitL4jRoJagphCZC723F6rHWa2YncwqD2UQxqX/13bHtOAfd9uJb0Pbk88PFa3vh5F/3aRHJPWlv8fy2URETEdbh8j9CIESN44oknuPTSS0+4fvr06dx0002MHz+ejh07MmPGDAICAnj99dfP6HilpaXk5+dXe3mEVgMc019mmJvDzbWOCubjW89l8oj2+Hpb2XQgn1cX7eShz9ard0hExAW5fCF0KmVlZaxcuZK0tN96MKxWK2lpaSxZsuSM9jl16lRCQ0Odr/j4+LqK69r63g1Wb9j+HazTlXVnw8tq4eYBSSy8fyB3p7UB4ONVe2nzt2+4csYSlu86YnJCERGp4taF0KFDh6isrCQ6Orra8ujoaLKyspzv09LSuOKKK/j666+Ji4s7ZZE0efJk8vLynK89e/acdNtGpUkS9JjgmP/4Blj8L3PzNALNQv25O60tk0e0x9tqocJusGzXEW6YuZwtWQVmxxMREdxgjFBdmDdvXo239fX1xdfXtx7TuLDznwAvGyx5Eb6fCp0uhfBEs1O5vZsHJDG+b0v2HC3m/g/XsCozl2HPLiK5eQjDO8Xg5+OFj5cVm7cVHy8rPl4WbL97HxFoo3VUEH4+GmMkIlLX3LoQioyMxMvLi+zs7GrLs7OziYmJMSmVG/P2dRRDWesc9xX69m/wp3fNTtUo2LytJDUN4uVru3P3rHSWZhxm/b581u+r2Ri0JoE23rqhF51iQ+s5qYiIZ3HrQshms9G9e3fmz5/PqFGjALDb7cyfP5+JEyeaG85dWSww4h/wcl/Y/CXsXAitBpqdqtGIDvHj/T+fw5GiMj5YsYcdOYWUV9oprzQorbD/Ou94lVXYKa2wcyCvhMNFZYx7fRnJzUPx9bZy+6DWdIkLM7s5IiJuz+ULocLCQrZv3+58n5GRQXp6OhEREbRo0YJJkyYxbtw4evToQa9evXj22WcpKipi/PjxJqZ2c1EdoOeNsOwVmPco/Hmg2YkanYhAG7cMSKrRtnnHyhn975/YcbCIhVscD8bdnlPInLv74+Pl1sP8RERMZzFc/JrehQsXMmjQoOOWjxs3jpkzZwLw4osv8vTTT5OVlUVqairPP/88vXv3rpPj5+fnExoaSl5eHiEhIXWyT7dQkA3/bAtY4MFM8POgtruggpJyvt9ykNLySqZ9s5nDRWW0jAzkL8PbMzxZp4FFRP6opn+/Xb4QMpvHFkIA0ztB/l64/itIPM/sNPKrd5fu5m+frgfA19vKgvsG0jxMj/MQEfm9mv79Vr+6nFxsqmO6P93MFPIHV/dqwSvXdSci0EZphZ1h/1rEu0t364aNIiJnQIWQnFyzVMf0QLqZKeQPLBYLwzrF8NaEXlgsUFhawd8+Xc8LC7ZTVmE3O56IiFtRISQn1yzFMVWPkEtKbh7Kuzf0Zlgnxw1Fp3+3lZRH57Iq86jJyURE3IcKITm52K6O6eFtjnsLics5t3UkM67tzvXnJgJwrLySlxfuMDeUiIgbUSEkJxfUFDpe4pj/+gHQGBSXZLFY+PvFnZh7T38A5m/KZu/RYpNTiYi4BxVCcmrn/x94+0Pmz7B3hdlp5BTaRgdzXutI7Aa8umin2XFERNyCCiE5tbB4aDfcMb91jrlZ5LRuG+S4SeO7SzNZvy/P5DQiIq7P5e8sLS6g7QjY8KmjEBrysNlp5BTOTYpkeKcY5mzIYvS/f6Z9s2BC/Hxo0SSA3i0jiAnxIzzQRpuoICwWi9lxRURMp0JITq/NULBYIXs95O5x9BKJy3psVCfyjpWzZOdh1u79tVdoO7y3NNO5TVqHaJ65ogthATaTUoqIuAYVQnJ6ARGOS+n3r4b9q1QIubioYMeDXTcdyGd/7jHyjpWzdm8eG/bncaiwjL1Hi5m3KZvHvtzI9CtTzY4rImIqFUJSM2EtHIVQQZbZSaSGOjQLoUMzx23lR3eLcy5fuvMwY179hS/S93Pf+e2I1eM5RMSDqRCSmglu5pgWHDA3h5y13q2acE6rCH7ZeYRzpy2gXXQwkcE2WkQE0C46mGvPScBbT7UXEQ+hQkhqJvjXJ5znqxBqDO4c0oblu5ZRaTfYkl3Almz4icMALN5+iBev7oafj5fJKUVE6p8KIamZ4FjHVD1CjcK5SZGsfCiNnYeKKCqt4EBuCbuPFPGfHzOYtymHu2el89I13fCy6soyEWncVAhJzVT1CGmMUKMRFmCjW4vqV431bR3J9a8vZ86GLJ6bt5VJ57czKZ2ISMPQQACpGecYIRVCjdm5SZE8dXkXAF78fjtTv9lE5mE9rkNEGi8VQlIzVT1CpXlQVmRuFqlXo7o2Z3TX5tgNeOWHnYx++Sd2H9Z3LiKNkwohqRm/ELAFOebVK9ToTbusC1NHd6ZddDCHCsu4+e2VVFTazY4lIlLnVAhJzTnHCWnAdGNn87ZyVa8WvH1DL0L9fdicVcCHK/eaHUtEpM6pEJKa0zghjxMV4sedQ9oA8My3W9h7VOOFRKRxUSEkNRee6Jhm/mJqDGlY152TQIdmIRwuKuO8f3zPsH8tYtIH6eQWl5kdTUTkrKkQkprrNMoxXf8xVOiPoKeweVv577gexIb6AbAlu4BPVu3jf2v2m5xMROTsqRCSmms5EIKi4dgRWP02GIbZiaSBxIb5s+C+gXxy27mkxIUCsDW70ORUIiJnT4WQ1JyXN6Re7Zj/ahJ8ebeKIQ/i5+NFtxbhjO2TCMC2nAJzA4mI1AEVQlI7Ax6Ec+8EixVWzoSfnjM7kTSwNtGO2yhsz9G9hUTE/akQktrx8YPzH4fzn3C8XzPL3DzS4JKaOgqhQ4WlHC3SWDERcW8qhOTMJPR1TEtyTY0hDS/Q15vmYf4AbD+ocUIi4t5UCMmZ8QtxTEvyzc0hpmgd5egVumLGElbuPmJyGhGRM6dCSM6MX5hjWl4EleWmRpGG17VFmHN+8ifrsNs1aF5E3JMKITkzviG/zatXyOPcMiCJ6VemAI7L6F/9cScHC0opKq1QUSQibsXb7ADipry8HQ9hLSt0jBMKbGJ2ImlAfj5ejO4Wx5bsAl75YSfTvtnMtG82O9f7+3gRYPPC3+aFn48Xk4a25YLOzUxMLCJyYuoRkjPn57ixHiV55uYQ09zSP4kBbZsSHuBTbfmx8koOF5Wx9+gxtucU8vz8bSYlFBE5NfUIyZnzDQH2QalOjXmq8EAbb07oBYDdblBSUUlxWSXHyiopKqvgQG4J42cuZ3NWAftyjzmvNhMRcRXqEZIzpx4h+R2r1UKAzZvIIF/iIwJoHxPCoPZR9EgIB2DB5hyTE4qIHE+FkJw5FUJSA4PaRwHw8Gfr6fV/83jk8/UYejSLiLgIFUJy5lQISQ1cnBJLsK/jLHxOQSlvLtnNzkN6PIeIuAYVQnLmnIWQxgjJycVHBLD8oTSW/y2NpsG+AKRn5pobSkTkVyqE5Mw57y6tHiE5NT8fL5oG+3JxSiwA6XtyzQ0kIvIrFUJy5nRqTGopNT4MUCEkIq5DhZCcORVCUktVhdCmA/nkl+jRLCJiPhVCcuZUCEktxYX7ExPiR4Xd4Jwn5zN0+g9s2K/fHxExj0cUQpdeeinh4eFcfvnlZkdpXKqeN6YbKkoNWSwWpo9JoUVEAMVllWzLKeTdpZlmxxIRD+YRhdBdd93FW2+9ZXaMxqfqCfTqEZJaODcpkgX3DmDyiPYALNp6UPcVEhHTeEQhNHDgQIKDg82O0fgERDimBQcgZ/OptxX5HW8vK9eek4CPl4W9R4+x+3Cx2ZFExEOZXggtWrSIkSNHEhsbi8Vi4bPPPjtum5deeonExET8/Pzo3bs3y5Yta/igcrzwREgaDPYK+OQmsNvNTiRuJNDXm24tHI/fGPjMQn7eccjkRCLiiUwvhIqKikhJSeGll1464frZs2czadIkHnnkEVatWkVKSgrDhg0jJ+e35xalpqaSnJx83Gv//v0N1QzPZLHAqJfB2w+y1sLh7WYnEjcztGO0c/4f36hXUUQanulPnx8xYgQjRow46frp06dz0003MX78eABmzJjBV199xeuvv86DDz4IQHp6ep3lKS0tpbS01Pk+P18DgU8pOAaik2HfCsheB03bmp1I3MjYPolYLBYe/3IjG/bnU1JeiZ+Pl9mxRMSDmN4jdCplZWWsXLmStLQ05zKr1UpaWhpLliypl2NOnTqV0NBQ5ys+Pr5ejtOoxHR2TLPWmZtD3I7N28qEvok0Dfalwm6wbp8G3otIw3LpQujQoUNUVlYSHR1dbXl0dDRZWVk13k9aWhpXXHEFX3/9NXFxcacsoiZPnkxeXp7ztWfPnjPO7zFikh3TrPXm5hC3ZLFY6NYiDIBVu4+aG0ZEPI7pp8Yawrx582q8ra+vL76+vvWYphGK6eKYqkdIzlC3FuF8uyGbVZkqhESkYbl0j1BkZCReXl5kZ2dXW56dnU1MTIxJqeQ4UR0BCxRmQeFBs9OIG+r669VjegaZiDQ0ly6EbDYb3bt3Z/78+c5ldrud+fPn06dPHxOTSTW+QdAkyTGftcbcLOKWOsWGYLFAdn4pOQUlZscREQ9ieiFUWFhIenq688qvjIwM0tPTycx03HZ/0qRJvPbaa7z55pts2rSJW2+9laKiIudVZOIimqU4pgfWmptD3FKgrzdJTYMAWK8B0yLSgEwfI7RixQoGDRrkfD9p0iQAxo0bx8yZMxkzZgwHDx5kypQpZGVlkZqaypw5c44bQC0ma5YC6z+GA+lmJxE31aV5KNtzClm3N5/B7fXft4g0DNMLoYEDB572OUMTJ05k4sSJDZRIzoizR0inxuTMJDcP5ZPV+3QJvYg0KNMLIWkkqq4cO7oLjh0F/3BT44j76RwXCsCP2w4y7vVldIwNYUj7KJKaBhEW4IPFYjE5oYg0RiqEpG4EREBYC8jNhIwfoePFZicSN9O5eSgxIX5k5Zfww9aD/LD1IC8v3AFAgM2L5mH++Nu8CA+wMaxTDFf3bmFyYhFpDFQISd1pOwKWvQL/u9Nxqiw8wexE4kb8fLxYcN8ANu7PZ2t2IT/vOMQvO49wqLCU4rJKtuUUOrddtO0gF3ZpRqi/j4mJRaQxUCEkdWfoo7BnqWPA9MqZkPaI2YnEzQTYvOmRGEGPxAhnj09JeSX7c49xIK+E0opKJsxcgWHA9pwCuidEmJxYRNyd6ZfPSyPi4w89b3DM71lmbhZpNPx8vGjVNIi+rSMZ3D6afm0iAdiaXXiaT4qInJ4KIalb8b0d030robLc3CzSKLWNDgZgmwohEakDKoSkbjVp47hirOKYbq4o9aJttOPGi9tyCkxOIiKNgQohqVtW62+9QnuWmptFGqU2v/YIbc1WISQiZ0+DpaXutegDW+fA1m+gz21mp5FGpk2Uo0coO7+U8//1A4G+3gTYvAiwOaZ+3l7ER/gzrFOMs2gSETkZFUJS9zpdCvMecdxPKHcPhMWbnUgakWA/H7rEhbJ2b94pB0z/e+EOVj40FH+bVwOmExF3c8aF0Pbt29mxYwf9+/fH398fwzB051dxCE+AxH6w60dYOxv632d2ImlkPri5D5sO5FNcVklRaQXFZZW/vhzz07/bSnFZJfvzjjkf5ioiciK1LoQOHz7MmDFjWLBgARaLhW3bttGqVStuuOEGwsPD+ec//1kfOcXddLnSUQhtnaNCSOqcn48XXVuc/DEuX6zZz/acQrLzSlQIicgp1Xqw9D333IO3tzeZmZkEBAQ4l48ZM4Y5c+bUaThxYy3OdUyz1ukyemlw0SG+AGQXlJicRERcXa17hObOncu3335LXFxcteVt2rRh9+7ddRZM3FxEK/ANhdI8yNn429PpRRpAdIgfAFl5pSYnERFXV+seoaKiomo9QVWOHDmCr69vnYSSRsBqhdhUx/y+VaZGEc9TVQhl56tHSEROrdaFUL9+/Xjrrbec7y0WC3a7naeeeopBgwbVaThxc827Oab7VQhJw4pRISQiNVTrU2NPPfUUQ4YMYcWKFZSVlfHAAw+wYcMGjhw5wk8//VQfGcVdxVYVQqvNzSEexzlGSIWQiJxGrXuEkpOT2bp1K+eddx6XXHIJRUVFjB49mtWrV5OUlFQfGcVdNW3vmB7ZBYZhahTxLL+dGtMYIRE5tTO6j1BoaCh/+9vf6jqLNDZVN1IsK4BjRyEgwtw84jGqCqGcghLsdgOrVfc4E5ETq3UhtGjRolOu79+//xmHkUbGxx8Co6AoB3IzVQhJg2ka7IvFAuWVBp+s3kezUD8iAm20iw5WUSQi1dS6EBo4cOBxy35/R+nKysqzCiSNTFiL3wqhqqvIROqZj5eV6GA/svJLuO/DNc7lbaKCePSSTpybFGliOhFxJbUeI3T06NFqr5ycHObMmUPPnj2ZO3dufWQUdxbWwjHNzTQ3h3icv1/ckbQO0fRMDKdtdBABNi+25RRy/evL+WTVXux2jVsTkTPoEQoNDT1u2dChQ7HZbEyaNImVK1fWSTBpJFQIiUmGJzdjeHIz5/v8knLu/3AN327IZtIHa5j8yTrCA2xEh/rRMyGcG/u1IibUz8TEImKGWvcInUx0dDRbtmypq91JY6FCSFxEiJ8PL17djbuGtCHYz5vSCjtZ+SWs2ZPLfxZnMPifC1m7N9fsmCLSwGrdI7R27dpq7w3D4MCBA0ybNo3U1NS6yiWNRViCY6pCSFyAj5eVe4a25bZBSeTkl5JbXM7OQ4W89uNO1u/L5/1le+gSF2Z2TBFpQLUuhFJTU7FYLBh/uC/MOeecw+uvv15nwaSRcPYI7XbcS8iiK3bEfL7eXsRHBBAfAZ3jQvH1tnLLO6tYsyfX7Ggi0sBqXQhlZGRUe2+1WmnatCl+fjq3LicQnghevlBWCEd2QhPddFNcT0p8GABbsgs4VlaJv83L3EAi0mBqXQglJCTURw5prLxt0KwL7F0Oe1eoEBKXFBPiR9NgXw4WlLLxQB7dE3TPKxFPUaNC6Pnnn6/xDu+8884zDiONVPMejkJo3wpIGWN2GpHjWCwWUuLCmLcpm/eW7qGgpIKkpkE0D/PXDRhFGrkaFUL/+te/arQzi8WiQkiOF9cDluLoERJxUV1bOAqhj1ft5eNVewEI8vWmeZg/XlYLof4+jO+bSGp8GFEhGgog0ljUqBD647ggkVpp3t0xzVoHZUVgCzQ3j8gJXHtOAvkl5ezIKSTzSDEZh4ooLK1gS3aBc5slOw8DcOeQNkwa2tasqCJSh87ooasitRKe6LiMPnc3/PwCDHzQ7EQixwn192HyiA7O9+WVdnYeLOJgQSmVhsHP2w/xyqKdAHyyai/3pLWp9nghEXFPFuOP18HXwN69e/niiy/IzMykrKys2rrp06fXWThXkJ+fT2hoKHl5eYSEhJgdx32t/wQ+Gg/efnDXGgiOMTuRSK0Vl1WQ+uh3lFXamX/vAJKaBpkdSUROoqZ/v2vdIzR//nwuvvhiWrVqxebNm0lOTmbXrl0YhkG3bt3OKrQ0Yp0uhZ+ehQNrYPs86Hqt2YlEai3A5k2vlhEs3n6IbzdkcU3vBHy8LPh6e+GlQdUibqnWj9iYPHky9913H+vWrcPPz4+PP/6YPXv2MGDAAK644or6yCiNgcUCSYMd87uXmJtF5CwMaNsUgKfmbCHl0bl0nPItHafM4erXfuFwYanJ6USktmpdCG3atImxY8cC4O3tzbFjxwgKCuKxxx7jH//4R50HlEakxbmOaebP5uYQOQsXpTQjOsS32rLSCjs/7zjMm0t2m5RKRM5UrU+NBQYGOscFNWvWjB07dtCpUycADh06VLfppHGJ7wVYHHeYLsiG4GizE4nUWrNQf5b+NQ273aDCblBpN3hzyS6mfbOZ2cszuXNwa7y96ux51iJSz2r9X+s555zD4sWLAbjgggu49957+b//+z8mTJjAOeecU+cBpRHxD4PoZMd8pk6PiXuzWi3YvK3427yY0LclTQJtZOeXMn7mcv766TqW7zpidkQRqYFa9whNnz6dwsJCAB599FEKCwuZPXs2bdq0aXRXjEk9aHEOZK+DPcug0yiz04jUCZu3lYmDW/Po/zby4zZHz/js5XsY0j6KlpGBJDUNIj4igD5JTUxOKiJ/VOtC6Mknn+Taax1X/AQGBjJjxow6DyWNWHxvWP4a7FlqdhKROjW+b0vOadWEn7YfYlXmUb5el8XcjdnVtvn41nPpnhBuUkIROZFaF0IHDx5k+PDhNG3alD/96U9ce+21pKSk1Ec2aYziezmmB9ZA+THw8Tc3j0gd6tAshA7NQjAMgx+3HWLnwUI2ZxUwa/keAGYvz1QhJOJiaj1G6PPPP+fAgQM8/PDDLF++nG7dutGpUyeefPJJdu3aVQ8Rz05ubi49evQgNTWV5ORkXnvtNbMjebawFhAUA/Zy2J9udhqRemGxWOjftinX923JtMu68MHNfQD4au0BCksrTE4nIr93RneW/r29e/fy/vvv8/rrr7Nt2zYqKlzrP/LKykpKS0sJCAigqKiI5ORkVqxYQZMmNTtXrztL14PZ18GmL2DIFOh3r9lpROqdYRgMeHohmUeKiQi00TzMH38fL/xtXgTYvDg3qQnX9Uk0O6ZIo1LTv99ndY1neXk5K1asYOnSpezatYvoaNe7HNrLy4uAgAAASktLMQyDs6z95Gy17O+Ybl9gbg6RBmKxWJg6ujMxIX4cKSpj3b48lu06wg9bD/LN+iwe/nwDK3frKjMRM5xRIfT9999z0003ER0dzfXXX09ISAhffvkle/furfW+Fi1axMiRI4mNjcVisfDZZ58dt81LL71EYmIifn5+9O7dm2XLltXqGLm5uaSkpBAXF8f9999PZGRkrXNKHWoz1DHNXALHck2NItJQ+raOZNEDg/jolj68cX1P/n1NN565IoUh7aMAeOCjtTzx5Ub+8+NOSsorTU4r4jlqPVi6efPmHDlyhOHDh/Pqq68ycuRIfH19T//BkygqKiIlJYUJEyYwevTo49bPnj2bSZMmMWPGDHr37s2zzz7LsGHD2LJlC1FRjn9AUlNTT3hKbu7cucTGxhIWFsaaNWvIzs5m9OjRXH755S7Ze+UxwhMhsh0c2gI7FkDy8d+7SGNk87bSIzGi2rL+bSIZ+MxCdhwsYsfBDADCAmxc3j3OjIgiHqfWY4Ree+01rrjiCsLCwuo+jMXCp59+yqhRo5zLevfuTc+ePXnxxRcBsNvtxMfHc8cdd/Dggw/W+hi33XYbgwcP5vLLLz/h+tLSUkpLf3teUH5+PvHx8RojVNe+/RsseRGShsC1HzueRSbioVZnHuXHbYeYvymbNXvzuGNwa+49v53ZsUTcWr2NEbrpppvqpQg6kbKyMlauXElaWppzmdVqJS0tjSVLanZn4uzsbAoKCgDIy8tj0aJFtGt38n9gpk6dSmhoqPMVHx9/do2QE+s+Hqw+sGM+bPqf2WlETNW1RTh3DmnD0I6OnuqsvBKTE4l4Dpd+IM6hQ4eorKw87jRWdHQ0WVlZNdrH7t276devHykpKfTr14877riDzp07n3T7yZMnk5eX53zt2bPnrNogJxHZGvre6ZhfqptyigBEh/gBkJWvQkikodR6jJC76dWrF+np6TXe3tfX96zGPEkttL8IfvwnHN5udhIRlxAT6iiEslUIiTQYl+4RioyMxMvLi+zs6repz87OJiYmxqRUUmfCEx3TwmzHXaZFPFxMVY+QTo2JNBiXLoRsNhvdu3dn/vz5zmV2u5358+fTp08fE5NJnfAPB1uwYz4309wsIi6gqkcov6SC4jLXujmtSGNleiFUWFhIenq68/RVRkYG6enpZGY6/jBOmjSJ1157jTfffJNNmzZx6623UlRUxPjx401MLXXCYoHwBMf80d3mZhFxAcF+PgTavAD1Cok0FNPHCK1YsYJBgwY530+aNAmAcePGMXPmTMaMGcPBgweZMmUKWVlZpKamMmfOHN0HqLEIS4Ds9ZCrQkgEIDrUj50Hi8jKL6FV0yCz44g0eqYXQgMHDjztIy8mTpzIxIkTGyiRNChnj9AuU2OIuIqYEEchpAHTIg3D9EJIPFzVgGn1CIkAv40Tmvr1Zl5fvIuU+FBS48PxskKryCBS4sPMDSjSyKgQEnOFaYyQyO+lxIXxyap95BSUklNQyrp9ebzzi2PMpNUCix4YRFx4gMkpRRoPFUJirqZtHdOcjVB0GAKbmJtHxGRj+yTQMzGC3GNl5B8rZ8HmHLLzS1mdeZT8kgq2ZBWoEBKpQyqExFwRrSCmC2SthY2fQs8bzU4kYiqLxULH2N+eizQ8uRkAt7y9kjkbssg8UmxWNJFGyfTL50XoMsYxXfuBuTlEXFh8hD8Ae47o5qMidUmFkJiv8+WABfYshYLs024u4olaRDhOh+05qh4hkbqkQkjMFxwDMb8+CHfXj+ZmEXFRcVWFkE6NidQpFULiGlr2d0x3LjQ1hoirig//rRA63b3XRKTmVAiJa2g5wDHNWGRuDhEXFRfuGCNUVFbJ0eJyk9OINB4qhMQ1JPQBq7fjxoqZv5idRsTl+Pl4ER3iC6Arx0TqkAohcQ2+wb9dPfbZbVCmf+hF/qhVpOPZY4u3HTQ5iUjjoUJIXMew/4PgWDiyA5a9YnYaEZczpmc8AP/8bivr9+VprJBIHVAhJK7DPxzSHnHML34WjuWamUbE5VzYpRnNw/wxDLjohcVc+9+l5OjhrCJnRYWQuJbOV0BkOyjJhY2fmZ1GxKX4eFl5+KKOtIoMxOZl5afth5n43mr1DImcBRVC4lqsXtDq1yvI9CBWkeMMT45hwX0D+erO8/D1trJs1xEWbtGYIZEzpUJIXE9IrGOav8/cHCIurE10MNefmwjAU99uwW5Xr5DImVAhJK4nJM4xzd9vbg4RF3frwCSC/bzZdCCf/63Vfy8iZ0KFkLieqh6hvL3m5hBxcWEBNm7u3wqA6d9tpVK9QiK1pkJIXE9oc8c0fz9oEKjIKY3v25KwAB92Hy7mu416aLFIbakQEtcT3MwxrSyF4iPmZhFxcYG+3lzdqwUAt7yzkreW7NJVZCK1oEJIXI+3LwQ2dczn6/SYyOmM7ZOIt9UCwJTPN/DLTv0PhEhNqRAS1xTyu9NjInJKMaF+vDauh/P9T9sPmZhGxL2oEBLXVFUIacC0SI0MahfF05d3AeDnHSqERGpKhZC4pqoB0zmbzM0h4kb6JDUBYM3ePDIOFZF3rFxXkomchrfZAUROqM0wWPYqpL8L/e+HkGZmJxJxeXHhASQ0CWD34WIGPbPQuTzA5sWVPeL5+8WdzAsn4qLUIySuqfUQiD8HKkrg5+fNTiPiNm7un0RkkA0fL4tzWXFZJTN/3sX6fXkmJhNxTRZD11meUn5+PqGhoeTl5RESEmJ2HM+y+WuYdRWEJcDda81OI+J2SisqKSypYMrnG/hq3QGah/lz68AkrBYLEYE+DO0Yg5fVcvodibihmv791qkxcV0t+4PVG3J3w5EMiGhpdiIRt+Lr7YVvkBd3p7VhzoYs9uUe46HP1jvXP/enVC5JbW5iQhHz6dSYuC7fIIjr5Zjf+b25WUTcWJvoYGb/+RyuOyeB8ztG0yYqCED3GxJBhZC4uqRBjumOBebmEHFzPRIjeHxUMq+O7cG957cFYM2eXHNDibgAFULi2tqc75hu/RYK9BwlkbqQEh8GwJbsAo6VVZobRsRkKoTEtcWmOk6PVZbBitfNTiPSKDQL9Sc6xJdKu8HavblmxxExlQZLi+s751b4aBmsehMGPggWXeUicrZS4sKYuzGbMa/+QrvoYMICfPCyWuiREM49Q9ti0X9n4iFUCInrazscsEDBASjMgeBosxOJuL1LuzZn4ZaDlFXa2ZJd4Fz+847DdE0IZ1C7KBPTiTQcFULi+mwB0CQJDm+HnA0qhETqwIjOzRjSIZqcghK2ZRdSVFbBvI3ZfJa+nymfr+e81pGEBdi4dWASIX4+ZscVqTcqhMQ9RHV0FELZGyBpsNlpRBoFm7eVuPAA4sIDADivdSQLNuew58gx3l+2x7ndX4a3NyuiSL3TYGlxD9HJjmn2RnNziDRiYQE23rvpHP4yvD1X9YoH4OOVe6motJucTKT+qEdI3EN0R8c0e/2ptxORs5LcPJTk5qGUVdiZuyGbnIJSbnlnJY+M7ER8RIDZ8UTqnHqExD1E/VoIHdwClRXmZhHxADZvK5d3jwNg3qYchj+7iLtmrebz9H0mJxOpWyqExD2EtwRvP6gsdTx7TETq3V1pbXjskk50TwinqKySz9P3c9+HaygoKTc7mkidUSEk7sFqhYgkx/zhHeZmEfEQATZvxvZJ5IOb+/Di1V0BKK80WLT1kMnJROqORxRCiYmJdOnShdTUVAYNGmR2HDlTTaoKoe3m5hDxMF5WCxd1ieXP/VsBMH+THncjjYdHFEIAP//8M+np6Xz/vZ5i7raatHZMVQiJmGJIe8dNFr/bmM0HK/ZQUq7nlIn701Vj4j5UCImYqntCOG2igtiWU8gDH63lb5+uI9DXm94tI4gO8cPmZaV1VBBDO0bTJMjX7LgiNWJ6j9CiRYsYOXIksbGxWCwWPvvss+O2eemll0hMTMTPz4/evXuzbNmyWh3DYrEwYMAAevbsybvvvltHyaXBOQshjRESMYO3l5UPb+nDX4a3p3mYP+WVBrnF5Xy7IZu3luzmP4szePCTdaRN/4FVmUfJO1ZOUWkF5boPkbgw03uEioqKSElJYcKECYwePfq49bNnz2bSpEnMmDGD3r178+yzzzJs2DC2bNlCVJSjmzY1NZWKiuMvqZ47dy6xsbEsXryY5s2bc+DAAdLS0ujcuTNdunSp97ZJHasqhPL3Qvkx8PE3N4+IB6p67Maf+7fiQN4xDhaUsjTjCMWlFRSXVbJgSw47DxYx+t8/Oz/jZbUwpH0Uj49KJjrEz8T0IsezGIZhmB2iisVi4dNPP2XUqFHOZb1796Znz568+OKLANjtduLj47njjjt48MEHa32M+++/n06dOnH99defcH1paSmlpaXO9/n5+cTHx5OXl0dISEitjyd1yDDgqZZw7CicNwmGTNGT6EVcTN6xcia+t4oftx1/ZdnwTjHMuK67CanEE+Xn5xMaGnrav9+mnxo7lbKyMlauXElaWppzmdVqJS0tjSVLltRoH0VFRRQUOJ6sXFhYyIIFC+jUqdNJt586dSqhoaHOV3x8/Nk1QuqOxQL973fML54OW74xN4+IHCfU34e3b+jNzicvYOsTI9j02HDenNALgB+3HaSsQqfJxLW4dCF06NAhKisriY6u/rTx6OhosrKyarSP7OxszjvvPFJSUjjnnHMYO3YsPXv2POn2kydPJi8vz/nas2fPSbcVE/S5HXrd7JhfO8vcLCJyUlarBZu3FX+bF/1aR9Ik0EZRWSUrdx81O5pINaaPEapvrVq1Ys2aNTXe3tfXF19fXe3g0rpeA8tega3fQkk++OmUpYgrs1ot9G/blE9X7+PF77exZOdh2scEM7xTDFarTm+LuVy6EIqMjMTLy4vs7Oo378rOziYmJsakVGK6mC7QpA0c3uYohrpcYXYiETmNge0chdBP2w/z0/bDALSJCqJDsxDCAny4uncL2sfof2qk4bl0IWSz2ejevTvz5893DqC22+3Mnz+fiRMnmhtOzGOxQKuBjkLo4Caz04hIDVzYuRl7jx4jO7+E0nI7X67dz7acQrblFALw1pLd+PlYmTyiA+POTTQ3rHgU0wuhwsJCtm//7QZ5GRkZpKenExERQYsWLZg0aRLjxo2jR48e9OrVi2effZaioiLGjx9vYmoxXWhzxzR/v7k5RKRGvL2s3D6otfP9A8PbsTTjCPtzj7Eq8yhfr8uipNzO8/O3cVWvFti8XXoIqzQiphdCK1asqPb8r0mTJgEwbtw4Zs6cyZgxYzh48CBTpkwhKyuL1NRU5syZc9wAavEwIVWF0D5zc4jIGWkS5MsFnZs53+cVl9P3Hws4XFTG3I1ZXNQl1sR04klc6j5Crqim9yGQBrZrMcy80HGTxTtWmp1GROrA9LlbeH7BdlLjw/j41nPx0kBqOQuN4j5CIicV/Ov/SeYfcNxoUUTc3tW9Ewi0eZG+J5cxryzh3aW7yc4vMTuWNHIqhMQ9hfzabV5eBCV55mYRkToRE+rHIyMdN7xdsfsof/t0Pf2f+p49R4pNTiaNmQohcU8+/uAf4ZjXgGmRRuOKHnE8PiqZsX0SaB7mT2mFnS/XHjA7ljRiKoTEfVX1CqkQEmk0LBYL152TwGOXJHPboCQA5qxXIST1R4WQuC9nIaQrx0Qao/M7xmC1wJq9eUyYuZw73l/N+n06FS51S4WQuC/1CIk0ak2DfenbOhKABZtz+N+a/fzp1V/4z4872Z97zOR00liYfh8hkTMWGu+YHtxsbg4RqTfTr0xl4ZYc7IbBu0szWbs3jye+2sRz87bx94s7Mbh9FOGBNrNjihtTISTuK6GvY5qxCOx2sKqDU6SxaRrsyxU9HP/TM6h9FNPnbmXl7qNsyynk3g/XEOrvw5LJgwmw6c+ZnBn95RD3FdcDbMFw7AhkrTE7jYjUs6hgP6Zd1oUv7zyPWwY4BlLnHStn31GdJpMzp0JI3JeXD7Ts55jf8b25WUSkwfh6e/HgiPYkNgkAIPdYucmJxJ2pEBL31urX59Tt/sncHCLS4EIDHGODcotVCMmZUyEk7q1ZF8c0RwOmRTxNmL8PALnFZSYnEXemQkjcW2RbxzR/L5QWmptFRBpUWICjEMrTqTE5CyqExL0FREBgU8f8oa3mZhGRBhXqr0JIzp4KIXF/ke0cUxVCIh7lt1NjKoTkzKkQEvfX9NfTYwe3mJtDRBqUc7C0eoTkLKgQEvenHiERj6TB0lIXVAiJ+4tq75ju/gkKc8zNIiINpmqwdL56hOQsqBAS99fiXIjqBMeOwpf3mJ1GRBpI1WBpnRqTs6FCSNyftw1Gv+qY3/wlFGSbm0dEGkRVj5AGS8vZUCEkjUNMMsT8enPFnQtNjSIiDSPU3zFYOr+knEq7YXIacVcqhKTxSPr1cRs79dwxEU9QdWrMMKCgRL1CcmZUCEnjkTTYMd3xveNfRhFp1GzeVgJtXoBuqihnToWQNB7x54BPIBRmwYr/mp1GRBpAVa/QgKcX8t/FGZSUV5qcSNyNCiFpPHz8YPBDjvlvH4Jdi83NIyL1rmtCuHP+8S83MvKFxew6VGRiInE3KoSkcel9C7QeChXH4J3LIHuD2YlEpB698KeuzJvUnycv7UxkkC/bcgr51zzdXFVqToWQNC5WK4x5GxL7QUUJrHjd7EQiUo+sVguto4K5uncLnr7CceXohv35JqcSd6JCSBofH384727H/IZPoVKDKEU8QYeYEAAyDhVRWqGxQlIzKoSkcWo5EAIiofiw7isk4iGiQ3wJ8fOm0m6wI0fjhKRmVAhJ4+TlDR0ucsxnLDI3i4g0CIvFQvtfe4W2ZheYnEbchQohabyiOjqmR3aam0NEGkzbmCAAtqgQkhpSISSNV0SSY3p4h7k5RKTBtIsOBmDpzsPY9dgNqQFvswOI1JsmrRzToxlgtzuuKBORRq1fm6bYvKysyszlvo/WMLxTDL4+Xti8rAT6ehET6oe/jxfBfj5mRxUXoUJIGq/QFmD1dlxGX7AfQuPMTiQi9SwxMpAnR3fmvg/X8MmqfXyyat8Jt3twRHtuGZDUwOnEFakQksbLyxvCE+HwdsfpMRVCIh7h8u5xRAbZ+HjVPvYcKaaswk5pRSWFpRVk55cCMG9jtgohAVQISWMXkeQohI7sgFYDzE4jIg1kYLsoBraLOm752r25XPziT+w6rMvrxUGDJqRxi/h1nJAGTIsIjlNnAIcKyygo0c1WRYWQNHaRbRzTQ3r2kIhAiJ8PTQJtAOw+XGxyGnEFKoSkcYtOdkyz1pubQ0RcRlWvkE6PCagQksYu+tebKhbsh+Ij5mYREZeQ0CQAUI+QOKgQksbNN9hx5RhA1jpTo4iIa0hs4ugRWrztECXlejirp2v0V41t2bKFMWPGVHv//vvvM2rUKPNCScOKToajuyB7va4cExHnqbElOw+T+thcWkUGYbWCj5eV6GA/YkL9CLB54evthbeXBW+rBS+rhSBfb1o0CSAyyJekpkF4WS0mt0TqQqMvhNq1a0d6ejoAhYWFJCYmMnToUHNDScOKTobNX2qckIgA0DepCa0iA9mbe4yScjsbD+TXeh/hAT5EBNqwWixYLPw6tRDk60Wb6GBsXo4TLhaL47Ef7WKC8bJasFocRVXVvNXiKMDiwv2xWFRYmaHRF0K/98UXXzBkyBACAwPNjiINqXk3x3TT/2DIwxASa24eETFVkyBfFtw3EMMw2JZTyL7cYwCUlleSlVdCdkEpJeWVlFbYqai0U2E3qLQbHC0uZ++RYrLzSzhaXM7R4hNffr9819FaZzqnVQQzx/fCz8frrNomtWd6IbRo0SKefvppVq5cyYEDB/j000+PO2310ksv8fTTT5OVlUVKSgovvPACvXr1qvWxPvjgA8aOHVtHycVttE6D5j1g3wqY+xBc/rrZiUTEBVgsFtpGB9P21we11lR5pZ2N+/MpKa/EboBhGI4pBocLy9h5sJCq572WVdpZlnGEgwWlGIZBpWFQaQe74Siu7HaD4vJKftl5hIc+W88zV6TUQ0vlVEwvhIqKikhJSWHChAmMHj36uPWzZ89m0qRJzJgxg969e/Pss88ybNgwtmzZQlSU466hqampVFRUHPfZuXPnEhvr+L///Px8fv75Z2bNmlW/DRLXY/WCYU/C6+fD9nlmpxERN+fjZSUlPqzO9vfT9kNc85+lfLJqL0+MSlavUAMzvRAaMWIEI0aMOOn66dOnc9NNNzF+/HgAZsyYwVdffcXrr7/Ogw8+COAcA3Qqn3/+Oeeffz5+fn6n3K60tJTS0lLn+/z82p87FhfUtJ1jWpIHZcVgCzA3j4jIr85NakKwnzcFJRVkHimudQ+VnB2Xvny+rKyMlStXkpaW5lxmtVpJS0tjyZIltdrXBx98UO3qsZOZOnUqoaGhzld8fHytc4sL8gsFn1+Ln4ID5mYREfkdi8VCq1+vZNt5UDd5bGguXQgdOnSIyspKoqOjqy2Pjo4mKyurxvvJy8tj2bJlDBs27LTbTp48mby8POdrz549tc4tLshigeBmjnkVQiLiYlpWFUKHCk1O4nlMPzXWEEJDQ8nOzq7Rtr6+vvj6+tZzIjFFSKzjKfT5KoRExLW0jAwCIEM9Qg3OpXuEIiMj8fLyOq6Iyc7OJiYmxqRU4raCf/2dUY+QiLiYVk0dPUIZh1QINTSXLoRsNhvdu3dn/vz5zmV2u5358+fTp08fE5OJW9KpMRFxUVWnxlbsPsqeI3oGWkMyvRAqLCwkPT3deeVXRkYG6enpZGZmAjBp0iRee+013nzzTTZt2sStt95KUVGR8yoykRqrupFi/n5zc4iI/EFVIQTQ76nvSd+Ta14YD2P6GKEVK1YwaNAg5/tJkyYBMG7cOGbOnMmYMWM4ePAgU6ZMISsri9TUVObMmXPcAGqR03KeGqv5QHsRkYYQ6OvNDee15L+LMwB45PP1TL6gAz5eVmxeVny8LbSICCDAZvqf7UbHYhiGYXYIV5afn09oaCh5eXmEhISYHUfORuZSx00Vw1rA3XoSvYi4npz8EgY9s5Cissrj1vl6W2kTHYSXxUKIvw/nJkXSr00kMaF+RATYsOohsNXU9O+3SkvxHCFVY4SywF7puOO0iIgLiQrx48nRnXl10U5KK+yUV9opr7BTXF5JbnE56/f9dpPfH7cd4h9zHPO+3lYCfb0ZnhzDk5d2Nim9e1IhJJ4jOBZ8Q6E0D/atgvieZicSETnOJanNuSS1ebVlhmGwJbuAA7klGBjsOXKMhVtyWLH7KIWlFZRW2CmtKOO9pZlc3asFyc1DTUrvflQIiefw8oakQbDxM9g2V4WQiLgNi8VC+5gQ2sf8dopn3LmJAFRU2tmfW8Lds1ezKjOX1xdnMH1MqjlB3ZDpV42JNKi2v95dfNtcc3OIiNQRby8rLZoE8MjITgD8b+1+jp1gjJGcmAoh8Sytf31u3YF0KDxoahQRkbqUEh9GsK835ZUG+3KPmR3HbagQEs8SFAXhLR3zh7aYm0VEpI7FhvkDsF+FUI2pEBLPE/FrIXQkw9wcIiJ1LDbMD1AhVBsqhMTzVPUIHVUhJCKNi3qEak+FkHie8ETHVD1CItLIVBVC+3JLTE7iPlQIieeJUI+QiDROzdUjVGsqhMTzOE+N7TI1hohIXXOeGstTIVRTKoTE81SdGjt2FI7lmplERKROVQ2WPpBbgt2uR4nWhAoh8Ty+QRDY1DG/6i2oKDM3j4hIHYkO8cNqgbJKO+NnLie/pNzsSC5PhZB4prhejul3D8PzqZCxyNQ4IiJ1wcfLSuuoIAB+2HqQFxdsNzmR61MhJJ7psv/A+f8HQTGQvw/eGwN7V5idSkTkrL1+fU8u7x4HwMyfdzFn/QEqKu0mp3JdFsMwdBLxFPLz8wkNDSUvL4+QkJDTf0DcS/kxmHUN7JgPzXvATfPNTiQictYMw2DMq7+wLOMIAH1aNeHGfi2xWCC5eShRwX4mJ6x/Nf37rULoNFQIeYDCgzC9A9jL4eZF0CzF7EQiImftcGEpL36/nQ+W76Hodw9hjQv358cHBmGxWExMV/9q+vdbp8ZEgppCx4sd8yteNzeLiEgdaRLkyyMjO/HRrecysF1TUuJCsVhg79FjHC7SRSJVVAiJAKRe7ZjuXGhqDBGRutahWQgzx/fi84nnERvquM/QrkNFJqdyHSqERABifj0ddnQ3lBWbm0VEpJ4kRgYAkKFCyEmFkAg4To8FNAEMOLzN7DQiIvUisUkgALsOqxCqokJIpErT9o7pwS3m5hARqSctI38thA6p57uKCiGRKk3bOaYHN5ubQ0SknlT1COnU2G9UCIlUqeoRylEhJCKNU2Lkb6fGdPccBxVCIlWqCqE9v0DmUnOziIjUgxYRAdi8rBSXVTLx/dVk55eYHcl0uqHiaeiGih6ktBD+fQ7k7QGfALhrrWMQtYhII/LOL7t55IsNVP76dHovq4WWkYEE2Lyc28SG+tM5LhTrKW662DTYlz5JTWge5l/vmc+E7ixdR1QIeZiiQ/DmxZCzAYZNhT63mZ1IRKTOrd2by9+/2MCqzNyz3ldkkC82Lwv+Ni/uGNyGUV2bn33AOqBCqI6oEPJAS1+Fb+53zI/7HyT0BavXqT8jIuKGcovLKCytIONQEeW/PpjVboeNB/LZc+TkV5YZwI6Dhazdm+fsWaqS2CTghI/v8PW2Mjw5hqt6tSA6pP6fdaZCqI6oEPJAxUfgmbaOZ48BhCdC6zSISAIfP4hoBXG9wBZgakwREbMVlJSz+3AxhgGfpe/jv4szTvsZL6uF8ACfass+u70vceF1+29qTf9+e9fpUUUag4AIGPYkrP/YcSn90V2w/D/Vt7H6wL1bILCJKRFFRFxBsJ8Pyc1DAegcF8rVvVtw9CTPMdt79BjvLt3N8l1HOVRYfRszu2TUI3Qa6hHycGVFsO072LcCcvdARQkcWOvoGbpztdnpRETczr7cYxSWVFRb1jIyEJt33V7Irh4hkbpgC4ROoxyvKoYBx46alUhExK252lVmuo+QSG1ZLI7TZyIi4vZUCImIiIjHUiEkIiIiHkuFkIiIiHgsFUIiIiLisVQIiYiIiMdSISQiIiIeS4WQiIiIeCwVQiIiIuKxVAiJiIiIx1IhJCIiIh5LhZCIiIh4LBVCIiIi4rFUCImIiIjH8jY7gKszDAOA/Px8k5OIiIhITVX93a76O34yKoROo6CgAID4+HiTk4iIiEhtFRQUEBoaetL1FuN0pZKHs9vt7N+/n+DgYCwWS53tNz8/n/j4ePbs2UNISEid7deVNPY2Nvb2QeNvY2NvHzT+Njb29kHjb2N9tc8wDAoKCoiNjcVqPflIIPUInYbVaiUuLq7e9h8SEtIof7F/r7G3sbG3Dxp/Gxt7+6Dxt7Gxtw8afxvro32n6gmqosHSIiIi4rFUCImIiIjHUiFkEl9fXx555BF8fX3NjlJvGnsbG3v7oPG3sbG3Dxp/Gxt7+6Dxt9Hs9mmwtIiIiHgs9QiJiIiIx1IhJCIiIh5LhZCIiIh4LBVCIiIi4rFUCJnkpZdeIjExET8/P3r37s2yZcvMjnRG/v73v2OxWKq92rdv71xfUlLC7bffTpMmTQgKCuKyyy4jOzvbxMSnt2jRIkaOHElsbCwWi4XPPvus2nrDMJgyZQrNmjXD39+ftLQ0tm3bVm2bI0eOcM011xASEkJYWBg33HADhYWFDdiKkztd+66//vrjvtPhw4dX28aV2zd16lR69uxJcHAwUVFRjBo1ii1btlTbpia/l5mZmVx44YUEBAQQFRXF/fffT0VFRUM25aRq0saBAwce9z3ecsst1bZx1Ta+/PLLdOnSxXmDvT59+vDNN98417v79wenb6M7f38nMm3aNCwWC3fffbdzmct8j4Y0uFmzZhk2m814/fXXjQ0bNhg33XSTERYWZmRnZ5sdrdYeeeQRo1OnTsaBAwecr4MHDzrX33LLLUZ8fLwxf/58Y8WKFcY555xjnHvuuSYmPr2vv/7a+Nvf/mZ88sknBmB8+umn1dZPmzbNCA0NNT777DNjzZo1xsUXX2y0bNnSOHbsmHOb4cOHGykpKcYvv/xi/Pjjj0br1q2Nq666qoFbcmKna9+4ceOM4cOHV/tOjxw5Um0bV27fsGHDjDfeeMNYv369kZ6eblxwwQVGixYtjMLCQuc2p/u9rKioMJKTk420tDRj9erVxtdff21ERkYakydPNqNJx6lJGwcMGGDcdNNN1b7HvLw853pXbuMXX3xhfPXVV8bWrVuNLVu2GH/9618NHx8fY/369YZhuP/3Zxinb6M7f39/tGzZMiMxMdHo0qWLcddddzmXu8r3qELIBL169TJuv/125/vKykojNjbWmDp1qompzswjjzxipKSknHBdbm6u4ePjY3z44YfOZZs2bTIAY8mSJQ2U8Oz8sVCw2+1GTEyM8fTTTzuX5ebmGr6+vsb7779vGIZhbNy40QCM5cuXO7f55ptvDIvFYuzbt6/BstfEyQqhSy655KSfcaf2GYZh5OTkGIDxww8/GIZRs9/Lr7/+2rBarUZWVpZzm5dfftkICQkxSktLG7YBNfDHNhqG4w/p7//o/JG7tTE8PNz4z3/+0yi/vypVbTSMxvP9FRQUGG3atDG+++67am1ype9Rp8YaWFlZGStXriQtLc25zGq1kpaWxpIlS0xMdua2bdtGbGwsrVq14pprriEzMxOAlStXUl5eXq2t7du3p0WLFm7b1oyMDLKysqq1KTQ0lN69ezvbtGTJEsLCwujRo4dzm7S0NKxWK0uXLm3wzGdi4cKFREVF0a5dO2699VYOHz7sXOdu7cvLywMgIiICqNnv5ZIlS+jcuTPR0dHObYYNG0Z+fj4bNmxowPQ188c2Vnn33XeJjIwkOTmZyZMnU1xc7FznLm2srKxk1qxZFBUV0adPn0b5/f2xjVUaw/d3++23c+GFF1b7vsC1/jvUQ1cb2KFDh6isrKz2xQJER0ezefNmk1Kdud69ezNz5kzatWvHgQMHePTRR+nXrx/r168nKysLm81GWFhYtc9ER0eTlZVlTuCzVJX7RN9f1bqsrCyioqKqrff29iYiIsIt2j18+HBGjx5Ny5Yt2bFjB3/9618ZMWIES5YswcvLy63aZ7fbufvuu+nbty/JyckANfq9zMrKOuF3XLXOlZyojQBXX301CQkJxMbGsnbtWv7yl7+wZcsWPvnkE8D127hu3Tr69OlDSUkJQUFBfPrpp3Ts2JH09PRG8/2drI3g/t8fwKxZs1i1ahXLly8/bp0r/XeoQkjOyogRI5zzXbp0oXfv3iQkJPDBBx/g7+9vYjI5U3/605+c8507d6ZLly4kJSWxcOFChgwZYmKy2rv99ttZv349ixcvNjtKvTlZG//85z875zt37kyzZs0YMmQIO3bsICkpqaFj1lq7du1IT08nLy+Pjz76iHHjxvHDDz+YHatOnayNHTt2dPvvb8+ePdx111189913+Pn5mR3nlHRqrIFFRkbi5eV13Mj47OxsYmJiTEpVd8LCwmjbti3bt28nJiaGsrIycnNzq23jzm2tyn2q7y8mJoacnJxq6ysqKjhy5IhbtrtVq1ZERkayfft2wH3aN3HiRL788ku+//574uLinMtr8nsZExNzwu+4ap2rOFkbT6R3794A1b5HV26jzWajdevWdO/enalTp5KSksJzzz3XqL6/k7XxRNzt+1u5ciU5OTl069YNb29vvL29+eGHH3j++efx9vYmOjraZb5HFUINzGaz0b17d+bPn+9cZrfbmT9/frVzw+6qsLCQHTt20KxZM7p3746Pj0+1tm7ZsoXMzEy3bWvLli2JiYmp1qb8/HyWLl3qbFOfPn3Izc1l5cqVzm0WLFiA3W53/mPmTvbu3cvhw4dp1qwZ4PrtMwyDiRMn8umnn7JgwQJatmxZbX1Nfi/79OnDunXrqhV83333HSEhIc5TF2Y6XRtPJD09HaDa9+jKbfwju91OaWlpo/j+TqaqjSfibt/fkCFDWLduHenp6c5Xjx49uOaaa5zzLvM91tmwa6mxWbNmGb6+vsbMmTONjRs3Gn/+85+NsLCwaiPj3cW9995rLFy40MjIyDB++uknIy0tzYiMjDRycnIMw3BcHtmiRQtjwYIFxooVK4w+ffoYffr0MTn1qRUUFBirV682Vq9ebQDG9OnTjdWrVxu7d+82DMNx+XxYWJjx+eefG2vXrjUuueSSE14+37VrV2Pp0qXG4sWLjTZt2rjM5eWnal9BQYFx3333GUuWLDEyMjKMefPmGd26dTPatGljlJSUOPfhyu279dZbjdDQUGPhwoXVLj0uLi52bnO638uqy3bPP/98Iz093ZgzZ47RtGlTl7k0+XRt3L59u/HYY48ZK1asMDIyMozPP//caNWqldG/f3/nPly5jQ8++KDxww8/GBkZGcbatWuNBx980LBYLMbcuXMNw3D/788wTt1Gd//+TuaPV8K5yveoQsgkL7zwgtGiRQvDZrMZvXr1Mn755RezI52RMWPGGM2aNTNsNpvRvHlzY8yYMcb27dud648dO2bcdtttRnh4uBEQEGBceumlxoEDB0xMfHrff/+9ARz3GjdunGEYjkvoH374YSM6Otrw9fU1hgwZYmzZsqXaPg4fPmxcddVVRlBQkBESEmKMHz/eKCgoMKE1xztV+4qLi43zzz/faNq0qeHj42MkJCQYN91003FFuiu370RtA4w33njDuU1Nfi937dpljBgxwvD39zciIyONe++91ygvL2/g1pzY6dqYmZlp9O/f34iIiDB8fX2N1q1bG/fff3+1+9AYhuu2ccKECUZCQoJhs9mMpk2bGkOGDHEWQYbh/t+fYZy6je7+/Z3MHwshV/keLYZhGHXXvyQiIiLiPjRGSERERDyWCiERERHxWCqERERExGOpEBIRERGPpUJIREREPJYKIREREfFYKoRERETEY6kQEhG3N3PmzOOeYl1frr/+ekaNGtUgxxKR+qenz4uInMCuXbto2bIlq1evJjU11bn8ueeeQ/ehFWk8VAiJiNRCaGio2RFEpA7p1JiI1Du73c7UqVNp2bIl/v7+pKSk8NFHH2G324mLi+Pll1+utv3q1auxWq3s3r0bgOnTp9O5c2cCAwOJj4/ntttuo7Cw8KTHO9Hpq7vvvpuBAwc638+ZM4fzzjuPsLAwmjRpwkUXXcSOHTuc66ue6N61a1csFovzs3/cd2lpKXfeeSdRUVH4+flx3nnnsXz5cuf6hQsXYrFYmD9/Pj169CAgIIBzzz2XLVu2OLdZs2YNgwYNIjg4mJCQELp3786KFStq9LMVkbOjQkhE6t3UqVN56623mDFjBhs2bOCee+7h2muv5ccff+Sqq67ivffeq7b9u+++S9++fUlISADAarXy/PPPs2HDBt58800WLFjAAw88cFaZioqKmDRpEitWrGD+/PlYrVYuvfRS7HY7AMuWLQNg3rx5HDhwgE8++eSE+3nggQf4+OOPefPNN1m1ahWtW7dm2LBhHDlypNp2f/vb3/jnP//JihUr8Pb2ZsKECc5111xzDXFxcSxfvpyVK1fy4IMP4uPjc1btE5EaqtNHuIqI/EFJSYkREBBg/Pzzz9WW33DDDcZVV11lrF692rBYLMbu3bsNwzCMyspKo3nz5sbLL7980n1++OGHRpMmTZzv33jjDSM0NNT5fty4ccYll1xS7TN33XWXMWDAgJPu8+DBgwZgrFu3zjAMw8jIyDAAY/Xq1dW2+/2+CwsLDR8fH+Pdd991ri8rKzNiY2ONp556yjAMw/j+++8NwJg3b55zm6+++soAjGPHjhmGYRjBwcHGzJkzT5pNROqPeoREpF5t376d4uJihg4dSlBQkPP11ltvsWPHDlJTU+nQoYOzV+iHH34gJyeHK664wrmPefPmMWTIEJo3b05wcDDXXXcdhw8fpri4+Ixzbdu2jauuuopWrVoREhJCYmIiAJmZmTXex44dOygvL6dv377OZT4+PvTq1YtNmzZV27ZLly7O+WbNmgGQk5MDwKRJk7jxxhtJS0tj2rRp1U7RiUj9UiEkIvWqaizPV199RXp6uvO1ceNGPvroI8BxaqiqEHrvvfcYPnw4TZo0ARxXb1100UV06dKFjz/+mJUrV/LSSy8BUFZWdsJjWq3W467sKi8vr/Z+5MiRHDlyhNdee42lS5eydOnSU+7zbP3+VJfFYgFwnob7+9//zoYNG7jwwgtZsGABHTt25NNPP62XHCJSnQohEalXHTt2xNfXl8zMTFq3bl3tFR8fD8DVV1/N+vXrWblyJR999BHXXHON8/MrV67Ebrfzz3/+k3POOYe2bduyf//+Ux6zadOmHDhwoNqy9PR05/zhw4fZsmULDz30EEOGDKFDhw4cPXq02vY2mw2AysrKkx4nKSkJm83GTz/95FxWXl7O8uXL6dix46l/MH/Qtm1b7rnnHubOncvo0aN54403avV5ETkzunxeROpVcHAw9913H/fccw92u53zzjuPvLw8fvrpJ0JCQhg3bhyJiYmce+653HDDDVRWVnLxxRc7P9+6dWvKy8t54YUXGDlyJD/99BMzZsw45TEHDx7M008/zVtvvUWfPn145513WL9+PV27dgUgPDycJk2a8Oqrr9KsWTMyMzN58MEHq+0jKioKf39/5syZQ1xcHH5+fsddOh8YGMitt97K/fffT0REBC1atOCpp56iuLiYG264oUY/n2PHjnH//fdz+eWX07JlS/bu3cvy5cu57LLLavR5ETk76hESkXr3+OOP8/DDDzN16lQ6dOjA8OHD+eqrr5yXqIPj9NiaNWu49NJL8ff3dy5PSUlh+vTp/OMf/yA5OZl3332XqVOnnvJ4w4YN4+GHH+aBBx6gZ8+eFBQUMHbsWOd6q9XKrFmzWLlyJcnJydxzzz08/fTT1fbh7e3N888/zyuvvEJsbCyXXHLJCY81bdo0LrvsMq677jq6devG9u3b+fbbbwkPD6/Rz8bLy4vDhw8zduxY2rZty5VXXsmIESN49NFHa/R5ETk7FuOPJ9JFREREPIR6hERERMRjqRASERERj6VCSERERDyWCiERERHxWCqERERExGOpEBIRERGPpUJIREREPJYKIREREfFYKoRERETEY6kQEhEREY+lQkhEREQ8lgohERER8Vj/D6AUCdRl9+7HAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from optilab.plotting import plot_box_plot, plot_convergence_curve, plot_ecdf_curves\n", + "\n", + "# plot convergence curve for the two optimizers\n", + "plot_convergence_curve({run.model_metadata.name: run.logs for run in [cmaes_runs, lmm_cmaes_runs]})" + ] + }, + { + "cell_type": "markdown", + "id": "f8c7346b-bb57-42ac-b0d8-65a5c816bea3", + "metadata": {}, + "source": [ + "As you can see the LMM-CMA-ES converges much quicker than regular CMA-ES. Now let's plot the ECDF curve:" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "a792d285-351b-49bd-a327-252805d8eebf", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjcAAAHLCAYAAAA0kLlRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAABo00lEQVR4nO3dd3xT5f4H8E+SpnsBhZZRWpACZRVkFmQohQqCDAUu8GMpKlcqYPWK9SpTBRyAIMqV6UW54gBUZFWgoICyrIi0ZRXKbNmlO02e3x8hh4ambVKSnDT5vF8vXknOOTnne06eNF+edRRCCAEiIiIiJ6GUOwAiIiIia2JyQ0RERE6FyQ0RERE5FSY3RERE5FSY3BAREZFTYXJDREREToXJDRERETkVJjdERETkVJjcEBERkVNhckNEREROhckNkYtbvXo1FApFmf9+++03o+0LCgqwYMECdOzYEQEBAfD09ETjxo0RFxeHEydOSNvNmDHDaD/e3t6oX78++vfvj1WrVqGwsLBULGPHji0zjq1bt1Z4LlqtFqtWrUKPHj1QvXp1eHh4IDw8HOPGjcOhQ4ce/GIRUZXgJncAROQYZs2ahQYNGpRa3qhRI+n5tWvX8Pjjj+Pw4cPo168fRowYAV9fX6SlpeGrr77CZ599hqKiIqP3f/rpp/D19UVhYSEuXryIbdu24ZlnnsHChQuxadMmhIaGGm3v4eGB5cuXl4ojKiqq3Pjz8/MxePBgbN26Fd26dcMbb7yB6tWr4+zZs/j666/x+eefIyMjA/Xq1bPkshBRVSSIyKWtWrVKABAHDx6scNsnnnhCKJVK8e2335ZaV1BQIF555RXp9fTp0wUAcfXq1VLbfvHFF0KpVIqOHTsaLR8zZozw8fGpxFkIMXHiRAFALFiwoNS64uJi8f7774vz589Xat8labVakZ+f/8D7ISLbYbMUEZnl999/x08//YRnn30WTz31VKn1Hh4e+OCDD8za18iRIzF+/Hj8/vvvSExMfODYLly4gP/85z/o1asXpkyZUmq9SqXCq6++KtXajB07FuHh4aW2MzSllaRQKBAXF4cvv/wSzZs3h4eHB3788UdUr14d48aNK7WP7OxseHp64tVXX5WWFRYWYvr06WjUqBE8PDwQGhqK1157rVTTXGJiIh555BEEBgbC19cXTZo0wRtvvFGJK0Lk2tgsRUQAgNu3b+PatWtGyxQKBWrUqAEA+OGHHwAAo0aNssrxRo0ahc8++wzbt29Hr169jNbdH4darUZAQECZ+9qyZQuKi4utFtv9du7cia+//hpxcXEICgpCREQEBg0ahPXr1+M///kP3N3dpW03btyIwsJC/OMf/wAA6HQ6PPnkk/j111/x/PPPIzIyEn/99RcWLFiAEydOYOPGjQCAv//+G/369UOrVq0wa9YseHh44NSpU9i7d69NzonImTG5ISIAQExMTKllHh4eKCgoAACkpKQAAFq2bGmV47Vo0QIAcPr0aaPlubm5qFmzptGy7t27Iykpqcx9WTu2+6WlpeGvv/5Cs2bNpGXDhg3DypUrsX37dvTr109avm7dOjRs2BDt2rUDAKxduxY///wzdu/ejUceeUTarkWLFpgwYQL27duHzp07IzExEUVFRdiyZQuCgoJsch5EroLJDREBAJYsWYLGjRsbLVOpVNLz7OxsAICfn59Vjufr6wsAuHPnjtFyT09P/Pjjj0bLqlWrVu6+rB3b/bp3726U2ADAY489hqCgIKxbt05Kbm7evInExESjJqlvvvkGkZGRaNq0qVGN1GOPPQYA2LVrFzp37ozAwEAAwPfff49x48ZBqWSvAaLKYnJDRACADh06SLUNpvj7+wPQJyOGH+IHkZOTA6B0QqJSqUzWIpWnZGy2YGoUmZubG5566imsXbsWhYWF8PDwwPr166HRaDBs2DBpu5MnTyIlJaVUbZRBVlYWAH1N0PLlyzF+/Hi8/vrr6NmzJwYPHoynn36aiQ6RhZjcEJFZmjZtCgD466+/0LVr1wfe37FjxwAYDzWvrJKxtW7dusLt7+80bKDVak0u9/LyMrn8H//4B/7zn/9gy5YtGDhwIL7++ms0bdrUaNi6TqdDy5YtMX/+fJP7MAyF9/Lywp49e7Br1y789NNP2Lp1K9atW4fHHnsM27dvN6pFI6Ly8b8DRGSW/v37AwC++OILq+xvzZo1AIDY2NgH3lefPn2gUqnMjq1atWq4detWqeXnzp2z6LjdunVD7dq1sW7dOly7dg07d+40qrUBgIceegg3btxAz549ERMTU+pfkyZNpG2VSiV69uyJ+fPn4/jx43jnnXewc+dO7Nq1y6K4iFwdkxsiMkt0dDQef/xxLF++XBrhU1JRUZFRX5PyrF27FsuXL0d0dDR69uz5wLGFhobiueeew/bt27F48eJS63U6HT788ENcuHABgD7huH37No4ePSptc/nyZWzYsMGi4yqVSjz99NP48ccfsWbNGhQXF5dKboYOHYqLFy9i2bJlpd6fn5+P3NxcAMCNGzdKrTfUQpmazZmIyqYQQgi5gyAi+axevRrjxo0rc4bizp07o2HDhgCAq1evonfv3vjzzz/Rv39/9OzZEz4+Pjh58iS++uorXL58WfohnjFjBmbOnCnNUFxUVCTNULx3715ERUXhp59+Qt26daVjjR07Ft9++63UH8cSeXl5GDhwIBITE9GjRw/069cP1apVQ0ZGBr755hukpqYiIyMDdevWxfXr1xEWFobg4GBMmjQJeXl5+PTTT1GzZk0cOXIEJf8sKhQKTJw4ER9//LHJ4+7duxePPPII/Pz8EB4ebpQwAfrEqn///tiyZQuGDRuGLl26QKvVIjU1FV9//TW2bduGdu3aYcqUKdizZw+eeOIJhIWFISsrC5988gkUCgWOHTtW7lB4IrqPzJMIEpHMDDMUl/Vv1apVRtvn5eWJDz74QLRv3174+voKd3d3ERERIV566SVx6tQpaTvDDMWGf56enqJevXqiX79+YuXKlaKgoKBULA8yQ7EQ+pmIly9fLrp27SoCAgKEWq0WYWFhYty4ceKPP/4w2nb79u2iRYsWwt3dXTRp0kR88cUXUswlARATJ04s85g6nU6EhoYKAOLtt982uU1RUZGYN2+eaN68ufDw8BDVqlUTbdu2FTNnzhS3b98WQgixY8cOMWDAAFGnTh3h7u4u6tSpI4YPHy5OnDhR6etB5KpYc0NEREROhX1uiIiIyKkwuSEiIiKnwuSGiIiInAqTGyIiInIqTG6IiIjIqTC5ISIiIqficveW0ul0uHTpEvz8/Mq8vwwRERE5FiEE7ty5gzp16lR4M1mXS24uXbok3aiOiIiIqpbz58+jXr165W7jcsmNn58fAP3F8ff3lzkax6bRaLB9+3b07t0barVa7nCIKsQyS1URy615srOzERoaKv2Ol8flkhtDU5S/vz+TmwpoNBp4e3vD39+fXziqElhmqSpiubWMOV1K2KGYiIiInAqTGyIiInIqLtcsZS6tVguNRiN3GLLSaDRwc3NDQUEBtFqt3OHISq1WQ6VSyR0GERGZgcnNfYQQuHLlCm7duiV3KLITQiAkJATnz5/nsHkAgYGBCAkJ4bUgInJwTG7uY0hsatWqBW9vb5f+IdPpdMjJyYGvr2+Fcwo4MyEE8vLykJWVBQCoXbu2zBEREVF5mNyUoNVqpcSmRo0acocjO51Oh6KiInh6erp0cgMAXl5eAICsrCzUqlWLTVRERA7MtX+x7mPoY+Pt7S1zJOSIDOXC1ftiERE5OiY3JrhyUxSVjeWCiKhqYHJDREREToXJDRERETkVWZObPXv2oH///qhTpw4UCgU2btxY4XuSkpLw8MMPw8PDA40aNcLq1attHicRERFVHbImN7m5uYiKisKSJUvM2j49PR1PPPEEHn30USQnJ2PKlCkYP348tm3bZuNIiYiIqKqQdSh4nz590KdPH7O3X7p0KRo0aIAPP/wQABAZGYlff/0VCxYsQGxsrMn3FBYWorCwUHqdnZ0NQD/i5f5RLxqNBkII6HQ66HQ6S0/H6QghpEdeD/3QeCEENBoNh4I7KMN3miPanJMifQ+Uez+ECHsEuq7/kjscq3Gmcrvp6GVsSL6EhkE++HffplbdtyXXp0rNc7N//37ExMQYLYuNjcWUKVPKfM+cOXMwc+bMUsu3b99easi3m5sbQkJCkJOTg6KiIgD6H/YCjTw/7J5qpdkjdHQ6HRYvXozPP/8cFy9eRM2aNTF27FgMHToUUVFRWLlyJT777DMkJycjMjISn332GbKzs/HKK6/g5MmT6NSpE5YuXYqgoCAAwJEjRzB79mwcPXoUGo0GLVu2xLvvvouoqKhy47hw4QLeeust7Ny5E0qlEtHR0Zg7dy7q168PAPj1118xffp0pKamws3NDU2bNsWyZcuk9Y6sqKgI+fn52LNnD4qLi+UOh8qRmJgodwhkAw2ubkerC3tx6XYRDt5pLnc4VucM5fbtIypcL1Tg6LlraIMzVt13Xl6e2dtWqeTmypUrCA4ONloWHByM7Oxs5OfnSxOtlZSQkID4+HjpdXZ2NkJDQ9G7d2/4+/sbbVtQUIDz58/D19cXnp6eAIC8omK0mSdPgTs2oxe83c37iF5//XUsX74cH374IR555BFcvnwZqamp8PX1BQC89957mD9/PurXr4/x48djwoQJ8PPzw6JFi+Dt7Y1//OMf+OCDD/DJJ58A0CdL48aNQ9OmTeHt7Y0FCxZg2LBhSEtLg5+fn8kYNBoNhg4dik6dOmHPnj1wc3PDO++8g6FDhyI5ORlKpRL/93//h/Hjx+Orr75CUVERDhw4AH9//1KfhSMqKCiAl5cXunXrJpUPciwajQaJiYno1asX1Gq13OGQlSl3/A5cAIIbP4y+vfrKHY7VOFO5nXl0F1CowQuPNkHfR8Ktum9Dy4s5qlRyUxkeHh7w8PAotVytVpcqRFqtFgqFAkqlUpqRV86ZeUvGUZ47d+5g0aJF+PjjjzFu3DgAQEREBLp164azZ88CAF599VWpCXDy5MkYPnw4duzYga5duwIAnn32WaxevVo6XkxMDHQ6HbKzs+Hv749ly5YhMDAQv/zyC/r162cyjm+++QY6nQ4rVqyQapxWr16NwMBA7NmzB+3atcPt27fRv39/REREAACaN686//tSKvU1aabKDjkWfkZO6s5lAICqWhhUTvj5OkO51RTruzM83rKO1c/Fkv1VqeQmJCQEmZmZRssyMzPh7+9vstbGGrzUKhyfZbo/j615qc3r15GSkoLCwkL07NmzzG1atWolPTfUfrVs2dJomeHeSYD+uv773//Grl27cO3aNWi1WuTl5SEjIwMAMGHCBHzxxRfS9jk5Ofjzzz9x6tSpUjU7BQUFOH36NHr37o2xY8ciNjYWvXr1QkxMDIYOHcp7NRGReW5f0D8G1JM3DipTYbG+G4e7m7wzzVSp5CY6OhqbN282WpaYmIjo6GibHVOhUJjdNCQXcxK7khmvoVbl/mUlOw2PGTMG169fx5w5cxAZGQkvLy9ER0dLfZFmzZqFV1991egYOTk5aNu2Lb788stSx69ZsyYAYNWqVZg0aRK2bt2KdevW4c0330RiYiI6depkwRkTkUsyJDf+TG4ckRACRVr974iHzMmNrEfPyclBcnIykpOTAeiHeicnJ0u1AwkJCRg9erS0/YQJE3DmzBm89tprSE1NxSeffIKvv/4aL7/8shzhO4yIiAh4eXlhx44dVtvn3r17ERcXh969e6N58+bw8PDAtWvXpPW1atVCo0aNpH8A8PDDD+PkyZOl1jVq1AgBAQHSe9u0aYOEhATs27cPLVq0wNq1a60WNxE5ISGAG+nAnUv616y5cUiGxAaQv+ZG1qMfOnQIbdq0QZs2bQAA8fHxaNOmDaZNmwYAuHz5spToAECDBg3w008/ITExEVFRUfjwww+xfPnyMoeBuwpPT09MnToVr732Gv773//i9OnT+O2337BixYpK7zMiIgJffPEF0tLS8Pvvv2PkyJEV1hCNHDkSQUFBGDBgAH755Rekp6cjKSkJkyZNwoULF5Ceno6EhATs378f586dw/bt23Hy5ElERkZWOk4icgE7ZgKLWuufq9wBn5qyhkOmGZqkAMBd5cLNUj169JDmUjHF1OzDPXr0wB9//GHDqKqmt956C25ubpg2bRouXbqE2rVrY8KECZXe34oVK/D888+jR48eCA0NxbvvvluqGep+3t7e2LNnD6ZOnYrBgwfjzp07qFu3Lnr27Al/f3/k5+cjNTUVn3/+Oa5fv47atWtj4sSJeOGFFyodJxG5gDNJ954/PBqQcaAHla2oRHIjd7OUQpSXXTih7OxsBAQE4Pbt2yaHgqenp6NBgwYc6gsYjZaSc9SYo2D5cHwajQabN29G3759q/yoEyrh/UZA7lXghV+A2q0q3r6KcZZye/FWPrrM3Ql3lRIn3jF/gl5zlff7fT/+YhERkePS5OsTGwAIDJU3FiqXoeZG7lobgMkNERE5MsMIKXdfwDNQ1lCofEUOMgwcYHJDRESO7PZ5/WNAKGDm7WhIHoXFWgCOkdw49gQuRETkuo59B2y6e/scNkk5tN0nrmL9EX0tmyM0SzG5ISIix3MnE/j2WQB3x7xUf0jWcKhsQghM/PIIcgr1NxQO8HaXOSImN0RE5IhunIGU2HR/HWg3TtZwqGwarZASm2cfaYBBberKHBGTGyIickS37k7gGt4VeDRB3lioXCVnJv5XbBN4mnlfRFuSv2GMiIjofobkJjBM3jioQoUarfRc7pmJDRwjCiIiopJuG5Kb+vLGQRWS7gSuUkKpdIwRbUxunESPHj0wZcoUucMgIrIOqeaGo6QcnSPNb2PAPjdEROR4bt2d34Y1Nw7l2MXbeP6/h3ArXyMt0929i5MjDAE3YHJDRESORacznryPHMYvJ6/h0u0Ck+tahwbaN5hyOE6a5aiEAIpy5flXyXuahoeH4+2338bo0aPh6+uLsLAw/PDDD7h69SoGDBgAX19ftGrVCocOHZLes3r1agQGBmLTpk1o0qQJvL29MWTIEOTl5eHzzz9HeHg4qlWrhkmTJkGr1ZZzdODWrVt44YUXEBwcDE9PT7Ro0QKbNm0q8zhPP/10hcdZs2YN2rVrBz8/P4SEhGDEiBHIysqq8Fr8+uuv6Nq1K7y8vBAaGopJkyYhNzdXWv/JJ58gIiICnp6eCA4OxtNPP23p5SYia8vJBLRFgEIF+Ms/rJjuMcxCPLB1Hfzy2qNG/5aNbidzdPew5qYimjzg3TryHPuNS4C7T6XeumDBArz77rt46623sGDBAowaNQqdO3fGM888g/fffx9Tp07F6NGj8ffff0Nxd0rzvLw8LFq0CF999RXu3LmDwYMHY9SoUahRowY2b96MM2fO4KmnnkKXLl0wbNgwk8fV6XTo06cP7ty5gy+++AIPPfQQjh8/DpXq3tBAU8cZNGgQAgMDyzyORqPB7Nmz0aRJE2RlZSE+Ph5jx47F5s2by7wGp0+fxuOPP463334bK1euxNWrVxEXF4e4uDisWrUKhw4dwqRJk7BmzRp07twZN27cwC+//FKp601EVmTobxNQF1DxZ8qRGDoP1/D1QGh1b5mjKRtLjZPq27cvXnjhBQDAtGnT8Omnn6J9+/YYMmQIAGDq1KmIjo5GZmYmQkJCAOgTiE8//RQPPaSfCfSpp57CF198gcuXL8Pf3x/NmjXDo48+il27dpWZ3Pz88884cOAAUlJS0LhxYwBAw4YNjba5/zhPP/001qxZg8zMTPj6+po8zjPPPCO9v2HDhli0aBHat2+PnJwc+Pr6moxlzpw5GDlypNTROiIiAosWLUL37t3x6aefIiMjAz4+PujXrx/8/PwQFhaGNm3aWHyticjKOAzcYRVqHOfO3+VhclMRtbe+BkWuY1dSq1atpOfBwcEAgJYtW5ZalpWVJSU33t7eUsJh2KZ+/fpGyUNwcLDUHPTuu+/i3XffldYdP34cycnJqFevnpTYmGLqOOHh4WUeBwAOHz6MGTNm4M8//8TNmzeh0+m/YBkZGWjWrBmaN2+Oc+fOAQC6du2KLVu24M8//8TRo0fx5ZdfSvsRQkCn0yE9PR29evVCWFgYGjZsiMcffxyPP/44Bg0aBG9vx/3fCJFLuKX/LrMzseMxNEt5uMk/UV95mNxURKGodNOQnNRqtfTc0OxkapkhSbh/vWEbNze3UssM75kwYQKGDh0qratTpw68vLwsis2wT1PLDMfJzc1FbGwsYmNj8eWXX6JmzZrIyMhAbGwsioqKAACbN2+GRqPvvW+IIScnBy+88AImTZpUKob69evD3d0dR44cQVJSErZv345p06ZhxowZOHjwIAIDAys8DyKykVuc48YRCSGQlHYVAOChZs0NOanq1aujevXqRstatWqFCxcu4MSJE+XW3lgiNTUV169fx9y5cxEaqh85UbIzNACEhZWuvn744Ydx/PhxNGrUqMx9u7m5ISYmBjExMZg+fToCAwOxc+dODB482CqxE1ElcKSUQ/rr4m1cvJUPAPBxd+yaG8dOvajK6d69O7p164annnoKiYmJSE9Px5YtW7B169ZK79NQy7J48WKcOXMGP/zwA2bPnl3h+6ZOnYp9+/YhLi4OycnJOHnyJL7//nvExcUBADZt2oRFixYhOTkZ586dw3//+1/odDo0adKk0rESkRXk39Q/+gTJGwcZuXQ3sQGAx1vUljGSijG5Iav77rvv0L59ewwfPhzNmjXDa6+9VuHw8fLUrFkTq1evxjfffINmzZph7ty5+OCDDyp8X6tWrbB7926cOHECXbt2RZs2bTBt2jTUqaMf/RYYGIj169fjscceQ2RkJJYuXYr//e9/aN68eaVjJSIryL+lf/QMkDUMMmYYKdWlUQ3U9POQOZryKYSo5GQqVVR2djYCAgJw+/Zt+Pv7G60rKChAeno6GjRoAE9PT5kidBw6nQ7Z2dnw9/eHUsk8mOXD8Wk0GmzevBl9+/Yt1Y+LqpB5DYD8G8CLvwG1IuWOxuaqSrlddzADU7/7Cz2b1sKKse3tfvzyfr/vx18sIiJyHEIABbf1zz0DZQ2FjBlqbhy9MzHA5IaIiBxJUS4g7jZjs1nKodyb48axOxMDTG6IiMiRGGptlGpAXfHUEmQfN3OL8M7mFACAJ2tuiIiILCA1SQXo5xkjh5B04t6kqvWqOf5Ep0xuTHCxPtZkJpYLIjsomdyQw8gr0jcV+nq44fluDSvYWn5Mbkow9FLPy8uTORJyRIZy4cijGYiqPCY3DsnQ3+axprWgVjl+6sAZiktQqVQIDAyU7mnk7e0t3abAFel0OhQVFaGgoMClh4ILIZCXl4esrCwEBgYa3eGciKys4Jb+kcmNQymQ7ilVNX4LmNzcx3ATyZI3bXRVQgjk5+fDy8vLpZM8g8DAQKl8EJGNsObGIUkjpapAZ2KAyU0pCoUCtWvXRq1ataQbMboqjUaDPXv2oFu3bi7fFKNWq1ljQ2QPmcf0j16BsoZB92QXaKR7SnlWgWHgAJObMqlUKpf/MVOpVCguLoanp6fLJzdEZAdndgNH/qt/7lH+DLRkHwUaLR59PwnXc4sAVJ2am6oRJREROb+slHvPmw+ULQy65+qdQimxeaimD2KbV42medbcEBGRY9AW6h+jhgN128obCwG4d8uFAC81drzSQ95gLMCaGyIicgzF+hoCqNgM7igKNPpRUlVhVuKSqla0RETkvLSG5MZD3jhIYqi58VRXrT6oTG6IiMgxGJql3JjcOIpCTdWa38aAfW6IiMgxaO9Ov8FmKVntTM1E4vFMAMClWwUAql7NDZMbIiJyDMV3a27YLCWr1779C9dyCo2W1fBxlymaymFyQ0REjsHQLMWaG1ndztf3fXqhe0P4ebhBpVTiiZa1ZY7KMkxuiIjIMRiapdjnRjZanYBGKwAAE7o9hGpVrMbGoGr1ECIiIufFZinZGYZ+A1Wvn01JTG6IiMgxsEOx7EomN1VthFRJVTdyIiJyLhwKLruCu/PauLspoVQqZI6m8pjcEBGRY5CapapmPw9nIM1IXIVrbQAmN0RE5CikZikmN3K5d7uFqtvfBmByQ0REjkLLmhu5FWiq5u0W7sfkhoiIHIPhxpluTG7kUlhFb5R5v6odPREROQ/eOFN2BcVsliIiIrIeNkvJTmqWcmNyQ0RE9OCkGYqZ3MhBpxPY8MdFAIAHm6WIiIisgEPBZbXv9HXpbuD+nlV7IkUmN0RE5BikPjdMbuRwNadAev7PHg/JGMmDY3JDRESOwZDccIZiWRTfvWFmjyY10aJugMzRPBgmN0REJD8hWHMjs2KdPrlxq8K3XTBgckNERPIzJDYAkxuZFGv1I6XclFU/NZD9DJYsWYLw8HB4enqiY8eOOHDgQLnbL1y4EE2aNIGXlxdCQ0Px8ssvo6CgoNz3EBGRg2NyIzup5kbFmpsHsm7dOsTHx2P69Ok4cuQIoqKiEBsbi6ysLJPbr127Fq+//jqmT5+OlJQUrFixAuvWrcMbb7xh58iJiMiqikskN+xzIwtDnxs2Sz2g+fPn47nnnsO4cePQrFkzLF26FN7e3li5cqXJ7fft24cuXbpgxIgRCA8PR+/evTF8+PAKa3uIiMjBGWpuFCpAWbUnkKuq7tXcyN6o88Dc5DpwUVERDh8+jISEBGmZUqlETEwM9u/fb/I9nTt3xhdffIEDBw6gQ4cOOHPmDDZv3oxRo0aVeZzCwkIUFhZKr7OzswEAGo0GGo3GSmfjnAzXh9eJqgqW2SqsMBdqAELljmIX+/wcpdwWaooBACqFkD0WUyyJSbbk5tq1a9BqtQgODjZaHhwcjNTUVJPvGTFiBK5du4ZHHnkEQggUFxdjwoQJ5TZLzZkzBzNnziy1fPv27fD29n6wk3ARiYmJcodAZBGW2arHt+ASegLQCAW2bN4sdziykLvcpp5XAlDiwvnz2Lz5nKyxmJKXl2f2trIlN5WRlJSEd999F5988gk6duyIU6dOYfLkyZg9ezbeeustk+9JSEhAfHy89Do7OxuhoaHo3bs3/P397RV6laTRaJCYmIhevXpBra7as1WSa2CZrcIy/wZSALWnD/r27St3NHblKOU2NfEkcCEdDzUIR9++TWWLoyyGlhdzyJbcBAUFQaVSITMz02h5ZmYmQkJCTL7nrbfewqhRozB+/HgAQMuWLZGbm4vnn38e//73v6E0MXzNw8MDHh6lO6ep1Wr+8TMTrxVVNSyzVZBCfzdqhcrDZT87ucutTqHvSOzupnLIz8CSmGTrNeTu7o62bdtix44d0jKdTocdO3YgOjra5Hvy8vJKJTAqlb7jmRDCdsESEZFtGW6aqXK8H1VXIY2WYofiBxMfH48xY8agXbt26NChAxYuXIjc3FyMGzcOADB69GjUrVsXc+bMAQD0798f8+fPR5s2baRmqbfeegv9+/eXkhwiIqqCDDfN5DBw2dybxK/qDwWXNbkZNmwYrl69imnTpuHKlSto3bo1tm7dKnUyzsjIMKqpefPNN6FQKPDmm2/i4sWLqFmzJvr374933nlHrlMgIiJr0OTrH9Ve8sbhwgqL9cmNhxtrbh5YXFwc4uLiTK5LSkoyeu3m5obp06dj+vTpdoiMiIjsRpOrf1T7yBuHCyvQ6Ps9eblX/ZaQqp+eERFR1Vd0d5ivO6fokEuB5m7NjZrJDRER0YPT3E1u1Exu5FJQrK+58XSCZqmqfwZERFT1FeXoH93ZLCUXQ7OUpxPU3Mje54aIiEhqlmLNjd0dybiJz/edRdqVOwAALyY3REREVmBolmLNjd199PNJ7D5xVXod7O8pYzTWweSGiIjkV3R3tBSTG7vLL9I3Rw1rF4pezYLRom7VvzURkxsiIpIfOxTLRqPTj5LqGVkLMc2CK9i6amCHYiIikh+HgsvGcNsFtRPcdsHAec6EiIiqLk7iJxuN4bYLqqp/2wUDJjdERCQ/1tzIRqvT19yonOCeUgZMboiISH4cLSWbYh2bpYiIiKyviM1SctE40d3ADZjcEBGR/DRslpKLoUOxm9J5UgLnORMiIqq6pJobJjf2Vqxjh2IiIiLrEoKT+MnoXp8bJjdERETWUVwAQP8Dy5ob+zM0S6nYLEVERGQlhmHgAGtuZMAOxURERNZmmMDPzRNQVv07Ulc1HApORERkbfm39I+eAbKG4Yo0Wp00iZ+n2nlSAuc5EyIiqpoKbukfPQPljMIlFWi00nNPtfPUmjG5ISIieRlqbrwC5YzCJeXfTW4UCsDDzXlSAuc5EyIiqppYcyObQo2+M7GnmwoKBTsUExERWQdrbmRjqLlxpv42AJMbIiKSG2tuZPP1wfMAAC8n6m8DMLkhIiK5cbSUbNIy7wAAcgqLZY7EupjcEBGRvApu6x/ZLGV3htmJZw5oLnMk1sXkhoiI5MVmKdkYbprp6cZmKSIiIuthh2LZaO7W3Lg50ezEAJMbIiKSG2tuZGOouXFzojuCA0xuiIhIbqy5kY2hz43aie4IDjC5ISIiOQnBmhsZGW6aqXKiO4IDTG6IiEhORbmA7u4wZNbc2F2xVt8spWazFBERkZUYam2UakDtLWsorogdiomIiKwt/6b+0StQf/dGsiupQzGbpYiIiKxEmp04UM4oXJb2bp8bNWtuiIiIrMTQLMX+NrIwNEuxQzEREZG1sOZGNpdv5+N2vgYAOxSbdOvWLWvshoiIXI1Uc1NN1jBc0Q/Jl6Tngd7uMkZifRYnN/PmzcO6deuk10OHDkWNGjVQt25d/Pnnn1YNjoiInBwn8JNN7t07gbcLq4YAL7XM0ViXxcnN0qVLERoaCgBITExEYmIitmzZgj59+uBf//qX1QMkIiInxgn8ZJOv0QIA2oY5X62Zm6VvuHLlipTcbNq0CUOHDkXv3r0RHh6Ojh07Wj1AIiJyYqy5kU2BRj8M3EPtXHcEBypRc1OtWjWcP38eALB161bExMQAAIQQ0Gq11o2OiIicm2GeG9bc2J2h5sbLCZMbi2tuBg8ejBEjRiAiIgLXr19Hnz59AAB//PEHGjVqZPUAiYjIiXEouGwMfW481c43cNri5GbBggVo0KABMjIy8N5778HX1xcAcPnyZbz44otWD5CIiJwYh4LLZsuxKwAAT1evudFoNHjhhRfw1ltvoUGDBkbrXn75ZasGRkRELkCTp3/08JU3DhcjhIBSAegEEFHL+a69RXVRarUa3333na1iISIiV6PJ1z+6ecobh4vRaAXu3nkBEcF+8gZjAxY3tA0cOBAbN260QShERORyigv0j0xu7MrQmRhgh2IAQEREBGbNmoW9e/eibdu28PHxMVo/adIkqwVHREROTIh7yY3aS95YXEzB3eRGpVQ43a0XgEokNytWrEBgYCAOHz6Mw4cPG61TKBRMboiIyDzFhfeeu3nIF4cLyi+6NwxcoWByg/T0dFvEQURErqY4/95zN9bc2FNBsT65ccZh4ADvCk5ERHIx1NwolIDKue5t5OgMNTfOOAwcMLPmJj4+HrNnz4aPjw/i4+PL3Xb+/PlWCYyIiJxcyZFSTtg04siceXZiwMzk5o8//oBGo5Gel8UZ2+2IiMhGOFJKNoV37yvl0jU3u3btMvmciIio0jhSSjbOXnPDPjdERCQPjaHmhiOl7E3qc+PunMmNxaOlAODQoUP4+uuvkZGRgaKiIqN169evt0pgRETk5AyjpThSyu7u1dw4Zx2HxWf11VdfoXPnzkhJScGGDRug0Wjw999/Y+fOnQgICLA4gCVLliA8PByenp7o2LEjDhw4UO72t27dwsSJE1G7dm14eHigcePG2Lx5s8XHJSIimRlGS6nZ58ae/si4iTc3HgPgvH1uLE5u3n33XSxYsAA//vgj3N3d8dFHHyE1NRVDhw5F/fr1LdrXunXrEB8fj+nTp+PIkSOIiopCbGwssrKyTG5fVFSEXr164ezZs/j222+RlpaGZcuWoW7dupaeBhERyY33lZJFUtpV6Xmb0ED5ArEhi5Ob06dP44knngAAuLu7Izc3FwqFAi+//DI+++wzi/Y1f/58PPfccxg3bhyaNWuGpUuXwtvbGytXrjS5/cqVK3Hjxg1s3LgRXbp0QXh4OLp3746oqChLT4OIiOTG0VKyMNx6YUDrOhjbpYHM0diGxX1uqlWrhjt37gAA6tati2PHjqFly5a4desW8vLyzN5PUVERDh8+jISEBGmZUqlETEwM9u/fb/I9P/zwA6KjozFx4kR8//33qFmzJkaMGIGpU6dCpTJdtVZYWIjCwntTfGdnZwMANBqNNLydTDNcH14nqipYZqsWZWEuVAB0Kg9oXfgzs3e5zS3UH6deoGeV+q5YEqvFyU23bt2QmJiIli1bYsiQIZg8eTJ27tyJxMRE9OzZ0+z9XLt2DVqtFsHBwUbLg4ODkZqaavI9Z86cwc6dOzFy5Ehs3rwZp06dwosvvgiNRoPp06ebfM+cOXMwc+bMUsu3b98Ob29vs+N1ZYmJiXKHQGQRltmqoWHWEbQEcCnrBg6z76Tdyu3JM0oASpw7cxKbC0/Y5ZjWYEkFisXJzccff4yCAn1V4r///W+o1Wrs27cPTz31FN58801Ld2cRnU6HWrVq4bPPPoNKpULbtm1x8eJFvP/++2UmNwkJCUazKmdnZyM0NBS9e/eGv7+/TeOt6jQaDRITE9GrVy+o1ZwanRwfy2zVotx3ErgI1KnfEMF9+8odjmzsXW63rzsKXL2C1i2aoW90mM2PZy2GlhdzWJzcVK9eXXquVCrx+uuvW7oLAEBQUBBUKhUyMzONlmdmZiIkJMTke2rXrg21Wm3UBBUZGYkrV66gqKgI7u7upd7j4eEBD4/Scyio1Wr+8TMTrxVVNSyzVYROP5WI0sMbSn5ediu3aVk5AAAfT/cq9T2xJNZKDXDXarX49ttvMXv2bMyePRvfffcdiouLLdqHu7s72rZtix07dkjLdDodduzYgejoaJPv6dKlC06dOgWdTictO3HiBGrXrm0ysSEiIgfG0VJ2J4TAqbvJjbPeERyoRHLz999/o3HjxhgzZgw2bNiADRs2YMyYMYiIiMCxY8cs2ld8fDyWLVuGzz//HCkpKfjnP/+J3NxcjBs3DgAwevRoow7H//znP3Hjxg1MnjwZJ06cwE8//YR3330XEydOtPQ0iIhIboZ5bpjc2E2R9l7lQMcGNWSMxLYsbpYaP348mjdvjkOHDqFatWoAgJs3b2Ls2LF4/vnnsW/fPrP3NWzYMFy9ehXTpk3DlStX0Lp1a2zdulXqZJyRkQGl8l7+FRoaim3btuHll19Gq1atULduXUyePBlTp0619DSIiEhuhhmKOYmf3RQU3Utuavo5720vLE5ukpOTjRIbQD88/J133kH79u0tDiAuLg5xcXEm1yUlJZVaFh0djd9++83i4xARkYPRcJ4bezPcdkGtUkCtYrOUpHHjxqU6AQNAVlYWGjVqZJWgiIjIBXASP7vLK9L3j3XW2y4YWJzczJkzB5MmTcK3336LCxcu4MKFC/j2228xZcoUzJs3D9nZ2dI/IiKiMjG5sbt7N8x07uTG4mapfv36AQCGDh0KhUIBQN/7GgD69+8vvVYoFNBqtdaKk4iInI1htJSadwW3F8OtF7zcmdwY2bVrly3iICIiV6O5O+Mskxu7yb/boZg1N/fp3r27LeIgIiJXk39L/+hVrdzNyHoMzVLsc0NERGQL+Tf1j56BsobhSlylzw2TGyIisj+dDii4pX/Omhu7KShyjT43TG6IiMj+iu4A4u6Ecl6BsobiSlhzQ0REZCuG/jZunuxQbEc5hZznxqTHHnsMt27dKrU8Ozsbjz32mDViIiIiZ2fob8MmKbtas/8cAOe+aSZQieQmKSkJRUVFpZYXFBTgl19+sUpQRETk5Az9bdiZ2K6C/NwBANV93GWOxLbMHgp+9OhR6fnx48dx5coV6bVWq8XWrVtRt25d60ZHRETOiTU3ssi/26G4S6MgmSOxLbOTm9atW0OhUEChUJhsfvLy8sLixYutGhwRETkpaY6bQDmjcDkFGk7iZyQ9PR1CCDRs2BAHDhxAzZo1pXXu7u6oVasWVCrnvlhERGQlrLmRRT5vv2AsLCwMAKDT6WwWDBERuQj2uZGFoVmKNTcmnDx5Ert27UJWVlapZGfatGlWCYyIiJwYa27sTqPVucztFyxObpYtW4Z//vOfCAoKQkhIiHRncABQKBRMboiIqGLsc2N3+05fl577eVaqbqPKsPjs3n77bbzzzjuYOnWqLeIhIiJXwJobu8vO1wAA3FVKp6+5sXiem5s3b2LIkCG2iIWIiFyFdF+pQDmjcCmGJqkujWrIHIntWZzcDBkyBNu3b7dFLERE5CoMzVKerLmxlwIXGSkFVKJZqlGjRnjrrbfw22+/oWXLllCr1UbrJ02aZLXgiIjISUnNUoGyhuFKDCOlnL1JCqhEcvPZZ5/B19cXu3fvxu7du43WKRQKJjdERFQ+rQYoytE/Z58bu3GVO4IDlUhu0tPTbREHERG5CkOTFAB4BsgWhqv54jf9TTO9XaBZyrlvC0pERI7H0CTlGQAonf+H1lH4e+m7kXi7O/cwcMDMmpv4+HjMnj0bPj4+iI+PL3fb+fPnWyUwIiJyUhwGLouCu31uekbWkjkS2zMrufnjjz+g0Wik52UpOaEfERGRSVLNTaCsYbga9rm5z65du0w+JyIispg0xw1rbuwpr8h1hoI/UJ+bCxcu4MKFC9aKhYiIXAGbpexOpxMoLNbfC9IVam4sTm50Oh1mzZqFgIAAhIWFISwsDIGBgZg9ezbvGE5ERBVjcmN3BcVa6bkr1NxY3GX63//+N1asWIG5c+eiS5cuAIBff/0VM2bMQEFBAd555x2rB0lERE5Emp2Yw8DtxdAkBQCebkxuSvn888+xfPlyPPnkk9KyVq1aoW7dunjxxReZ3BARUfk0efpHd29543Ah92YnVkKpdP7BPxY3S924cQNNmzYttbxp06a4ceOGVYIiIiInVlygf3TzkjcOF1LgQiOlgEokN1FRUfj4449LLf/4448RFRVllaCIiMiJafL1j2pPeeNwIYZh4K4wgR9QiWap9957D0888QR+/vlnREdHAwD279+P8+fPY/PmzVYPkIiInAxrbuwur0SzlCuw+Cy7d++OEydOYNCgQbh16xZu3bqFwYMHIy0tDV27drVFjERE5Ew0d5Mb1tzYjTSBnwuMlAIqUXMDAHXq1GHHYSIiqpziu81SrLmxuZTL2UhY/xcys/UJpbeazVJlunnzJlasWIGUlBQAQLNmzTBu3DhUr17dqsEREZETYs2N3Ww6egnJ529JrxvW9JEvGDuyuFlqz549CA8Px6JFi3Dz5k3cvHkTixYtQoMGDbBnzx5bxEhERM6ENTd2k1uob44a1KYuvp0QjdkDW8gckX1YXHMzceJEDBs2DJ9++ilUKn3bnVarxYsvvoiJEyfir7/+snqQRETkRFhzYzeG+W0a1fJFu3DXaV2xuObm1KlTeOWVV6TEBgBUKhXi4+Nx6tQpqwZHREROiDU3dmPoSOzpIvPbGFic3Dz88MNSX5uSUlJSOM8NERFVjDU3dmMYAu7tIqOkDCxulpo0aRImT56MU6dOoVOnTgCA3377DUuWLMHcuXNx9OhRadtWrVpZL1IiIqr6hGDNjR3la4oBMLmp0PDhwwEAr732msl1CoUCQggoFApotdpS2xARkQsrLrz3nDU3Nndv8j4mN+VKT0+3RRxEROQKDLU2AGtu7CCfzVLmCQsLs0UcRETkCgz9bRRKQKWWNxYXkO9iN8w0cI2bTBARkWMo2d9GoZA3FhdgaJZyldsuGDC5ISIi++FIKbsqKHKtu4EbMLkhIiL74UgpuxFCII/NUkRERDbGmhu7KdLqoNUJAGyWKtPKlStRWFhY8YZERERlKb6b3LDmxuYKinTSc1cbLWV2cvPcc8/h9u3b0us6derg7NmztoiJiIicVTFrbuwl7+4Efm5KBdQq12qoMftshRBGr+/cuQOdTlfG1kRERCZoDH1umNzYmquOlALY54aIiOxJqrlhs5StueoEfoAFyY1CoYCixJwE978mIiKqEGtu7MZVJ/ADLJihWAiBxo0bSwlNTk4O2rRpA6XSOD+6ceOGdSMkIiLnIXUoZnJja/eapVxrjhvAguRm1apVtoyDiIhcAYeC282mPy8BALzUrtcDxezkZsyYMTYLYsmSJXj//fdx5coVREVFYfHixejQoUOF7/vqq68wfPhwDBgwABs3brRZfEREZCWaXP2j2lveOFyAYRhQTmGxrHHIweK6KiEEDh8+jLNnz0KhUKBBgwZo06ZNpfvfrFu3DvHx8Vi6dCk6duyIhQsXIjY2FmlpaahVq1aZ7zt79ixeffVVdO3atVLHJSIiGRTdTW7cfeWNwwXo7k7gN6RtqMyR2J9FdVW7du3CQw89hI4dO2Lo0KEYMmQI2rdvj4iICOzZs6dSAcyfPx/PPfccxo0bh2bNmmHp0qXw9vbGypUry3yPVqvFyJEjMXPmTDRs2LBSxyUiIhkYkhsPJje2Vnw3uVEqXW/wj9k1N6dOnUK/fv3QsWNHLFiwAE2bNoUQAsePH8eiRYvQt29fHD161KJko6ioCIcPH0ZCQoK0TKlUIiYmBvv37y/zfbNmzUKtWrXw7LPP4pdffin3GIWFhUYzK2dnZwMANBoNNBqN2bG6IsP14XWiqoJl1vGpCu5ACUCr8oKOnxMA25VbTbG+Q7FC6JziO2HJOZid3CxcuBCdOnXCjh07jJY3bdoUgwYNQkxMDBYsWIDFixebffBr165Bq9UiODjYaHlwcDBSU1NNvufXX3/FihUrkJycbNYx5syZg5kzZ5Zavn37dnh7s83XHImJiXKHQGQRllnH1eniWQQD+DPlFM5nbZY7HIdi7XJ78bISgBIpx//G5hvHrLpvOeTl5Zm9rdnJTVJSEubMmWNynUKhwJQpU4xqYGzhzp07GDVqFJYtW4agoCCz3pOQkID4+HjpdXZ2NkJDQ9G7d2/4+/vbKlSnoNFokJiYiF69ekGtVssdDlGFWGYdn+q/nwB3gFbtO6Nl075yh+MQbFVuf7j5B3DjKqJatUTfdvWstl+5GFpezGF2cpORkYGWLVuWub5FixY4d+6c2QcGgKCgIKhUKmRmZhotz8zMREhISKntT58+jbNnz6J///7SMsMtINzc3JCWloaHHnrI6D0eHh7w8PAotS+1Ws0/fmbitaKqhmXWgd0dLeXm5Q/wMzJi7XJ7t8sN3NVuTvF9sOQczO5QnJOTU24zjre3t0VVRgDg7u6Otm3bGjV16XQ67NixA9HR0aW2b9q0Kf766y8kJydL/5588kk8+uijSE5ORmio6/UIJyKqUgpz9I8cLWVz2rvJjcoF7yZg0VDw48eP48qVKybXXbt2rVIBxMfHY8yYMWjXrh06dOiAhQsXIjc3F+PGjQMAjB49GnXr1sWcOXPg6emJFi1aGL0/MDAQAEotJyIiByQNBfeRNw4XoDW0bKiY3JSrZ8+epe4ODuj73AghKjXXzbBhw3D16lVMmzYNV65cQevWrbF161apk3FGRkapWzwQEVEVxXlu7EZrGArOmpuypaen2yyIuLg4xMXFmVyXlJRU7ntXr15t/YCIiMj6dLp7MxQzubE5Q3LjxnluyhYWFmbLOIiIyNlpSvTLZLOUzRkm8VO5YHJjdnvPyZMnMXz4cJNDsW7fvo0RI0bgzJkzVg2OiIiciKFJCgpA7SVrKK5Ax+SmYu+//z5CQ0NNzg0TEBCA0NBQvP/++1YNjoiInEhRiZFSLtgPxN5Yc2OG3bt3Y8iQIWWuHzp0KHbu3GmVoIiIyAlJyQ2bpOxBy+SmYhkZGeXepTsoKAjnz5+3SlBEROSEOAzcrpjcmCEgIACnT58uc/2pU6d4OwMiIiob7whuV/dGS7nedCpmn3G3bt3KvSnmokWL0LVrV6sERURETqiIsxPb070+NzIHIgOzTzkhIQFbtmzB008/jQMHDuD27du4ffs2fv/9dzz11FPYtm2bzW+cSUREVRibpezqXrOU62U3Zs9z06ZNG3z77bd45plnsGHDBqN1NWrUwNdff42HH37Y6gESEZGTMCQ36rLvU0jWIyU3LjgyzaLbL/Tr1w/nzp3D1q1bcerUKQgh0LhxY/Tu3bvcm2oSERFJk/ix5sYu8jVaAICnmjU3FfLy8sKgQYNsEQsRETkzTb7+0c1T3jhchCG58XJXyRyJ/ZmdzvXt2xe3b9+WXs+dOxe3bt2SXl+/fh3NmjWzanBEROREDMkNZye2Oa1OoKhYf1dwb3eL6zGqPLOTm23btqGwsFB6/e677+LGjRvS6+LiYqSlpVk3OiIich7FBfpH1tzYXF5RsfTcmzU3ZRNClPuaiIioXFLNDZMbW8sv0jdJKRSAh5vr9blxvTMmIiJ5GGpuOFrK5vLuJjfeahUULjhayuzkRqFQlLpArnjBiIioktih2G4MyY2XC/a3ASwYLSWEwNixY+Hh4QEAKCgowIQJE+Djox/SV7I/DhERUSnsUGw3+Rp9nxtX7G8DWJDcjBkzxuj1//3f/5XaZvTo0Q8eEREROSd2KLab/CL9SCkvNZObcq1atcqWcRARkbNjzY3dGEZLueIcNwA7FBMRkb1IHYqZ3NiaYQI/V22WYnJDRET2IXUoZnJja9JoKSY3RERENsR5buzG1UdLMbkhIiL7KGbNjb0Ybr3gihP4AUxuiIjIXjSGPjesubE1jVaf3KhVrvkz75pnTURE9qXTAdq786Gx5sbmiqXkxjUn22VyQ0REtmcYKQVwtJQdFGn1939kzQ0REZGtGDoTA0xu7MBQc+PGmhsiIiIbMXQmVqoBpWsOT7anYp2+5sadNTdEREQ2ouEEfvZUZKi5Ubrmz7xrnjUREdlXMe8Ibk9Sh2I3NksRERHZBoeB25XG0KGYNTdEREQ2Yqi5UXvLG4eL0LBDMRERkY1p2CxlT5zEj4iIyNak+0qxQ7E9FEvz3LDmhoiIyDYMk/ix5sYudqRmAWDNDRERke0U3tE/uvvIG4cLKNBopedhNVzzejO5ISIi28u/qX/0ri5vHC4gr+hectM6NFC+QGTE5IaIiGwv74b+0YvJja3lFRUDANzdlFAp2eeGiIjINlhzYzf5d2tuvN1d9zYXTG6IiMj28g01N9XkjcMFGJqlvNVMboiIiGyHzVJ283v6dQCAt4ebzJHIh8kNERHZHpul7OLqnUK8uzkVAODnyeSGiIjIdtgsZReZ2QXS80k9I2SMRF5MboiIyLZ0OiD/lv45m6VsytDfpkGQDx5tUkvmaOTD5IaIiGyr4BYA/e0AWHNjW4Zh4F4u3JkYYHJDRES2Zuhv4+4LuLnLG4uTMwwD9/FgckNERGQ7huSGTVI2Z2iW8nJ33c7EAJMbIiKyNcMwcG82SdlanoZz3ABMboiIyNZyrugffWrKG4cLyCvU97lx5dmJASY3RERka9dP6R+rPyRvHC7gXrMUkxsiIiLbuX5a/1iDyY2t5WsMHYrZ54aIiMh2mNzYDYeC6zG5ISIi29HpgBtn9M9rNJI3FheQxzuCAwBcu96KiIhs584V4MIhQFsIqNyBgFC5I3J613OKADC5YXJDRETWd/UE8ElHQOj0r6s1AJSu/YNrazqdwO4TVwFwnhs2SxERkfVdOXovsfGvB3R8Xt54XEDu3f42ANA+3LXnFHKI5GbJkiUIDw+Hp6cnOnbsiAMHDpS57bJly9C1a1dUq1YN1apVQ0xMTLnbExGRDPKu6x+bDQDi/wbaj5c3HhdguPWCQgHUr+4tczTykj25WbduHeLj4zF9+nQcOXIEUVFRiI2NRVZWlsntk5KSMHz4cOzatQv79+9HaGgoevfujYsXL9o5ciIiKpM0K3ENeeNwIVJnYrUKCoVC5mjkJXtyM3/+fDz33HMYN24cmjVrhqVLl8Lb2xsrV640uf2XX36JF198Ea1bt0bTpk2xfPly6HQ67Nixw86RExFRmQw1N7yflN1IyY2Lz3EDyNyhuKioCIcPH0ZCQoK0TKlUIiYmBvv37zdrH3l5edBoNKhe3fQXqLCwEIWFhdLr7OxsAIBGo4FGo3mA6J2f4frwOlFVwTLrOFS516AEoPUMhI6fR7msVW6z8woAAF5qpVN+Byw5J1mTm2vXrkGr1SI4ONhoeXBwMFJTU83ax9SpU1GnTh3ExMSYXD9nzhzMnDmz1PLt27fD29u12yTNlZiYKHcIRBZhmZVf9PkTqAUg+cR5XLi2We5wqoQHLbf7MhUAVCguyMPmzc53zfPy8szetkrXXc2dOxdfffUVkpKS4OnpaXKbhIQExMfHS6+zs7Olfjr+/v72CrVK0mg0SExMRK9evaBWq+UOh6hCLLOOw235+8AdIKrTo2jVyPR/PknPWuU28eujAK7A3csHffs+Yr0AHYSh5cUcsiY3QUFBUKlUyMzMNFqemZmJkJCQct/7wQcfYO7cufj555/RqlWrMrfz8PCAh4dHqeVqtZp//MzEa0VVDcusA8i/CQBw868F8LMwy4OWW4VS3422R5NaTln+LTknWTsUu7u7o23btkadgQ2dg6Ojo8t833vvvYfZs2dj69ataNeunT1CJSIiS+TfHS3FDsV2k1eon+emaYifzJHIT/Zmqfj4eIwZMwbt2rVDhw4dsHDhQuTm5mLcuHEAgNGjR6Nu3bqYM2cOAGDevHmYNm0a1q5di/DwcFy5cgUA4OvrC19fX9nOg4iI7irKAzR3+0dwKLjdcLTUPbJfgWHDhuHq1auYNm0arly5gtatW2Pr1q1SJ+OMjAwolfcqmD799FMUFRXh6aefNtrP9OnTMWPGDHuGTkREphhqbZRugAdrEezFcEdwbxe/IzjgAMkNAMTFxSEuLs7kuqSkJKPXZ8+etX1ARERUeSUn8HPxyeTsKZd3BJfIPokfERE5GcMEfmySspvfz1zHqawcAGyWApjcEBGRtd0+r3/0K3/UK1nP/jPXpecP1fSRMRLHwOSGiIis69oJ/WNQY3njcCEarf4O7KM6hcHP0/mGgVuKyQ0REVnXtZP6x6AIeeNwIRqtAMD+NgZMboiIyLpYc2N3RcX6mhs3FTtwA0xuiIjImooLgZtn9c+Z3NiNoVlKreLPOsDkhoiIrOlGOiB0gIc/4Btc8fZkFcV3m6WY3OjxKhARkfXcOK1/rPEQ57ixI0PNjTuTGwBMboiIyJoK7+gfvarJG4eLKdKyz01JTG6IiMh6DPeUUnvLG4eLYZ8bY7wKRERkPZp8/aPaS944XIyhzw2bpfR4FYiIyHpYcyMLQ7OU2o3NUgCTGyIisiap5obJjT0ZmqXclPxZB5jcEBGRNbFZyu52pmbitzP6O7Gzz40erwIREVkPm6Xsbu3v56Xn4UG87gCTGyIisqYiQ3LDmht7yS0sBgC80qsxmob4yxyNY2ByQ0RE1qNhcmNveUX65KZZHSY2BkxuiIjIetih2O5yi7QAAG93N5kjcRxMboiIyHrYodju8u8mNz4eKpkjcRxMboiIyHrYodiuLtzMw8Vb+oSSNTf3MLkhIiLrYc2N3WRlF6DH+0nSa9bc3MPkhoiIrId9buzm7PU8FOv0t10Y2q4eQvw9ZY7IcbAOi4iIrIejpewm9+4oqRZ1/fHe01EyR+NYWHNDRETWY6i5cWfNja3lFXKUVFmY3BARkXUIwQ7FdmSoufFxZ1+b+zG5ISIi6yguBKDvA8JmKdvLuzszsbcHa27ux+SGiIisw1BrAwBuTG5sbUdqFgDAW82am/sxuSEiIuswJDcqd0DF2gRby7ihv94KhcyBOCAmN0REZB2c48auirX6JsAno+rKHInjYXJDRETWwc7EdmW4YWawv4fMkTgeJjdERGQdrLmxK+mGmexQXAqTGyIisg7W3NiNRqtDUbEOAIeCm8LkhoiIHlxRLnD7ov45a25sLu9urQ3ASfxM4RUhIqIHc+pnYO0/AJ1G/5rJjc3tPXUNAKBWKeDuxnqK+/GKEBHRg0nfcy+xUaiAxn3kjccFnL2eCwDQ3B0xRcZYc0NERA8m97r+sccbQLdXASX7gNia4b5SYzuHyxuIg2LNDRERPZg8fRMJfGsxsbET6b5SHrzepjC5ISKiB5N7N7nxCZI3DheSa7ivFDsTm8TkhoiIHoyh5sanprxxuBDDHDccBm4akxsiInowhj433qy5sZddhptmcgI/k5jcEBFR5RUXAkV39M99asgbi4vILSyW5rmp5u0uczSOickNERFVnqG/jdIN8AyUNRRXcStfIz3v1pi1ZaYwuSEiosoz9LfxrgEoFPLG4iLy7nYmruathocb+9yYwuSGiIgqz1Bzw/42dpPDkVIVYnJDRESVl3e3MzH729iNob8N57gpG5MbIiKqPNbc2J1hjhsfjpQqE68MERFVXh4n8LOn6d8fw8bkSwAAHzZLlYlXhoiIKu/Wef2jX4i8cbgAnU7g8/3npNfN6vjLGI1jY3JDRESVdzVF/1izqbxxuIA8jVZ6/tOkR9CsNpObsrDPDRERVY5OC1w7qX/O5MbmDH1tVEoFmtX2h4JD78vE5IaIiCrn5lmguABw8wSqhcsdjdO7d7NMFRObCjC5ISKiyrmaqn8MigCUHJZsa7mF+mYpX46SqhCvEBERWa7gNvDrQv3zmpGyhuKsrtwuwNELt6TXJ7NyAOhrbqh8TG6IiMhy618ALhzQP6/F/jbWJoTAoE/24vLtglLr/DzVMkRUtTC5ISIiywgBZOzXPw8IBVr9Q954nFBhsU5KbKLqBUCp1PexUSkUGN+1gZyhVQlMboiIyDJ3rgAFtwCFEog7BKg95Y7I6RjuHwUAG17sIiU3ZB52KCYiIstkHdc/Vn+IiY2N5BTcvcWCu4qJTSU4RHKzZMkShIeHw9PTEx07dsSBAwfK3f6bb75B06ZN4enpiZYtW2Lz5s12ipSIiJB1d+K+WuxIbCs5vH/UA5E9uVm3bh3i4+Mxffp0HDlyBFFRUYiNjUVWVpbJ7fft24fhw4fj2WefxR9//IGBAwdi4MCBOHbsmJ0jJyJyUYZZiWs1kzcOJ2aY08bXk8lNZSiEEELOADp27Ij27dvj448/BgDodDqEhobipZdewuuvv15q+2HDhiE3NxebNm2SlnXq1AmtW7fG0qVLS21fWFiIwsJC6XV2djZCQ0Nx7do1+Ptbb+rqjLRk5P441Wr7cxRFRUVwd3eXOwwis7HM2l6rwsMAgEXV38RB764yR1P1CSFw7do1BAUFSZPz3czT4NilbLSs64/1EzrJHKFjyM7ORlBQEG7fvl3h77esKWFRUREOHz6MhIQEaZlSqURMTAz2799v8j379+9HfHy80bLY2Fhs3LjR5PZz5szBzJkzSy3fvn07vL29Kx/8fQqunsKwu194p1NY8SZEDoVl1ua0QoF1l2riIq7LHYqTUAK3b5Ra6lZwi10v7srLyzN7W1mTm2vXrkGr1SI4ONhoeXBwMFJTU02+58qVKya3v3LlisntExISjJIhQ81N7969rVpzc+PqZfz+u4fV9ucIhE6HCxcuoF69elAoZW/BJKoQy6z95PiEYUq1VnKH4RS0Wi2OHTuGFi1aQKW6N0GfSqlA10ZBCPTmvDaA/vfbXE7fmOfh4QEPj9JJh1qthlptvQITXKc+ggfFWW1/jkCj0SBz82a07dvXqteKyFZYZqkq0mg08Mr8C33bhrLclsOSayPrf22CgoKgUqmQmZlptDwzMxMhISEm3xMSEmLR9kRERORaZE1u3N3d0bZtW+zYsUNaptPpsGPHDkRHR5t8T3R0tNH2AJCYmFjm9kRERORaZG+Wio+Px5gxY9CuXTt06NABCxcuRG5uLsaNGwcAGD16NOrWrYs5c+YAACZPnozu3bvjww8/xBNPPIGvvvoKhw4dwmeffSbnaRAREZGDkD25GTZsGK5evYpp06bhypUraN26NbZu3Sp1Gs7IyICyRMfAzp07Y+3atXjzzTfxxhtvICIiAhs3bkSLFi3kOgUiIiJyILInNwAQFxeHuDjTnXGTkpJKLRsyZAiGDBli46iIiIioKuJYSSIiInIqTG6IiIjIqTC5ISIiIqfC5IaIiIicCpMbIiIicipMboiIiMipMLkhIiIip8LkhoiIiJyKQ0ziZ09CCACW3TrdVWk0GuTl5SE7O5t3qqUqgWWWqiKWW/MYfrcNv+Plcbnk5s6dOwCA0NBQmSMhIiIiS925cwcBAQHlbqMQ5qRATkSn0+HSpUvw8/ODQqEwWte+fXscPHjwgfZf2X1Y8j5ztzVnu/K2yc7ORmhoKM6fPw9/f3+zYnN01viMHenYLLPGWGarxrEfdL/2KLOWbM9yW5otyo4QAnfu3EGdOnWM7jlpisvV3CiVStSrV8/kOpVK9cAFq7L7sOR95m5rznbmbOPv7+80XzhrfMaOdGyWWdNYZh372A+6X3uUWUu2Z7ktzVZlp6IaGwN2KC5h4sSJsu3DkveZu60521njnKsSOc/XFsdmmXV+zlZmrbFfe5RZS7ZnuS1N7vN1uWYpMl92djYCAgJw+/Ztp/nfBDk3llmqilhurY81N1QmDw8PTJ8+HR4eHnKHQmQWllmqilhurY81N0RERORUWHNDREREToXJDRERETkVJjdERETkVJjcEBERkVNhckNEREROhckNVcqmTZvQpEkTREREYPny5XKHQ2SWQYMGoVq1anj66aflDoWoQufPn0ePHj3QrFkztGrVCt98843cIVUZHApOFisuLkazZs2wa9cuBAQEoG3btti3bx9q1Kghd2hE5UpKSsKdO3fw+eef49tvv5U7HKJyXb58GZmZmWjdujWuXLmCtm3b4sSJE/Dx8ZE7NIfHmhuy2IEDB9C8eXPUrVsXvr6+6NOnD7Zv3y53WEQV6tGjB/z8/OQOg8gstWvXRuvWrQEAISEhCAoKwo0bN+QNqopgcuOC9uzZg/79+6NOnTpQKBTYuHFjqW2WLFmC8PBweHp6omPHjjhw4IC07tKlS6hbt670um7durh48aI9QicX9qDllsjerFlmDx8+DK1Wi9DQUBtH7RyY3Lig3NxcREVFYcmSJSbXr1u3DvHx8Zg+fTqOHDmCqKgoxMbGIisry86REt3DcktVjbXK7I0bNzB69Gh89tln9gjbOQhyaQDEhg0bjJZ16NBBTJw4UXqt1WpFnTp1xJw5c4QQQuzdu1cMHDhQWj958mTx5Zdf2iVeIiEqV24Ndu3aJZ566il7hEkkqWyZLSgoEF27dhX//e9/7RWqU2DNDRkpKirC4cOHERMTIy1TKpWIiYnB/v37AQAdOnTAsWPHcPHiReTk5GDLli2IjY2VK2Qis8otkSMxp8wKITB27Fg89thjGDVqlFyhVklMbsjItWvXoNVqERwcbLQ8ODgYV65cAQC4ubnhww8/xKOPPorWrVvjlVde4UgpkpU55RYAYmJiMGTIEGzevBn16tVj4kOyMafM7t27F+vWrcPGjRvRunVrtG7dGn/99Zcc4VY5bnIHQFXTk08+iSeffFLuMIgs8vPPP8sdApHZHnnkEeh0OrnDqJJYc0NGgoKCoFKpkJmZabQ8MzMTISEhMkVFVD6WW6pqWGZti8kNGXF3d0fbtm2xY8cOaZlOp8OOHTsQHR0tY2REZWO5paqGZda22CzlgnJycnDq1CnpdXp6OpKTk1G9enXUr18f8fHxGDNmDNq1a4cOHTpg4cKFyM3Nxbhx42SMmlwdyy1VNSyzMpJ7uBbZ365duwSAUv/GjBkjbbN48WJRv3594e7uLjp06CB+++03+QImEiy3VPWwzMqH95YiIiIip8I+N0RERORUmNwQERGRU2FyQ0RERE6FyQ0RERE5FSY3RERE5FSY3BAREZFTYXJDREREToXJDRERETkVJjdERETkVJjcOIizZ89CoVAgOTlZ7lAkqamp6NSpEzw9PdG6dWuT2wgh8Pzzz6N69eqyx++I17CykpKSoFAocOvWLZsfa8aMGWV+vnIKDw/HwoULpdcKhQIbN26s9PtNsXSfpowdOxYDBw4sc/3q1asRGBj4QMcwlz2PZS89evTAlClT5A5DUtm/efd/No76vbufo11/czG5uWvs2LFQKBSYO3eu0fKNGzdCoVDIFJW8pk+fDh8fH6SlpRndubakrVu3YvXq1di0aRMuX76MFi1a2CU2Uz8ooaGhdo2hKjL1Y/7qq6+W+fk6ksuXL6NPnz5mb3/w4EE8//zzNoxIXuYkb2R91vqbV1W+d+vXr8fs2bPlDsNiTG5K8PT0xLx583Dz5k25Q7GaoqKiSr/39OnTeOSRRxAWFoYaNWqUuU3t2rXRuXNnhISEwM1NvhvNq1Qq2WOoinx9fcv8fB1JSEgIPDw8zN6+Zs2a8Pb2tmFEVFVptVrodLpKvddaf/OqyveuevXq8PPzkzsMizG5KSEmJgYhISGYM2dOmduYqkpcuHAhwsPDpdeGWoV3330XwcHBCAwMxKxZs1BcXIx//etfqF69OurVq4dVq1aV2n9qaio6d+4MT09PtGjRArt37zZaf+zYMfTp0we+vr4IDg7GqFGjcO3aNWl9jx49EBcXhylTpiAoKAixsbEmz0On02HWrFmoV68ePDw80Lp1a2zdulVar1AocPjwYcyaNQsKhQIzZswotY+xY8fipZdeQkZGBhQKhXQNTP2PsnXr1kb7UCgUWL58OQYNGgRvb29ERETghx9+MHrP33//jX79+sHf3x9+fn7o2rUrTp8+jRkzZuDzzz/H999/D4VCAYVCgaSkJJPNUrt370aHDh3g4eGB2rVr4/XXX0dxcbHR9Zo0aRJee+01VK9eHSEhISbP9X7Lly9HZGQkPD090bRpU3zyySfSus6dO2Pq1KlG21+9ehVqtRp79uwBAKxZswbt2rWDn58fQkJCMGLECGRlZZV5PHPK3cGDB9GrVy8EBQUhICAA3bt3x5EjR6T1hm0HDRpk9Hndv++KyobhOq9fvx6PPvoovL29ERUVhf3790vbnDt3Dv3790e1atXg4+OD5s2bY/PmzWWeX1ZWFvr37w8vLy80aNAAX375ZaltStY6mXON7y+HJ0+eRLdu3eDp6YlmzZohMTGx1DHOnz+PoUOHIjAwENWrV8eAAQNw9uxZab1Wq0V8fDwCAwNRo0YNvPbaazD33sMbN25EREQEPD09ERsbi/PnzwPQX0+lUolDhw4Zbb9w4UKEhYWZ/BHu0aMHzp07h5dffln6DpS0bds2REZGwtfXF48//jguX75stL688mtKRd8TU9+9W7duSd9N4F5T67Zt29CmTRt4eXnhscceQ1ZWFrZs2YLIyEj4+/tjxIgRyMvLMzp+cXEx4uLiEBAQgKCgILz11ltG172wsBCvvvoq6tatCx8fH3Ts2FE6LnCvSeiHH35As2bN4OHhgYyMDJPnWt7fjLL+5pmyevVq1K9fH97e3hg0aBCuX79utP7+711lfzcqKrOG/X7wwQeoXbs2atSogYkTJ0Kj0UjbfPLJJ1LZDA4OxtNPPy2tu79Z6ubNmxg9ejSqVasGb29v9OnTBydPnix1rcsrg0lJSejQoQN8fHwQGBiILl264Ny5c2Vey0qR85bkjmTMmDFiwIABYv369cLT01OcP39eCCHEhg0bRMnLNH36dBEVFWX03gULFoiwsDCjffn5+YmJEyeK1NRUsWLFCgFAxMbGinfeeUecOHFCzJ49W6jVauk46enpAoCoV6+e+Pbbb8Xx48fF+PHjhZ+fn7h27ZoQQoibN2+KmjVrioSEBJGSkiKOHDkievXqJR599FHp2N27dxe+vr7iX//6l0hNTRWpqakmz3f+/PnC399f/O9//xOpqanitddeE2q1Wpw4cUIIIcTly5dF8+bNxSuvvCIuX74s7ty5U2oft27dErNmzRL16tUTly9fFllZWUIIIcLCwsSCBQuMto2KihLTp0+XXhvOde3ateLkyZNi0qRJwtfXV1y/fl0IIcSFCxdE9erVxeDBg8XBgwdFWlqaWLlypUhNTRV37twRQ4cOFY8//ri4fPmyuHz5sigsLJSu4R9//CHtw9vbW7z44osiJSVFbNiwQQQFBRnF0b17d+Hv7y9mzJghTpw4IT7//HOhUCjE9u3bTV43IYT44osvRO3atcV3330nzpw5I7777jtRvXp1sXr1aiGEEB9//LGoX7++0Ol00nsWL15stGzFihVi8+bN4vTp02L//v0iOjpa9OnTR9p+165dAoC4efOmEMK8crdjxw6xZs0akZKSIo4fPy6effZZERwcLLKzs4UQQmRlZQkAYtWqVUaf1/37rqhsGK5z06ZNxaZNm0RaWpp4+umnRVhYmNBoNEIIIZ544gnRq1cvcfToUXH69Gnx448/it27d5d5Tfv06SOioqLE/v37xaFDh0Tnzp2Fl5eXUTkCIDZs2GD2NS5ZDrVarWjRooXo2bOnSE5OFrt37xZt2rQx2mdRUZGIjIwUzzzzjDh69Kg4fvy4GDFihGjSpIkoLCwUQggxb948Ua1aNfHdd99J19jPz08MGDCgzHNbtWqVUKvVol27dmLfvn3i0KFDokOHDqJz587SNr169RIvvvii0ftatWolpk2bZnKf169fF/Xq1ROzZs2SvgMljxUTEyMOHjwoDh8+LCIjI8WIESOk91ZUfk2p6Hty/3dPCP3fKwBi165dQoh7ZbpTp07i119/FUeOHBGNGjUS3bt3F7179xZHjhwRe/bsETVq1BBz5841Oravr6+YPHmySE1NFV988YXw9vYWn332mbTN+PHjRefOncWePXvEqVOnxPvvvy88PDykMmu4Lp07dxZ79+4VqampIjc3t9R5VvQ3o6y/eff77bffhFKpFPPmzRNpaWnio48+EoGBgSIgIEDa5v7vXWV+N8wps2PGjBH+/v5iwoQJIiUlRfz4449G1+/gwYNCpVKJtWvXirNnz4ojR46Ijz76yOj6T548WXr95JNPisjISLFnzx6RnJwsYmNjRaNGjURRUZHRtS6rDGo0GhEQECBeffVVcerUKXH8+HGxevVqce7cOZPXsrKY3NxlSG6EEKJTp07imWeeEUJUPrkJCwsTWq1WWtakSRPRtWtX6XVxcbHw8fER//vf/4QQ9/44lPxSazQaUa9ePTFv3jwhhBCzZ88WvXv3Njr2+fPnBQCRlpYmhNAXxDZt2lR4vnXq1BHvvPOO0bL27dsb/YG9PyEx5f5zF8L85ObNN9+UXufk5AgAYsuWLUIIIRISEkSDBg2kL8z9Sn5eBvf/gX3jjTdEkyZNjH4AlyxZInx9faXPpnv37uKRRx4x2k/79u3F1KlTyzznhx56SKxdu9Zo2ezZs0V0dLQQQp9EuLm5iT179kjro6Ojy93nwYMHBQApiaxMcnM/rVYr/Pz8xI8//igtK/ljbnD/visqG4brvHz5cmn933//LQCIlJQUIYQQLVu2FDNmzCgztpLS0tIEAHHgwAFpWUpKigBQZnJjzjUuWQ63bdsm3NzcxMWLF6X1W7ZsMdrnmjVrSpWXwsJC4eXlJbZt2yaEEKJ27drivffek9YbvqMVJTcAxG+//Vbq/H7//XchhBDr1q0T1apVEwUFBUIIIQ4fPiwUCoVIT08vc7+mvmeGY506dUpatmTJEhEcHCy9rqj8mlLR98SS5Obnn3+WtpkzZ44AIE6fPi0te+GFF0RsbKzRsSMjI40+l6lTp4rIyEghhBDnzp0TKpXK6LMVQoiePXuKhIQEo+uSnJxc5jkKYd7fjIq+d0IIMXz4cNG3b1+jZcOGDaswubH0d8OcMmvYb3FxsbTNkCFDxLBhw4QQQnz33XfC399f+k/Q/UomNydOnBAAxN69e6X1165dE15eXuLrr78WQlRcBq9fvy4AiKSkpDKunnWwWcqEefPm4fPPP0dKSkql99G8eXMolfcub3BwMFq2bCm9VqlUqFGjRqmmiOjoaOm5m5sb2rVrJ8Xx559/YteuXfD19ZX+NW3aFIC+Hdigbdu25caWnZ2NS5cuoUuXLkbLu3Tp8kDnbKlWrVpJz318fODv7y9dj+TkZHTt2hVqtbrS+09JSUF0dLRRlX2XLl2Qk5ODCxcumIwDAGrXrl1mE1Fubi5Onz6NZ5991uhzePvtt6XPoGbNmujdu7fUtJKeno79+/dj5MiR0n4OHz6M/v37o379+vDz80P37t0BoMyqcnNkZmbiueeeQ0REBAICAuDv74+cnByL9mlJ2Sh53WrXrg0A0nWbNGkS3n77bXTp0gXTp0/H0aNHyzxmSkoK3NzcjMpt06ZNyx31Y841vv8YoaGhqFOnjrSs5HcN0H+/Tp06BT8/P+lzrV69OgoKCnD69Gncvn0bly9fRseOHaX3GL6jFXFzc0P79u1LnZ/hmg4cOBAqlQobNmwAoK/af/TRR8tt9iiLt7c3HnroIel1yfJsTvktiyXfE3P3ExwcDG9vbzRs2NBo2f377dSpk9H3ODo6GidPnoRWq8Vff/0FrVaLxo0bG53T7t27jc7J3d291Dncz9y/GRVJSUkxKieGmCti6e9GRWW25H5VKpX0uuRn16tXL4SFhaFhw4YYNWoUvvzyy1LNgiXPy83NzejcatSogSZNmhj9fSivDFavXh1jx45FbGws+vfvj48++qhUs6k1sOelCd26dUNsbCwSEhIwduxYo3VKpbJUG3vJtkuD+3+UFQqFyWWWdGrLyclB//79MW/evFLrDD8ugD5RkNODXCPD9fDy8rJdgBbEcb+cnBwAwLJly0r98Sr5x2PkyJGYNGkSFi9ejLVr16Jly5bSH6nc3FzExsYiNjYWX375JWrWrImMjAzExsaW2QHcnGs6ZswYXL9+HR999BHCwsLg4eGB6OjoB+pUXp6S183wY2C4buPHj0dsbCx++uknbN++HXPmzMGHH36Il156yWrHL+8aV0ZOTg7atm1rsr9PzZo1HyTUCrm7u2P06NFYtWoVBg8ejLVr1+Kjjz6q1L5MlWdD2TG3/Jq7X8PnbfhBLllGTX3n79+Ptf4uqlQqHD58uNQ5+Pr6Ss+9vLwcfuSrpb8b5pbZ8vbh5+eHI0eOICkpCdu3b8e0adMwY8YMHDx4sNLTCpRXBgFg1apVmDRpErZu3Yp169bhzTffRGJiIjp16lSp45nCmpsyzJ07Fz/++KNRJ0lAX2CuXLli9EFZc16V3377TXpeXFyMw4cPIzIyEgDw8MMP4++//0Z4eDgaNWpk9M+ShMbf3x916tTB3r17jZbv3bsXzZo1e+BzqFmzplEmnp2djfT0dIv20apVK/zyyy9l/oF0d3eHVqstdx+RkZHYv3+/0We1d+9e+Pn5oV69ehbFYxAcHIw6dergzJkzpT6DBg0aSNsNGDAABQUF2Lp1K9auXWtUo5Camorr169j7ty56Nq1K5o2bVrh/4DNKXd79+7FpEmT0LdvXzRv3hweHh5Gnc0B/R+d8q6bNctGaGgoJkyYgPXr1+OVV17BsmXLTG7XtGlTqawbpKWlVTjHT3nX+H6RkZE4f/68Ubks+V0D9N+vkydPolatWqU+24CAAAQEBKB27dr4/fffpffcH3dZiouLjToMG87P8N0G9Anhzz//jE8++QTFxcUYPHhwufs05ztwP3PLr6UMP6Qlr681/y6WvOaA/rOLiIiASqVCmzZtoNVqkZWVVeqcQkJCLDqOtf5mREZGmozZ2ioqs+Zyc3NDTEwM3nvvPRw9ehRnz57Fzp07S20XGRmJ4uJio3O7fv060tLSLP770KZNGyQkJGDfvn1o0aIF1q5da9H7K8LkpgwtW7bEyJEjsWjRIqPlPXr0wNWrV/Hee+/h9OnTWLJkCbZs2WK14y5ZsgQbNmxAamoqJk6ciJs3b+KZZ54BAEycOBE3btzA8OHDcfDgQZw+fRrbtm3DuHHjLP4j969//Qvz5s3DunXrkJaWhtdffx3JycmYPHnyA5/DY489hjVr1uCXX37BX3/9hTFjxlT4v8L7xcXFITs7G//4xz9w6NAhnDx5EmvWrEFaWhoA/UiYo0ePIi0tDdeuXTOZBL344os4f/48XnrpJaSmpuL777/H9OnTER8fb1T1a6mZM2dizpw5WLRoEU6cOIG//voLq1atwvz586VtfHx8MHDgQLz11ltISUnB8OHDpXX169eHu7s7Fi9ejDNnzuCHH36ocB4Jc8pdREQE1qxZg5SUFPz+++8YOXJkqRqw8PBw7NixA1euXClzygNrlI0pU6Zg27ZtSE9Px5EjR7Br1y6jH/KSmjRpgscffxwvvPACfv/9dxw+fBjjx4+vsPauvGt8v5iYGDRu3BhjxozBn3/+iV9++QX//ve/jbYZOXIkgoKCMGDAAPzyyy9IT09HUlISJk2aJDVJTJ48GXPnzsXGjRuRmpqKF1980ayJFtVqNV566SXp/MaOHYtOnTqhQ4cO0jaRkZHo1KkTpk6diuHDh1d4/uHh4dizZw8uXrxYKoktjznl11JeXl7o1KkT5s6di5SUFOzevRtvvvlmpfd3v4yMDMTHxyMtLQ3/+9//sHjxYqk8Nm7cGCNHjsTo0aOxfv16pKen48CBA5gzZw5++ukni45jrb8ZhlqJDz74ACdPnsTHH39sNOLQWswpsxXZtGkTFi1ahOTkZJw7dw7//e9/odPp0KRJk1LbRkREYMCAAXjuuefw66+/4s8//8T//d//oW7duhgwYIBZx0tPT0dCQgL279+Pc+fOYfv27Th58qT09+HAgQNo2rQpLl68aP6FMIHJTTlmzZpVqno0MjISn3zyCZYsWYKoqCgcOHAAr776qtWOOXfuXMydOxdRUVH49ddf8cMPPyAoKAgApP9Ra7Va9O7dGy1btsSUKVMQGBho8Y/1pEmTEB8fj1deeQUtW7bE1q1b8cMPPyAiIuKBzyEhIQHdu3dHv3798MQTT2DgwIFG7a/mqFGjBnbu3ImcnBx0794dbdu2xbJly6Tqzueeew5NmjRBu3btULNmzVI1DQBQt25dbN68GQcOHEBUVBQmTJiAZ5999oH/6I4fPx7Lly/HqlWr0LJlS3Tv3h2rV68u9T/fkSNH4s8//0TXrl1Rv359aXnNmjWxevVqfPPNN2jWrBnmzp2LDz74oNxjmlPuVqxYgZs3b+Lhhx/GqFGjMGnSJNSqVctomw8//BCJiYkIDQ1FmzZtTB7LGmVDq9Vi4sSJiIyMxOOPP47GjRuXO9x41apVqFOnDrp3747Bgwfj+eefLxW7KWVd4/splUps2LAB+fn56NChA8aPH4933nnHaBtvb2/s2bMH9evXx+DBgxEZGYlnn30WBQUF8Pf3BwC88sorGDVqFMaMGYPo6Gj4+flh0KBBFcbp7e2NqVOnYsSIEejSpQt8fX2xbt26Uts9++yzKCoqkv5DU55Zs2bh7NmzeOihhyxqNjO3/Fpq5cqVKC4uRtu2bTFlyhS8/fbbD7S/kkaPHi19dhMnTsTkyZONJmhctWoVRo8ejVdeeQVNmjTBwIEDcfDgwXLLhCnW+pvRqVMnLFu2DB999BGioqKwfft2qyZ7BuaU2YoEBgZi/fr1eOyxxxAZGYmlS5fif//7H5o3b25y+1WrVqFt27bo168foqOjIYTA5s2bze4f6e3tjdTUVDz11FNo3Lgxnn/+eUycOBEvvPACACAvLw9paWll1tqbSyHub8gnIiJZzJ49G9988025HbCJqGKsuSEikllOTg6OHTuGjz/+2KqdrolcFZMbIiKZxcXFoW3btujRo4dZTVJEVD42SxEREZFTYc0NERERORUmN0RERORUmNwQERGRU2FyQ0RERE6FyQ0RERE5FSY3RERE5FSY3BAREZFTYXJDRERETuX/Afkd6Tf5O+bBAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot_ecdf_curves({run.model_metadata.name: run.logs for run in [cmaes_runs, lmm_cmaes_runs]}, DIM, TOLERANCE)" + ] + }, + { + "cell_type": "markdown", + "id": "ab0ff345-f254-4ced-a010-f595f83876bf", + "metadata": {}, + "source": [ + "Lastly, let's plot the box plot of optimization results:" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "eca95650-aabb-4526-a62b-e5822781d107", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAGsCAYAAAAPJKchAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAkpUlEQVR4nO3de1TUdf7H8RdQDBA3iRzCUCovTN7BVHL9WbsUWYfVU22slzRT0zbWim1LKrGbUm2abVlspmGlafetdKmWlsyk1SCrXUHEROgCaq2gQNAy398fHacluThc/Dj4fJwzJ/3O9/IeOgNPv/OdwcuyLEsAAACGeJseAAAAnNyIEQAAYBQxAgAAjCJGAACAUcQIAAAwihgBAABGESMAAMAoYgQAABhFjAAAAKOIEQAAYJRHxcimTZuUlJSkyMhIeXl56fXXX+/S4zU2NmrBggU6++yz5e/vr3PPPVf33Xef+AR9AAA6zymmB3BHTU2Nhg4dquuuu05XXHFFlx/vwQcf1JNPPqnVq1dr4MCB+vjjjzVjxgyFhIRo3rx5XX58AABOBh4VI+PHj9f48eNbvL++vl533nmnXnjhBR08eFCDBg3Sgw8+qAsvvLBdx9uyZYsmTJigyy+/XJIUHR2tF154QVu3bm3X/gAAwNE86mWatqSkpCgvL0/r1q3TZ599pt/85je69NJLtWvXrnbt74ILLlBOTo6Ki4slSZ9++qk2b97cahABAAD3eNSZkdaUlZXpmWeeUVlZmSIjIyVJt956q7Kzs/XMM89o8eLFbu9z/vz5qq6uVkxMjHx8fNTY2KhFixZpypQpnT0+AAAnrW5zZuTzzz9XY2Oj+vfvr8DAQNft/fff1+7duyVJRUVF8vLyavU2f/581z5ffPFFrVmzRmvXrlVBQYFWr16thx9+WKtXrzb1MAEA6Ha6zZmRw4cPy8fHR/n5+fLx8WlyX2BgoCTpnHPOUWFhYav7Of30011//uMf/6j58+frt7/9rSRp8ODB2rt3rzIyMjR9+vROfgQAAJycuk2MDB8+XI2Njdq3b5/Gjh3b7Dq+vr6KiYk55n3W1tbK27vpySMfHx85nc4OzQoAAH7iUTFy+PBhlZSUuP6+Z88ebd++XWFhYerfv7+mTJmiadOmacmSJRo+fLj279+vnJwcDRkyxPWOGHckJSVp0aJF6t27twYOHKhPPvlES5cu1XXXXdeZDwsAgJOal+VBn+CVm5uriy666Kjl06dPV1ZWln744Qfdf//9evbZZ/XVV18pPDxco0eP1j333KPBgwe7fbxDhw5pwYIFeu2117Rv3z5FRkZq0qRJSk9Pl6+vb2c8JAAATnoeFSMAAKD76TbvpgEAAJ6JGAEAAEZ5xAWsTqdTX3/9tYKCguTl5WV6HAAAcAwsy9KhQ4cUGRl51LtT/5dHxMjXX3+tqKgo02MAAIB2KC8v11lnndXi/R4RI0FBQZJ+fDDBwcGGpwEAAMeiurpaUVFRrp/jLfGIGDny0kxwcDAxAgCAh2nrEgsuYAUAAEYRIwAAwChiBAAAGEWMAAAAo9yOkU2bNikpKUmRkZHy8vLS66+/3uY2ubm5io2Nlc1mU9++fZWVldWOUQEAQHfkdozU1NRo6NChWr58+TGtv2fPHl1++eW66KKLtH37dt18882aNWuW3n77bbeHBQAA3Y/bb+0dP368xo8ff8zrZ2Zm6uyzz9aSJUskSQ6HQ5s3b9YjjzyixMREdw8PAAC6mS6/ZiQvL08JCQlNliUmJiovL6/Fberr61VdXd3kBgAAuqcuj5GKigrZ7fYmy+x2u6qrq1VXV9fsNhkZGQoJCXHd+Ch4AAC6rxPy3TRpaWmqqqpy3crLy02PBAAAukiXfxx8RESEKisrmyyrrKxUcHCw/P39m93GZrPJZrN19WgAAOAE0OVnRuLj45WTk9Nk2bvvvqv4+PiuPjQAAPAAbp8ZOXz4sEpKSlx/37Nnj7Zv366wsDD17t1baWlp+uqrr/Tss89KkubOnavHH39ct912m6677jq99957evHFF7Vhw4bOexQAAKNqa2tVVFTU5np1dXUqLS1VdHR0i2fHj4iJiVFAQEBnjYgTmNsx8vHHH+uiiy5y/T01NVWSNH36dGVlZembb75RWVmZ6/6zzz5bGzZs0C233KJHH31UZ511lp5++mne1gsA3UhRUZHi4uI6dZ/5+fmKjY3t1H3ixORlWZZleoi2VFdXKyQkRFVVVQoODjY9DgDgZ471zEhhYaGmTp2q559/Xg6Ho9V1OTPi+Y7153eXX8AKAOj+AgIC3DqL4XA4OOsBF2IExw2vKQMAmkOM4LjhNWUAQHOIERw3MTExys/Pb3M9d19TBgB4NmIExw2vKQMAmnNCfhw8AAA4eRAjAADAKGIEAAAYRYwAAACjiBEAAGAUMQIAAIwiRgAAgFHECAAAMIoYAQAARhEjAADAKGIEAAAYRYwAAACjiBEAAGAUMQIAAIwiRgAAgFHECAAAMIoYAQAARhEjAADAKGIEAAAYRYwAAACjiBEAAGAUMQIAAIwiRgAAgFHECAAAMIoYAQAARhEjAADAKGIEAAAYRYwAAACjiBEAAGAUMQIAAIwiRgAAgFHECAAAMIoYAQAARhEjAADAKGIEAAAYRYwAAACjiBEAAGAUMQIAAIwiRgAAgFHECAAAMIoYAQAARhEjAADAKGIEAAAYRYwAAACjiBEAAGAUMQIAAIwiRgAAgFHECAAAMIoYAQAARhEjAADAKGIEAAAYRYwAAACjiBEAAGAUMQIAAIxqV4wsX75c0dHR8vPz06hRo7R169ZW11+2bJkGDBggf39/RUVF6ZZbbtH333/froEBAED34naMrF+/XqmpqVq4cKEKCgo0dOhQJSYmat++fc2uv3btWs2fP18LFy5UYWGhVq5cqfXr1+uOO+7o8PAAAMDzuR0jS5cu1ezZszVjxgydd955yszMVEBAgFatWtXs+lu2bNGYMWM0efJkRUdH65JLLtGkSZPaPJsCAABODm7FSENDg/Lz85WQkPDTDry9lZCQoLy8vGa3ueCCC5Sfn++Kjy+++EIbN27UZZdd1uJx6uvrVV1d3eQGAAC6p1PcWfnAgQNqbGyU3W5vstxut6uoqKjZbSZPnqwDBw7oF7/4hSzL0n//+1/NnTu31ZdpMjIydM8997gzGgAA8FBd/m6a3NxcLV68WE888YQKCgr06quvasOGDbrvvvta3CYtLU1VVVWuW3l5eVePCQAADHHrzEh4eLh8fHxUWVnZZHllZaUiIiKa3WbBggW65pprNGvWLEnS4MGDVVNTo+uvv1533nmnvL2P7iGbzSabzebOaAAAwEO5dWbE19dXcXFxysnJcS1zOp3KyclRfHx8s9vU1tYeFRw+Pj6SJMuy3J0XAAB0M26dGZGk1NRUTZ8+XSNGjNDIkSO1bNky1dTUaMaMGZKkadOmqVevXsrIyJAkJSUlaenSpRo+fLhGjRqlkpISLViwQElJSa4oAQAAJy+3YyQ5OVn79+9Xenq6KioqNGzYMGVnZ7suai0rK2tyJuSuu+6Sl5eX7rrrLn311Vc644wzlJSUpEWLFnXeowAAAB7Ly/KA10qqq6sVEhKiqqoqBQcHmx4HXaygoEBxcXHKz89XbGys6XEAdCKe3yeXY/35ze+mAQAARhEjAADAKGIEAAAYRYwAAACjiBEAAGAUMQIAAIwiRgAAgFHECAAAMIoYAQAARhEjAADAKGIEAAAYRYwAAACjiBEAAGAUMQIAAIwiRgAAgFHECAAAMIoYAQAARhEjAADAKGIEAAAYRYwAAACjiBEAAGAUMQIAAIwiRgAAgFHECAAAMIoYAQAARhEjAADAKGIEAAAYRYwAAACjiBEAAGAUMQIAAIwiRgAAgFHECAAAMIoYAQAARhEjAADAKGIEAAAYRYwAAACjiBEAAGAUMQIAAIwiRgAAgFHECAAAMIoYAQAARhEjAADAqFNMD4DuY9euXTp06FCH91NYWNjkvx0RFBSkfv36dXg/AICuQ4ygU+zatUv9+/fv1H1OnTq1U/ZTXFxMkADACYwYQac4ckbk+eefl8Ph6NC+6urqVFpaqujoaPn7+7d7P4WFhZo6dWqnnK0BAHQdYgSdyuFwKDY2tsP7GTNmTCdMAwDwBFzACgAAjCJGAACAUbxMAwBoE++WQ1ciRgAAreLdcuhqxAgAoFW8Ww5djRgBABwT3i2HrsIFrAAAwChiBAAAGEWMAAAAo4gRAABgFDECAACMIkYAAIBRxAgAADCqXTGyfPlyRUdHy8/PT6NGjdLWrVtbXf/gwYO68cYbdeaZZ8pms6l///7auHFjuwYGAADdi9sferZ+/XqlpqYqMzNTo0aN0rJly5SYmKidO3eqZ8+eR63f0NCgiy++WD179tTLL7+sXr16ae/evQoNDe2M+QEAgIdzO0aWLl2q2bNna8aMGZKkzMxMbdiwQatWrdL8+fOPWn/VqlX67rvvtGXLFp166qmSpOjo6I5NDQAAug23XqZpaGhQfn6+EhISftqBt7cSEhKUl5fX7DZvvPGG4uPjdeONN8put2vQoEFavHixGhsbWzxOfX29qqurm9wAAED35FaMHDhwQI2NjbLb7U2W2+12VVRUNLvNF198oZdfflmNjY3auHGjFixYoCVLluj+++9v8TgZGRkKCQlx3aKiotwZEwAAeJAufzeN0+lUz5499dRTTykuLk7Jycm68847lZmZ2eI2aWlpqqqqct3Ky8u7ekwAAGCIW9eMhIeHy8fHR5WVlU2WV1ZWKiIiotltzjzzTJ166qny8fFxLXM4HKqoqFBDQ4N8fX2P2sZms8lms7kzGgAA8FBunRnx9fVVXFyccnJyXMucTqdycnIUHx/f7DZjxoxRSUmJnE6na1lxcbHOPPPMZkMEAACcXNx+mSY1NVUrVqzQ6tWrVVhYqBtuuEE1NTWud9dMmzZNaWlprvVvuOEGfffdd7rppptUXFysDRs2aPHixbrxxhs771EAAACP5fZbe5OTk7V//36lp6eroqJCw4YNU3Z2tuui1rKyMnl7/9Q4UVFRevvtt3XLLbdoyJAh6tWrl2666SbdfvvtnfcoAACAx3I7RiQpJSVFKSkpzd6Xm5t71LL4+Hh99NFH7TkUAADo5vjdNAAAwChiBAAAGEWMAAAAo4gRAABgFDECAACMIkYAAIBRxAgAADCKGAEAAEYRIwAAwChiBAAAGEWMAAAAo4gRAABgFDECAACMIkYAAIBRxAgAADCKGAEAAEYRIwAAwChiBAAAGEWMAAAAo4gRAABgFDECAACMIkYAAIBRxAgAADCKGAEAAEYRIwAAwChiBAAAGEWMAAAAo4gRAABgFDECAACMIkYAAIBRxAgAADCKGAEAAEYRIwAAwChiBAAAGEWMAAAAo4gRAABgFDECAACMIkYAAIBRxAgAADCKGAEAAEYRIwAAwKhTTA+A7iMi0Ev+B4ulr0+MxvU/WKyIQC/TYwAA2kCMoNPMifOVY9McaZPpSX7k0I8zAQBObMQIOs1f8huUnJ4lR0yM6VEkSYVFRfrLksn6telBAACtIkbQaSoOW6oL7S9FDjM9iiSprsKpisOW6TEAAG04MV7cBwAAJy1iBAAAGEWMAAAAo4gRAABgFDECAACMIkYAAIBRxAgAADCKGAEAAEYRIwAAwChiBAAAGEWMAAAAo4gRAABgFL8oDwDQpohAL/kfLJa+PjH+Det/sFgRgV6mx0AnaVeMLF++XH/6059UUVGhoUOH6rHHHtPIkSPb3G7dunWaNGmSJkyYoNdff709hwYAGDAnzleOTXOkTaYn+ZFDP86E7sHtGFm/fr1SU1OVmZmpUaNGadmyZUpMTNTOnTvVs2fPFrcrLS3VrbfeqrFjx3ZoYADA8feX/AYlp2fJERNjehRJUmFRkf6yZLJ+bXoQdAq3Y2Tp0qWaPXu2ZsyYIUnKzMzUhg0btGrVKs2fP7/ZbRobGzVlyhTdc889+uCDD3Tw4MEODQ0AOL4qDluqC+0vRQ4zPYokqa7CqYrDlukx0EncevGvoaFB+fn5SkhI+GkH3t5KSEhQXl5ei9vde++96tmzp2bOnHlMx6mvr1d1dXWTGwAA6J7cipEDBw6osbFRdru9yXK73a6Kiopmt9m8ebNWrlypFStWHPNxMjIyFBIS4rpFRUW5MyYAAPAgXXpZ9KFDh3TNNddoxYoVCg8PP+bt0tLSVFVV5bqVl5d34ZQAAMAkt64ZCQ8Pl4+PjyorK5ssr6ysVERExFHr7969W6WlpUpKSnItczqdPx74lFO0c+dOnXvuuUdtZ7PZZLPZ3BkNAAB4KLfOjPj6+iouLk45OTmuZU6nUzk5OYqPjz9q/ZiYGH3++efavn276/brX/9aF110kbZv387LLwAAwP1306Smpmr69OkaMWKERo4cqWXLlqmmpsb17ppp06apV69eysjIkJ+fnwYNGtRk+9DQUEk6ajkAADg5uR0jycnJ2r9/v9LT01VRUaFhw4YpOzvbdVFrWVmZvL1PjE/oAwAAJ752fQJrSkqKUlJSmr0vNze31W2zsrLac0gAANBNcQoDAAAYRYwAAACjiBEAAGAUMQIAAIwiRgAAgFHECAAAMIoYAQAARhEjAADAKGIEAAAYRYwAAACjiBEAAGAUMQIAAIwiRgAAgFHECAAAMIoYAQAARhEjAADAKGIEAAAYRYwAAACjiBEAAGAUMQIAAIwiRgAAgFHECAAAMIoYAQAARhEjAADAKGIEAAAYRYwAAACjiBEAAGAUMQIAAIwiRgAAgFHECAAAMIoYAQAARhEjAADAKGIEAAAYRYwAAACjiBEAAGAUMQIAAIwiRgAAgFHECAAAMIoYAQAARhEjAADAKGIEAAAYRYwAAACjiBEAAGAUMQIAAIwiRgAAgFHECAAAMIoYAQAARhEjAADAqFNMDwAAOLHV1tZKkgoKCjq8r7q6OpWWlio6Olr+/v7t3k9hYWGHZ8GJgxgBALSqqKhIkjR79mzDkxwtKCjI9AjoBMQIAKBVEydOlCTFxMQoICCgQ/sqLCzU1KlT9fzzz8vhcHRoX0FBQerXr1+H9oETAzECAGhVeHi4Zs2a1an7dDgcio2N7dR9wnNxASsAADCKGAEAAEYRIwAAwChiBAAAGEWMAAAAo4gRAABgFDECAACMaleMLF++XNHR0fLz89OoUaO0devWFtddsWKFxo4dqx49eqhHjx5KSEhodX0AAHBycTtG1q9fr9TUVC1cuFAFBQUaOnSoEhMTtW/fvmbXz83N1aRJk/SPf/xDeXl5ioqK0iWXXKKvvvqqw8MDAADP53aMLF26VLNnz9aMGTN03nnnKTMzUwEBAVq1alWz669Zs0a/+93vNGzYMMXExOjpp5+W0+lUTk5Oh4cHAACez60YaWhoUH5+vhISEn7agbe3EhISlJeXd0z7qK2t1Q8//KCwsLAW16mvr1d1dXWTGwAA6J7cipEDBw6osbFRdru9yXK73a6Kiopj2sftt9+uyMjIJkHzcxkZGQoJCXHdoqKi3BkTAAB4kOP6bpoHHnhA69at02uvvSY/P78W10tLS1NVVZXrVl5efhynBAAAx5Nbv7U3PDxcPj4+qqysbLK8srJSERERrW778MMP64EHHtDf//53DRkypNV1bTabbDabO6MBAAAP5daZEV9fX8XFxTW5+PTIxajx8fEtbvfQQw/pvvvuU3Z2tkaMGNH+aQEAQLfj1pkRSUpNTdX06dM1YsQIjRw5UsuWLVNNTY1mzJghSZo2bZp69eqljIwMSdKDDz6o9PR0rV27VtHR0a5rSwIDAxUYGNiJDwUAAHgit2MkOTlZ+/fvV3p6uioqKjRs2DBlZ2e7LmotKyuTt/dPJ1yefPJJNTQ06Kqrrmqyn4ULF+ruu+/u2PQAAMDjuR0jkpSSkqKUlJRm78vNzW3y99LS0vYcAh6mtrZWklRQUNDhfdXV1am0tFTR0dHy9/dv934KCws7PAsAoOu1K0aAnysqKpIkzZ492/AkRwsKCjI9AgCgFcQIOsXEiRMlSTExMQoICOjQvgoLCzV16lQ9//zzcjgcHdpXUFCQ+vXr16F9AAC6FjGCThEeHq5Zs2Z16j4dDodiY2M7dZ8AgBPPcf3QMwAAgJ8jRgAAgFHECAAAMIoYAQAARhEjAADAKGIEAAAYRYwAAACjiBEAAGAUMQIAAIwiRgAAgFHECAAAMIoYAQAARhEjAADAKGIEAAAYRYwAAACjiBEAAGAUMQIAAIwiRgAAgFHECAAAMIoYAQAARhEjAADAKGIEAAAYRYwAAACjiBEAAGAUMQIAAIwiRgAAgFHECAAAMIoYAQAARhEjAADAKGIEAAAYRYwAAACjiBEAAGAUMQIAAIwiRgAAgFHECAAAMIoYAQAARhEjAADAKGIEAAAYRYwAAACjiBEAAGAUMQIAAIwiRgAAgFHECAAAMIoYAQAARhEjAADAKGIEAAAYRYwAAACjiBEAAGDUKaYHAAB4vtraWhUVFbW5XmFhYZP/tiYmJkYBAQEdng0nPmIEANBhRUVFiouLO+b1p06d2uY6+fn5io2N7chY8BDECACgw2JiYpSfn9/menV1dSotLVV0dLT8/f3b3CdODsQIjhtO4wLdV0BAwDGfxRgzZkwXTwNPQ4zguOE0LgCgOcQIjhtO4wIAmuNlWZZleoi2VFdXKyQkRFVVVQoODjY9DgAAOAbH+vObzxkBAABGtStGli9frujoaPn5+WnUqFHaunVrq+u/9NJLiomJkZ+fnwYPHqyNGze2a1gAAND9uB0j69evV2pqqhYuXKiCggINHTpUiYmJ2rdvX7Prb9myRZMmTdLMmTP1ySefaOLEiZo4caL+9a9/dXh4AADg+dy+ZmTUqFE6//zz9fjjj0uSnE6noqKi9Pvf/17z588/av3k5GTV1NTorbfeci0bPXq0hg0bpszMzGM6JteMAADgebrkmpGGhgbl5+crISHhpx14eyshIUF5eXnNbpOXl9dkfUlKTExscX1Jqq+vV3V1dZMbAADontyKkQMHDqixsVF2u73JcrvdroqKima3qaiocGt9ScrIyFBISIjrFhUV5c6YAADAg5yQ76ZJS0tTVVWV61ZeXm56JAAA0EXc+tCz8PBw+fj4qLKyssnyyspKRURENLtNRESEW+tLks1mk81mc2c0AADgodw6M+Lr66u4uDjl5OS4ljmdTuXk5Cg+Pr7ZbeLj45usL0nvvvtui+sDAICTi9sfB5+amqrp06drxIgRGjlypJYtW6aamhrNmDFDkjRt2jT16tVLGRkZkqSbbrpJ48aN05IlS3T55Zdr3bp1+vjjj/XUU0917iMBAAAeye0YSU5O1v79+5Wenq6KigoNGzZM2dnZrotUy8rK5O390wmXCy64QGvXrtVdd92lO+64Q/369dPrr7+uQYMGdd6jAAAAHovfTQMAALoEv5sGAAB4BLdfpjHhyMkbPvwMAADPceTndlsvwnhEjBw6dEiS+PAzAAA80KFDhxQSEtLi/R5xzYjT6dTXX3+toKAgeXl5mR4HXay6ulpRUVEqLy/nGiGgm+H5fXKxLEuHDh1SZGRkkze3/JxHnBnx9vbWWWedZXoMHGfBwcF8swK6KZ7fJ4/WzogcwQWsAADAKGIEAAAYRYzghGOz2bRw4UJ+PxHQDfH8RnM84gJWAADQfXFmBAAAGEWMAAAAo4gRAABgFDECAN3EhRdeqJtvvtn0GIDbiBEAAGAUMQIAAIwiRtAuTqdTDz30kPr27SubzabevXtr0aJFKi0tlZeXl1588UWNHTtW/v7+Ov/881VcXKxt27ZpxIgRCgwM1Pjx47V//37X/rZt26aLL75Y4eHhCgkJ0bhx41RQUNDmHOXl5br66qsVGhqqsLAwTZgwQaWlpa77c3NzNXLkSJ122mkKDQ3VmDFjtHfv3q74kgAnlOjoaN1///2aNm2aAgMD1adPH73xxhvav3+/JkyYoMDAQA0ZMkQff/yxa5usrCyFhobqrbfe0oABAxQQEKCrrrpKtbW1Wr16taKjo9WjRw/NmzdPjY2NrR7/4MGDmjNnjux2u/z8/DRo0CC99dZbHTrOc889pxEjRigoKEgRERGaPHmy9u3b1+bXYvPmza7vR1FRUZo3b55qampc9z/xxBPq16+f/Pz8ZLfbddVVV7n75UZHWUA73HbbbVaPHj2srKwsq6SkxPrggw+sFStWWHv27LEkWTExMVZ2dra1Y8cOa/To0VZcXJx14YUXWps3b7YKCgqsvn37WnPnznXtLycnx3ruueeswsJCa8eOHdbMmTMtu91uVVdXtzhDQ0OD5XA4rOuuu8767LPPrB07dliTJ0+2BgwYYNXX11s//PCDFRISYt16661WSUmJtWPHDisrK8vau3fv8fgSAcfduHHjrJtuusmyLMvq06ePFRYWZmVmZlrFxcXWDTfcYAUHB1uXXnqp9eKLL1o7d+60Jk6caDkcDsvpdFqWZVnPPPOMdeqpp1oXX3yxVVBQYL3//vvW6aefbl1yySXW1Vdfbf373/+23nzzTcvX19dat25di3M0NjZao0ePtgYOHGi988471u7du60333zT2rhxY4eOs3LlSmvjxo3W7t27rby8PCs+Pt4aP358q1+TkpIS67TTTrMeeeQRq7i42Prwww+t4cOHW9dee61lWZa1bds2y8fHx1q7dq1VWlpqFRQUWI8++mhH/jegHYgRuK26utqy2WzWihUrjrrvSIw8/fTTrmUvvPCCJcnKyclxLcvIyLAGDBjQ4jEaGxutoKAg680332xxneeee84aMGCA6xupZVlWfX295e/vb7399tvWt99+a0mycnNz3X2IgEf6eYxMnTrVdd8333xjSbIWLFjgWpaXl2dJsr755hvLsn6MBElWSUmJa505c+ZYAQEB1qFDh1zLEhMTrTlz5rQ4x9tvv215e3tbO3fubPb+zjrOtm3bLElNtvm5mTNnWtdff32TZR988IHl7e1t1dXVWa+88ooVHBzc6j980PV4mQZuKywsVH19vX71q1+1uM6QIUNcf7bb7ZKkwYMHN1n2v6dXKysrNXv2bPXr108hISEKDg7W4cOHVVZWJkmaO3euAgMDXTdJ+vTTT1VSUqKgoCDX8rCwMH3//ffavXu3wsLCdO211yoxMVFJSUl69NFH9c0333Tq1wI4kR3L81BSk+diQECAzj333CbrREdHu553R5Yd2Wbx4sVNnptlZWXavn27zjrrLPXv37/F2dw9jiTl5+crKSlJvXv3VlBQkMaNGydJru8TAwcOdM0xfvx4ST9+n8jKymoyY2JiopxOp/bs2aOLL75Yffr00TnnnKNrrrlGa9asUW1tbZtfW3SuU0wPAM/j7+/f5jqnnnqq689eXl7NLnM6na6/T58+Xd9++60effRR9enTRzabTfHx8WpoaJAk3Xvvvbr11lubHOPw4cOKi4vTmjVrjjr+GWecIUl65plnNG/ePGVnZ2v9+vW666679O6772r06NFuPGLAMx3L81BSk+fi/95/ZJ3mlh3ZZu7cubr66qtd90VGRrr9PeJYjlNTU6PExEQlJiZqzZo1OuOMM1RWVqbExETX94mNGzfqhx9+kPTT96nDhw9rzpw5mjdv3lEz9O7dW76+viooKFBubq7eeecdpaen6+6779a2bdsUGhra5uNA5yBG4LZ+/frJ399fOTk5mjVrVqfs88MPP9QTTzyhyy67TNKPF6YeOHDAdX/Pnj3Vs2fPJtvExsZq/fr16tmzp4KDg1vc9/DhwzV8+HClpaUpPj5ea9euJUaAThIWFqawsLAmy4YMGaIvv/xSxcXFrZ4dcUdRUZG+/fZbPfDAA4qKipKkJhffSlKfPn2O2i42NlY7duxQ3759W9z3KaecooSEBCUkJGjhwoUKDQ3Ve++9pyuuuKJTZkfbeJkGbvPz89Ptt9+u2267Tc8++6x2796tjz76SCtXrmz3Pvv166fnnntOhYWF+uc//6kpU6a0+a+rKVOmKDw8XBMmTNAHH3ygPXv2KDc3V/PmzdOXX36pPXv2KC0tTXl5edq7d6/eeecd7dq1Sw6Ho91zAmjbuHHj9H//93+68sor9e6772rPnj3629/+puzs7Hbv88hZjMcee0xffPGF3njjDd13331tbnf77bdry5YtSklJ0fbt27Vr1y799a9/VUpKiiTprbfe0p///Gdt375de/fu1bPPPiun06kBAwa0e1a4jxhBuyxYsEB/+MMflJ6eLofDoeTk5GN6i11LVq5cqf/85z+KjY3VNddco3nz5h11JuTnAgICtGnTJvXu3VtXXHGFHA6HZs6cqe+//17BwcEKCAhQUVGRrrzySvXv31/XX3+9brzxRs2ZM6fdcwI4Nq+88orOP/98TZo0Seedd55uu+22Nt8O3JozzjhDWVlZeumll3TeeefpgQce0MMPP9zmdkOGDNH777+v4uJijR07VsOHD1d6eroiIyMlSaGhoXr11Vf1y1/+Ug6HQ5mZmXrhhRc0cODAds8K93lZlmWZHgIAAJy8ODMCAACMIkYAAIBRxAgAADCKGAEAAEYRIwAAwChiBAAAGEWMAAAAo4gRAABgFDECAACMIkYAAIBRxAgAADCKGAEAAEb9P3m7SP12hU9oAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "plot_box_plot(data={run.model_metadata.name: run.bests_y() for run in [cmaes_runs, lmm_cmaes_runs]})" + ] + }, + { + "cell_type": "markdown", + "id": "9b71e00a-3f5f-4bb4-8e5a-fa2269656358", + "metadata": {}, + "source": [ + "Plotting functions also alow for saving the plots to an image file. Use keyword argument `savepath=` to specify where to save the image." + ] + }, + { + "cell_type": "markdown", + "id": "feb6e473-3fd6-45c4-b5dc-9dbf7bbf35e1", + "metadata": {}, + "source": [ + "## Save results to a pickle file and analyze it using optilab's CLI tool\n", + "To save the results of an experiment you can dump optimization runs to a pickle file and then read it and plot is using optilab's CLI functionality. Firstly, pack all OptimizationRun objects into a list. Then use a utility function to save it to a pickle file." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "a23ac042-18a0-40da-94da-f7704be2b4b3", + "metadata": {}, + "outputs": [], + "source": [ + "from optilab.utils import dump_to_pickle\n", + "\n", + "runs = [cmaes_runs, lmm_cmaes_runs]\n", + "\n", + "SAVEFILE_NAME = 'tutorial.pkl'\n", + "\n", + "dump_to_pickle(runs, SAVEFILE_NAME)" + ] + }, + { + "cell_type": "markdown", + "id": "8fc57ab4-c8b2-4527-83c6-2afc5e3235f8", + "metadata": {}, + "source": [ + "Now that you saved the results to a pickle file, you can read it into the CLI tool to get various information about the results. The CLI tool also allows to perform statistical testing on the results to determine if the difference in results is statistically significant." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "1266ec91-4670-4883-94cd-751e5f549d11", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "# File tutorial.pkl\n", + "Figure(640x480)\n", + "Figure(640x480)\n", + "Figure(640x480)\n", + "| | model | function | runs | dim | popsize | bounds | tolerance |\n", + "|----|------------|------------|--------|-------|-----------|----------|-------------|\n", + "| 0 | cma-es | sphere | 51 | 2 | 4 | -100 100 | 1e-08 |\n", + "| 1 | lmm-cma-es | sphere | 51 | 2 | 4 | -100 100 | 1e-08 | \n", + "\n", + "| | y_min | y_max | y_mean | y_std | y_median | y_iqr |\n", + "|----|-------------|-------------|-------------|-------------|-------------|-------------|\n", + "| 0 | 2.03327e-10 | 9.74329e-09 | 4.6452e-09 | 2.97277e-09 | 4.59554e-09 | 4.7739e-09 |\n", + "| 1 | 3.16116e-10 | 9.85303e-09 | 4.34819e-09 | 2.68483e-09 | 4.11367e-09 | 3.99832e-09 | \n", + "\n", + "| | evals_min | evals_max | evals_mean | evals_std |\n", + "|----|-------------|-------------|--------------|-------------|\n", + "| 0 | 220 | 400 | 292.706 | 40.388 |\n", + "| 1 | 67 | 99 | 82.549 | 7.07748 | \n", + "\n", + "## Mann Whitney U test on optimization results (y).\n", + "p-values for alternative hypothesis row < column\n", + "| | 0 | 1 |\n", + "|----|--------|--------|\n", + "| 0 | - | 0.6658 |\n", + "| 1 | 0.3366 | - | \n", + "\n", + "## Mann Whitney U test on number of objective function evaluations.\n", + "p-values for alternative hypothesis row < column\n", + "| | 0 | 1 |\n", + "|----|--------|--------|\n", + "| 0 | - | 1.0000 |\n", + "| 1 | 0.0000 | - | \n", + "\n" + ] + } + ], + "source": [ + "!python -m optilab $SAVEFILE_NAME --test_y --test_eval" + ] + }, + { + "cell_type": "markdown", + "id": "66a083f2-a583-440b-8d87-aea1cc433004", + "metadata": {}, + "source": [ + "## Closing remarks\n", + "Thank you for choosing optilab for your project. If you wish to learn more checkout the projects repo on [github](https://github.com/mlojek/optilab) and the project's documentation on [readthedocs](https://optilab.readthedocs.io). Feel free to use optilab in your research and work. If you wish to contribute to the project, feel free to do so yourself or leave an issue in the repo. Best of luck, Marcin." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.1" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}