From 5c65fae6df1fe6b596e1af09c483f24208bce4b3 Mon Sep 17 00:00:00 2001 From: Lindon Roberts Date: Tue, 12 Sep 2017 11:30:51 +0100 Subject: [PATCH] Initial commit --- .gitignore | 174 + LICENSE.txt | 674 ++ README.md | 2 - README.rst | 93 + dfogn/__init__.py | 50 + dfogn/alternative_move.py | 188 + dfogn/dfogn.py | 965 ++ dfogn/dfogn_resampling.py | 1265 +++ dfogn/linear_altmov.py | 154 + dfogn/tests/test_dfogn.py | 105 + dfogn/tests/test_linear_altmov.py | 181 + dfogn/tests/test_trust_region.py | 237 + dfogn/tests/test_util.py | 106 + dfogn/trust_region.py | 362 + dfogn/util.py | 175 + dfogn/version.py | 27 + docs/Makefile | 20 + docs/_build/doctrees/environment.pickle | Bin 0 -> 11644 bytes docs/_build/doctrees/index.doctree | Bin 0 -> 17780 bytes docs/_build/doctrees/info.doctree | Bin 0 -> 30366 bytes docs/_build/doctrees/install.doctree | Bin 0 -> 14924 bytes docs/_build/doctrees/userguide.doctree | Bin 0 -> 60660 bytes docs/_build/html/.buildinfo | 4 + docs/_build/html/_images/data_fitting.png | Bin 0 -> 29893 bytes docs/_build/html/_sources/index.rst.txt | 50 + docs/_build/html/_sources/info.rst.txt | 94 + docs/_build/html/_sources/install.rst.txt | 89 + docs/_build/html/_sources/userguide.rst.txt | 402 + docs/_build/html/_static/ajax-loader.gif | Bin 0 -> 673 bytes docs/_build/html/_static/alabaster.css | 693 ++ docs/_build/html/_static/basic.css | 639 ++ docs/_build/html/_static/comment-bright.png | Bin 0 -> 756 bytes docs/_build/html/_static/comment-close.png | Bin 0 -> 829 bytes docs/_build/html/_static/comment.png | Bin 0 -> 641 bytes docs/_build/html/_static/custom.css | 1 + docs/_build/html/_static/doctools.js | 287 + docs/_build/html/_static/down-pressed.png | Bin 0 -> 222 bytes docs/_build/html/_static/down.png | Bin 0 -> 202 bytes docs/_build/html/_static/file.png | Bin 0 -> 286 bytes docs/_build/html/_static/jquery-3.1.0.js | 10074 +++++++++++++++++ docs/_build/html/_static/jquery.js | 4 + docs/_build/html/_static/minus.png | Bin 0 -> 90 bytes docs/_build/html/_static/plus.png | Bin 0 -> 90 bytes docs/_build/html/_static/pygments.css | 69 + docs/_build/html/_static/searchtools.js | 758 ++ docs/_build/html/_static/underscore-1.3.1.js | 999 ++ docs/_build/html/_static/underscore.js | 31 + docs/_build/html/_static/up-pressed.png | Bin 0 -> 214 bytes docs/_build/html/_static/up.png | Bin 0 -> 203 bytes docs/_build/html/_static/websupport.js | 808 ++ docs/_build/html/genindex.html | 107 + docs/_build/html/index.html | 163 + docs/_build/html/info.html | 179 + docs/_build/html/install.html | 202 + docs/_build/html/objects.inv | 5 + docs/_build/html/search.html | 120 + docs/_build/html/searchindex.js | 1 + docs/_build/html/userguide.html | 497 + docs/_build/latex/DFO-GN.pdf | Bin 0 -> 262930 bytes docs/_build/latex/DFO-GN.tex | 801 ++ docs/_build/latex/Makefile | 68 + docs/_build/latex/data_fitting.png | Bin 0 -> 29893 bytes docs/_build/latex/footnotehyper-sphinx.sty | 268 + docs/_build/latex/latexmkjarc | 7 + docs/_build/latex/latexmkrc | 9 + docs/_build/latex/sphinx.sty | 1518 +++ docs/_build/latex/sphinxhighlight.sty | 105 + docs/_build/latex/sphinxhowto.cls | 94 + docs/_build/latex/sphinxmanual.cls | 114 + docs/_build/latex/sphinxmulticell.sty | 317 + docs/conf.py | 183 + docs/data_fitting.png | Bin 0 -> 29893 bytes docs/index.rst | 50 + docs/info.rst | 94 + docs/install.rst | 89 + docs/make.bat | 36 + docs/userguide.rst | 402 + examples/data_fitting.py | 29 + examples/nonlinear_system.py | 30 + examples/rosenbrock_basic.py | 32 + examples/rosenbrock_bounds.py | 28 + examples/rosenbrock_noisy.py | 46 + manual.pdf | Bin 0 -> 262930 bytes setup.py | 68 + 84 files changed, 25440 insertions(+), 2 deletions(-) create mode 100644 .gitignore create mode 100644 LICENSE.txt delete mode 100644 README.md create mode 100644 README.rst create mode 100644 dfogn/__init__.py create mode 100644 dfogn/alternative_move.py create mode 100644 dfogn/dfogn.py create mode 100644 dfogn/dfogn_resampling.py create mode 100644 dfogn/linear_altmov.py create mode 100644 dfogn/tests/test_dfogn.py create mode 100644 dfogn/tests/test_linear_altmov.py create mode 100644 dfogn/tests/test_trust_region.py create mode 100644 dfogn/tests/test_util.py create mode 100644 dfogn/trust_region.py create mode 100644 dfogn/util.py create mode 100644 dfogn/version.py create mode 100644 docs/Makefile create mode 100644 docs/_build/doctrees/environment.pickle create mode 100644 docs/_build/doctrees/index.doctree create mode 100644 docs/_build/doctrees/info.doctree create mode 100644 docs/_build/doctrees/install.doctree create mode 100644 docs/_build/doctrees/userguide.doctree create mode 100644 docs/_build/html/.buildinfo create mode 100644 docs/_build/html/_images/data_fitting.png create mode 100644 docs/_build/html/_sources/index.rst.txt create mode 100644 docs/_build/html/_sources/info.rst.txt create mode 100644 docs/_build/html/_sources/install.rst.txt create mode 100644 docs/_build/html/_sources/userguide.rst.txt create mode 100644 docs/_build/html/_static/ajax-loader.gif create mode 100644 docs/_build/html/_static/alabaster.css create mode 100644 docs/_build/html/_static/basic.css create mode 100644 docs/_build/html/_static/comment-bright.png create mode 100644 docs/_build/html/_static/comment-close.png create mode 100644 docs/_build/html/_static/comment.png create mode 100644 docs/_build/html/_static/custom.css create mode 100644 docs/_build/html/_static/doctools.js create mode 100644 docs/_build/html/_static/down-pressed.png create mode 100644 docs/_build/html/_static/down.png create mode 100644 docs/_build/html/_static/file.png create mode 100644 docs/_build/html/_static/jquery-3.1.0.js create mode 100644 docs/_build/html/_static/jquery.js create mode 100644 docs/_build/html/_static/minus.png create mode 100644 docs/_build/html/_static/plus.png create mode 100644 docs/_build/html/_static/pygments.css create mode 100644 docs/_build/html/_static/searchtools.js create mode 100644 docs/_build/html/_static/underscore-1.3.1.js create mode 100644 docs/_build/html/_static/underscore.js create mode 100644 docs/_build/html/_static/up-pressed.png create mode 100644 docs/_build/html/_static/up.png create mode 100644 docs/_build/html/_static/websupport.js create mode 100644 docs/_build/html/genindex.html create mode 100644 docs/_build/html/index.html create mode 100644 docs/_build/html/info.html create mode 100644 docs/_build/html/install.html create mode 100644 docs/_build/html/objects.inv create mode 100644 docs/_build/html/search.html create mode 100644 docs/_build/html/searchindex.js create mode 100644 docs/_build/html/userguide.html create mode 100644 docs/_build/latex/DFO-GN.pdf create mode 100644 docs/_build/latex/DFO-GN.tex create mode 100644 docs/_build/latex/Makefile create mode 100644 docs/_build/latex/data_fitting.png create mode 100644 docs/_build/latex/footnotehyper-sphinx.sty create mode 100644 docs/_build/latex/latexmkjarc create mode 100644 docs/_build/latex/latexmkrc create mode 100644 docs/_build/latex/sphinx.sty create mode 100644 docs/_build/latex/sphinxhighlight.sty create mode 100644 docs/_build/latex/sphinxhowto.cls create mode 100644 docs/_build/latex/sphinxmanual.cls create mode 100644 docs/_build/latex/sphinxmulticell.sty create mode 100644 docs/conf.py create mode 100644 docs/data_fitting.png create mode 100644 docs/index.rst create mode 100644 docs/info.rst create mode 100644 docs/install.rst create mode 100644 docs/make.bat create mode 100644 docs/userguide.rst create mode 100644 examples/data_fitting.py create mode 100644 examples/nonlinear_system.py create mode 100644 examples/rosenbrock_basic.py create mode 100644 examples/rosenbrock_bounds.py create mode 100644 examples/rosenbrock_noisy.py create mode 100644 manual.pdf create mode 100644 setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..3c6343e --- /dev/null +++ b/.gitignore @@ -0,0 +1,174 @@ +## Python ignores +*/__pycache__ +*.egg-info +*/*.eggs +*.pyc + +## LaTeX standard ignores +*.pdf # exclude all pdfs unless specifically included +*~ +*.backup +*.o +*.swp +*.swo +*.pyc + +## Core latex/pdflatex auxiliary files: +*.aux +*.lof +*.log +*.lot +*.fls +*.out +*.toc +*.fmt + +## Intermediate documents: +*.dvi +*-converted-to.* +# these rules might exclude image files for figures etc. +# *.ps +# *.eps +# *.pdf + +## Bibliography auxiliary files (bibtex/biblatex/biber): +*.bbl +*.bcf +*.blg +*-blx.aux +*-blx.bib +*.brf +*.run.xml + +## Build tool auxiliary files: +*.fdb_latexmk +*.synctex +*.synctex.gz +*.synctex.gz(busy) +*.pdfsync + +## Auxiliary and intermediate files from other packages: +# algorithms +*.alg +*.loa + +# achemso +acs-*.bib + +# amsthm +*.thm + +# beamer +*.nav +*.snm +*.vrb + +# cprotect +*.cpt + +#(e)ledmac/(e)ledpar +*.end +*.[1-9] +*.[1-9][0-9] +*.[1-9][0-9][0-9] +*.[1-9]R +*.[1-9][0-9]R +*.[1-9][0-9][0-9]R +*.eledsec[1-9] +*.eledsec[1-9]R +*.eledsec[1-9][0-9] +*.eledsec[1-9][0-9]R +*.eledsec[1-9][0-9][0-9] +*.eledsec[1-9][0-9][0-9]R + +# glossaries +*.acn +*.acr +*.glg +*.glo +*.gls + +# gnuplottex +*-gnuplottex-* + +# hyperref +*.brf + +# knitr +*-concordance.tex +*.tikz +*-tikzDictionary + +# listings +*.lol + +# makeidx +*.idx +*.ilg +*.ind +*.ist + +# minitoc +*.maf +*.mtc +*.mtc[0-9] +*.mtc[1-9][0-9] + +# minted +_minted* +*.pyg + +# morewrites +*.mw + +# mylatexformat +*.fmt + +# nomencl +*.nlo + +# sagetex +*.sagetex.sage +*.sagetex.py +*.sagetex.scmd + +# sympy +*.sout +*.sympy +sympy-plots-for-*.tex/ + +# pdfcomment +*.upa +*.upb + +#pythontex +*.pytxcode +pythontex-files-*/ + +# Texpad +.texpadtmp + +# TikZ & PGF +*.dpth +*.md5 +*.auxlock + +# todonotes +*.tdo + +# xindy +*.xdy + +# xypic precompiled matrices +*.xyc + +# WinEdt +*.bak +*.sav + +# endfloat +*.ttt +*.fff + +# Latexian +TSWLatexianTemp* diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..94a9ed0 --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,674 @@ + GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU General Public License is a free, copyleft license for +software and other kinds of works. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +the GNU General Public License is intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. We, the Free Software Foundation, use the +GNU General Public License for most of our software; it applies also to +any other work released this way by its authors. You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + To protect your rights, we need to prevent others from denying you +these rights or asking you to surrender the rights. Therefore, you have +certain responsibilities if you distribute copies of the software, or if +you modify it: responsibilities to respect the freedom of others. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must pass on to the recipients the same +freedoms that you received. You must make sure that they, too, receive +or can get the source code. And you must show them these terms so they +know their rights. + + Developers that use the GNU GPL protect your rights with two steps: +(1) assert copyright on the software, and (2) offer you this License +giving you legal permission to copy, distribute and/or modify it. + + For the developers' and authors' protection, the GPL clearly explains +that there is no warranty for this free software. For both users' and +authors' sake, the GPL requires that modified versions be marked as +changed, so that their problems will not be attributed erroneously to +authors of previous versions. + + Some devices are designed to deny users access to install or run +modified versions of the software inside them, although the manufacturer +can do so. This is fundamentally incompatible with the aim of +protecting users' freedom to change the software. The systematic +pattern of such abuse occurs in the area of products for individuals to +use, which is precisely where it is most unacceptable. Therefore, we +have designed this version of the GPL to prohibit the practice for those +products. If such problems arise substantially in other domains, we +stand ready to extend this provision to those domains in future versions +of the GPL, as needed to protect the freedom of users. + + Finally, every program is threatened constantly by software patents. +States should not allow patents to restrict development and use of +software on general-purpose computers, but in those that do, we wish to +avoid the special danger that patents applied to a free program could +make it effectively proprietary. To prevent this, the GPL assures that +patents cannot be used to render the program non-free. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Use with the GNU Affero General Public License. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU Affero General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the special requirements of the GNU Affero General Public License, +section 13, concerning interaction through a network will apply to the +combination as such. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If the program does terminal interaction, make it output a short +notice like this when it starts in an interactive mode: + + Copyright (C) + This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, your program's commands +might be different; for a GUI interface, you would use an "about box". + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. diff --git a/README.md b/README.md deleted file mode 100644 index 6a6f11c..0000000 --- a/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# DFO-GN -A derivative-free solver for nonlinear least-squares minimization and solving systems of nonlinear equations. DFO-GN stands for Derivative-Free Optimization using Gauss-Newton. diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..f0178d1 --- /dev/null +++ b/README.rst @@ -0,0 +1,93 @@ +==================================================================================== +DFO-GN: Derivative-Free Nonlinear Least-Squares Solver |Build Status| |PyPI Version| +==================================================================================== +DFO-GN is a package for solving nonlinear least-squares minimisation, without requiring derivatives of the objective. + +This is an implementation of the algorithm from our paper: +A Derivative-Free Gauss-Newton Method, C. Cartis and L. Roberts, submitted (2017). + +Documentation +------------- +Documentation for DFO-GN is available at [URL]. + +Requirements +------------ +DFO-GN requires the following software to be installed: + +* `Python 2.7 or Python 3 `_ + +Additionally, the following python packages should be installed (these will be installed automatically if using `pip `_, see `Installation using pip`_): + +* `NumPy 1.11 or higher `_ +* `SciPy 0.18 or higher `_ + + +Installation using pip +---------------------- +For easy installation, use `pip `_ as root: + + .. code-block:: bash + + $ [sudo] pip install --pre dfogn + +If you do not have root privileges or you want to install DFO-GN for your private use, you can use: + + .. code-block:: bash + + $ pip install --pre --user dfogn + +which will install DFO-GN in your home directory. + +Note that if an older install of DFO-GN is present on your system you can use: + + .. code-block:: bash + + $ [sudo] pip install --pre --upgrade dfogn + +to upgrade DFO-GN to the latest version. + +Manual installation +------------------- +The source code for DFO-GN is `available on Github `_: + + .. code-block:: bash + + $ git clone https://github.com/numericalalgorithmsgroup/dfogn + $ cd dfogn + +or through the `Python Package Index `_: + + .. code-block:: bash + + $ wget http://pypi.python.org/packages/source/d/dfogn/dfogn-X.X.tar.gz + $ tar -xzvf dfogn-X.X.tar.gz + $ cd dfogn-X.X + +DFO-GN is written in pure Python and requires no compilation. It can be installed using: + + .. code-block:: bash + + $ [sudo] pip install --pre . + +If you do not have root privileges or you want to install DFO-GN for your private use, you can use: + + .. code-block:: bash + + $ pip install --pre --user . + +instead. + +Testing +------- +If you installed DFO-GN manually, you can test your installation by running: + + .. code-block:: bash + + $ python setup.py test + +Alternatively, the HTML documentation provides some simple examples of how to run DFO-GN. + +.. |Build Status| image:: https://travis-ci.org/numericalalgorithmsgroup/DFOGN.svg?branch=master + :target: https://travis-ci.org/numericalalgorithmsgroup/DFOGN +.. |PyPI Version| image:: https://img.shields.io/pypi/v/DFOGN.svg + :target: https://pypi.python.org/pypi/DFOGN \ No newline at end of file diff --git a/dfogn/__init__.py b/dfogn/__init__.py new file mode 100644 index 0000000..fdea6b3 --- /dev/null +++ b/dfogn/__init__.py @@ -0,0 +1,50 @@ +""" +DFO-GN +==== + +Derivative-free nonlinear least-squares solver (with optional bound constraints). + +It solves the nonlinear least-squares problem: + min_{x} f(x) = r1(x)**2 + ... + rm(x)**2, +subject to the (optional) bounds + lb <= x <= ub, +where each function ri(x) is differentiable, possibly nonconvex. +Since the derivatives of ri(x) are never required or approximated, +the solver works when the evaluation of ri(x) is noisy. + +---- + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +The development of this software was sponsored by NAG Ltd. (http://www.nag.co.uk) +and the EPSRC Centre For Doctoral Training in Industrially Focused Mathematical +Modelling (EP/L015803/1) at the University of Oxford. Please contact NAG for +alternative licensing. + +Copyright 2017, Lindon Roberts + +""" + +# Ensure compatibility with Python 2 +from __future__ import absolute_import, division, print_function, unicode_literals + +from .version import __version__ +__all__ = ['__version__'] + +# Main solver & exit flags +from .dfogn import * +__all__ += ['solve', 'EXIT_SUCCESS', 'EXIT_INPUT_ERROR', 'EXIT_MAXFUN_WARNING', 'EXIT_TR_INCREASE_ERROR', + 'EXIT_LINALG_ERROR', 'EXIT_ALTMOV_MEMORY_ERROR'] + + diff --git a/dfogn/alternative_move.py b/dfogn/alternative_move.py new file mode 100644 index 0000000..3adb246 --- /dev/null +++ b/dfogn/alternative_move.py @@ -0,0 +1,188 @@ +""" +alternative_move +================== +A geometry-improving step calculation. +Based on the routine ALTMOV from BOBYQA (Powell, 2009). + + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +The development of this software was sponsored by NAG Ltd. (http://www.nag.co.uk) +and the EPSRC Centre For Doctoral Training in Industrially Focused Mathematical +Modelling (EP/L015803/1) at the University of Oxford. Please contact NAG for +alternative licensing. + +Copyright 2017, Lindon Roberts + +""" +# Ensure compatibility with Python 2 +from __future__ import absolute_import, division, print_function, unicode_literals + +from math import sqrt +import numpy as np + +__all__ = ['altmov'] + + +# Exact interpolation for n+1 points (i.e. linear models only) +# Also works for inexact interpolation and n+1 points (when still linear models) - see above function definition +def altmov(xpt, sl, su, kopt, xopt, knew, adelt, H_knew): + npt, n = xpt.shape + assert sl.size == n, "sl and xpt have incompatible sizes" + assert su.size == n, "su and xpt have incompatible sizes" + assert xopt.size == n, "xopt and xpt have incompatible sizes" + assert 0 <= kopt <= npt-1, "kopt must be in range 0, ..., npt-1" + assert adelt > 0.0, "adelt must be strictly positive" + assert H_knew.size == n, "H_knew and xpt have incompatible sizes" + # H_knew = knew-th column of H (used to construct knew-th Lagrange polynomial) + # Outputs: + #xnew = np.zeros((n,)) + xalt = np.zeros((n,)) + cauchy = 0.0 + + # Calculate the gradient of the KNEW-th Lagrange function at XOPT. + glag = H_knew # 0th entry is constant term, rest is gradient + yt = xpt[knew,:] # interpolation point to replace + + # Searching along lines {xk + alpha*(yj-xk)} to get xnew = xopt + stpsav * (xpt[jsav, :] - xopt) + jsav = None + stpsav = None + best_abs_phi = -1.0 # way to compare different values of j + + for j in range(npt): + if j==kopt: + continue + yj = xpt[j, :] + + # Get bounds on alpha_j from sl, su and adelt + alpha_lower = -adelt / np.linalg.norm(yj - xopt) + alpha_upper = adelt / np.linalg.norm(yj - xopt) + for i in range(n): + if yj[i]-xopt[i] > 0.0: + alpha_lower = max(alpha_lower, (sl[i] - xopt[i]) / (yj[i] - xopt[i])) + alpha_upper = min(alpha_upper, (su[i] - xopt[i]) / (yj[i] - xopt[i])) + elif yj[i]-xopt[i] < 0.0: + alpha_lower = max(alpha_lower, (su[i] - xopt[i]) / (yj[i] - xopt[i])) + alpha_upper = min(alpha_upper, (sl[i] - xopt[i]) / (yj[i] - xopt[i])) + + # Since the Lagrange poly is linear, optimal alpha is either lower or upper value + phi_lower = 1.0 + np.dot(xopt - yt, glag) + alpha_lower * np.dot(yj - xopt, glag) + phi_upper = 1.0 + np.dot(xopt - yt, glag) + alpha_upper * np.dot(yj - xopt, glag) + if abs(phi_lower) > abs(phi_upper): + alpha_j = alpha_lower + abs_phi_j = abs(phi_lower) + else: + alpha_j = alpha_lower + abs_phi_j = abs(phi_lower) + + # Compare to best so far + if stpsav is None or abs_phi_j > best_abs_phi: + jsav = j + stpsav = alpha_j + best_abs_phi = abs_phi_j + + # Finish up this search + xnew = xopt + stpsav * (xpt[jsav, :] - xopt) + xnew = np.maximum(sl, np.minimum(xnew, su)) + + # Alternative (xalt) - try a standard Cauchy step for + # Output 'cauchy' is set to |Lambda_t(xalt)| for later comparison against |Lambda_t(xnew)| + # In BOBYQA, there were two steps: generate sk from linear model, then scale to get ck (using curvature) + # Here, only first step needed, and the code is mostly reused + + # Prepare for the iterative method that assembles the constrained Cauchy + # step in W. The sum of squares of the fixed components of W is formed in + # WFIXSQ, and the free components of W are set to BIGSTP. + bigstp = 2.0 * adelt + xalt_backup = np.zeros((n,)) # w(n+i) for i in range(n) + iflag = False + csave = -1.0 + + while True: # loop label 100 + wfixsq = 0.0 + ggfree = 0.0 + w_vec = np.zeros((n,)) # first n components of w + for i in range(n): + w_vec[i] = 0.0 + tempa = min(xopt[i] - sl[i], glag[i]) + tempb = max(xopt[i] - su[i], glag[i]) + if tempa > 0.0 or tempb < 0.0: + w_vec[i] = bigstp + ggfree += glag[i] ** 2 + if ggfree == 0.0: + cauchy = 0.0 + return xnew, xalt, cauchy, best_abs_phi + + # Investigate whether more components of W can be fixed. + while True: # loop label 120 + temp = adelt ** 2 - wfixsq + if temp > 0.0: + wsqsav = wfixsq + step = sqrt(temp / ggfree) + ggfree = 0.0 + for i in range(n): + if w_vec[i] == bigstp: + temp = xopt[i] - step * glag[i] + if temp <= sl[i]: + w_vec[i] = sl[i] - xopt[i] + wfixsq += w_vec[i] ** 2 + elif temp >= su[i]: + w_vec[i] = su[i] - xopt[i] + wfixsq += w_vec[i] ** 2 + else: + ggfree += glag[i] ** 2 + if wfixsq > wsqsav and ggfree > 0.0: + continue # next iteration loop label 120 + else: + break # quit loop label 120 + else: + break # quit loop label 120 + + # Set the remaining free components of W and all components of XALT, + # except that W may be scaled later. + for i in range(n): + if w_vec[i] == bigstp: + w_vec[i] = -step * glag[i] + xalt[i] = max(sl[i], min(xopt[i] + w_vec[i], su[i])) + elif w_vec[i] == 0.0: + xalt[i] = xopt[i] + elif glag[i] > 0.0: + xalt[i] = sl[i] + else: + xalt[i] = su[i] + #gw = np.dot(glag, w_vec) + + # Skip the curvature scaling step, 'cauchy' is just abs(linear objective) + #cauchy = abs(gw) + cauchy = abs(1.0 + np.dot(xalt-yt, glag)) + + # If IFLAG is zero, then XALT is calculated as before after reversing + # the sign of GLAG. Thus two XALT vectors become available. The one that + # is chosen is the one that gives the larger value of CAUCHY. + # xalt_backup = np.zeros((n,)) # w(n+i) for i in range(n) + if not iflag: + glag = -glag + xalt_backup = xalt.copy() + csave = cauchy + iflag = True + continue # another iteration of loop label 100 + else: + break # quit loop label 100 + # end loop label 100 + + # Choose either xalt or xalt_backup depending on cauchy v csave + if csave > cauchy: + xalt = xalt_backup.copy() + cauchy = csave + return xnew, xalt, cauchy, best_abs_phi diff --git a/dfogn/dfogn.py b/dfogn/dfogn.py new file mode 100644 index 0000000..f1db7a8 --- /dev/null +++ b/dfogn/dfogn.py @@ -0,0 +1,965 @@ +""" +DFO-GN +==================== +A derivative-free solver for least squares minimisation with bound constraints + +Call structure is: + x, f, nf, exit_flag, exit_str = dfogn(objfun, x0, lower, upper, + maxfun, init_tr_radius, rhoend=1e-8) + +Required inputs: + objfun Objective function, callable as: residual_vector = objfun(x) + x0 Initial starting point, NumPy ndarray +Optional inputs: + lower, upper Lower and upper bound constraints (lower <= x <= upper), + must be NumPy ndarrays of same size as x0 (default +/-1e20) + maxfun Maximum number of allowable function evalutions (default 1000) + init_tr_radius Initial trust region radius (default 0.1*max(1, ||x0||_infty) + rhoend Termination condition on trust region radius (default 1e-8) + +Outputs: + x Estimate of minimiser + f Value of least squares objective at x (f = ||objfun(x)||^2) + nf Number of objective evaluations used to find x + exit_flag Integer flag indicating termination criterion (see list below imports) + exit_str String with more detailed termination message + + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +The development of this software was sponsored by NAG Ltd. (http://www.nag.co.uk) +and the EPSRC Centre For Doctoral Training in Industrially Focused Mathematical +Modelling (EP/L015803/1) at the University of Oxford. Please contact NAG for +alternative licensing. + +Copyright 2017, Lindon Roberts + +""" +# Ensure compatibility with Python 2 +from __future__ import absolute_import, division, print_function, unicode_literals + +import logging +from math import sqrt +import numpy as np +import scipy.linalg as sp_linalg +import warnings + +from .util import * +from .trust_region import * +from .alternative_move import * +from .linear_altmov import * + +__all__ = ['solve', 'EXIT_SUCCESS', 'EXIT_INPUT_ERROR', 'EXIT_MAXFUN_WARNING', 'EXIT_TR_INCREASE_ERROR', + 'EXIT_LINALG_ERROR', 'EXIT_ALTMOV_MEMORY_ERROR'] + +####################### +# Exit codes +EXIT_SUCCESS = 0 # successful finish (rho=rhoend or sufficient objective reduction) +EXIT_INPUT_ERROR = 1 # error, bad inputs +EXIT_MAXFUN_WARNING = 2 # warning, reached max function evals +EXIT_TR_INCREASE_ERROR = 3 # error, trust region step increased model value +EXIT_LINALG_ERROR = 4 # error, linalg error (singular matrix encountered) +EXIT_ALTMOV_MEMORY_ERROR = 5 # error, stpsav issue in ALTMOV +####################### + + +class OptimResults: + def __init__(self, xmin, rmin, fmin, jacmin, nf, exit_flag, exit_msg): + self.x = xmin + self.resid = rmin + self.f = fmin + self.jacobian = jacmin + self.nf = nf + self.flag = exit_flag + self.msg = exit_msg + # Set standard names for exit flags + self.EXIT_MAXFUN_WARNING = EXIT_MAXFUN_WARNING + self.EXIT_SUCCESS = EXIT_SUCCESS + self.EXIT_INPUT_ERROR = EXIT_INPUT_ERROR + self.EXIT_TR_INCREASE_ERROR = EXIT_TR_INCREASE_ERROR + self.EXIT_LINALG_ERROR = EXIT_LINALG_ERROR + self.EXIT_ALTMOV_MEMORY_ERROR = EXIT_ALTMOV_MEMORY_ERROR + + +class Model: + def __init__(self, n, m, npt, x0, xl, xu): + assert npt==n+1, "Require strictly linear model" + # Problem sizes + self.n = n + self.m = m + self.npt = npt + + # Actual model info + # Here, the model for each residual is centred around xbase + # m(x) = model_const_term + gqv*(x-xbase) + + self.kbase = 0 # index of base point + self.xbase = x0 # base point + self.xl = xl # lower bounds (absolute terms) + self.xu = xu # upper bounds (absolute terms) + self.sl = xl - x0 # lower bounds (adjusted for xbase), should be -ve (actually < -rhobeg) + self.su = xu - x0 # upper bounds (adjusted for xbase), should be +ve (actually > rhobeg) + self.xpt = np.zeros((npt, n)) # interpolation points + self.fval_v = np.zeros((npt, m)) # residual vectors at each xpt(+xbase) + self.fval = np.zeros((npt, )) # total sum of squares at each xpt(+xbase) + self.model_const_term_v = np.zeros((m,)) # constant term of each mini-model + self.gqv = np.zeros((n, m)) # interpolated gradients for each mini-model + + self.kopt = None # index of current best x + + self.fbeg = None # initial sum of squares at x0 + + self.xsave = None # possible final return value (abs coords) + self.rsave = None # residuals for possible final return value + self.fsave = None # sum of squares for final return value + self.jacsave = None # approximate Jacobian at possible final return value + + self.lu = None # LU decomp of interp matrix + self.piv = None # pivots for LU decomposition of interp matrix + self.lu_current = False # whether current LU factorisation of interp matrix is up-to-date or not + + self.EXACT_CONST_TERM = True # use exact c=r(xopt) for interpolation (improve conditioning) + # Affects mini-model interpolation / interpolation matrix, but also geometry updating + + def x_within_bounds(self, k=None, x=None): + # Get x value for k-th point or x vector (in absolute terms, force within bounds) + if k is not None: + return np.minimum(np.maximum(self.xl, self.xbase + self.xpt[k, :]), self.xu) + elif x is not None: + return np.minimum(np.maximum(self.xl, self.xbase + x), self.xu) + else: + return None + + def xopt(self): + # Current best x (relative to xbase) + return self.xpt[self.kopt, :].copy() + + def fval_v_opt(self): + return self.fval_v[self.kopt,:] + + def fval_opt(self): + return self.fval[self.kopt] + + def update_point(self, knew, xnew, v_err, f): + # Add point xnew with objective vector v_err (full objective f) at the knew-th index + self.xpt[knew,:] = xnew + self.fval_v[knew, :] = v_err + self.fval[knew] = f + + # Update XOPT, GOPT and KOPT if the new calculated F is less than FOPT. + if f < self.fval_opt(): + self.kopt = knew + + self.lu_current = False + return + + def gqv_at_xopt(self): + return self.gqv + + def shift_base(self, xbase_shift): + for m1 in range(self.m): + self.model_const_term_v[m1] += np.dot(self.gqv[:, m1], xbase_shift) + + # The main updates + for k in range(self.npt): + self.xpt[k, :] = self.xpt[k, :] - xbase_shift + self.xbase += xbase_shift + self.sl = self.sl - xbase_shift + self.su = self.su - xbase_shift + + self.lu_current = False + self.factorise_LU() + + return + + def interpolate_mini_models(self): + # Build interpolation matrix and factorise (in self.lu, self.piv) + try: + self.factorise_LU() + if self.EXACT_CONST_TERM: + idx_to_use = [k for k in range(self.npt) if k != self.kopt] + for m1 in range(self.m): + rhs = np.zeros((self.n,)) + for i in range(self.n): + k = idx_to_use[i] + rhs[i] = self.fval_v[k, m1] - self.fval_v[self.kopt, m1] - \ + np.dot(self.gqv[:, m1], self.xpt[k, :] - self.xopt()) + soln = sp_linalg.lu_solve((self.lu, self.piv), rhs) + self.gqv[:, m1] += soln # whole solution is gradient + + # shift constant term back + self.model_const_term_v = self.fval_v[self.kopt, :] - np.dot(self.gqv.T, self.xopt()) + return True # flag ok + else: + model_values_v = np.zeros((self.npt, self.m)) + for k in range(self.npt): + model_values_v[k, :] = self.predicted_values(self.xpt[k, :], d_based_at_xopt=False, + with_const_term=True) + + # Sometimes when things get too close to a solution, we can get NaNs in model_values - flag error & quit + if np.any(np.isnan(model_values_v)): + self.gqv = None + return False # flag error + + for m1 in range(self.m): + rhs = self.fval_v[:, m1] - model_values_v[:, m1] + soln = sp_linalg.lu_solve((self.lu, self.piv), rhs) + self.model_const_term_v[m1] += soln[0] + self.gqv[:, m1] += soln[1:] # first term is constant, rest is gradient term + + return True # flag ok + except np.linalg.LinAlgError: + self.gqv = None + return False # flag error + except ValueError: # happens when LU decomposition has Inf or NaN + self.gqv = None + return False # flag error + + def factorise_LU(self): + if not self.lu_current: + Wmat = self.build_interp_matrix() + self.lu, self.piv = sp_linalg.lu_factor(Wmat) # LU has L and U parts, piv indicates row swaps for pivoting + self.lu_current = True + return + + def solve_LU(self, rhs): + # If lu_current, use that, otherwise revert to generic solver + if self.lu_current: + if self.EXACT_CONST_TERM: + return sp_linalg.lu_solve((self.lu, self.piv), rhs) # only get gradient (no const term) + else: + return sp_linalg.lu_solve((self.lu, self.piv), rhs)[1:] # only return gradient (1st term is constant) + else: + logging.warning("model.solve_LU not using factorisation") + Wmat = self.build_interp_matrix() + if self.EXACT_CONST_TERM: + return np.linalg.solve(Wmat, rhs) # only get gradient (no const term) + else: + return np.linalg.solve(Wmat, rhs)[1:] # only return gradient (1st term is constant) + + def get_final_results(self): + # Called when about to exit BOBYQB + # Return x and fval for optimal point (either from xsave+fsave or kopt) + if self.fval_opt() <= self.fsave: # optimal has changed since xsave+fsave were last set + x = self.x_within_bounds(k=self.kopt) + rvec = self.fval_v_opt() + f = self.fval_opt() + jacmin = self.gqv_at_xopt().T + else: + x = self.xsave + rvec = self.rsave + f = self.fsave + jacmin = self.jacsave + + return x, rvec, f, jacmin + + def build_full_model(self): + # Build full least squares objective model from mini-models + # Centred around xopt = xpt[kopt, :] + v_temp = self.fval_v_opt() # m-vector + gqv_xopt = self.gqv_at_xopt() # J^T (transpose of Jacobian) at xopt, rather than xbase + + # Use the gradient at xopt to formulate \sum_i (2*f_i \nabla f_i) = 2 J^t m(x_opt) + gopt = np.dot(gqv_xopt, v_temp) # n-vector (gqv = J^T) + + # Gauss-Newton part of Hessian + hq = to_upper_triangular_vector(np.dot(gqv_xopt, gqv_xopt.T)) + + # Apply scaling based on convention for objective - this code uses sumsq(r_i) not 0.5*sumsq(r_i) + gopt = 2.0 * gopt + hq = 2.0 * hq + + return gopt, hq + + def build_interp_matrix(self): + if self.EXACT_CONST_TERM: + Wmat = np.zeros((self.n, self.n)) + idx_to_use = [k for k in range(self.npt) if k != self.kopt] + for i in range(self.n): + Wmat[i,:] = self.xpt[idx_to_use[i], :] - self.xopt() + else: + Wmat = np.zeros((self.n + 1, self.n + 1)) + Wmat[:, 0] = 1.0 + Wmat[:, 1:] = self.xpt # size npt * n + return Wmat + + def predicted_values(self, d, d_based_at_xopt=True, with_const_term=False): + if d_based_at_xopt: + Jd = np.dot(self.gqv.T, d + self.xopt()) # J^T * d (where Jacobian J = self.gqv^T) + else: # d based at xbase + Jd = np.dot(self.gqv.T, d) # J^T * d (where Jacobian J = self.gqv^T) + return Jd + (self.model_const_term_v if with_const_term else 0.0) + + def square_distances_to_xopt(self): + sq_distances = np.zeros((self.npt,)) + for k in range(self.npt): + sq_distances[k] = sumsq(self.xpt[k, :] - self.xopt()) + return sq_distances + + def min_objective_value(self, abs_tol=1.0e-12, rel_tol=1.0e-20): + # Set a minimum value so that if the full objective falls below it, we immediately finish + return max(abs_tol, rel_tol*self.fbeg) + + +def build_initial_set(objfun, x0, xl, xu, rhobeg, maxfun): + # Evaluate at initial point (also gets us m) + v_err0, f0 = eval_least_squares_objective(objfun, x0, eval_num=1) + + # Get dimension of problem and number of sample points from x0 and v_err0 information + n = np.size(x0) + npt = n + 1 + m = np.size(v_err0) + + # Initialise model (sets x0 as base point and xpt = zeros, so xpt[0,:] = x0) + model = Model(n, m, npt, x0, xl, xu) + + # Build initial sample set + at_upper_boundary = (model.su < 0.01 * rhobeg) # su = xu - x0, should be +ve, actually > rhobeg + for k in range(n): + step_size = (rhobeg if not at_upper_boundary[k] else -rhobeg) + model.xpt[k+1, k] = step_size + + # Add results of objective evaluation at x0 + model.fval_v[0, :] = v_err0 + model.fval[0] = f0 + model.kopt = 0 + model.fbeg = f0 + model.xsave = x0.copy() + model.rsave = v_err0.copy() + model.fsave = f0 + model.jacmin = np.zeros((m, n)) + + # Evaluate objective at each point in the initial sample set + for nf in range(1, min(npt, maxfun)): + x = model.x_within_bounds(k=nf) + v_err, f = eval_least_squares_objective(objfun, x, eval_num=nf+1) # nf is one behind because of f(x0) + + model.fval[nf] = f + model.fval_v[nf, :] = v_err + + if f < model.fval_opt(): # update optimal point + model.kopt = nf + + return model + + +def altmov_wrapper(model, knew, adelt): + model.factorise_LU() + # First need to get knew-th column of H matrix + if model.EXACT_CONST_TERM: + if knew == model.kopt: + ek = -np.ones((model.n, )) # matrix based on (y-xk), so different geom structure for kopt + else: + ek = np.zeros((model.n, )) + if knew < model.kopt: + ek[knew] = 1.0 + else: + ek[knew - 1] = 1.0 + H_knew = model.solve_LU(ek) + else: + ek = np.zeros((model.n + 1,)) + ek[knew] = 1.0 + H_knew = model.solve_LU(ek) + xnew, xalt, cauchy, abs_denom = altmov(model.xpt, model.sl, model.su, model.kopt, + model.xopt(), knew, adelt, H_knew) + # abs_denom is Lagrange_knew evaluated at xnew + return xnew, xalt, cauchy, abs_denom + + +def altmov_wrapper_v2(model, knew, adelt): + model.factorise_LU() + # First need to get knew-th column of H matrix + if model.EXACT_CONST_TERM: + if knew == model.kopt: + ek = -np.ones((model.n, )) # matrix based on (y-xk), so different geom structure for kopt + else: + ek = np.zeros((model.n, )) + if knew < model.kopt: + ek[knew] = 1.0 + else: + ek[knew - 1] = 1.0 + g = model.solve_LU(ek) # H_knew + else: + ek = np.zeros((model.n + 1,)) + ek[knew] = 1.0 + g = model.solve_LU(ek) # H_knew + + c = 1 if knew == model.kopt else 0 # c, g are for knew-th Lagrange polynomial, based at xopt (c + g*(x-xopt)) + xnew = max_step_in_box_and_ball(model.xopt(), c, g, model.sl, model.su, adelt) + return xnew + + +def choose_knew(model, delta, xnew, skip_kopt=True): + # in model, uses: n, npt, xpt, kopt/xopt, build_interp_matrix() + # model unchanged by this method + + # Criteria is to maximise: max(1, ||yt-xk||^4/Delta^4) * abs(Lagrange_t(xnew)) + # skip_kopt determines whether to check t=kopt as a possible candidate or not + + model.factorise_LU() # Prep for linear solves + + delsq = delta ** 2 + scaden = -1.0 + knew = None # may knew never be set here? + + try: + for k in range(model.npt): + if skip_kopt and k == model.kopt: + continue # next k in this inner loop + if model.EXACT_CONST_TERM: + if k == model.kopt: + ek = -np.ones((model.n,)) # matrix based on (y-xk), so different geom structure for kopt + else: + ek = np.zeros((model.n, )) + if k < model.kopt: + ek[k] = 1.0 + else: + ek[k-1] = 1.0 + Hk = model.solve_LU(ek) + else: + ek = np.zeros((model.n + 1,)) + ek[k] = 1.0 + Hk = model.solve_LU(ek) # k-th column of H, except 1st entry (i.e. Lagrange polynomial gradient) + lagrange_k_at_d = 1.0 + np.dot(xnew-model.xpt[k, :], Hk) + distsq = sumsq(model.xpt[k, :] - model.xopt()) + temp = max(1.0, (distsq / delsq) ** 2) + if temp * abs(lagrange_k_at_d) > scaden: + scaden = temp * abs(lagrange_k_at_d) + knew = k + + linalg_error = False + except np.linalg.LinAlgError: + linalg_error = True + + return knew, linalg_error + + +def trust_region_subproblem_least_squares(model, delta): + # in model, uses: n, npt, xpt, kopt/xopt, sl, su, build_full_model() + # model unchanged by this method + + # Build model for full least squares objectives + gopt, hq = model.build_full_model() + # Call original BOBYQA trsbox function + d, gnew, crvmin = trsbox(model.xopt(), gopt, hq, model.sl, model.su, delta) + return d, gopt, hq, gnew, crvmin + + +def done_with_current_rho(model, nf, nfsav, rho, diffs, xnew, gnew, hq, crvmin): + # in model, uses: n, sl, su + # model unchanged by this method + + if nf <= nfsav + 2: + return False + + errbig = max(diffs) + frhosq = 0.125 * rho ** 2 + if crvmin > 0.0 and errbig > frhosq * crvmin: + return False + + bdtol = errbig / rho + for j in range(model.n): + bdtest = bdtol + if xnew[j] == model.sl[j]: + bdtest = gnew[j] + if xnew[j] == model.su[j]: + bdtest = -gnew[j] + if bdtest < bdtol: + curv = get_hessian_element(model.n, hq, j, j) # curv = Hessian(j, j) + bdtest += 0.5 * curv * rho + if bdtest < bdtol: + return False + + return True + + +def reduce_rho(old_rho, rhoend): + ratio = old_rho/rhoend + if ratio <= 16.0: + new_rho = rhoend + elif ratio <= 250.0: + new_rho = sqrt(ratio)*rhoend + else: + new_rho = 0.1*old_rho + delta = max(0.5*old_rho, new_rho) + return delta, new_rho + + +def check_and_fix_geometry(model, objfun, distsq, delta, rho, dnorm, diffs, nf, nfsav, maxfun, rounding_error_const, + update_delta=True): + # [Fortran label 650] + # If any xpt more than distsq away from xopt, fix geometry + knew_tmp, distsq_tmp = get_vector_max(all_square_distances(model.xpt, model.xopt())) + if distsq_tmp > distsq: # fix geometry and quit + knew = knew_tmp + distsq = distsq_tmp + + dist = sqrt(distsq) + if update_delta: # optional + delta = max(min(0.1 * delta, 0.5 * dist), 1.5 * rho) # use 0.5*dist, within range [0.1*delta, 1.5*rho] + + adelt = max(min(0.1 * dist, delta), rho) + if adelt ** 2 <= rounding_error_const * sumsq(model.xopt()): + model.shift_base(model.xopt()) + + model, nf, nfsav, diffs, return_to_new_tr_iteration, exit_flag, exit_str \ + = fix_geometry(model, objfun, knew, adelt, rho, dnorm, diffs, nf, nfsav, maxfun) + + return model, delta, nf, nfsav, diffs, return_to_new_tr_iteration, exit_flag, exit_str + else: + # Do nothing, just quit + # return_to_new_tr_iteration = None when didn't fix geometry + return model, delta, nf, nfsav, diffs, None, None, None + + +def fix_geometry(model, objfun, knew, adelt, rho, dnorm, diffs, nf, nfsav, maxfun): + # in model, uses: n, npt, xpt, sl, su, kopt/xopt, build_interp_metrix, and others + # model is changed by this function: gqv from interp_mini_models, and others + + USE_OLD_ALTMOV = False + try: + if USE_OLD_ALTMOV: + xnew, xalt, cauchy, denom = altmov_wrapper(model, knew, adelt) + else: + xnew = altmov_wrapper_v2(model, knew, adelt) + xalt = None + cauchy = None + denom = None + except np.linalg.LinAlgError: + exit_flag = EXIT_LINALG_ERROR + exit_str = "Singular matrix encountered in ALTMOV" + return_to_new_tr_iteration = False # return and quit + return model, nf, nfsav, diffs, return_to_new_tr_iteration, exit_flag, exit_str + + if xnew is None: # issue with stpsav occurred, quit DFOGN + exit_flag = EXIT_ALTMOV_MEMORY_ERROR + exit_str = "Error in ALTMOV - stpsav undefined" + return_to_new_tr_iteration = False # return and quit + return model, nf, nfsav, diffs, return_to_new_tr_iteration, exit_flag, exit_str + + if USE_OLD_ALTMOV and denom < cauchy and cauchy > 0.0: + xnew = xalt.copy() + + d = xnew - model.xopt() + + # [Fortran label 360] + x = model.x_within_bounds(x=xnew) + if nf >= maxfun: + exit_flag = EXIT_MAXFUN_WARNING + exit_str = "Objective has been called MAXFUN times" + return_to_new_tr_iteration = False # return and quit + return model, nf, nfsav, diffs, return_to_new_tr_iteration, exit_flag, exit_str + + nf += 1 + v_err, f = eval_least_squares_objective(objfun, x, eval_num=nf) + + if f <= model.min_objective_value(): + # Force model.get_final_results() to return this new point if it's better than xopt, then quit + model.xsave = x + model.rsave = v_err.copy() + model.fsave = f + model.jacsave = model.gqv_at_xopt().T + exit_flag = EXIT_SUCCESS + exit_str = "Sufficient reduction in objective value" + return_to_new_tr_iteration = False # return and quit + return model, nf, nfsav, diffs, return_to_new_tr_iteration, exit_flag, exit_str + + # Use the quadratic model to predict the change in F due to the step D, + # and set DIFF to the error of this prediction. + gopt, hq = model.build_full_model() + if gopt is None: # Use this to indicate linalg error + if f < model.fval_opt(): + # Force model.get_final_results() to return this new point if it's better than xopt, then quit + model.xsave = x + model.rsave = v_err.copy() + model.fsave = f + model.jacsave = model.gqv_at_xopt().T + exit_flag = EXIT_LINALG_ERROR + exit_str = "Singular matrix encountered in FIX_GEOMETRY (full model interpolation step)" + return_to_new_tr_iteration = False # return and quit + return model, nf, nfsav, diffs, return_to_new_tr_iteration, exit_flag, exit_str + + pred_reduction = - calculate_model_value(gopt, hq, d) + actual_reduction = model.fval_opt() - f + diffs = [abs(pred_reduction - actual_reduction), diffs[0], diffs[1]] + + if dnorm > rho: + nfsav = nf + + # Update bmat, zmat, gopt, etc. (up to label ~560) + model.update_point(knew, xnew, v_err, f) + + exit_flag = None + exit_str = None + return_to_new_tr_iteration = True # return and start new trust region iteration (label 60) + return model, nf, nfsav, diffs, return_to_new_tr_iteration, exit_flag, exit_str + + +def dfogn_main(objfun, x0, xl, xu, rhobeg, rhoend, maxfun): + exit_flag = None + exit_str = None + + # One variable in BOBYQB depends on which code form we are using + if zhang_code_structure: + rounding_error_const = 0.1 # Zhang code + else: + rounding_error_const = 1.0e-3 # BOBYQA + + ########################################################### + # Set up initial interpolation set + ########################################################### + model = build_initial_set(objfun, x0, xl, xu, rhobeg, maxfun) + + if maxfun < model.npt: + exit_flag = EXIT_MAXFUN_WARNING + exit_str = "Objective has been called MAXFUN times" + x, rvec, f, jacmin = model.get_final_results() + return x, rvec, f, jacmin, maxfun, exit_flag, exit_str + # return x, f, maxfun, exit_flag, exit_str + + ########################################################### + # Set other variables before begin iterations + ########################################################### + finished_main_loop = False + + (rho, delta) = (rhobeg, rhobeg) + nf = min(maxfun, model.npt) # number of function evaluations so far + nfsav = nf + diffs = [0.0, 0.0, 0.0] # (diffa, diffb, diffc) in Fortran code, used in done_with_current_rho() + + ########################################################### + # Start of main loop [Fortran label 60] + ########################################################### + while not finished_main_loop: + # Interpolate each mini-model + interp_ok = model.interpolate_mini_models() + if not interp_ok: + exit_flag = EXIT_LINALG_ERROR + exit_str = "Singular matrix in mini-model interpolation (main loop)" + finished_main_loop = True + break # quit + + # Solve trust region subproblem to get tentative step d + # Model for full least squares objective is given by (gopt, hq) + d, gopt, hq, gnew, crvmin = trust_region_subproblem_least_squares(model, delta) + logging.debug("Trust region step is d = " + str(d)) + xnew = model.xopt() + d + dsq = sumsq(d) + dnorm = min(delta, sqrt(dsq)) + + if dnorm < 0.5 * rho: + ################### + # Start failed TR step + ################### + logging.debug("Failed trust region step") + + if not done_with_current_rho(model, nf, nfsav, rho, diffs, xnew, gnew, hq, crvmin): + # [Fortran label 650] + distsq = (10.0 * rho) ** 2 + model, delta, nf, nfsav, diffs, return_to_new_tr_iteration, geom_exit_flag, geom_exit_str = \ + check_and_fix_geometry(model, objfun, distsq, delta, rho, dnorm, diffs, nf, nfsav, maxfun, + rounding_error_const, update_delta=True) + + if return_to_new_tr_iteration is not None: # i.e. if we did actually fix geometry + if return_to_new_tr_iteration: + finished_main_loop = False + continue # next trust region step + else: # quit + exit_flag = geom_exit_flag + exit_str = geom_exit_str + finished_main_loop = True + break # quit + # If we didn't fix geometry, reduce rho as below + # otherwise, if we are done with current rho, reduce rho as below + + # Reduce rho and continue [Fortran label 680] + if rho > rhoend: + delta, rho = reduce_rho(rho, rhoend) + logging.info("New rho = %g after %i function evaluations" % (rho, nf)) + logging.debug("Best so far: f = %.15g at x = " % (model.fval_opt()) + str(model.xbase + model.xopt())) + nfsav = nf + finished_main_loop = False + continue # next trust region step + else: + # Cannot reduce rho, so check xnew and quit + x = model.x_within_bounds(x=xnew) + if nf >= maxfun: # quit + exit_flag = EXIT_MAXFUN_WARNING + exit_str = "Objective has been called MAXFUN times" + finished_main_loop = True + break # quit + + nf += 1 + v_err, f = eval_least_squares_objective(objfun, x, eval_num=nf) # v_err not used here + + # Force model.get_final_results() to return this new point if it's better than xopt, then quit + model.xsave = x + model.rsave = v_err.copy() + model.fsave = f + model.jacsave = model.gqv_at_xopt().T + exit_flag = EXIT_SUCCESS + exit_str = "rho has reached rhoend" + finished_main_loop = True + break # quit + ################### + # End failed TR step + ################### + else: + ################### + # Start successful TR step + ################### + logging.debug("Successful trust region step") + + # Severe cancellation is likely to occur if XOPT is too far from XBASE. [Fortran label 90] + if dsq <= rounding_error_const * sumsq(model.xopt()): + model.shift_base(model.xopt()) # includes a re-factorisation of the interpolation matrix + xnew = xnew - model.xopt() + + # Set KNEW to the index of the next interpolation point to be deleted to make room for a trust + # region step. Again RESCUE may be called if rounding errors have damaged + # the chosen denominator, which is the reason for attempting to select + # KNEW before calculating the next value of the objective function. + knew, linalg_error = choose_knew(model, delta, xnew, skip_kopt=True) + + if linalg_error: + exit_flag = EXIT_LINALG_ERROR + exit_str = "Singular matrix when finding knew (in main loop)" + finished_main_loop = True + break # quit + + # Calculate the value of the objective function at XBASE+XNEW, unless + # the limit on the number of calculations of F has been reached. + # [Fortran label 360, with ntrits > 0] + x = model.x_within_bounds(x=xnew) + + if nf >= maxfun: + exit_flag = EXIT_MAXFUN_WARNING + exit_str = "Objective has been called MAXFUN times" + finished_main_loop = True + break # quit + + nf += 1 + v_err, f = eval_least_squares_objective(objfun, x, eval_num=nf) + + if f <= model.min_objective_value(): + # Force model.get_final_results() to return this new point if it's better than xopt, then quit + model.xsave = x + model.rsave = v_err.copy() + model.fsave = f + model.jacsave = model.gqv_at_xopt().T + exit_flag = EXIT_SUCCESS + exit_str = "Objective is sufficiently small" + finished_main_loop = True + break # quit + + # Use the quadratic model to predict the change in F due to the step D, + # and set DIFF to the error of this prediction. + pred_reduction = - calculate_model_value(gopt, hq, d) + actual_reduction = model.fval_opt() - f + diffs = [abs(pred_reduction - actual_reduction), diffs[0], diffs[1]] + + if dnorm > rho: + nfsav = nf + + if pred_reduction < 0.0: + exit_flag = EXIT_TR_INCREASE_ERROR + exit_str = "Trust region step gave model increase" + finished_main_loop = True + break # quit + + # Pick the next value of DELTA after a trust region step. + # Update trust region radius + ratio = actual_reduction / pred_reduction + if ratio <= 0.1: + delta = min(0.5 * delta, dnorm) + elif ratio <= 0.7: + delta = max(0.5 * delta, dnorm) + else: # (ratio > 0.7) Different updates depending on which code version we're using + if zhang_code_structure: + delta = min(max(2.0 * delta, 4.0 * dnorm), 1.0e10) # DFBOLS code version + elif bbqtr: + delta = max(0.5 * delta, 2.0 * dnorm) # BOBYQA version + else: + delta = max(delta, 2.0 * dnorm) # Zhang paper version + if delta <= 1.5 * rho: # cap trust region radius at rho + delta = rho + logging.debug("New delta = %g (rho = %g) from ratio %g" % (delta, rho, ratio)) + + # Recalculate KNEW and DENOM if the new F is less than FOPT. + if actual_reduction > 0.0: # f < model.fval_opt() + knew, linalg_error = choose_knew(model, delta, xnew, skip_kopt=False) + + if linalg_error: + exit_flag = EXIT_LINALG_ERROR + exit_str = "Singular matrix when finding knew (in main loop, second time)" + finished_main_loop = True + break # quit + + # Updating... + logging.debug("Updating with knew = %i" % knew) + model.update_point(knew, xnew, v_err, f) + + # If a trust region step has provided a sufficient decrease in F, then + # branch for another trust region calculation. + if ratio >= 0.1: + finished_main_loop = False + continue # next trust region step + + # Alternatively, find out if the interpolation points are close enough + # to the best point so far. + # [Fortran label 650] + distsq = max((2.0 * delta) ** 2, (10.0 * rho) ** 2) + model, delta, nf, nfsav, diffs, return_to_new_tr_iteration, geom_exit_flag, geom_exit_str = \ + check_and_fix_geometry(model, objfun, distsq, delta, rho, dnorm, diffs, nf, nfsav, maxfun, + rounding_error_const, update_delta=False) # don't update delta when ntrits > 0 + + if return_to_new_tr_iteration is not None: # i.e. if we did actually fix geometry + if return_to_new_tr_iteration: + finished_main_loop = False + continue # next trust region step + else: # quit + exit_flag = geom_exit_flag + exit_str = geom_exit_str + finished_main_loop = True + break # quit + # If we didn't fix geometry, reduce rho [Fortran label 680] + + if ratio > 0.0: + finished_main_loop = False + continue # next trust region step + + if max(delta, dnorm) > rho: + finished_main_loop = False + continue # next trust region step + + # Reduce rho and continue [Fortran label 680] + if rho > rhoend: + delta, rho = reduce_rho(rho, rhoend) + logging.info("New rho = %g after %i function evaluations" % (rho, nf)) + logging.debug("Best so far: f = %.15g at x = " % (model.fval_opt()) + str(model.xbase + model.xopt())) + nfsav = nf + finished_main_loop = False + continue # next trust region step + else: + # Cannot reduce rho further + exit_flag = EXIT_SUCCESS + exit_str = "rho has reached rhoend" + finished_main_loop = True + break # quit + ################### + # End successful TR step + ################### + ############################# + # End this iteration of main loop - take next TR step + ############################# + ########################################################### + # End of main loop [Fortran label 720] + ########################################################### + + x, rvec, f, jacmin = model.get_final_results() + logging.debug("At return from DFOGN, number of function evals = %i" % nf) + logging.debug("Smallest objective value = %.15g at x = " % f + str(x)) + return x, rvec, f, jacmin, nf, exit_flag, exit_str + # return x, f, nf, exit_flag, exit_str + + +def solve(objfun, x0, lower=None, upper=None, maxfun=1000, rhobeg=None, rhoend=1e-8): + # If bounds not provided, set to something large + xl = (lower if lower is not None else -1.0e20 * np.ones(x0.shape)) + xu = (upper if upper is not None else 1.0e20 * np.ones(x0.shape)) + + # Set default value of rhobeg to something sensible + rhobeg = (rhobeg if rhobeg is not None else 0.1 * max(np.max(np.abs(x0)), 1.0)) + + n = np.size(x0) + + # Input & parameter checks + input_error_msg = None + if rhobeg < 0.0: + input_error_msg = "Input error: rhobeg must be strictly positive" + + if rhoend < 0.0: + input_error_msg = "Input error: rhoend must be strictly positive" + + if rhobeg <= rhoend: + input_error_msg = "Input error: rhobeg must be > rhoend" + + if maxfun <= 0: + input_error_msg = "Input error: maxfun must be strictly positive" + + if np.shape(x0) != (n,): + input_error_msg = "Input error: x0 must be a vector" + + if np.shape(x0) != np.shape(xl): + input_error_msg = "Input error: lower bounds must have same shape as x0" + + if np.shape(x0) != np.shape(xu): + input_error_msg = "Input error: upper bounds must have same shape as x0" + + if np.min(xu - xl) < 2.0 * rhobeg: + input_error_msg = "Input error: gap between lower and upper must be at least 2*rhobeg" + + # Process input errors + if input_error_msg is not None: + results = OptimResults(x0, None, None, None, 0, EXIT_INPUT_ERROR, "Input error: " + input_error_msg) + return results + + if maxfun <= n + 1: + warnings.warn("maxfun <= npt: Are you sure your budget is large enough?", RuntimeWarning) + + # Enforce lower bounds on x0 (ideally with gap of at least rhobeg) + idx = (xl < x0) & (x0 <= xl+rhobeg) + if np.any(idx): + warnings.warn("Some entries of x0 too close to lower bound, adjusting", RuntimeWarning) + x0[idx] = xl[idx] + rhobeg + + idx = (x0 <= xl) + if np.any(idx): + warnings.warn("Some entries of x0 below lower bound, adjusting", RuntimeWarning) + x0[idx] = xl[idx] + + # Enforce upper bounds on x0 (ideally with gap of at least rhobeg) + idx = (xu-rhobeg <= x0) & (x0 < xu) + if np.any(idx): + warnings.warn("Some entries of x0 too close to upper bound, adjusting", RuntimeWarning) + x0[idx] = xu[idx] - rhobeg + + idx = (x0 >= xu) + if np.any(idx): + warnings.warn("Some entries of x0 above upper bound, adjusting", RuntimeWarning) + x0[idx] = xu[idx] + + x, rvec, f, jacmin, nf, exit_flag, exit_str = dfogn_main(objfun, x0.copy(), xl, xu, rhobeg, rhoend, maxfun) + + # Clean up exit_str to have better information: + if exit_flag == EXIT_SUCCESS: + exit_str = "Success: " + exit_str + elif exit_flag == EXIT_MAXFUN_WARNING: + exit_str = "Warning: " + exit_str + elif exit_flag == EXIT_INPUT_ERROR: + exit_str = "Input error: " + exit_str + elif exit_flag == EXIT_TR_INCREASE_ERROR: + exit_str = "Trust region subproblem error: " + exit_str + elif exit_flag == EXIT_LINALG_ERROR: + exit_str = "Linear algebra error: " + exit_str + elif exit_flag == EXIT_ALTMOV_MEMORY_ERROR: + exit_str = "ALTMOV memory error: " + exit_str + else: + exit_str = "Unknown exit flag " + str(exit_flag) + " with message " + exit_str + + # Build solution object + results = OptimResults(x, rvec, f, jacmin, nf, exit_flag, exit_str) + # return x, f, nf, exit_flag, exit_str + return results + diff --git a/dfogn/dfogn_resampling.py b/dfogn/dfogn_resampling.py new file mode 100644 index 0000000..924d5ab --- /dev/null +++ b/dfogn/dfogn_resampling.py @@ -0,0 +1,1265 @@ +""" +DFO-GN +==================== +A derivative-free solver for least squares minimisation with bound constraints. +This version has resampling (not part of main package). + +This file is a modified version of DFOGN which allows resampling and restarts, +to better cope with noisy problems. + +Lindon Roberts, 2017 + +Call structure is: + x, f, nf, exit_flag, exit_str = dfogn(objfun, x0, lower, upper, + maxfun, init_tr_radius, rhoend=1e-8) + +Required inputs: + objfun Objective function, callable as: residual_vector = objfun(x) + x0 Initial starting point, NumPy ndarray +Optional inputs: + lower, upper Lower and upper bound constraints (lower <= x <= upper), + must be NumPy ndarrays of same size as x0 (default +/-1e20) + maxfun Maximum number of allowable function evalutions (default 1000) + init_tr_radius Initial trust region radius (default 0.1*max(1, ||x0||_infty) + rhoend Termination condition on trust region radius (default 1e-8) + +Outputs: + x Estimate of minimiser + f Value of least squares objective at x (f = ||objfun(x)||^2) + nf Number of objective evaluations used to find x + exit_flag Integer flag indicating termination criterion (see list below imports) + exit_str String with more detailed termination message + + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +The development of this software was sponsored by NAG Ltd. (http://www.nag.co.uk) +and the EPSRC Centre For Doctoral Training in Industrially Focused Mathematical +Modelling (EP/L015803/1) at the University of Oxford. Please contact NAG for +alternative licensing. + +Copyright 2017, Lindon Roberts + +""" +# Ensure compatibility with Python 2 +from __future__ import absolute_import, division, print_function, unicode_literals + +import numpy as np +import scipy.linalg as sp_linalg +from math import sqrt +import logging +from .util import * +from .trust_region import * +from .alternative_move import * + +####################### +# Exit codes +EXIT_SUCCESS = 0 # successful finish (rho=rhoend or sufficient objective reduction) +EXIT_INPUT_ERROR = 1 # error, bad inputs +EXIT_MAXFUN_WARNING = 2 # warning, reached max function evals +EXIT_TR_INCREASE_ERROR = 3 # error, trust region step increased model value +EXIT_LINALG_ERROR = 4 # error, linalg error (singular matrix encountered) +EXIT_ALTMOV_MEMORY_ERROR = 5 # error, stpsav issue in ALTMOV + +# Errors for which we can do a restart (not including rho=rhoend in EXIT_SUCCESS) +DO_RESTART_ERRORS = [EXIT_TR_INCREASE_ERROR, EXIT_LINALG_ERROR, EXIT_ALTMOV_MEMORY_ERROR] +####################### + +####################### +# Sampling scenarios +SCEN_PRELIM = 1 # during prelim +SCEN_GROWING_NEW_DIRECTION = 2 # adding new direction while growing +SCEN_TR_STEP = 3 # sampling xk+sk from successful trust region step +SCEN_GEOM_UPDATE = 4 # calling altmov for geometry fixing +SCEN_RHOEND_REACHED = 5 # reached rhoend in unsuccessful TR step +####################### + + +class Model: + def __init__(self, n, m, npt, x0, xl, xu): + assert npt==n+1, "Require strictly linear model" + # Problem sizes + self.n = n + self.m = m + self.npt = npt + self.npt_so_far = 0 # how many points have we added so far (for growing initial set) + + # Actual model info + # Here, the model for each residual is centred around xbase + # m(x) = model_const_term + gqv*(x-xbase) + + self.kbase = 0 # index of base point + self.xbase = x0 # base point + self.xl = xl # lower bounds (absolute terms) + self.xu = xu # upper bounds (absolute terms) + self.sl = xl - x0 # lower bounds (adjusted for xbase), should be -ve (actually < -rhobeg) + self.su = xu - x0 # upper bounds (adjusted for xbase), should be +ve (actually > rhobeg) + self.xpt = np.zeros((npt, n)) # interpolation points + self.fval_v = np.zeros((npt, m)) # residual vectors at each xpt(+xbase) + self.fval = np.zeros((npt, )) # total sum of squares at each xpt(+xbase) + self.model_const_term_v = np.zeros((m,)) # constant term of each mini-model + self.gqv = np.zeros((n, m)) # interpolated gradients for each mini-model + + self.kopt = None # index of current best x + + self.fbeg = None # initial sum of squares at x0 + + self.xsave = None # possible final return value (abs coords) + self.fsave = None # sum of squares for final return value + + self.lu = None # LU decomp of interp matrix + self.piv = None # pivots for LU decomposition of interp matrix + self.lu_current = False # whether current LU factorisation of interp matrix is up-to-date or not + + self.EXACT_CONST_TERM = True # use exact c=r(xopt) for interpolation (improve conditioning) + # Affects mini-model interpolation / interpolation matrix, but also geometry updating + + self.nsamples = np.zeros((npt,), dtype=np.int) # how many samples we have averaged to get fval_v, where fval = sumsq(avg fval_v) + + def x_within_bounds(self, k=None, x=None): + # Get x value for k-th point or x vector (in absolute terms, force within bounds) + if k is not None: + return np.minimum(np.maximum(self.xl, self.xbase + self.xpt[k, :]), self.xu) + elif x is not None: + return np.minimum(np.maximum(self.xl, self.xbase + x), self.xu) + else: + return None + + def xopt(self): + # Current best x (relative to xbase) + return self.xpt[self.kopt, :].copy() + + def fval_v_opt(self): + return self.fval_v[self.kopt,:] + + def fval_opt(self): + return self.fval[self.kopt] + + def update_point(self, knew, xnew, v_err, f): + if knew >= self.npt_so_far and self.npt_so_far < self.npt: + # when still growing, need to append in correct order + assert knew == self.npt_so_far, "Updating new index too far along (%g when should be %g)" % (knew, self.npt_so_far) + self.npt_so_far += 1 + + # Add point xnew with objective vector v_err (full objective f) at the knew-th index + self.xpt[knew,:] = xnew + self.fval_v[knew, :] = v_err + self.fval[knew] = f + self.nsamples[knew] = 1 + + # Update XOPT, GOPT and KOPT if the new calculated F is less than FOPT. + if f < self.fval_opt(): + self.kopt = knew + + self.lu_current = False + return + + def add_point_resample(self, knew, v_err_new): + # We have resampled point knew and got a new fval_v = v_err_new + # Update our estimates of fval_v + assert knew < self.npt_to_use(), "Invalid knew" + t = float(self.nsamples[knew]) / float(self.nsamples[knew] + 1) + self.fval_v[knew, :] = t * self.fval_v[knew, :] + (1 - t) * v_err_new + self.fval[knew] = sumsq(self.fval_v[knew, :]) + self.nsamples[knew] += 1 + + if self.fval[knew] < self.fval_opt(): + self.kopt = knew + + return + + def npt_to_use(self): + # Number of points to use when building interpolation system + return min(self.npt_so_far, self.npt) # depends on whether we have a full set yet (or not) + + def gqv_at_xopt(self): + return self.gqv + + def shift_base(self, xbase_shift): + for m1 in range(self.m): + self.model_const_term_v[m1] += np.dot(self.gqv[:, m1], xbase_shift) + + # The main updates + for k in range(self.npt): + self.xpt[k, :] = self.xpt[k, :] - xbase_shift + self.xbase += xbase_shift + self.sl = self.sl - xbase_shift + self.su = self.su - xbase_shift + + self.lu_current = False + self.factorise_LU() + + return + + def interpolate_mini_models(self): + # Build interpolation matrix and factorise (in self.lu, self.piv) + self.factorise_LU() + try: + if self.EXACT_CONST_TERM: + idx_to_use = [k for k in range(self.npt) if k != self.kopt] + for m1 in range(self.m): + rhs = np.zeros((self.n,)) + for i in range(self.n): + k = idx_to_use[i] + rhs[i] = self.fval_v[k, m1] - self.fval_v[self.kopt, m1] - \ + np.dot(self.gqv[:, m1], self.xpt[k, :] - self.xopt()) + soln = sp_linalg.lu_solve((self.lu, self.piv), rhs) + self.gqv[:, m1] += soln # whole solution is gradient + + # shift constant term back + self.model_const_term_v = self.fval_v[self.kopt, :] - np.dot(self.gqv.T, self.xopt()) + return True # flag ok + else: + model_values_v = np.zeros((self.npt, self.m)) + for k in range(self.npt): + model_values_v[k, :] = self.predicted_values(self.xpt[k, :], d_based_at_xopt=False, + with_const_term=True) + + # Sometimes when things get too close to a solution, we can get NaNs in model_values - flag error & quit + if np.any(np.isnan(model_values_v)): + self.gqv = None + return False # flag error + + for m1 in range(self.m): + rhs = self.fval_v[:, m1] - model_values_v[:, m1] + soln = sp_linalg.lu_solve((self.lu, self.piv), rhs) + self.model_const_term_v[m1] += soln[0] + self.gqv[:, m1] += soln[1:] # first term is constant, rest is gradient term + + return True # flag ok + except np.linalg.LinAlgError: + self.gqv = None + return False # flag error + except ValueError: # happens when LU decomposition has Inf or NaN + self.gqv = None + return False # flag error + + def factorise_LU(self): + if not self.lu_current: + Wmat = self.build_interp_matrix() + self.lu, self.piv = sp_linalg.lu_factor(Wmat) # LU has L and U parts, piv indicates row swaps for pivoting + self.lu_current = True + return + + def solve_LU(self, rhs): + # If lu_current, use that, otherwise revert to generic solver + if self.lu_current: + if self.EXACT_CONST_TERM: + return sp_linalg.lu_solve((self.lu, self.piv), rhs) # only get gradient (no const term) + else: + return sp_linalg.lu_solve((self.lu, self.piv), rhs)[1:] # only return gradient (1st term is constant) + else: + logging.warning("model.solve_LU not using factorisation") + Wmat = self.build_interp_matrix() + if self.EXACT_CONST_TERM: + return np.linalg.solve(Wmat, rhs) # only get gradient (no const term) + else: + return np.linalg.solve(Wmat, rhs)[1:] # only return gradient (1st term is constant) + + def get_final_results(self): + # Called when about to exit BOBYQB + # Return x and fval for optimal point (either from xsave+fsave or kopt) + if self.fsave is None or self.fval_opt() <= self.fsave: # optimal has changed since xsave+fsave were last set + x = self.x_within_bounds(k=self.kopt) + f = self.fval_opt() + else: + x = self.xsave + f = self.fsave + + return x, f + + def build_full_model(self): + # Build full least squares objective model from mini-models + # Centred around xopt = xpt[kopt, :] + v_temp = self.fval_v_opt() # m-vector + gqv_xopt = self.gqv_at_xopt() # J^T (transpose of Jacobian) at xopt, rather than xbase + + # Use the gradient at xopt to formulate \sum_i (2*f_i \nabla f_i) = 2 J^t m(x_opt) + gopt = np.dot(gqv_xopt, v_temp) # n-vector (gqv = J^T) + + # Gauss-Newton part of Hessian + hq = to_upper_triangular_vector(np.dot(gqv_xopt, gqv_xopt.T)) + + # Apply scaling based on convention for objective - this code uses sumsq(r_i) not 0.5*sumsq(r_i) + gopt = 2.0 * gopt + hq = 2.0 * hq + + return gopt, hq + + def build_interp_matrix(self): + if self.EXACT_CONST_TERM: + Wmat = np.zeros((self.n, self.n)) + idx_to_use = [k for k in range(self.npt) if k != self.kopt] + for i in range(self.n): + Wmat[i,:] = self.xpt[idx_to_use[i], :] - self.xopt() + else: + Wmat = np.zeros((self.n + 1, self.n + 1)) + Wmat[:, 0] = 1.0 + Wmat[:, 1:] = self.xpt # size npt * n + return Wmat + + def predicted_values(self, d, d_based_at_xopt=True, with_const_term=False): + if d_based_at_xopt: + Jd = np.dot(self.gqv.T, d + self.xopt()) # J^T * d (where Jacobian J = self.gqv^T) + else: # d based at xbase + Jd = np.dot(self.gqv.T, d) # J^T * d (where Jacobian J = self.gqv^T) + return Jd + (self.model_const_term_v if with_const_term else 0.0) + + def square_distances_to_xopt(self): + sq_distances = np.zeros((self.npt,)) + for k in range(self.npt): + sq_distances[k] = sumsq(self.xpt[k, :] - self.xopt()) + return sq_distances + + def min_objective_value(self, abs_tol=1.0e-12, rel_tol=1.0e-20): + # Set a minimum value so that if the full objective falls below it, we immediately finish + if self.fbeg is not None: + return max(abs_tol, rel_tol * self.fbeg) + else: + return abs_tol + + +def sample_objective(m, objfun, x, nf, nx, maxfun, min_obj_value, nsamples=1): + # Sample from objective function several times, keeping track of maxfun and min_obj_value throughout + if m is None: + # Don't initialise v_err_list yet + v_err_list = None + else: + v_err_list = np.zeros((nsamples, m)) + f_list = np.zeros((nsamples,)) + exit_flag = None + exit_str = None + nsamples_run = 0 + + for i in range(nsamples): + if nf >= maxfun: + exit_flag = EXIT_MAXFUN_WARNING + exit_str = "Objective has been called MAXFUN times" + break # quit + + nf += 1 + this_v_err, f_list[i] = eval_least_squares_objective_v2(objfun, x, eval_num=nf, pt_num=nx+1, full_x_thresh=6) + if m is None: + m = len(this_v_err) + v_err_list = np.zeros((nsamples, m)) + v_err_list[i, :] = this_v_err + + nsamples_run += 1 + + if f_list[i] <= min_obj_value: + # Force model.get_final_results() to return this new point if it's better than xopt, then quit + exit_flag = EXIT_SUCCESS + exit_str = "Objective is sufficiently small" + break # quit + + return v_err_list, f_list, nf, nx+1, nsamples_run, exit_flag, exit_str + + +def build_initial_set(objfun, x0, xl, xu, rhobeg, maxfun, nsamples, nf_so_far, nx_so_far, ndirs_initial, nruns_so_far, + m=None, random_initial_directions=False): + n = np.size(x0) + npt = n + 1 + if m is not None: + # Initialise model (sets x0 as base point and xpt = zeros, so xpt[0,:] = x0) + model = Model(n, m, npt, x0, xl, xu) + model.kopt = 0 + minval = model.min_objective_value() + else: + # If we don't yet have m, wait until we have done a function evaluation before initialising model + model = None + minval = -1.0 + + assert 1 <= ndirs_initial < np.size(x0)+1, "build_inital_set: must have 1 <= ndirs_initial < n+1" + nx = nx_so_far + nf = nf_so_far + + # For calling nsamples: + delta = rhobeg + rho = rhobeg + current_iter = 0 + + # Evaluate at initial point (also gets us m in the first run through) + nsamples_to_use = nsamples(delta, rho, current_iter, nruns_so_far, SCEN_PRELIM) + v_err_list, f_list, nf, nx, nsamples_run, exit_flag, exit_str = sample_objective(m, objfun, x0, + nf, nx, maxfun, + minval, + nsamples=nsamples_to_use) + + # If we have just learned m, initialise model (sets x0 as base point and xpt = zeros, so xpt[0,:] = x0) + if model is None: + # Now we know m = v_err_list.shape[1] + model = Model(n, v_err_list.shape[1], npt, x0, xl, xu) + model.kopt = 0 + + f0 = sumsq(np.mean(v_err_list[:nsamples_run, :], axis=0)) # estimate actual objective value + # Handle exit conditions (f < min obj value or maxfun reached) + if exit_flag is not None: # then exit_str is also set + if nsamples_run > 0: + fmin = np.min(f_list[:nsamples_run]) + if model.fsave is None or fmin < model.fsave: + model.xsave = x0 + model.fsave = fmin + return_to_new_tr_iteration = False # return and quit + return model, nf, nx, return_to_new_tr_iteration, exit_flag, exit_str + + # Otherwise, add new results (increments model.npt_so_far) + model.update_point(0, model.xpt[0, :], v_err_list[0, :], f_list[0]) + for i in range(1, nsamples_run): + model.add_point_resample(0, v_err_list[i, :]) # add new info + + # Add results of objective evaluation at x0 + model.fbeg = f0 + model.xsave = x0.copy() + model.fsave = f0 + + # Build initial sample set either using random orthogonal directions, or coordinate directions + if random_initial_directions: + # Get ndirs_initial random orthogonal directions + A = np.random.randn(n, ndirs_initial) # Standard Gaussian n*ndirs_initial + Q = np.linalg.qr(A)[0] # Q is n*ndirs_initial with orthonormal columns + + # Now add the random directions + for ndirns in range(ndirs_initial): + dirn = Q[:, ndirns] + # Scale direction to ensure the new point lies within initial trust region, satisfies constraints + scale_factor = rhobeg / np.linalg.norm(dirn) + for j in range(n): + if dirn[j] < 0.0: + scale_factor = min(scale_factor, model.sl[j] / dirn[j]) + elif dirn[j] > 0.0: + scale_factor = min(scale_factor, model.su[j] / dirn[j]) + model.xpt[1 + ndirns, :] = scale_factor * dirn + + else: + at_upper_boundary = (model.su < 0.01 * rhobeg) # su = xu - x0, should be +ve, actually > rhobeg + for k in range(ndirs_initial): + step_size = (rhobeg if not at_upper_boundary[k] else -rhobeg) + model.xpt[k+1, k] = step_size + + # Evaluate objective at each point in the initial sample set + for k in range(1, ndirs_initial): + x = model.x_within_bounds(k=k) + nsamples_to_use = nsamples(delta, rho, current_iter, nruns_so_far, SCEN_PRELIM) + v_err_list, f_list, nf, nx, nsamples_run, exit_flag, exit_str = sample_objective(model.m, objfun, x, + nf, nx, maxfun, + model.min_objective_value(), + nsamples=nsamples_to_use) + + # f = sumsq(np.mean(v_err_list[:nsamples_run, :], axis=0)) # estimate actual objective value + # Handle exit conditions (f < min obj value or maxfun reached) + if exit_flag is not None: # then exit_str is also set + if nsamples_run > 0: + fmin = np.min(f_list[:nsamples_run]) + if model.fsave is None or fmin < model.fsave: + model.xsave = x + model.fsave = fmin + return_to_new_tr_iteration = False # return and quit + return model, nf, nx, return_to_new_tr_iteration, exit_flag, exit_str + + # Otherwise, add new results (increments model.npt_so_far) + model.update_point(k, model.xpt[k, :], v_err_list[0, :], f_list[0]) + for i in range(1, nsamples_run): + model.add_point_resample(k, v_err_list[i, :]) # add new info + + return_to_new_tr_iteration = True # return and continue + exit_flag = None + exit_str = None + return model, nf, nx, return_to_new_tr_iteration, exit_flag, exit_str + + +def get_new_orthogonal_directions(model, adelt, num_steps=1): + # Step from xopt along a random direction orthogonal to other yt (or multiple mutually orthogonal steps) + for i in range(20): # allow several tries, in case we choosing a point in the subspace of (yt-xk) [very unlucky] + A = np.random.randn(model.n, num_steps) + # (modified) Gram-Schmidt to orthogonalise + for k in range(min(model.npt_so_far, model.npt)): + if k == model.kopt: + continue + yk = model.xpt[k,:] - model.xopt() + for j in range(num_steps): + A[:,j] = A[:,j] - (np.dot(A[:,j], yk) / np.dot(yk, yk)) * yk + # continue if every column sufficiently large + all_cols_ok = True + for j in range(num_steps): + if np.linalg.norm(A[:,j]) < 1e-8: + all_cols_ok = False + if all_cols_ok: + break + # Scale appropriately so within bounds and ||d|| <= adelt + Q = np.linalg.qr(A)[0] # Q is n*ndirs with orthonormal columns + for j in range(num_steps): + scale_factor = adelt / np.linalg.norm(Q[:,j]) + for i in range(model.n): + if Q[i,j] < 0.0: + scale_factor = min(scale_factor, (model.sl[i] - model.xopt()[i]) / Q[i,j]) + elif Q[i,j] > 0.0: + scale_factor = min(scale_factor, (model.su[i] - model.xopt()[i]) / Q[i,j]) + Q[:,j] = Q[:,j] * scale_factor + # Finished! + return Q + + +def altmov_wrapper(model, knew, adelt): + model.factorise_LU() + # First need to get knew-th column of H matrix + if model.EXACT_CONST_TERM: + if knew == model.kopt: + ek = -np.ones((model.n, )) # matrix based on (y-xk), so different geom structure for kopt + else: + ek = np.zeros((model.n, )) + if knew < model.kopt: + ek[knew] = 1.0 + else: + ek[knew - 1] = 1.0 + H_knew = model.solve_LU(ek) + else: + ek = np.zeros((model.n + 1,)) + ek[knew] = 1.0 + H_knew = model.solve_LU(ek) + xnew, xalt, cauchy, abs_denom = altmov(model.xpt, model.sl, model.su, model.kopt, + model.xopt(), knew, adelt, H_knew) + # abs_denom is Lagrange_knew evaluated at xnew + return xnew, xalt, cauchy, abs_denom + + +def choose_knew(model, delta, xnew, skip_kopt=True): + # in model, uses: n, npt, xpt, kopt/xopt, build_interp_matrix() + # model unchanged by this method + + # Criteria is to maximise: max(1, ||yt-xk||^4/Delta^4) * abs(Lagrange_t(xnew)) + # skip_kopt determines whether to check t=kopt as a possible candidate or not + + model.factorise_LU() # Prep for linear solves + + delsq = delta ** 2 + scaden = -1.0 + knew = None # may knew never be set here? + + try: + for k in range(model.npt): + if skip_kopt and k == model.kopt: + continue # next k in this inner loop + if model.EXACT_CONST_TERM: + if k == model.kopt: + ek = -np.ones((model.n,)) # matrix based on (y-xk), so different geom structure for kopt + else: + ek = np.zeros((model.n, )) + if k < model.kopt: + ek[k] = 1.0 + else: + ek[k-1] = 1.0 + Hk = model.solve_LU(ek) + else: + ek = np.zeros((model.n + 1,)) + ek[k] = 1.0 + Hk = model.solve_LU(ek) # k-th column of H, except 1st entry (i.e. Lagrange polynomial gradient) + lagrange_k_at_d = 1.0 + np.dot(xnew-model.xpt[k, :], Hk) + distsq = sumsq(model.xpt[k, :] - model.xopt()) + temp = max(1.0, (distsq / delsq) ** 2) + if temp * abs(lagrange_k_at_d) > scaden: + scaden = temp * abs(lagrange_k_at_d) + knew = k + + linalg_error = False + except np.linalg.LinAlgError: + linalg_error = True + + return knew, linalg_error + + +def trust_region_subproblem_least_squares(model, delta): + # in model, uses: n, npt, xpt, kopt/xopt, sl, su, build_full_model() + # model unchanged by this method + + # Build model for full least squares objectives + gopt, hq = model.build_full_model() + # Call original BOBYQA trsbox function + d, gnew, crvmin = trsbox(model.xopt(), gopt, hq, model.sl, model.su, delta) + return d, gopt, hq, gnew, crvmin + + +def done_with_current_rho(model, current_iter, last_successful_iter, rho, diffs, xnew, gnew, hq, crvmin): + # in model, uses: n, sl, su + # model unchanged by this method + + if current_iter <= last_successful_iter + 2: + return False + + errbig = max(diffs) + frhosq = 0.125 * rho ** 2 + if crvmin > 0.0 and errbig > frhosq * crvmin: + return False + + bdtol = errbig / rho + for j in range(model.n): + bdtest = bdtol + if xnew[j] == model.sl[j]: + bdtest = gnew[j] + if xnew[j] == model.su[j]: + bdtest = -gnew[j] + if bdtest < bdtol: + curv = get_hessian_element(model.n, hq, j, j) # curv = Hessian(j, j) + bdtest += 0.5 * curv * rho + if bdtest < bdtol: + return False + + return True + + +def reduce_rho(old_rho, rhoend): + ratio = old_rho/rhoend + if ratio <= 16.0: + new_rho = rhoend + elif ratio <= 250.0: + new_rho = sqrt(ratio)*rhoend + else: + new_rho = 0.1*old_rho + delta = max(0.5*old_rho, new_rho) + return delta, new_rho + + +def check_and_fix_geometry(model, objfun, distsq, delta, rho, dnorm, diffs, nf, nx, current_iter, last_successful_iter, + maxfun, nsamples, rounding_error_const, nruns_so_far, update_delta=True): + # [Fortran label 650] + # If any xpt more than distsq away from xopt, fix geometry + knew_tmp, distsq_tmp = get_vector_max(all_square_distances(model.xpt, model.xopt())) + if distsq_tmp > distsq: # fix geometry and quit + knew = knew_tmp + distsq = distsq_tmp + + dist = sqrt(distsq) + if update_delta: # optional + delta = max(min(0.1 * delta, 0.5 * dist), 1.5 * rho) # use 0.5*dist, within range [0.1*delta, 1.5*rho] + + adelt = max(min(0.1 * dist, delta), rho) + if adelt ** 2 <= rounding_error_const * sumsq(model.xopt()): + model.shift_base(model.xopt()) + + model, nf, nx, last_successful_iter, diffs, return_to_new_tr_iteration, exit_flag, exit_str \ + = fix_geometry(model, objfun, knew, delta, adelt, rho, dnorm, diffs, nf, nx, current_iter, + last_successful_iter, maxfun, nsamples, nruns_so_far) + + return model, delta, nf, nx, last_successful_iter, diffs, return_to_new_tr_iteration, exit_flag, exit_str + else: + # Do nothing, just quit + # return_to_new_tr_iteration = None when didn't fix geometry + return model, delta, nf, nx, last_successful_iter, diffs, None, None, None + + +def fix_geometry(model, objfun, knew, delta, adelt, rho, dnorm, diffs, nf, nx, current_iter, last_successful_iter, + maxfun, nsamples, nruns_so_far): + # in model, uses: n, npt, xpt, sl, su, kopt/xopt, build_interp_metrix, and others + # model is changed by this function: gqv from interp_mini_models, and others + + try: + xnew, xalt, cauchy, denom = altmov_wrapper(model, knew, adelt) + except np.linalg.LinAlgError: + exit_flag = EXIT_LINALG_ERROR + exit_str = "Singular matrix encountered in ALTMOV" + return_to_new_tr_iteration = False # return and quit + return model, nf, nx, last_successful_iter, diffs, return_to_new_tr_iteration, exit_flag, exit_str + + if xnew is None: # issue with stpsav occurred, quit DFOGN + exit_flag = EXIT_ALTMOV_MEMORY_ERROR + exit_str = "Error in ALTMOV - stpsav undefined" + return_to_new_tr_iteration = False # return and quit + return model, nf, nx, last_successful_iter, diffs, return_to_new_tr_iteration, exit_flag, exit_str + + if denom < cauchy and cauchy > 0.0: + xnew = xalt.copy() + + d = xnew - model.xopt() + + # [Fortran label 360] + x = model.x_within_bounds(x=xnew) + nsamples_to_use = nsamples(delta, rho, current_iter, nruns_so_far, SCEN_GEOM_UPDATE) + v_err_list, f_list, nf, nx, nsamples_run, exit_flag, exit_str = sample_objective(model.m, objfun, x, nf, nx, maxfun, + model.min_objective_value(), + nsamples=nsamples_to_use) + + # Handle exit conditions (f < min obj value or maxfun reached) + if exit_flag is not None: # then exit_str is also set + if nsamples_run > 0: + fmin = np.min(f_list[:nsamples_run]) + if fmin < model.fsave: + model.xsave = x + model.fsave = fmin + return_to_new_tr_iteration = False # return and quit + return model, nf, nx, last_successful_iter, diffs, return_to_new_tr_iteration, exit_flag, exit_str + + # Otherwise, add new results + model.update_point(knew, xnew, v_err_list[0, :], f_list[0]) # increments model.npt_so_far, if still growing + for i in range(1, nsamples_run): + model.add_point_resample(knew, v_err_list[i, :]) # add new info + + # Estimate actual reduction to add to diffs vector + f = sumsq(np.mean(v_err_list[:nsamples_run, :], axis=0)) # estimate actual objective value + + # Use the quadratic model to predict the change in F due to the step D, + # and set DIFF to the error of this prediction. + gopt, hq = model.build_full_model() + if gopt is None: # Use this to indicate linalg error + if f < model.fval_opt(): + # Force model.get_final_results() to return this new point if it's better than xopt, then quit + model.xsave = x + model.fsave = f + exit_flag = EXIT_LINALG_ERROR + exit_str = "Singular matrix encountered in FIX_GEOMETRY (full model interpolation step)" + return_to_new_tr_iteration = False # return and quit + return model, nf, nx, last_successful_iter, diffs, return_to_new_tr_iteration, exit_flag, exit_str + + pred_reduction = - calculate_model_value(gopt, hq, d) + actual_reduction = model.fval_opt() - f + diffs = [abs(pred_reduction - actual_reduction), diffs[0], diffs[1]] + + if dnorm > rho: + last_successful_iter = current_iter + + exit_flag = None + exit_str = None + return_to_new_tr_iteration = True # return and start new trust region iteration (label 60) + return model, nf, nx, last_successful_iter, diffs, return_to_new_tr_iteration, exit_flag, exit_str + + +def dfogn_main(objfun, x0, xl, xu, rhobeg, rhoend, maxfun, nsamples, m=None, delta_scale_for_new_dirns_when_growing=1.0, + use_random_initial_directions=False, ndirs_initial=None, num_geom_steps_when_growing=1, nf_so_far=0, + nx_so_far=0, nruns_so_far=0): + exit_flag = None + exit_str = None + + # One variable in BOBYQB depends on which code form we are using + if zhang_code_structure: + rounding_error_const = 0.1 # Zhang code + else: + rounding_error_const = 1.0e-3 # BOBYQA + + ########################################################### + # Set up initial interpolation set + ########################################################### + # It shouldn't ever happen, but make sure ndirs_initial is not None + if ndirs_initial is None: + ndirs_initial = np.size(x0) + + model, nf, nx, return_to_new_tr_iteration, exit_flag, exit_str = \ + build_initial_set(objfun, x0, xl, xu, rhobeg, maxfun, nsamples, nf_so_far, nx_so_far, ndirs_initial, + nruns_so_far, m=m, random_initial_directions=use_random_initial_directions) + + if not return_to_new_tr_iteration: + x, f = model.get_final_results() + return x, f, nf, nx, exit_flag, exit_str, model.m + + ########################################################### + # Set other variables before begin iterations + ########################################################### + finished_main_loop = False + + (rho, delta) = (rhobeg, rhobeg) + diffs = [0.0, 0.0, 0.0] # (diffa, diffb, diffc) in Fortran code, used in done_with_current_rho() + + ########################################################### + # Start of main loop [Fortran label 60] + ########################################################### + current_iter = -1 + last_successful_iter = 0 + while not finished_main_loop: + current_iter += 1 + logging.debug("Iter %g (last successful %g) with delta = %g and rho = %g" % ( + current_iter, last_successful_iter, delta, rho)) + + # Interpolate each mini-model + interp_ok = model.interpolate_mini_models() + if not interp_ok: + exit_flag = EXIT_LINALG_ERROR + exit_str = "Singular matrix in mini-model interpolation (main loop)" + finished_main_loop = True + break # quit + + # Solve trust region subproblem to get tentative step d + # Model for full least squares objective is given by (gopt, hq) + d, gopt, hq, gnew, crvmin = trust_region_subproblem_least_squares(model, delta) + logging.debug("Trust region step is d = " + str(d)) + xnew = model.xopt() + d + dsq = sumsq(d) + dnorm = min(delta, sqrt(dsq)) + + if dnorm < 0.5 * rho and model.npt_so_far < model.n + 1: + # Failed TR step during initial phase - add a point and see if that helps + logging.debug("Failed trust region step during growing phase - adding new directions") + dnew_matrix = get_new_orthogonal_directions(model, delta_scale_for_new_dirns_when_growing * delta, + num_steps=num_geom_steps_when_growing) + break_main_loop = False # the internal breaks only quit this inner loop! + for j in range(num_geom_steps_when_growing): + xnew = model.xopt() + dnew_matrix[:, j] + logging.debug("Growing: compulsory geometry improving step xnew = %s" % str(xnew)) + x = model.x_within_bounds(x=xnew) + + nsamples_to_use = nsamples(delta, rho, current_iter, nruns_so_far, SCEN_GROWING_NEW_DIRECTION) + v_err_list, f_list, nf, nx, nsamples_run, exit_flag, exit_str = \ + sample_objective(model.m, objfun, x, nf, nx, maxfun, model.min_objective_value(), + nsamples=nsamples_to_use) + + # Handle exit conditions (f < min obj value or maxfun reached) + if exit_flag is not None: # then exit_str is also set + if nsamples_run > 0: + fmin = np.min(f_list[:nsamples_run]) + if fmin < model.fsave: + model.xsave = x + model.fsave = fmin + break_main_loop = True + break # quit inner loop over j, then quit main iteration + + if model.npt_so_far < model.npt: # still growing + kmin = model.npt_so_far + logging.debug("Updating point kmin=%g, since still growing" % kmin) + else: # full set + kmin, linalg_error = choose_knew(model, delta, xnew, skip_kopt=True) + + if linalg_error: + exit_flag = EXIT_LINALG_ERROR + exit_str = "Singular matrix when finding kmin (in main loop)" + break_main_loop = True + break # quit inner loop over j, then quit main iteration + logging.debug("Updating point kmin=%g, chosen in usual way" % kmin) + + # Otherwise, add new results, incrementing model.npt_so_far (if still growing) + model.update_point(kmin, xnew, v_err_list[0, :], f_list[0]) + for i in range(1, nsamples_run): + model.add_point_resample(kmin, v_err_list[i, :]) # add new info + + # Finished adding new directions - restart main trust region iteration (if no errors encountered) + if break_main_loop: + finished_main_loop = True + break # quit + else: + finished_main_loop = False + continue # next trust region step + + elif dnorm < 0.5 * rho: + ################### + # Start failed TR step + ################### + logging.debug("Failed trust region step (main phase)") + + if not done_with_current_rho(model, current_iter, last_successful_iter, rho, diffs, xnew, gnew, hq, + crvmin): + # [Fortran label 650] + distsq = (10.0 * rho) ** 2 + model, delta, nf, nx, last_successful_iter, diffs, return_to_new_tr_iteration, geom_exit_flag, geom_exit_str = \ + check_and_fix_geometry(model, objfun, distsq, delta, rho, dnorm, diffs, nf, nx, current_iter, + last_successful_iter, maxfun, nsamples, rounding_error_const, nruns_so_far, update_delta=True) + + if return_to_new_tr_iteration is not None: # i.e. if we did actually fix geometry + if return_to_new_tr_iteration: + finished_main_loop = False + continue # next trust region step + else: # quit + exit_flag = geom_exit_flag + exit_str = geom_exit_str + finished_main_loop = True + break # quit + # If we didn't fix geometry, reduce rho as below + # otherwise, if we are done with current rho, reduce rho as below + + # Reduce rho and continue [Fortran label 680] + if rho > rhoend: + delta, rho = reduce_rho(rho, rhoend) + logging.info("New rho = %g after %i function evaluations" % (rho, nf)) + logging.debug("Best so far: f = %.15g at x = " % (model.fval_opt()) + str(model.xbase + model.xopt())) + last_successful_iter = current_iter + finished_main_loop = False + continue # next trust region step + else: + # Cannot reduce rho, so check xnew and quit + x = model.x_within_bounds(x=xnew) + nsamples_to_use = nsamples(delta, rho, current_iter, nruns_so_far, SCEN_RHOEND_REACHED) + v_err_list, f_list, nf, nx, nsamples_run, exit_flag, exit_str = \ + sample_objective(model.m, objfun, x, nf, nx, maxfun, model.min_objective_value(), + nsamples=nsamples_to_use) + + # Handle exit conditions (f < min obj value or maxfun reached) + if exit_flag is not None: # then exit_str is also set + if nsamples_run > 0: + fmin = np.min(f_list[:nsamples_run]) + if fmin < model.fsave: + model.xsave = x + model.fsave = fmin + finished_main_loop = True + break # quit + + # Force model.get_final_results() to return this new point if it's better than xopt, then quit + model.xsave = x + model.fsave = np.min(f_list[:nsamples_run]) + exit_flag = EXIT_SUCCESS + exit_str = "rho has reached rhoend" + finished_main_loop = True + break # quit + ################### + # End failed TR step + ################### + else: + ################### + # Start successful TR step + ################### + logging.debug("Successful trust region step") + + # Severe cancellation is likely to occur if XOPT is too far from XBASE. [Fortran label 90] + if dsq <= rounding_error_const * sumsq(model.xopt()): + model.shift_base(model.xopt()) # includes a re-factorisation of the interpolation matrix + xnew = xnew - model.xopt() + + # Set KNEW to the index of the next interpolation point to be deleted to make room for a trust + # region step. Again RESCUE may be called if rounding errors have damaged + # the chosen denominator, which is the reason for attempting to select + # KNEW before calculating the next value of the objective function. + knew, linalg_error = choose_knew(model, delta, xnew, skip_kopt=True) + + if linalg_error: + exit_flag = EXIT_LINALG_ERROR + exit_str = "Singular matrix when finding knew (in main loop)" + finished_main_loop = True + break # quit + + # Calculate the value of the objective function at XBASE+XNEW, unless + # the limit on the number of calculations of F has been reached. + # [Fortran label 360, with ntrits > 0] + x = model.x_within_bounds(x=xnew) + + nsamples_to_use = nsamples(delta, rho, current_iter, nruns_so_far, SCEN_TR_STEP) + v_err_list, f_list, nf, nx, nsamples_run, exit_flag, exit_str = \ + sample_objective(model.m, objfun, x, nf, nx, maxfun, model.min_objective_value(), + nsamples=nsamples_to_use) + # Estimate f in order to compute 'actual reduction' + f = sumsq(np.mean(v_err_list[:nsamples_run, :], axis=0)) # estimate actual objective value + + # Handle exit conditions (f < min obj value or maxfun reached) + if exit_flag is not None: # then exit_str is also set + if nsamples_run > 0: + fmin = np.min(f_list[:nsamples_run]) + if fmin < model.fsave: + model.xsave = x + model.fsave = fmin + finished_main_loop = True + break # quit + + # Use the quadratic model to predict the change in F due to the step D, + # and set DIFF to the error of this prediction. + pred_reduction = - calculate_model_value(gopt, hq, d) + actual_reduction = model.fval_opt() - f + diffs = [abs(pred_reduction - actual_reduction), diffs[0], diffs[1]] + + if dnorm > rho: + last_successful_iter = current_iter + + if pred_reduction < 0.0: + exit_flag = EXIT_TR_INCREASE_ERROR + exit_str = "Trust region step gave model increase" + finished_main_loop = True + break # quit + + # Pick the next value of DELTA after a trust region step. + # Update trust region radius + ratio = actual_reduction / pred_reduction + if ratio <= 0.1: + delta = min(0.5 * delta, dnorm) + elif ratio <= 0.7: + delta = max(0.5 * delta, dnorm) + else: # (ratio > 0.7) Different updates depending on which code version we're using + if zhang_code_structure: + delta = min(max(2.0 * delta, 4.0 * dnorm), 1.0e10) # DFBOLS code version + elif bbqtr: + delta = max(0.5 * delta, 2.0 * dnorm) # BOBYQA version + else: + delta = max(delta, 2.0 * dnorm) # Zhang paper version + if delta <= 1.5 * rho: # cap trust region radius at rho + delta = rho + logging.debug("New delta = %g (rho = %g) from ratio %g" % (delta, rho, ratio)) + + # Recalculate KNEW and DENOM if the new F is less than FOPT. + if actual_reduction > 0.0: # f < model.fval_opt() + knew, linalg_error = choose_knew(model, delta, xnew, skip_kopt=False) + + if linalg_error: + exit_flag = EXIT_LINALG_ERROR + exit_str = "Singular matrix when finding knew (in main loop, second time)" + finished_main_loop = True + break # quit + + # Updating... + logging.debug("Updating with knew = %i" % knew) + model.update_point(knew, xnew, v_err_list[0, :], f_list[0]) # increments model.npt_so_far, if still growing + for i in range(1, nsamples_run): + model.add_point_resample(knew, v_err_list[i, :]) # add new info + + # When growing and don't yet have a full set of directions, we always need a geometry improving step + if model.npt_so_far <= model.n + 1: # even after npt function evaluations, still one direction short + dnew_matrix = get_new_orthogonal_directions(model, delta_scale_for_new_dirns_when_growing * delta, + num_steps=num_geom_steps_when_growing) + # breaks below only stop num_geom_steps_when_growing loop, check if need to quit main loop too + break_main_loop = False + for j in range(num_geom_steps_when_growing): + xnew = model.xopt() + dnew_matrix[:, j] + logging.debug("Growing: compulsory geometry improving step xnew = %s" % str(xnew)) + x = model.x_within_bounds(x=xnew) + + nsamples_to_use = nsamples(delta, rho, current_iter, nruns_so_far, SCEN_GROWING_NEW_DIRECTION) + v_err_list, f_list, nf, nx, nsamples_run, exit_flag, exit_str = \ + sample_objective(model.m, objfun, x, nf, nx, maxfun, model.min_objective_value(), + nsamples=nsamples_to_use) + + # Handle exit conditions (f < min obj value or maxfun reached) + if exit_flag is not None: # then exit_str is also set + if nsamples_run > 0: + fmin = np.min(f_list[:nsamples_run]) + if fmin < model.fsave: + model.xsave = x + model.fsave = fmin + finished_main_loop = True + break_main_loop = True + break # quit + + if model.npt_so_far < model.npt: # still growing + kmin = model.npt_so_far + logging.debug("Updating point kmin=%g, since still growing" % kmin) + else: # full set + kmin, linalg_error = choose_knew(model, delta, xnew, skip_kopt=True) + + if linalg_error: + exit_flag = EXIT_LINALG_ERROR + exit_str = "Singular matrix when finding kmin (in main loop)" + finished_main_loop = True + break_main_loop = True + break # quit + logging.debug("Updating point kmin=%g, chosen in usual way" % kmin) + + # Otherwise, add new results, incrementing model.npt_so_far (if still growing) + model.update_point(kmin, xnew, v_err_list[0, :], f_list[0]) + for i in range(1, nsamples_run): + model.add_point_resample(kmin, v_err_list[i, :]) # add new info + + # Finished adding new directions - restart main trust region iteration (if no errors encountered) + if break_main_loop: + finished_main_loop = True + break # quit + + # If a trust region step has provided a sufficient decrease in F, then + # branch for another trust region calculation. + if ratio >= 0.1: + finished_main_loop = False + continue # next trust region step + + # Alternatively, find out if the interpolation points are close enough + # to the best point so far. + # [Fortran label 650] + distsq = max((2.0 * delta) ** 2, (10.0 * rho) ** 2) + model, delta, nf, nx, last_successful_iter, diffs, return_to_new_tr_iteration, geom_exit_flag, geom_exit_str = \ + check_and_fix_geometry(model, objfun, distsq, delta, rho, dnorm, diffs, nf, nx, current_iter, + last_successful_iter, maxfun, nsamples, rounding_error_const, nruns_so_far, + update_delta=False) # don't update delta when ntrits > 0 + + if return_to_new_tr_iteration is not None: # i.e. if we did actually fix geometry + if return_to_new_tr_iteration: + finished_main_loop = False + continue # next trust region step + else: # quit + exit_flag = geom_exit_flag + exit_str = geom_exit_str + finished_main_loop = True + break # quit + # If we didn't fix geometry, reduce rho [Fortran label 680] + + if ratio > 0.0: + finished_main_loop = False + continue # next trust region step + + if max(delta, dnorm) > rho: + finished_main_loop = False + continue # next trust region step + + # Reduce rho and continue [Fortran label 680] + if rho > rhoend: + delta, rho = reduce_rho(rho, rhoend) + logging.info("New rho = %g after %i function evaluations" % (rho, nf)) + logging.debug("Best so far: f = %.15g at x = " % (model.fval_opt()) + str(model.xbase + model.xopt())) + last_successful_iter = current_iter + finished_main_loop = False + continue # next trust region step + else: + # Cannot reduce rho further + exit_flag = EXIT_SUCCESS + exit_str = "rho has reached rhoend" + finished_main_loop = True + break # quit + ################### + # End successful TR step + ################### + ############################# + # End this iteration of main loop - take next TR step + ############################# + ########################################################### + # End of main loop [Fortran label 720] + ########################################################### + + x, f = model.get_final_results() + logging.debug("At return from DFOGN, number of function evals = %i" % nf) + logging.debug("Smallest objective value = %.15g at x = " % f + str(x)) + return x, f, nf, nx, exit_flag, exit_str, model.m + + +def dfogn_resampling(objfun, x0, lower=None, upper=None, maxfun=1000, nsamples=None, init_tr_radius=None, rhoend=1e-8, + delta_scale_for_new_dirns_when_growing=1.0, use_random_initial_directions=False, + ndirs_initial='n', num_geom_steps_when_growing=1, use_restarts=True, + max_unsuccessful_restarts=10): + # If bounds not provided, set to something large + xl = (lower if lower is not None else -1.0e20 * np.ones(x0.shape)) + xu = (upper if upper is not None else 1.0e20 * np.ones(x0.shape)) + + # Set default value of rhobeg to something sensible + rhobeg = (init_tr_radius if init_tr_radius is not None else 0.1 * max(np.max(np.abs(x0)), 1.0)) + + # Set default number of samples to be 1 for every evaluation + if nsamples is None: + nsamples_to_use = lambda delta, rho, iter, nruns_so_far, scenario : 1 + else: + nsamples_to_use = nsamples + + n = np.size(x0) + assert (rhobeg > 0.0), "rhobeg must be strictly positive" + assert (rhoend > 0.0), "rhoend must be strictly positive" + assert (rhoend < rhobeg), "rhoend must be less than rhobeg" + assert (maxfun > 0), "maxfun must be strictly positive" + assert (np.shape(x0) == (n,)), "x0 must be a vector" + assert (np.shape(x0) == np.shape(xl)), "xl must have same shape as x0" + assert (np.shape(x0) == np.shape(x0)), "xu must have same shape as x0" + assert (np.all(xu-xl >= 2.0*rhobeg)), "gap between xl and xu must be at least 2*rhobeg" + + if maxfun <= n+1: + logging.warning("Warning (maxfun <= n+1): Are you sure your budget is large enough?") + + # Parse string arguments: number of geometry steps to take at each growing iteration of main TR loop + n_extra_steps_to_use = None + if type(num_geom_steps_when_growing) == int: + n_extra_steps_to_use = num_geom_steps_when_growing + elif type(num_geom_steps_when_growing) == str: + if num_geom_steps_when_growing == 'tenthn': + n_extra_steps_to_use = int(x0.size // 10) + elif num_geom_steps_when_growing == 'fifthn': + n_extra_steps_to_use = int(x0.size // 5) + elif num_geom_steps_when_growing == 'qtrn': + n_extra_steps_to_use = int(x0.size // 4) + + assert n_extra_steps_to_use is not None, "Unknown num_geom_steps_when_growing: " + str( + num_geom_steps_when_growing) + n_extra_steps_to_use = max(n_extra_steps_to_use, 1) # floor at 1 + + # Parse string arguments: number of initial directions to add before beginning main TR loop + ndirs_initial_val = None + if type(ndirs_initial) == int: + ndirs_initial_val = ndirs_initial + elif type(ndirs_initial) == str: + if ndirs_initial == 'tenthn': + ndirs_initial_val = int(n // 10) + elif ndirs_initial == 'fifthn': + ndirs_initial_val = int(n // 5) + elif ndirs_initial == 'qtrn': + ndirs_initial_val = int(n // 4) + elif ndirs_initial == 'halfn': + ndirs_initial_val = int(n // 2) + elif ndirs_initial == 'n': + ndirs_initial_val = n + elif ndirs_initial == '2n': + ndirs_initial_val = 2 * n + elif ndirs_initial == 'nsq': + ndirs_initial_val = (n + 1) * (n + 2) // 2 - 1 + + assert ndirs_initial_val is not None, "Unknown ndirs_initial: " + str(ndirs_initial) + assert ndirs_initial_val == n, "Must have n initial directions (build_interp_matrix assumes this)" + ndirs_initial_val = max(ndirs_initial_val, 1) # floor at 1 + + # Enforce lower bounds on x0 (ideally with gap of at least rhobeg) + idx = (xl < x0) & (x0 <= xl+rhobeg) + x0[idx] = xl[idx] + rhobeg + + idx = (x0 <= xl) + x0[idx] = xl[idx] + + # Enforce upper bounds on x0 (ideally with gap of at least rhobeg) + idx = (xu-rhobeg <= x0) & (x0 < xu) + x0[idx] = xu[idx] - rhobeg + + idx = (x0 >= xu) + x0[idx] = xu[idx] + + # First run + x, f, nf, nx, exit_flag, exit_str, m = \ + dfogn_main(objfun, x0, xl, xu, rhobeg, rhoend, maxfun, nsamples_to_use, m=None, + delta_scale_for_new_dirns_when_growing=delta_scale_for_new_dirns_when_growing, + use_random_initial_directions=use_random_initial_directions, ndirs_initial=ndirs_initial_val, + num_geom_steps_when_growing=n_extra_steps_to_use, + nf_so_far=0, nx_so_far=0, nruns_so_far=0) + + # Now do repeats + nruns_so_far = 1 + reduction_last_run = True # did the last run give us a reduction? + rhobeg_to_use = rhobeg + rhoend_to_use = rhoend + last_successful_run = 1 + + while use_restarts and nf < maxfun and nruns_so_far - last_successful_run < max_unsuccessful_restarts and \ + ((exit_flag == EXIT_SUCCESS and 'rho' in exit_str) or exit_flag in DO_RESTART_ERRORS): + if reduction_last_run: + rhobeg_to_use = max(0.1 * max(np.max(np.abs(x)), 1.0), 10 * rhoend_to_use) + rhoend_to_use = 1.0 * rhoend_to_use + else: + # Reduce initial TR radius when things have been going badly + rhobeg_to_use = max(0.5 * rhobeg_to_use, 10 * rhoend_to_use) + logging.info( + "Restarting from finish point (f = %g) after %g function evals; new rhobeg = %g and rhoend = %g" % ( + f, nf, rhobeg_to_use, rhoend_to_use)) + x2, f2, nf, nx, exit_flag, exit_str, m_tmp = \ + dfogn_main(objfun, x, xl, xu, rhobeg_to_use, rhoend_to_use, maxfun, nsamples_to_use, m=m, + delta_scale_for_new_dirns_when_growing=delta_scale_for_new_dirns_when_growing, + use_random_initial_directions=use_random_initial_directions, ndirs_initial=ndirs_initial_val, + num_geom_steps_when_growing=n_extra_steps_to_use, + nf_so_far=nf, nx_so_far=nx, nruns_so_far=nruns_so_far) + + nruns_so_far += 1 + if f2 < f or np.isnan(f): + logging.info("Successful run with new f = %s compared to old f = %s" % (f2, f)) + last_successful_run = nruns_so_far + x = x2 + f = f2 + reduction_last_run = True + else: + logging.info("Unsuccessful run with new f = %s compared to old f = %s" % (f2, f)) + reduction_last_run = False + + logging.info("Finished after a total of %g runs" % nruns_so_far) + + # Clean up exit_str to have better information: + if exit_flag == EXIT_SUCCESS: + exit_str = "Success: " + exit_str + elif exit_flag == EXIT_MAXFUN_WARNING: + exit_str = "Warning: " + exit_str + elif exit_flag == EXIT_INPUT_ERROR: + exit_str = "Input error: " + exit_str + elif exit_flag == EXIT_TR_INCREASE_ERROR: + exit_str = "Trust region subproblem error: " + exit_str + elif exit_flag == EXIT_LINALG_ERROR: + exit_str = "Linear algebra error: " + exit_str + elif exit_flag == EXIT_ALTMOV_MEMORY_ERROR: + exit_str = "ALTMOV memory error: " + exit_str + else: + exit_str = "Unknown exit flag " + str(exit_flag) + " with message " + exit_str + + return x, f, nf, exit_flag, exit_str diff --git a/dfogn/linear_altmov.py b/dfogn/linear_altmov.py new file mode 100644 index 0000000..ef5eab4 --- /dev/null +++ b/dfogn/linear_altmov.py @@ -0,0 +1,154 @@ +""" +linear_altmov +================== +A geometry-improving step calculation (assuming linear models). + +Here, ALTMOV needs to maximise |l(y)| for linear Lagrange polynomial l(y) +in a box intersect ball. We can solve this exactly via an active set approach. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +The development of this software was sponsored by NAG Ltd. (http://www.nag.co.uk) +and the EPSRC Centre For Doctoral Training in Industrially Focused Mathematical +Modelling (EP/L015803/1) at the University of Oxford. Please contact NAG for +alternative licensing. + +Copyright 2017, Lindon Roberts + +""" +# Ensure compatibility with Python 2 +from __future__ import absolute_import, division, print_function, unicode_literals + +from math import sqrt +import numpy as np + + +__all__ = ["max_step_in_box_and_ball"] + +ZERO_THRESH = 1e-14 + + +def ball_step(x0, g, Delta): + # Given initial point x0, take largest step in direction g allowed by ||x|| <= Delta + # That is, solve + # ||x0 + alpha*g||^2 = Delta^2, alpha >= 0 + # Using this method, solution exists whenever ||x0|| <= Delta^2 [take alpha=0 if g=0] + gdotx0 = np.dot(g, x0) + gsqnorm = np.dot(g, g) + x0sqnorm = np.dot(x0, x0) + if sqrt(gsqnorm) < ZERO_THRESH: # Error catching: if g=0, make no step + return 0.0 + return (sqrt(gdotx0**2 + gsqnorm*(Delta**2 - x0sqnorm)) - gdotx0) / gsqnorm + + +def min_linear_in_box_and_ball(g, a, b, Delta): + # Solve the convex program: + # min_x g' * x + # s.t. a <= x <= b + # ||x||^2 <= Delta^2 + # using an active-set type approach + + n = g.size + x = np.zeros((n,)) + dirn = -g + cons_dirns = [] + + # If g[i] = 0, never step along this direction + constant_directions = np.where(np.abs(dirn) < ZERO_THRESH)[0] + dirn[constant_directions] = 0.0 + cons_dirns += list(constant_directions) + + for i in range(n): + if np.linalg.norm(dirn) < ZERO_THRESH: + return x + alpha_unc = ball_step(x, dirn, Delta) + xnew = x + alpha_unc * dirn + # Check if hit box bounds + on_box_bdry = False + hit_upper = None + idx_hit = None + for j in range(n): + if j in cons_dirns: + continue # only looking at unconstrained directions + if xnew[j] <= a[j]: + on_box_bdry = True + hit_upper = False + idx_hit = j + break + elif xnew[j] >= b[j]: + on_box_bdry = True + hit_upper = True + idx_hit = j + break + + if not on_box_bdry: + return xnew # unconstrained solution + else: + # Go as far as possible until hit box, then remove that direction from 'dirn' + cons_dirns.append(idx_hit) # new constrained direction + alpha_con = ((b[idx_hit] if hit_upper else a[idx_hit]) - x[idx_hit]) / dirn[idx_hit] + x = x + alpha_con * dirn + x[idx_hit] = b[idx_hit] if hit_upper else a[idx_hit] # force boundary exactly + dirn[idx_hit] = 0.0 # no more searching this direction + return x + + +def max_linear_in_box_and_ball(g, a, b, Delta): + # Solve the convex program: + # max_x g' * x + # s.t. a <= x <= b + # ||x||^2 <= Delta^2 + return min_linear_in_box_and_ball(-g, a, b, Delta) + + +def max_abs_linear_in_box_and_ball(g, a, b, Delta, c=0): + # Solve the program: + # max_x abs(c + g' * x) + # s.t. a <= x <= b + # ||x||^2 <= Delta^2 + # by maximising and minimising (g' * x) separately + xmin = min_linear_in_box_and_ball(g, a, b, Delta) + xmax = max_linear_in_box_and_ball(g, a, b, Delta) + + if abs(c + np.dot(g, xmin)) >= abs(c + np.dot(g, xmax)): + return xmin + else: + return xmax + + +def max_step_in_box_and_ball(xbase, c, g, lower, upper, Delta): + # Consider the problem of maximising the step. That is, solve + # min_x abs(c + g' * (x - xbase)) + # s.t. lower <= x <= upper + # ||x-xbase|| <= Delta + # Setting s = x-xbase (or x = xbase + s), this is equivalent to: + # min_s abs(c + g' * s) + # s.t. lower - xbase <= s <= upper - xbase + # ||s|| <= Delta + s = max_abs_linear_in_box_and_ball(g, lower-xbase, upper-xbase, Delta, c=c) + return xbase + s + +if __name__ == '__main__': + g = np.array([1.0, -1.0]) + a = np.array([-2.0, -2.0]) + b = np.array([1.0, 2.0]) + delta = 2.0 + c = -1.0 + xmin = min_linear_in_box_and_ball(g, a, b, delta) + xmax = max_linear_in_box_and_ball(g, a, b, delta) + xabs = max_abs_linear_in_box_and_ball(g, a, b, delta, c=c) + print("Min: x = %s, c + g'*x = %g" % (str(xmin), c + np.dot(g, xmin))) + print("Max: x = %s, c + g'*x = %g" % (str(xmax), c + np.dot(g, xmax))) + print("Max abs: take x = %s, abs(g'*x) = %g" % (str(xabs), abs(c + np.dot(g, xabs)))) + print("Done") diff --git a/dfogn/tests/test_dfogn.py b/dfogn/tests/test_dfogn.py new file mode 100644 index 0000000..e472637 --- /dev/null +++ b/dfogn/tests/test_dfogn.py @@ -0,0 +1,105 @@ +""" + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +The development of this software was sponsored by NAG Ltd. (http://www.nag.co.uk) +and the EPSRC Centre For Doctoral Training in Industrially Focused Mathematical +Modelling (EP/L015803/1) at the University of Oxford. Please contact NAG for +alternative licensing. + +Copyright 2017, Lindon Roberts + +""" + +# Ensure compatibility with Python 2 +from __future__ import absolute_import, division, print_function, unicode_literals + +import numpy as np +import unittest + +import dfogn + + +def array_compare(x, y, thresh=1e-14): + return np.max(np.abs(x - y)) < thresh + + +def rosenbrock(x): + return np.array([10.0 * (x[1] - x[0] ** 2), 1.0 - x[0]]) + +def rosenbrock_jacobian(x): + return np.array([[-20.0*x[0], 10.0], [-1.0, 0.0]]) + +class TestRosenbrockGeneric(unittest.TestCase): + # Minimise the (2d) Rosenbrock function + def runTest(self): + # n, m = 2, 2 + x0 = np.array([-1.2, 1.0]) + soln = dfogn.solve(rosenbrock, x0) + self.assertTrue(array_compare(soln.x, np.array([1.0, 1.0]), thresh=1e-4), "Wrong xmin") + self.assertTrue(array_compare(soln.resid, rosenbrock(soln.x), thresh=1e-10), "Wrong resid") + self.assertTrue(array_compare(soln.jacobian, rosenbrock_jacobian(soln.x), thresh=1e-2), "Wrong Jacobian") + self.assertTrue(abs(soln.f) < 1e-10, "Wrong fmin") + + +class TestRosenbrockBounds(unittest.TestCase): + # Minimise the (2d) Rosenbrock function, where x[1] hits the upper bound + def runTest(self): + # n, m = 2, 2 + x0 = np.array([-1.2, 0.7]) # standard start point does not satisfy bounds + lower = np.array([-2.0, -2.0]) + upper = np.array([1.0, 0.9]) + xmin = np.array([0.9486, 0.9]) # approximate + fmin = np.dot(rosenbrock(xmin), rosenbrock(xmin)) + soln = dfogn.solve(rosenbrock, x0, lower=lower, upper=upper) + print(soln.x) + self.assertTrue(array_compare(soln.x, xmin, thresh=1e-2), "Wrong xmin") + self.assertTrue(array_compare(soln.resid, rosenbrock(soln.x), thresh=1e-10), "Wrong resid") + self.assertTrue(array_compare(soln.jacobian, rosenbrock_jacobian(soln.x), thresh=1e-2), "Wrong Jacobian") + self.assertTrue(abs(soln.f - fmin) < 1e-4, "Wrong fmin") + + +class TestRosenbrockBounds2(unittest.TestCase): + # Minimise the (2d) Rosenbrock function, where x[0] hits upper bound + def runTest(self): + # n, m = 2, 2 + x0 = np.array([-1.2, 0.7]) # standard start point too close to upper bounds + lower = np.array([-2.0, -2.0]) + upper = np.array([0.9, 0.9]) + xmin = np.array([0.9, 0.81]) # approximate + fmin = np.dot(rosenbrock(xmin), rosenbrock(xmin)) + soln = dfogn.solve(rosenbrock, x0, lower=lower, upper=upper) + print(soln.x) + self.assertTrue(array_compare(soln.x, xmin, thresh=1e-2), "Wrong xmin") + self.assertTrue(array_compare(soln.resid, rosenbrock(soln.x), thresh=1e-10), "Wrong resid") + self.assertTrue(array_compare(soln.jacobian, rosenbrock_jacobian(soln.x), thresh=1e-2), "Wrong Jacobian") + self.assertTrue(abs(soln.f - fmin) < 1e-4, "Wrong fmin") + + +class TestLinear(unittest.TestCase): + # Solve min_x ||Ax-b||^2, for some random A and b + def runTest(self): + n, m = 2, 5 + np.random.seed(0) # (fixing random seed) + A = np.random.rand(m, n) + b = np.random.rand(m) + objfun = lambda x: np.dot(A, x) - b + xmin = np.linalg.lstsq(A, b)[0] + fmin = np.dot(objfun(xmin), objfun(xmin)) + x0 = np.zeros((n,)) + soln = dfogn.solve(objfun, x0) + self.assertTrue(array_compare(soln.x, xmin, thresh=1e-2), "Wrong xmin") + self.assertTrue(array_compare(soln.resid, objfun(soln.x), thresh=1e-10), "Wrong resid") + self.assertTrue(array_compare(soln.jacobian, A, thresh=1e-2), "Wrong Jacobian") + self.assertTrue(abs(soln.f - fmin) < 1e-4, "Wrong fmin") diff --git a/dfogn/tests/test_linear_altmov.py b/dfogn/tests/test_linear_altmov.py new file mode 100644 index 0000000..eee6876 --- /dev/null +++ b/dfogn/tests/test_linear_altmov.py @@ -0,0 +1,181 @@ +""" + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +The development of this software was sponsored by NAG Ltd. (http://www.nag.co.uk) +and the EPSRC Centre For Doctoral Training in Industrially Focused Mathematical +Modelling (EP/L015803/1) at the University of Oxford. Please contact NAG for +alternative licensing. + +""" + +# Ensure compatibility with Python 2 +from __future__ import absolute_import, division, print_function, unicode_literals + +from math import sqrt +import numpy as np +import unittest + +from dfogn.linear_altmov import max_step_in_box_and_ball +from dfogn.util import calculate_model_value + + +def cauchy_pt(g, hess, delta): + # General expression for the Cauchy point + crv = np.dot(g, hess.vec_mul(g)) + gnorm = np.linalg.norm(g) + if crv <= 0.0: + alpha = delta / gnorm + else: + alpha = min(delta / gnorm, gnorm**2 / crv) + s = -alpha * g + red = calculate_model_value(g, hess, s) + crvmin = np.dot(s, hess.vec_mul(s)) / np.dot(s, s) + if crvmin < 0.0: + crvmin = -1.0 + return s, red, crvmin + + +def cauchy_pt_box(g, hess, delta, lower, upper): + # General expression for the Cauchy point, lower <= s <= upper + crv = np.dot(g, hess.vec_mul(g)) + gnorm = np.linalg.norm(g) + if crv <= 0.0: + alpha = delta / gnorm + else: + alpha = min(delta / gnorm, gnorm**2 / crv) + # print("alpha = %g" % alpha) + # Then cap with bounds: + for i in range(len(g)): + if g[i] > 0: # s[i] negative, will hit lower + alpha = min(alpha, -lower[i] / g[i]) + elif g[i] < 0: # s[i] positive, will hit upper + alpha = min(alpha, -upper[i] / g[i]) + # print("alpha = %g after i=%g" % (alpha, i)) + s = -alpha * g + red = calculate_model_value(g, hess, s) + crvmin = np.dot(s, hess.vec_mul(s)) / np.dot(s, s) + if crvmin < 0.0: + crvmin = -1.0 + return s, red, crvmin + + +class TestGeom(unittest.TestCase): + def runTest(self): + xbase = np.array([0.0, 0.0]) + g = np.array([1.0, -1.0]) + a = np.array([-2.0, -2.0]) + b = np.array([1.0, 2.0]) + delta = 2.0 + c = -1.0 + x = max_step_in_box_and_ball(xbase, c, g, a, b, delta) + xtrue = np.array([-sqrt(2.0), sqrt(2.0)]) + self.assertTrue(np.max(np.abs(x - xtrue)) < 1e-10, 'Wrong step') + + +class TestGeom2(unittest.TestCase): + def runTest(self): + xbase = np.array([0.0, 0.0]) + g = np.array([1.0, -1.0]) + a = np.array([-2.0, -2.0]) + b = np.array([1.0, 2.0]) + delta = 5.0 + c = -1.0 + x = max_step_in_box_and_ball(xbase, c, g, a, b, delta) + xtrue = np.array([-2.0, 2.0]) + self.assertTrue(np.max(np.abs(x - xtrue)) < 1e-10, 'Wrong step') + + +class TestGeom3(unittest.TestCase): + def runTest(self): + xbase = np.array([0.0, 0.0]) + 1 + g = np.array([1.0, -1.0]) + a = np.array([-2.0, -2.0]) + 1 + b = np.array([1.0, 2.0]) + 1 + delta = 5.0 + c = 3.0 # may want to max instead + x = max_step_in_box_and_ball(xbase, c, g, a, b, delta) + xtrue = np.array([1.0, -2.0]) + 1 + self.assertTrue(np.max(np.abs(x - xtrue)) < 1e-10, 'Wrong step') + + +class TestGeomOldBug(unittest.TestCase): + def runTest(self): + xbase = np.array([0.0, 0.0]) + g = np.array([-1.0, -1.0]) + a = np.array([-2.0, -2.0]) + b = np.array([0.1, 0.9]) + delta = sqrt(2.0) + c = -1.0 # may want to max instead + x = max_step_in_box_and_ball(xbase, c, g, a, b, delta) + xtrue = b + print(x) + self.assertTrue(np.max(np.abs(x - xtrue)) < 1e-10, 'Wrong step') + # self.assertFalse(True, "bad") + + +class TestGeomOldBug2(unittest.TestCase): + def runTest(self): + xbase = np.array([0.0, 0.0, 0.0]) + g = np.array([-1.0, -1.0, -1.0]) + a = np.array([-2.0, -2.0, -2.0]) + b = np.array([0.9, 0.1, 5.0]) + delta = sqrt(3.0) + c = -1.0 # may want to max instead + x = max_step_in_box_and_ball(xbase, c, g, a, b, delta) + xtrue = np.array([0.9, 0.1, sqrt(3.0 - 0.81 - 0.01)]) + print(x) + self.assertTrue(np.max(np.abs(x - xtrue)) < 1e-10, 'Wrong step') + # self.assertFalse(True, "bad") + + +class TestGeom2WithZeros(unittest.TestCase): + def runTest(self): + xbase = np.array([0.0, 0.0]) + g = np.array([0.0, -1.0]) + a = np.array([-2.0, -2.0]) + b = np.array([1.0, 2.0]) + delta = 5.0 + c = 0.0 + x = max_step_in_box_and_ball(xbase, c, g, a, b, delta) + xtrue = np.array([0.0, 2.0]) + self.assertTrue(np.max(np.abs(x - xtrue)) < 1e-10, 'Wrong step') + + +class TestGeom2WithAlmostZeros(unittest.TestCase): + def runTest(self): + xbase = np.array([0.0, 0.0]) + g = np.array([1e-15, -1.0]) + a = np.array([-2.0, -2.0]) + b = np.array([1.0, 2.0]) + delta = 5.0 + c = 0.0 + x = max_step_in_box_and_ball(xbase, c, g, a, b, delta) + xtrue = np.array([0.0, 2.0]) + self.assertTrue(np.max(np.abs(x - xtrue)) < 1e-10, 'Wrong step') + + +class TestGeom2WithAlmostZeros2(unittest.TestCase): + def runTest(self): + xbase = np.array([0.0, 0.0]) + g = np.array([1e-15, 0.0]) + a = np.array([-2.0, -2.0]) + b = np.array([1.0, 2.0]) + delta = 5.0 + c = 0.0 + x = max_step_in_box_and_ball(xbase, c, g, a, b, delta) + xtrue = np.array([0.0, 0.0]) + # print(x) + # print(xtrue) + self.assertTrue(np.max(np.abs(x - xtrue)) < 1e-10, 'Wrong step') diff --git a/dfogn/tests/test_trust_region.py b/dfogn/tests/test_trust_region.py new file mode 100644 index 0000000..f2f2d60 --- /dev/null +++ b/dfogn/tests/test_trust_region.py @@ -0,0 +1,237 @@ +""" + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +The development of this software was sponsored by NAG Ltd. (http://www.nag.co.uk) +and the EPSRC Centre For Doctoral Training in Industrially Focused Mathematical +Modelling (EP/L015803/1) at the University of Oxford. Please contact NAG for +alternative licensing. + +""" + +# Ensure compatibility with Python 2 +from __future__ import absolute_import, division, print_function, unicode_literals + +from math import sqrt +import numpy as np +import unittest + +from dfogn.trust_region import trsbox +from dfogn.util import calculate_model_value, to_upper_triangular_vector, right_multiply_hessian + + +def cauchy_pt(g, hess, delta): + # General expression for the Cauchy point + crv = np.dot(g, right_multiply_hessian(hess, g)) + gnorm = np.linalg.norm(g) + if crv <= 0.0: + alpha = delta / gnorm + else: + alpha = min(delta / gnorm, gnorm**2 / crv) + s = -alpha * g + red = calculate_model_value(g, hess, s) + crvmin = np.dot(s, right_multiply_hessian(hess, s)) / np.dot(s, s) + if crvmin < 0.0: + crvmin = -1.0 + return s, red, crvmin + + +def cauchy_pt_box(g, hess, delta, lower, upper): + # General expression for the Cauchy point, lower <= s <= upper + crv = np.dot(g, right_multiply_hessian(hess, g)) + gnorm = np.linalg.norm(g) + if crv <= 0.0: + alpha = delta / gnorm + else: + alpha = min(delta / gnorm, gnorm**2 / crv) + # print("alpha = %g" % alpha) + # Then cap with bounds: + for i in range(len(g)): + if g[i] > 0: # s[i] negative, will hit lower + alpha = min(alpha, -lower[i] / g[i]) + elif g[i] < 0: # s[i] positive, will hit upper + alpha = min(alpha, -upper[i] / g[i]) + # print("alpha = %g after i=%g" % (alpha, i)) + s = -alpha * g + red = calculate_model_value(g, hess, s) + crvmin = np.dot(s, right_multiply_hessian(hess, s)) / np.dot(s, s) + if crvmin < 0.0: + crvmin = -1.0 + return s, red, crvmin + + +class TestUncInternal(unittest.TestCase): + def runTest(self): + n = 3 + g = np.array([1.0, 0.0, 1.0]) + H = np.array([[1.0, 0.0, 0.0], [0.0, 2.0, 0.0], [0.0, 0.0, 2.0]]) + Delta = 2.0 + hess = to_upper_triangular_vector(H) + xopt = np.ones((n,)) # trying nonzero (since bounds inactive) + sl = -1e20 * np.ones((n,)) + su = 1e20 * np.ones((n,)) + d, gnew, crvmin = trsbox(xopt, g, hess, sl, su, Delta) + true_d = np.array([-1.0, 0.0, -0.5]) + est_min = calculate_model_value(g, hess, d) + true_min = calculate_model_value(g, hess, true_d) + # Hope to get actual correct answer for internal minimum? + # self.assertTrue(np.all(d == true_d), 'Wrong answer') + # self.assertAlmostEqual(est_min, true_min, 'Wrong min value') + s_cauchy, red_cauchy, crvmin_cauchy = cauchy_pt(g, hess, Delta) + self.assertTrue(est_min <= red_cauchy, 'Cauchy reduction not achieved') + self.assertTrue(np.all(gnew == g + right_multiply_hessian(hess, d)), 'Wrong gnew') + self.assertAlmostEqual(crvmin, -1.0, 'Wrong crvmin') + + +class TestUncBdry(unittest.TestCase): + def runTest(self): + n = 3 + g = np.array([1.0, 0.0, 1.0]) + H = np.array([[1.0, 0.0, 0.0], [0.0, 2.0, 0.0], [0.0, 0.0, 2.0]]) + Delta = 5.0 / 12.0 + hess = to_upper_triangular_vector(H) + xopt = np.zeros((n,)) + sl = -1e20 * np.ones((n,)) + su = 1e20 * np.ones((n,)) + d, gnew, crvmin = trsbox(xopt, g, hess, sl, su, Delta) + true_d = np.array([-1.0 / 3.0, 0.0, -0.25]) + est_min = calculate_model_value(g, hess, d) + true_min = calculate_model_value(g, hess, true_d) + # Hope to get actual correct answer + # self.assertTrue(np.all(d == true_d), 'Wrong answer') + # self.assertAlmostEqual(est_min, true_min, 'Wrong min value') + s_cauchy, red_cauchy, crvmin_cauchy = cauchy_pt(g, hess, Delta) + self.assertTrue(est_min <= red_cauchy, 'Cauchy reduction not achieved') + self.assertTrue(np.all(gnew == g + right_multiply_hessian(hess, d)), 'Wrong gnew') + self.assertAlmostEqual(crvmin, 0.0, 'Wrong crvmin') + + +class TestUncBdry2(unittest.TestCase): + def runTest(self): + n = 3 + g = np.array([1.0, 0.0, 1.0]) + H = np.array([[-2.0, 0.0, 0.0], [0.0, -1.0, 0.0], [0.0, 0.0, -1.0]]) + Delta = 5.0 / 12.0 + hess = to_upper_triangular_vector(H) + xopt = np.zeros((n,)) + sl = -1e20 * np.ones((n,)) + su = 1e20 * np.ones((n,)) + d, gnew, crvmin = trsbox(xopt, g, hess, sl, su, Delta) + true_d = np.array([-1.0 / 3.0, 0.0, -0.25]) + est_min = calculate_model_value(g, hess, d) + true_min = calculate_model_value(g, hess, true_d) + # Hope to get actual correct answer + # self.assertTrue(np.all(d == true_d), 'Wrong answer') + # self.assertAlmostEqual(est_min, true_min, 'Wrong min value') + s_cauchy, red_cauchy, crvmin_cauchy = cauchy_pt(g, hess, Delta) + self.assertTrue(est_min <= red_cauchy, 'Cauchy reduction not achieved') + self.assertTrue(np.all(gnew == g + right_multiply_hessian(hess, d)), 'Wrong gnew') + self.assertAlmostEqual(crvmin, 0.0, 'Wrong crvmin') + + +class TestUncBdry3(unittest.TestCase): + def runTest(self): + n = 3 + g = np.array([0.0, 0.0, 1.0]) + H = np.array([[-2.0, 0.0, 0.0], [0.0, -1.0, 0.0], [0.0, 0.0, -1.0]]) + Delta = 0.5 + hess = to_upper_triangular_vector(H) + xopt = np.zeros((n,)) + sl = -1e20 * np.ones((n,)) + su = 1e20 * np.ones((n,)) + d, gnew, crvmin = trsbox(xopt, g, hess, sl, su, Delta) + true_d = np.array([0.0, 0.0, -0.5]) + est_min = calculate_model_value(g, hess, d) + true_min = calculate_model_value(g, hess, true_d) + # Hope to get actual correct answer + # self.assertTrue(np.all(d == true_d), 'Wrong answer') + # self.assertAlmostEqual(est_min, true_min, 'Wrong min value') + s_cauchy, red_cauchy, crvmin_cauchy = cauchy_pt(g, hess, Delta) + self.assertTrue(est_min <= red_cauchy, 'Cauchy reduction not achieved') + self.assertTrue(np.all(gnew == g + right_multiply_hessian(hess, d)), 'Wrong gnew') + self.assertAlmostEqual(crvmin, 0.0, 'Wrong crvmin') + # self.assertAlmostEqual(crvmin, crvmin_cauchy, 'Wrong crvmin') + + +class TestUncHard(unittest.TestCase): + def runTest(self): + n = 3 + g = np.array([0.0, 0.0, 1.0]) + H = np.array([[-2.0, 0.0, 0.0], [0.0, -1.0, 0.0], [0.0, 0.0, -1.0]]) + Delta = sqrt(2.0) + hess = to_upper_triangular_vector(H) + xopt = np.zeros((n,)) + sl = -1e20 * np.ones((n,)) + su = 1e20 * np.ones((n,)) + d, gnew, crvmin = trsbox(xopt, g, hess, sl, su, Delta) + true_d = np.array([1.0, 0.0, -1.0]) # non-unique solution + est_min = calculate_model_value(g, hess, d) + true_min = calculate_model_value(g, hess, true_d) + # Hope to get actual correct answer + # self.assertTrue(np.all(d == true_d), 'Wrong answer') + # self.assertAlmostEqual(est_min, true_min, 'Wrong min value') + s_cauchy, red_cauchy, crvmin_cauchy = cauchy_pt(g, hess, Delta) + self.assertTrue(est_min <= red_cauchy, 'Cauchy reduction not achieved') + self.assertTrue(np.all(gnew == g + right_multiply_hessian(hess, d)), 'Wrong gnew') + self.assertAlmostEqual(crvmin, 0.0, 'Wrong crvmin') + + +class TestConInternal(unittest.TestCase): + def runTest(self): + n = 3 + g = np.array([1.0, 0.0, 1.0]) + H = np.array([[1.0, 0.0, 0.0], [0.0, 2.0, 0.0], [0.0, 0.0, 2.0]]) + Delta = 2.0 + hess = to_upper_triangular_vector(H) + xopt = np.ones((n,)) # trying nonzero (since bounds inactive) + sl = xopt + np.array([-0.5, -10.0, -10.0]) + su = xopt + np.array([10.0, 10.0, 10.0]) + d, gnew, crvmin = trsbox(xopt, g, hess, sl, su, Delta) + true_d = np.array([-1.0, 0.0, -0.5]) + est_min = calculate_model_value(g, hess, d) + true_min = calculate_model_value(g, hess, true_d) + # Hope to get actual correct answer for internal minimum? + # self.assertTrue(np.all(d == true_d), 'Wrong answer') + # self.assertAlmostEqual(est_min, true_min, 'Wrong min value') + s_cauchy, red_cauchy, crvmin_cauchy = cauchy_pt_box(g, hess, Delta, sl-xopt, su-xopt) + # print(s_cauchy) + # print(d) + self.assertTrue(est_min <= red_cauchy, 'Cauchy reduction not achieved') + self.assertTrue(np.all(gnew == g + right_multiply_hessian(hess, d)), 'Wrong gnew') + self.assertAlmostEqual(crvmin, 1.5, 'Wrong crvmin') + + +class TestConBdry(unittest.TestCase): + def runTest(self): + n = 3 + g = np.array([1.0, 0.0, 1.0]) + H = np.array([[1.0, 0.0, 0.0], [0.0, 2.0, 0.0], [0.0, 0.0, 2.0]]) + Delta = 5.0 / 12.0 + hess = to_upper_triangular_vector(H) + xopt = np.zeros((n,)) + sl = xopt + np.array([-0.3, -0.01, -0.1]) + su = xopt + np.array([10.0, 1.0, 10.0]) + d, gnew, crvmin = trsbox(xopt, g, hess, sl, su, Delta) + true_d = np.array([-1.0 / 3.0, 0.0, -0.25]) + est_min = calculate_model_value(g, hess, d) + true_min = calculate_model_value(g, hess, true_d) + # Hope to get actual correct answer + # self.assertTrue(np.all(d == true_d), 'Wrong answer') + # self.assertAlmostEqual(est_min, true_min, 'Wrong min value') + s_cauchy, red_cauchy, crvmin_cauchy = cauchy_pt_box(g, hess, Delta, sl - xopt, su - xopt) + self.assertTrue(est_min <= red_cauchy, 'Cauchy reduction not achieved') + self.assertTrue(np.max(np.abs(gnew - g - right_multiply_hessian(hess, d))) < 1e-10, 'Wrong gnew') + self.assertAlmostEqual(crvmin, 1.0, 'Wrong crvmin') + # self.assertAlmostEqual(crvmin, crvmin_cauchy, 'Wrong crvmin') + diff --git a/dfogn/tests/test_util.py b/dfogn/tests/test_util.py new file mode 100644 index 0000000..12f85f8 --- /dev/null +++ b/dfogn/tests/test_util.py @@ -0,0 +1,106 @@ +""" + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +The development of this software was sponsored by NAG Ltd. (http://www.nag.co.uk) +and the EPSRC Centre For Doctoral Training in Industrially Focused Mathematical +Modelling (EP/L015803/1) at the University of Oxford. Please contact NAG for +alternative licensing. + +""" + +# Ensure compatibility with Python 2 +from __future__ import absolute_import, division, print_function, unicode_literals + +import numpy as np +import unittest + +from dfogn.util import * + + +def array_compare(x, y, thresh=1e-14): + return np.max(np.abs(x - y)) < thresh + + +class TestSumsq(unittest.TestCase): + def runTest(self): + n = 10 + x = np.sin(np.arange(n)) + normx = np.sum(x**2) + self.assertAlmostEqual(normx, sumsq(x), 'Wrong answer') + + +class TestEval(unittest.TestCase): + def runTest(self): + objfun = lambda x : np.array([10*(x[1]-x[0]**2), 1-x[0]]) + x = np.array([-1.2, 1.0]) + fvec, f = eval_least_squares_objective(objfun, x) + self.assertTrue(np.all(fvec == objfun(x)), 'Residuals wrong') + self.assertAlmostEqual(f, sumsq(fvec), 'Sum of squares wrong') + + +class TestModelValue(unittest.TestCase): + def runTest(self): + n = 5 + A = np.arange(n ** 2, dtype=np.float).reshape((n, n)) + H = np.sin(A + A.T) # force symmetric + hess = to_upper_triangular_vector(H) + vec = np.exp(np.arange(n, dtype=np.float)) + g = np.cos(3*np.arange(n, dtype=np.float) - 2.0) + mval = np.dot(g, vec) + 0.5 * np.dot(vec, np.dot(H, vec)) + self.assertAlmostEqual(mval, calculate_model_value(g, hess, vec), 'Wrong value') + + +class TestInitFromMatrix(unittest.TestCase): + def runTest(self): + n = 3 + nvals = n*(n+1)//2 + A = np.arange(n**2, dtype=np.float).reshape((n,n)) + hess = to_upper_triangular_vector(A+A.T) # force symmetric + self.assertEqual(len(hess), nvals, 'Wrong length') + self.assertTrue(np.all(hess == np.array([0.0, 4.0, 8.0, 8.0, 12.0, 16.0])), + 'Wrong initialised values') + + +class TestToFull(unittest.TestCase): + def runTest(self): + n = 7 + A = np.arange(n ** 2, dtype=np.float).reshape((n, n)) + H = A + A.T # force symmetric + hess = to_upper_triangular_vector(H) + self.assertTrue(np.all(to_full_matrix(n, hess) == H), 'Wrong values') + + +class TestGetElementGood(unittest.TestCase): + def runTest(self): + n = 3 + A = np.arange(n ** 2, dtype=np.float).reshape((n, n)) + H = A + A.T # force symmetric + hess = to_upper_triangular_vector(H) + for i in range(n): + for j in range(n): + self.assertEqual(get_hessian_element(n, hess, i, j), H[i,j], + 'Wrong value for (i,j)=(%g,%g): got %g, expecting %g' + % (i, j, get_hessian_element(n, hess, i, j), H[i,j])) + + +class TestMultGood(unittest.TestCase): + def runTest(self): + n = 5 + A = np.arange(n ** 2, dtype=np.float).reshape((n, n)) + H = np.sin(A + A.T) # force symmetric + hess = to_upper_triangular_vector(H) + vec = np.exp(np.arange(n, dtype=np.float)) + hs = np.dot(H, vec) + self.assertTrue(array_compare(right_multiply_hessian(hess, vec), hs, thresh=1e-12), 'Wrong values') diff --git a/dfogn/trust_region.py b/dfogn/trust_region.py new file mode 100644 index 0000000..20251b1 --- /dev/null +++ b/dfogn/trust_region.py @@ -0,0 +1,362 @@ +""" +trust_region +================== +A trust region subproblem solver. + +The solver trsbox is an implementation of the routine TRSBOX from BOBYQA (Powell, 2009). +Some modifications to the termination conditions are from the equivalent routine +from DFBOLS (Zhang et al, 2010). + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +The development of this software was sponsored by NAG Ltd. (http://www.nag.co.uk) +and the EPSRC Centre For Doctoral Training in Industrially Focused Mathematical +Modelling (EP/L015803/1) at the University of Oxford. Please contact NAG for +alternative licensing. + +Copyright 2017, Lindon Roberts + +""" +# Ensure compatibility with Python 2 +from __future__ import absolute_import, division, print_function, unicode_literals + +import numpy as np +from math import sqrt + +from .util import d_within_bounds, zhang_code_structure, right_multiply_hessian, sumsq + +__all__ = ['trsbox'] + + +def trsbox(xopt, gopt, hq, sl, su, delta): + n = xopt.size + assert gopt.size == n, "gopt and xopt have incompatible sizes" + assert hq.size == n * (n + 1) // 2, "hq and xopt have incompatible sizes" + assert sl.size == n, "sl and xopt have incompatible sizes" + assert su.size == n, "su and xopt have incompatible sizes" + assert delta > 0.0, "delta must be strictly positive" + # Assume gopt and hq have full quadratic model for objective + # i.e. skip straight to label 8 in DFBOLS version + + # The sign of GOPT(I) gives the sign of the change to the I-th variable + # that will reduce Q from its value at XOPT. Thus XBDI(I) shows whether + # or not to fix the I-th variable at one of its bounds initially, with + # NACT being set to the number of fixed variables. D and GNEW are also + # set for the first iteration. DELSQ is the upper bound on the sum of + # squares of the free variables. QRED is the reduction in Q so far. + + iterc = 0 + nact = 0 # number of fixed variables + + xbdi = np.zeros((n,), dtype=np.int) # fix x_i at bounds? [values -1, 0, 1 + xbdi[(xopt <= sl) & (gopt >= 0.0)] = -1 + xbdi[(xopt >= su) & (gopt <= 0.0)] = 1 + + d = np.zeros((n,)) + s = np.zeros((n,)) + gnew = gopt.copy() + qred = 0.0 + delsq = delta ** 2 + crvmin = -1.0 + beta = 0.0 # label 20 + + need_alt_trust_step = False # will either quit main CG loop to finish, or do alternative step + MAX_LOOP_ITERS = 100 * n**2 # avoid infinite loops + # while True: # main CG loop [label 30] + for ii in range(MAX_LOOP_ITERS): + s[xbdi != 0] = 0.0 + if beta == 0.0: + s[xbdi == 0] = -gnew[xbdi == 0] + else: + s[xbdi == 0] = beta * s[xbdi == 0] - gnew[xbdi == 0] + stepsq = sumsq(s) + + if stepsq == 0.0: + need_alt_trust_step = False + break # break and quit + + if beta == 0.0: + gredsq = stepsq + itermax = iterc + n - nact + + if iterc == 0: + gredsq0 = gredsq + + # Exit conditions TODO different from BOBYQA + if zhang_code_structure: + if gredsq <= min(1.0e-6 * gredsq0, 1.0e-18) or gredsq * delsq <= min(1.0e-6 * qred ** 2, 1.0e-18): # DFBOLS + need_alt_trust_step = False + break # break and quit + else: + if gredsq * delsq <= 1.0e-4 * qred ** 2: # BOBYQA + need_alt_trust_step = False + break # break and quit + + # Multiply the search direction by the second derivative matrix of Q and + # calculate some scalars for the choice of steplength. Then set BLEN to + # the length of the the step to the trust region boundary and STPLEN to + # the steplength, ignoring the simple bounds. + + hs = right_multiply_hessian(hq, s) + + # label 50 + ds = np.dot(s[xbdi == 0], d[xbdi == 0]) + shs = np.dot(s[xbdi == 0], hs[xbdi == 0]) + resid = delsq - sumsq(d[xbdi == 0]) + if resid <= 0.0: + need_alt_trust_step = True + break # break and calculate alt step instead + + temp = sqrt(stepsq * resid + ds ** 2) + blen = (resid / (temp + ds) if ds >= 0.0 else (temp - ds) / stepsq) + stplen = (blen if shs <= 0.0 else min(blen, gredsq / shs)) + + # Exit condition TODO different from BOBYQA + if zhang_code_structure: + if stplen <= 1.0e-30: # DFBOLS + need_alt_trust_step = False + break # break and quit + else: + if False: # BOBYQA (i.e. no exit condition here) + need_alt_trust_step = False + break # break and quit + + # Reduce STPLEN if necessary in order to preserve the simple bounds, + # letting IACT be the index of the new constrained variable. + iact = -1 + for i in range(n): + if s[i] != 0.0: + temp = (su[i] - xopt[i] - d[i] if s[i] > 0.0 else sl[i] - xopt[i] - d[i]) / s[i] + if temp < stplen: + stplen = temp + iact = i + + # Update CRVMIN, GNEW and D. Set SDEC to the decrease that occurs in Q. + sdec = 0.0 + if stplen > 0.0: + iterc += 1 + temp = shs / stepsq + if iact == 0 and temp > 0.0: + crvmin = (min(crvmin, temp) if crvmin != -1.0 else temp) + ggsav = gredsq + gnew += stplen * hs + d += stplen * s + gredsq = sumsq(gnew[xbdi == 0]) + sdec = max(stplen * (ggsav - 0.5 * stplen * shs), 0.0) + qred += sdec + + # Restart the conjugate gradient method if it has hit a new bound. + if iact > -1: + nact += 1 + xbdi[iact] = (1 if s[iact] >= 0.0 else -1) + delsq = delsq - d[iact] ** 2 + if delsq <= 0.0: + need_alt_trust_step = True + break # break and calculate alt step instead + beta = 0.0 # label 20 + continue # restart loop (new CG iteration) + + # If STPLEN is less than BLEN, then either apply another conjugate + # gradient iteration or RETURN. + if stplen >= blen: + need_alt_trust_step = True + break # break and calculate alt step instead + + # Exit condition TODO different from BOBYQA + if zhang_code_structure: + if iterc == itermax or sdec <= 1.0e-6 * qred: # DFBOLS + need_alt_trust_step = False + break # break and quit + else: + if iterc == itermax or sdec <= 1.0e-2 * qred: # BOBYQA + need_alt_trust_step = False + break # break and quit + + beta = gredsq / ggsav + continue # new CG iteration + # end of CG loop + + # either done or need to take and alternative step + if need_alt_trust_step: + crvmin = 0.0 + d, gnew = alt_trust_step(n, xopt, hq, sl, su, d, xbdi, nact, gnew, qred) + return d, gnew, crvmin + else: + return d_within_bounds(d, xopt, sl, su, xbdi), gnew, crvmin + + +""" +Alternative trust region step (starts at label 100 of Fortran code, where crvmin = 0) + +d, gnew = alt_trust_step(n, xopt, hq, sl, su, d, xbdi, nact, gnew, qred) + +""" + + +def alt_trust_step(n, xopt, hq, sl, su, d, xbdi, nact, gnew, qred): + MAX_LOOP_ITERS = 100 * n ** 2 # avoid infinite loops + # while True: # label 100 here + for ii in range(MAX_LOOP_ITERS): + if nact >= n - 1: + return d_within_bounds(d, xopt, sl, su, xbdi), gnew + + # Prepare for the alternative iteration by calculating some scalars + # and by multiplying the reduced D by the second derivative matrix of + # Q, where S holds the reduced D in the call of GGMULT. + s = np.zeros((n,)) + s[xbdi == 0] = d[xbdi == 0] + dredsq = sumsq(d[xbdi == 0]) + dredg = np.dot(d[xbdi == 0], gnew[xbdi == 0]) + gredsq = sumsq(gnew[xbdi == 0]) + + # Label 210 (crvmin = 0, itcsav = iterc) + hs = right_multiply_hessian(hq, s) + + hred = hs.copy() + # quit 210 by goto 120 + + # Let the search direction S be a linear combination of the reduced D + # and the reduced G that is orthogonal to the reduced D. + restart_alt_loop = False # once the below loop finishes, quit unless need to go again + # while True: # label 120 + for jj in range(MAX_LOOP_ITERS): + temp = gredsq * dredsq - dredg ** 2 + if temp <= 1.0e-4 * qred ** 2: + restart_alt_loop = False + break # quit inner label 120 loop and return results + temp = sqrt(temp) + s = np.zeros((n,)) + s[xbdi == 0] = (dredg * d[xbdi == 0] - dredsq * gnew[xbdi == 0]) / temp + sredg = -temp + + # By considering the simple bounds on the variables, calculate an upper + # bound on the tangent of half the angle of the alternative iteration, + # namely ANGBD, except that, if already a free variable has reached a + # bound, there is a branch back to label 100 after fixing that variable. + free_variable_reached_bound = False + angbd = 1.0 + iact = -1 + for i in range(n): + if xbdi[i] == 0: + tempa = xopt[i] + d[i] - sl[i] + tempb = su[i] - xopt[i] - d[i] + if tempa <= 0.0: + nact += 1 + xbdi[i] = -1 + free_variable_reached_bound = True + break # skip the rest of this for loop + elif tempb <= 0.0: + nact += 1 + xbdi[i] = 1 + free_variable_reached_bound = True + break # skip the rest of this for loop + ssq = d[i] ** 2 + s[i] ** 2 + temp = ssq - (xopt[i] - sl[i]) ** 2 + if temp > 0.0: + temp = sqrt(temp) - s[i] + if angbd * temp > tempa: + angbd = tempa / temp + iact = i + xsav = -1 + temp = ssq - (su[i] - xopt[i]) ** 2 + if temp > 0.0: + temp = sqrt(temp) + s[i] + if angbd * temp > tempb: + angbd = tempb / temp + iact = i + xsav = 1 + # End for loop + if free_variable_reached_bound: # deal with break conditions above + restart_alt_loop = True + break # quit inner label 120 loop and restart alt iteration loop (label 100) + + # Label 210 (crvmin = 0, itcsav < iterc since iterc+=1 earlier) + hs = right_multiply_hessian(hq, s) + + # Label 150 + # Calculate HHD and some curvatures for the alternative iteration. + shs = np.sum(s[xbdi == 0] * hs[xbdi == 0]) + dhs = np.sum(d[xbdi == 0] * hs[xbdi == 0]) + dhd = np.sum(d[xbdi == 0] * hred[xbdi == 0]) + + # Seek the greatest reduction in Q for a range of equally spaced values + # of ANGT in [0,ANGBD], where ANGT is the tangent of half the angle of + # the alternative iteration. + redmax = 0.0 + isav = -1 + redsav = 0.0 + temp = 0.0 # force scope outside i loop below since needed later + iu = int(17 * angbd + 3.1) + for i in range(iu): # i = 0, ..., iu-1 + angt = angbd * float(i + 1) / float(iu) + sth = 2.0 * angt / (1.0 + angt ** 2) + temp = shs + angt * (angt * dhd - 2.0 * dhs) + rednew = sth * (angt * dredg - sredg - 0.5 * sth * temp) + if rednew > redmax: + redmax = rednew + isav = i + rdprev = redsav + elif i == isav + 1: + rdnext = rednew + redsav = rednew + + # Return if the reduction is zero. Otherwise, set the sine and cosine + # of the angle of the alternative iteration, and calculate SDEC. + if isav == -1: + restart_alt_loop = False + break # quit inner label 120 loop and return results + + if isav < iu - 1: + temp = (rdnext - rdprev) / (2.0 * redmax - rdprev - rdnext) + angt = angbd * (float(isav + 1) + 0.5 * temp) / float(iu) + + cth = (1.0 - angt ** 2) / (1.0 + angt ** 2) + sth = 2.0 * angt / (1.0 + angt ** 2) + temp = shs + angt * (angt * dhd - 2.0 * dhs) + sdec = sth * (angt * dredg - sredg - 0.5 * sth * temp) + + if sdec <= 0.0: + restart_alt_loop = False + break # quit inner label 120 loop and return results + + # Update GNEW, D and HRED. If the angle of the alternative iteration + # is restricted by a bound on a free variable, that variable is fixed + # at the bound. + gnew += (cth - 1.0) * hred + sth * hs + d[xbdi == 0] = cth * d[xbdi == 0] + sth * s[xbdi == 0] + dredg = np.dot(d[xbdi == 0], gnew[xbdi == 0]) + gredsq = sumsq(gnew[xbdi == 0]) + hred = cth * hred + sth * hs + + qred += sdec + if iact > -1 and isav == iu - 1: + nact += 1 + xbdi[iact] = xsav + restart_alt_loop = True + break # quit inner label 120 loop and restart alt iteration loop (label 100) + + if (sdec <= 0.01 * qred): + restart_alt_loop = False + break # quit inner label 120 loop and return results + continue # back to inner label 120 loop + + # End inner label 120 loop + + if restart_alt_loop: + continue + else: + break # end outer loop and quit + + # End while True (label 100) + return d_within_bounds(d, xopt, sl, su, xbdi), gnew diff --git a/dfogn/util.py b/dfogn/util.py new file mode 100644 index 0000000..5dc0be4 --- /dev/null +++ b/dfogn/util.py @@ -0,0 +1,175 @@ +""" +Util +================== +A set of useful extra functions for DFO-GN. + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +The development of this software was sponsored by NAG Ltd. (http://www.nag.co.uk) +and the EPSRC Centre For Doctoral Training in Industrially Focused Mathematical +Modelling (EP/L015803/1) at the University of Oxford. Please contact NAG for +alternative licensing. + +Copyright 2017, Lindon Roberts + +""" +# Ensure compatibility with Python 2 +from __future__ import absolute_import, division, print_function, unicode_literals + +import numpy as np +import logging +import sys + +zhang_code_structure = True # True = Zhang version, False = NAG/BOBYQA version +bbqtr = False # BOBYQA trust region update or Zhang paper trust region update (only used when zhang_code_structure = False) + + +def sumsq(x): + # There are several ways to calculate sum of squares of a vector: + # np.dot(x,x) + # np.sum(x**2) + # np.sum(np.square(x)) + # etc. + # Using the timeit routine, it seems like dot(x,x) is ~3-4x faster than other methods + return np.dot(x, x) + + +def get_vector_max(x): + # Get k and x[k] with max value of x + idx = np.argmax(x) + return idx, x[idx] + + +def get_vector_min(x): + # Get k and x[k] with min value of x + idx = np.argmin(x) + return idx, x[idx] + + +def all_square_distances(xpt, xopt): + # Return vector of squared Euclidean distances between each row of xpt and xopt + npt, n = xpt.shape + assert xopt.size == n, "xpt and xopt have incompatible sizes" + all_sq_dist = np.zeros((npt,)) + for k in range(npt): + all_sq_dist[k] = sumsq(xpt[k, :]-xopt) + return all_sq_dist + + +def to_upper_triangular_vector(A): + n = A.shape[0] + assert A.shape == (n,n), "A must be a square matrix" + + hq = np.zeros((n*(n+1)//2,)) + ih = -1 + for j in range(n): # j = 0, ..., n-1 + for i in range(j + 1): # i = 0, ..., j + ih += 1 + hq[ih] = A[i,j] + return hq + + +def to_full_matrix(n, hq): + assert hq.size == n * (n + 1) // 2, "hq has wrong size given input n" + A = np.zeros((n,n)) + ih = -1 + for j in range(n): # j = 0, ..., n-1 + for i in range(j + 1): # i = 0, ..., j + ih += 1 + A[i, j] = hq[ih] + A[j, i] = hq[ih] + return A + + +def eval_least_squares_objective(objfun, x, verbose=True, eval_num=0): + # Evaluate least squares function + fvec = objfun(x) + f = sumsq(fvec) # objective = sum(ri^2) [no 1/2 factor at front] + + if verbose: + if len(x) < 6: + logging.info("Function eval %i has f = %.15g at x = " % (eval_num, f) + str(x)) + else: + logging.info("Function eval %i has f = %.15g at x = [...]" % (eval_num, f)) + + return fvec, f + + +def eval_least_squares_objective_v2(objfun, x, verbose=True, eval_num=0, pt_num=0, full_x_thresh=6): + # Evaluate least squares function + fvec = objfun(x) + + try: + if np.max(np.abs(fvec)) >= np.sqrt(sys.float_info.max): + f = sys.float_info.max + else: + f = sumsq(fvec) # objective = sum(ri^2) [no 1/2 factor at front] + except OverflowError: + f = sys.float_info.max + + if verbose: + if len(x) < full_x_thresh: + logging.info("Function eval %i at point %i has f = %.15g at x = " % (eval_num, pt_num, f) + str(x)) + else: + logging.info("Function eval %i at point %i has f = %.15g at x = [...]" % (eval_num, pt_num, f)) + + return fvec, f + + +def calculate_model_value(gopt, hq, s): + # Calculate model value (s^T * gopt + 0.5* s^T * H * s) = s^T * (gopt + 0.5 * H*s) + assert gopt.shape == s.shape, "gopt and s have incompatible sizes" + Hs = right_multiply_hessian(hq, s) + return np.dot(s, gopt + 0.5*Hs) + + +def right_multiply_hessian(hq, s): + # Multiply H*s where H is Hessian defined by hq + n = s.size + assert hq.size == n * (n + 1) // 2, "hq and s have incompatible sizes" + hs = np.zeros((n,)) + + ih = -1 + for j in range(n): # j = 0, ..., n-1 + for i in range(j + 1): # i = 0, ..., j + ih += 1 + if i < j: + hs[j] += hq[ih] * s[i] + hs[i] += hq[ih] * s[j] + + return hs + + +def get_hessian_element(n, hq, i, j): + # Get element (i,j) of Hessian, for i,j=0,...,n-1 + assert hq.size == n * (n + 1) // 2, "hq has wrong size given input n" + assert 0 <= i <= n-1, "i must be in 0, ..., n-1" + assert 0 <= j <= n-1, "j must be in 0, ..., n-1" + ih = -1 + for k1 in range(n): + for k2 in range(k1+1): + ih += 1 + if (k1==i and k2==j) or (k1==j and k2==i): + return hq[ih] + return None + + +def d_within_bounds(d, xopt, sl, su, xbdi): + # Used in TRSBOX, force d to be within bounds + # In Fortran code, is at label 190 + xnew = np.maximum(np.minimum(xopt + d, su), sl) + xnew[xbdi == -1] = sl[xbdi == -1] + xnew[xbdi == 1] = su[xbdi == 1] + d = xnew - xopt + return d diff --git a/dfogn/version.py b/dfogn/version.py new file mode 100644 index 0000000..274ff20 --- /dev/null +++ b/dfogn/version.py @@ -0,0 +1,27 @@ +""" +Version number +==== + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU General Public License for more details. + +You should have received a copy of the GNU General Public License +along with this program. If not, see . + +The development of this software was sponsored by NAG Ltd. (http://www.nag.co.uk) +and the EPSRC Centre For Doctoral Training in Industrially Focused Mathematical +Modelling (EP/L015803/1) at the University of Oxford. Please contact NAG for +alternative licensing. + +Copyright 2017, Lindon Roberts + +""" + +__version__ = '0.1' diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..55b714d --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = python3 -msphinx +SPHINXPROJ = DFO-GN +SOURCEDIR = . +BUILDDIR = _build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle new file mode 100644 index 0000000000000000000000000000000000000000..aced0d95a2023c5fbbea0f57de259744f3045b5e GIT binary patch literal 11644 zcmc&)Ylt0J9p5JVy8F6!w`mo$cI39{HuT<2+tk|7*QT4KWzD8$Q$%eX&Yd~;p1GMj zGnvQkUNLGvG)?QlL>y5+2%?`_ON&ZT2^5MEKM7hvP{fKN2-XjR;0ICsopa{3$+o=@ za7%A?X3jbP^FP1yfBeqli=$sVR~wQ4@rA%+ma|o-&W7c?PKP>S%qvGC%QlY|K7BR5 z^O^X3yvQd^yJ^`pi1~EVG72plO^)HKLpt;+$2zxS8DT7&T#6gK9J+??L>(QneG5&6 zoKeeZy7B5bpRk-DB(^P{OHn}mR%DqpUM=x4%Q5Lzyt*gilN}lolY|7_8m?_qBeYyc zC}GkjiR{o+J@Uy0G!G#%=A+5CXX8`xBAY!QvpM|A_*Krw5nph9t7SRF)~ya{Q9bf4 zD8{Eu*9d(|P0Nq@N0yk|p-V!mr4`qR1UkV|%L-Z4STdV#%UMG8U`ZFfn&MeBMc+*y4Bw&U#DfjWDc9addWLTk8P>hd{KX&S#gLmH_vspfF`0{b}>F44AgZ15( z>#6BS#o4r(?$hSBrf1NY1bm7(u9E}D?qT<`Ll@9PB9_gvQx{?k3d6$R<2=(JuMyBU zJWrxO?YnNM2i;D?g}tYKqb542i&}hqU6mHR#WjRe#A|#eq#X}aK$rbifOlA~VG~~t z8Lx9`@ss%05f#Qg;tOwF<5O@5C$yRt_3<`oFc=$(;xlp75cu&i(MMt|eh>OibI9__%wjk4cC!&0>n!~Q#~S_%Q6scMo9bIl*YBX> z6eD)1+jN5>)Qs_&phNsnKNqW3z{2y*B6{iw zP}CPTj9R2&Qxh88g5cQEn=mp5ItgOSL9 zub{J*>b$wRS@*gja~-L(NjH{IGl1T6B2*ROiTG60jT}=K@eGrJfXU`qnN=>sAACbb z!~MNQL%4wytxRlI_9b#eP{GXY5MS#p`_j36;$- z@7l_-leOb(g)Vj|L|<1`U**30nt3YNv77DR(Zidnhf3c))YK3TYs=aS^{ovAs|{K^ zh8(9ICs7d8*63#FI@-Ex1JJ@q`)mju|0s2g5SCpBF(6N3>HGL&svwK4Q^E)JkWqwJ zULpwbBnY7e(x04nJ&2l3YYW!EN9!Ug5ez%BZNvscaPXu46Fr{bTpvY85 z>MFbq{*dJH$uyWi!HJ}cxIJ>i$OxYkiDOGgw^C0L8cQDW1e^FQG@BrYxT?pXMaM-&vP(PVM!mj2vwUZ+9ns>(Sis(c;=}^tK@DD&Yd`o6< zQ4RTA@Sr;8?SvH=)CLOFje=NA*c2R=I)X9@ug;P#zSc!nYm+ThDl2ey7}=B>P#uGh z+w)MPy&wnNAx=bW*+YeH8Fpk+F;TD{+0P;$SBwrQpJvELLEUWNxxTWluZONrW6Dd0 z>vh2(8E}3~@RPf=;b7vk($)ePWrL{fx97kCF-r)F4L1twqCDnhhiq6aX>iGxIoOJP z57G~cfi>K%s}=xxbRUAT!G85Pr!jV(@CA zkS`O8iv%$g#TGydisBZe9lBtwEkz74XV3F-$@IZeCK1)y3nI&vFS8de#DREx;c|oT z+t$u8Es>+qvLe?!=mKe3a($7U+7H8+M9a$M_9OY@quC?YEm+v7l#2Rn_W1H|I&c4X zSzBjw>hZ`Qe#Xk`@z@*xW)=1L(4W461g38i-)TmUfp8$H5ok?;8kKd&3O&m>tAjBK zn*@qew(GQt$(Oy19Cj6X{7qIth~=XIDap&ohtrJ+ODT{d3(+x+tL&Bbr!ayZvfYck zBwFegB3rRJeedQ@`)N@LX?c;Clhq0sO+pan6D|i6^GVAm9xIAJ*Nga8A|9hGWlX9l z<#RnPe0uaj$tQQRC`ihVo%JRI(`N;Kr(gG1isU2TbdEaRVXCLV|Wot+ooBY==U?^^4tv>#ZT+x}zF^bt1sm#QGk@g7~$d z3kKK;YHqVe&qXRH_JrYIa}4(jGAnrXS}ImsS7HTQIhu*h9Gv&s0%-qZH__g^M}^$> z0Pd_+P3*v75ht((`%65!M(h^$(L&bg3FetZ1Nio>D@vXK74j1I)|NAQIlk{%6~NvE z$-CzDcMJjSbTvy)ASUUm@?G5hV2NQ;6NeWv)qv@U zbmUoHPmcU@j?~S`u8|Z2wjJ=OGCglS;clIj?j7|#BmPN#L1*xW^g?F1VFyEn}EDJ z1Ryt7)rmK;YXtWr)EPi8Z0yH5HVY@^mOFOraR8l&$YI*46o|dCn~42-2*l>A1m~?d zaHlo8Ma18p^1|)S*iGUof>&t-EUv?9#e+_}vhpO0m70eZ<42~Jph-2bD ziBD_F3wr`?WK?BvB-C4$r8Mn`l%_$Nb^_8g<(mag`yLemX$evb>2`p{=MWL>1Idtj z+Ag^!;8&}O)nRkW3c59Q*GC+l(hXv(ovEI1y)DPJn%LoC>`XzXc2eFSf=umbE;F~$ zo~i@m+HPX}$PgGG$S}_LO$Egj2a~Q-6L(Jf+DCmnhx$MUb-tl`G!wnF0?|8cu=djm zz<+!n2KooROGb2(lwl)qCzxo=#Zb~?_*wufDG+!|8H$Hhav zD?(FoO@~jW$HKU3Demmx1}WLXRW)43Slz>?a5=~zk`Lin9~Vn-$si04%ZSV;@GP&X z;&5_Jp7CCY*W>}O_$y9#FW~%tN*%M}jG!AoFHZ(1Qi<)WP=k+(Gj0^&k`SLsPPNs| zS;&=QrA#T1^4X*YvrOEjLo@6E^YQd8y9>xFIKvf(9r5aIxTT`*H}QkLUCdgN_e|tf z7;5k3x2M#)-DEkKopIK2H*smM)yKZ%pOY>@P%6vU3drnq vhcX!M+h|*$Az669`F=8OrZ|1aEaSd%YCW(*Hj6{!IsD4_Rd7)(YSjM++)EX_ literal 0 HcmV?d00001 diff --git a/docs/_build/doctrees/index.doctree b/docs/_build/doctrees/index.doctree new file mode 100644 index 0000000000000000000000000000000000000000..3836a760401fef01add57e43876cc4ae9bb90dd5 GIT binary patch literal 17780 zcmds9OKcoRde)l{@u7!huis_I-jHL?3@NSGHtjX^phPH=Mv7(c+K$%4p6QyY9`?XeDki!Bw<&^KQ>aOW| zh~&_+*S2AbobKws|N8&$f7D-p&4-76>|ETZ{*yDdUyUQj4GW%c^ROew=|YoxQRhdU z`9J8~?aayJRvh^?-;X>$;^lzX!p`?QaxAjyp4H?Xv0qMCoydwD-_v*R;x65C>`)On zW>wccf5YW=ofB!iX2_)(*YRxMV}W1c0gXv6a;lcgnoyGqNrAPV(fMR_K07*}8GX>NR60`sJJs?xK!wD% z_zLEzKSECQX{PE2mg`uoY6X!)dPpwh(+iJzKuDn2vD~^JIFV?EtRDDr3py}2^WB?y zeKVF*)i?+MfikARJ0*DpaMCc56DhZL6vvp zzK%T61I;i3$DJ$t<#<&%u8sHLkvXwXj#d3;52nSKD2V;ypq#WKh`SO;AWv97Ot{tf zuDVtj>c7V;KyV8DaXW6AQ3r{6;%l?%E{83P_!@5oJmkS0-ihTQHP0)D<&l)@#iH<= zya>$(#g)X?!eSjVj4MUJrS27Bf?*L*;G2aYj5=aoezJ1udVaOUQsY#{24I9hG^;r- zhfxIFih!0%dopYZ$J@-`j-Bedk_iJO-xM)B7uOj(!g)e&{j1Ey_fxYFysR3(IoB1_% zeG?(ep3k6ex57S};g25wI1%?)!nKi-7WmJ1A@GltznMVb;_$Z!O&Q$(v%$TJ!ExYp zN$=e?ALiWVTGam zQcjv#+sFvaWW|hS<}3cqc-VwxF$*s{FWF9bFia{$7e@t0IuC>Qktt|vfPP1{hFNV5 zZ@;jA-@YWLK)AQqjW$ffV@P1uk!m6Jq_o1b5$u`kQ{IUb73&1w3n>@KwO zkGvw_NL&JXv*oojX?95VMVWz5#Mhkvvf`;=jt_o&_8GBxp%X=4`F=C_o&q7S2;MV}*Zk^1{&p~q}n_nl+>iPr5 zg^`zXdX0TRZR+>_@=K1G0VDsiGi=aAK4qUgT8Uj`|5&JD3DON5jg!zpPf{k2=#d?{ zKQ|-@oV{{f`y7=8tL%|du?eDEc8~^SiL-v<{dq8Q*HRMDpS^*Ev5ECbz&N5YEs%Yz zovg@$>xv>+`m2Cu;}nskWkxoQGMV;3&S%(dUjh;WOSJxbxYar+CbD@FweF3`RkWtw%}8w2wb)ge1{LjSVxI4WZ3CTL?p5X#c!6yQB;AbPv0SoEjO$5VSS|Kc6@Y~t*h5kT z4+iN_3PS@!cs78r1#vX3Hmg(3GUb=#7shTB761;t9-qTrJR zuhr~&u(R9D^YrM3)CZ=*XohH%ps^-1SX=)hO*Hku1`A*4+q1eF)!jdmM64&Xr4P5g zU{+mJ6YMUHV8lC&>=#AjXXGlV4A}Tdsvdb#SFa1G2p13*L`WwRfbF&#MKNl-U+`sxn{WC;`|DEC+hnM4^f> zCG{u{%+XfGd~d*eXxLz9~JBxe~Vx8P(QZ*g|L;zKZB+tT8=|jI@-|? zNeX`|s{Qn?dEua-t4fMTIcEPvIQg7`;2V0^cGv?o=Pom#1+)G*j1;> zy$~g9Z0-6jRj%F;4(eO9`3H|sQ)M~K{OaWk7fUxU79zM{c0=u{u(fnoymW4ry(Xfl zwOlN2Y-|*;FJ8b-VUg}*Q?Iz;2d=&0*qpZEX&*hG$;7FWPAP6?+Z}s~#1=B-0=ukI zZrXQ*-BT$})eS2?iib&o^0aa($%iw&{-FIE6OO@$aXP<2UNqGHS7B6qlG=pm7W~D( z%9r=5KvVrs`yv?dFw7Uq%MT1OWScadWKq%c=G5l2VIDk?8s@LeyniV^&DQ@P>el}o zV;J5am&GZy1@P);z<8El6$P$6Sj?w$Heh?e^s@jO)mJ^6_^PAO*X0^D4KQJ0?MkKm z+cDMYXbS-zAdKs1>4<2L7R7h7?ysV~Wr6Oir&Rk(V%5rRP^N(@7SX|qI;q(cO%#LS zEmSuGmeXz4q`E!26;w4}&^|>i@SDWXGDA}Xa!T5NP;EXVZ&9(iwXFYAZ9@Y)Ms?at zBgY7r*O#~a>?K|j8YtH^StWenGmGGg5Q0rDqm)vN(d zJ{l!FN}DKy%Px;Mh>u~5M>(5xX60!Sid9?3N?VP(TP)q>sGLn!<(6$HjiO@-f$Z!4 zkX}bCCONKG;vAdr`_y85ylHLPX!{VID+jPSZ*5f3YzD`T*0~Eha}1YN9wEn&MFx&) zyqhb@LzF#s{s^stV-EI<(HN;dOi=$^Jw}>|k6ccw7N`)L((TSYh3v7Ee&ZuBB@fkI zzk*w6r;<~;xR1s|SHGq4Cm7QB5&iRH{KMsRGD0}EjV8i-vpHp^)b^CMO zb$g?YNA<#wDw@|!0@BypH8t4VH4R{#y^`60JT#RCOSF13 z$fO;N7xx5~K?k`N8vIAb7rb7IuR> zZyVfw4i3Yh_Jt9=+E7UUrnff2OZOC{lC*hy7k_d{ZOquiWA_hpL*ajB4{@;%lH0wq z7lD%p(`@a&&YWcsID_E>&iomLz=wy&>?~)>U+*8XJi)cBM;o<*w3sB1)GC#fe)_UT z)x;U`s_+~tDe212%BAzyuiivy=3RY842udnBX+azj9AyQY~LyPLA|(yF8rl4FBKPG zS$yfm7tbucO$8fOWj=rX{5!vU4kT}6PD>fVAPp_V)j}&O{=PbOR!o!Jf{2%rKvUl{CPvXrP^@?(m!r1E^)I+SsZ#;e#Z(Nq@f$@ zTs467WKd`JAhAI^+!Gb8^A{8Z@0UG2lSZ-o3f~yoQ`_rtsIgsqAj&6@(=?L&X(vW{ zO>lW<6FinN!PkeGAkJH&qVe2rpqk}X8RSeabO&|YKVUa%dp_}q+e6-{ZYpapLlyPX z9!)AcT?Fgi7xgOQ_jXpq6B$L6hbiJorHDCWmD|r>1<@2ZN*TowUBIBWMy9YEbv}`( zbEuOsT097sGgNuc@5)p4sY+4n)zAk!Yv{3zhCUpop~sbmo;Dg9I;BCy92@YXDQl+4 zHaH|JS_HM^e>p+@9#Ol6QW72Cn<|N!Epcp1oY)c%ZHb3V;u~Ayu`Th$mUwDQu&tQN zaX3<&9D_|xg_h(oEbP@0$+AApqE400paUZ0*#odB^v&q*-Sld<{`OOnQMTNdoM*!_aI1gnc2{K8wF&+c?4dj%J}*O>`JPJB@%3}Jo*KDj82q?IAJc(=H#kl;}cz$ zMNUqIO|(#zZ^!t~1!8K35M)>(`J+b#?ZHNMD5S?xE&p;pbO6m+G*b88J*@5mw4~A& z!9sr(LrFfaub`hqU4c#`QBF_4m*hzt+{T$k%n$7)D2G?6HV^fqF&;Cvnk_W9eGlvq zV0##x#ts1>TwRBOuEW|6Qw6o1VyPAK*oHcqXpWOlRt0*!s@U4qHUs73A$UG3SjU$N z+>_G@*U6cDLLbI+q7LDlBHTT^<38 zX$3yxBfwjqDEm~|_;+|0d7N)L5fEWTao9EcqY3s&0GQcl6G=>+R8Ut9KKlMi1sIK&vO2S;)I?wK(Q7paxp${i ziYw&UnibzQNLL{TdlB6)`%qUN>KmrN;{(BC9NH!?PJvGzHNHXhUz(uMS1im#oLEj9 z4S-Urh0o)SkLb#&#zVNTqFZBy zMn2m3Kl=4#!=>BLG#;m4CmVHKjOC`jhW-N=eu-xF_dcawwF zVDaLmXYir3CpnXTZ3nhjwR^o!2X^Q;H|<*Y+Fs|7b1XSxMN!~Z;t1f-@CPuWTkCb1 z9H3zZ4ArI;hT`YxidEfEz)jcUw*G3$t?}3R_fpr2lf!|v75Z@iD3ZfpdH!_*^i2~3 zv@O6E5yXe_qqQDbZASwN*dBBCI#bSR0Og#+#2&_<2l3}2{5k(-H%|69UC-`bTS*S8 z?iUsuzhy7Pp&cx|AQ2lbthQ$rax7tlRvJEUr zHE`R6TEnj!kz*S#y!dMArDelvuKNM{X&G*4th<}GXSnz#4veRSC%)=f%&y#(M&qDe^u2Xs57Yp*$FX8|+Y zL!c}QsscLuhM)E~?O@Zjw+4)pdPg1Z6CFBB=x-{St~zeBhNT0w&0wkrA~$OGC=4Tg zAv4kbS)_VQre9S+pzABn=drYVlKnUAZQ2j1w_oq_-f&9G+_mA{aLUVQ^2NAf#g*Zc zJhSCsr6S(|Nf-iK1E$Mgl_I}{uOvb z8TdLkVP8ymT}=2B8j3NS1gm#Yl7lBn83YVEch^0;Mwnr4z@+Z^9_=VAFq*a%My2p( zj2#*pE!T5f?pK-P812BXH0_o#vxVJY_`FxFrcv=@uZE4}g;8L+UKGxzXZS%6ox|R3 zfxVcqmz)jTj_YlgL1yKa6*(0v1aCT8bHjGi>d=$)w*6gA&B;M8ZdL4{yG#rzIncBs zd%K%BtH6+XYkSr8R=K;%t=DhA<=r-n$5(;Enqkyuwr7pYOU7y#x6E7a(xuyP-D=$q z@Z)W~d27*FT|KgjzM@-UIV#`Q?JQ%pY2P%q#Xl8D0kAKY0?B^jVGKjXZUP|18Hr1R zI|5XH3kRd1K3Q ztB%pKEiW|O$f)_?P0*1*GdE*5uo(>``iZN7O>Me0@vJuh%i0X>g7UI7I*%>`(Kp6` zZe+LG5%p+?kqesQE};c@ksWy03LRsn(t&t!y>+7wMoC(PTd%X)g=h0d9o@NLix$+d zKB7LB#zUTtvabc!zXz1>Nu~jmV})**=2PF-1fG|;MshlbXJXLblOpJ34@bNR8W^z^ zejudIO0u@wi=zitlKul2vWMA`J90F)U`#T%*l%FP^Yajo3yw8z_xDCKcbi~y>|)nP z)Hs@yznqr*<-0c=>^}p_ixF-4MhglFD&d>8S z`l+UIcJH1&ys(X<_-S%2_N-XjH@<<^FV~H^Il`(EdLgg1=euENZf>3s?vb8SjiI3t zW5$NG$z+o}oF3sD7kpl?KGa;RV^nMddWau}E-f+XB~~Ps%7CHL0VB7n)i{6{m@ONh z?LbywNJu7|#S%Kj=Wd;Tw>PaO2+r;kFGeeFMlP&P1JVTY3*Xa3LYUZ~k!EZYZAgi< zy~aJS+mLlqq>!)+(L-1^Za5$5gM*%a0!&-3`HSoMc^)gikM z!b0%9^_1jl%n)r1wew-gK0IQ%A;ucixs+rN4xtMT980vu9D_r)<4W@0eMT`FR4oJV zI|wstI#Ozmkz{)5cqJ|r+8%lQ6l_-<>b={;6z?p+*ihK(ySzj5^Zj6kEsm~<7Wv6Z zcuc<>NhWHYp24Kb#5pN&UIC{VB5;TqdLbFTWFJU|uLV#_*rdhGjL-|D(Gn{D>#eCC z;5B4B#kQb+^l@i0CLqZmiP}C)5E2$_iC`@C>Gs;J5@j%g9Jk_)NQZn^9H4QWBLmfWHl~Nse;&Fq-xr3m@jB5*pNCDli8Spn z-oACe@4Xy9z3(ATuhOJE{6WdX=dpfsbD#IOh<4^lL+|)86lZX5Ve;4BjI}0eXV0O-AZ6V06gbJQ9Hc6z!9x-SasW-vLCF|uM7O!0^6z- z{aaYKJy@-gT@!E0b_N6TOc!CXyuX&di9 zX*HFNmmI`5ao~27FYgW&ca!}hO!BeeJ7|z3Og8V$mg}SUQBW%mF39-5G#SIPj;sKt zNE+)u!|UXTUJwrY>qW?p!aPH9m#(FksPR9va(%)tp|;Q+{T9cexot^uCJLPz$*=7i zgG%Fyx6(c^EQJ+K@$@iFm=GN#J%)zEPg4p4^k9S!>TX0{qXB{9-bW?<5yjU&Cgco< zjqV^$ESo2%U(+zWC}98}d|;neP6vF@Kk>#B^f?8`8;|0r$sKJEOPTpLmi?;mdLcie zg^UPg1HhZg@(Zy|53zp^-M~XdL_qOl1ZP5pLu5xttilcI!yrI-2Dv9j&5A6zusUsW zDmCFji`bjRp&fRj!K!BQT!X}J`fc*88ZD&EAZ!Q|C`6VAfm5y8VW9vrt`JC3xa#WC8Tbqm1U&W3E4*End&y^Fz8;47!x(#*DbX&w>Tnjc~8 zZPT00_Ts^9N3l=JDxs_zfHVWYam}ql0M8l{YwQH>x7|9Kp4KiCsjKCuDCj?bxE3V8 zYd>rg6dSR`8*#Mq@bCz-93-iT*EFbO9i@JF1o7rEJ@V_~U_Z7!em@ige{u3%lb_X{ zbW(h>C$jsq*u3^$&Vz|(=}}2AeO-AsoL;_(^DZZvIKP)X)-jMhLMaVkT+Eto&!}IC zkAV*%k>U59KXBg5rGhvj2jqNDMrodcaVJRw`6bas^8FXVUj@fzmJH60zzIy8dfa4( zsAr4ZB4oJP6hc5O&In_O3W};!9pA0Otl>Vu#(NS%u~>?GaI8O40)i|u3iV@{8bbZm z2o{zSCS`*&_((!+Ashm}UxROqB0P{o_#etswILcB0Lg0ZaXB$l*SBk z+m&6KW}Y5)Z6rZuo8$SxS8N(6WF5o6e^Fih$ z)@BgvlAH_5X)Okqqt7bjC|B-p(u}DSydaklE5vfekw+v`EoFcjJhMfqzA6z@l7v(y zDM{H50zU|Idg3RFfShMT#hQu5{i_}F6l$+|N_=my)=#_1J%S^R(eb)~H8kCHWVY!{ z$8?R-C?&#lNsT}{>KIMp$(`@51-Z9ArITb}RMK9_ro})UdQhG`NMTQ;4 zDXmDe6>UX}EOwwrSfxn)-sY#Qnes#x#=B43X=uUp0(j@aX4a;~T^HgA7LUO$E_>ysvPA3HBV*1y| zS2xN1CEm2un=x|Znclw86g!~>dPQ{c#=pZkd7aWTrB5>CkR5B%9+Mjm*c;bT{@~swX&lAoVgz?Gx$SV94|h^7~$`N2K`SXq-{C1Dw_J@Q$ods|iCBos+=o zB|!8#t=Y&6K{^Flqz+OmC?#EIq_%vdrD~EV9vyq^rwlJsU{Vr)>p9Rnl==`BEe#|0 z4cIx(+1P{Pc` zhILD4&Bno;UN{axY9gT>{{0W-5AY^&fpk%D)!mA18o= zJNG1#5d}yfFO{S?;CEX3NvA_jen`o}CWaC|OP53>O#bdIEsEbEkYJ~f-%7p_TEfyQ zjz3XY?8$9&akjJw|1EiU?QM(BRhN%kN3scaI+96Ub-}t6H^D86CYqzAU^^5T+n=> zAwg`#hCInb_DC{6cgSWfz0&LLvIk!&|Uj@mWk^a}@)*Z~cr!E!@W&MON+X zfL&W`;1ldw)q6XSl>QHj&wg*K7zmFQ;3Ijs3biT@>=am3F>w2rs_`Ce#A54&Q3~#^ z5+Oxf4`bP5PF_yfK8>hJh2rzl^YFcw$*Z7&hWv3uO3{GBo#Y0lgKT}y)u;2gHV)>8 zD0ToNl7tn6i;@*Eyg2%Zb5()A&4+$r`3UEOi*7g!AUO9}r~l`6?!0+f?$Zm}=}(@GQ9gdP3$O_aHtJu*#=?Hk7>6UD zz)zE>!(Ed=N7#f&Vvw#RQ6PXB47-6Sc!B*+b5SQQx7=nEo}y643-KOjf9PN`bQXR~ z=f)({sV`lGIx7!@%0cp4=gKYtk>?P*Hj-G2r+F<#yF{melc5rkI?$}+%mB>w6GAfR zqckv+N}X2(VZ8BUm|}BtD=aY-d3+k@1VpGxxX4;QBEUF}OI8qQs2*+MY&sMlA`y10 zaY#AsTO-i{asz=3c9oY6F(S2re53X(arkT)off9(5(k=Pv{(S5Jd^yHIEc$mJu+G3 zkupJ|>~o}pz#KwC^@;$=i<~_p{i3~5>CiZ}8YKE(1miU}cK)P?}% zy~2}6J`Xp?+4-^o;-+I6DtU-;g6z-*ePmko&Kr%sMfELFj%c1s4?%ju(`r{8&%GIw zyq5>~*J^fMo`dF76F4-nwY5cOCg^m8NGK{JU6W2n@OcT2&Nc0Yw(Bljytufy_|!9R zJp0LCzIgHJr!V4wL^%_FnDB_i*#X2t{oRlh4PJM$kDB!M6gl5-X-2m4Mgr>l1*zRX zq^6fjC8Gagg**{LxnG!pbbruxiz|&EVgIPi%SPsmO5?xrD!EGMQ+C8J^ih<OUcp{*(@c zydVyQ>j4? zK~Y4MI{51z*3?slF+tsbtC5sWZ(KW_S^{+KellqW!eIiQ=J%ITz!w>XIq1>OMlHNC|XBslGwdDrK-kB`X zM%77u^o498R#}zpsw(-4S%7DhYu(($x5;%3xj;n)TBYusJUm5WUxOngKG-3I!Li&-lmYxI?Ja-$YatilE=!O zc2s;)9~rzMi_P;)e%t2fFzBP=@a=`_)Tel=x*Qple{>j|$OAO|)8b$q>9L(+@Ey-E zr#w)7?=z4TmCok|^1MFQaF$s^mM>~>1cz8naftIHS*C8N_wktTGxHj?+ZtB^M4^?x zP>fyoi|wRfyEMdPWlSR3uS?1PT}bw;5Uy13iKJT}dq8!56yBp3S_VPPqIyWLhBv20 zkUK0Zj50zM>9Z_9XW07|S>Xl=tDzz>j4RT&HiuNml~h5)tl(y!H9GR_E#W{Ejz*VG zFX1uzpgO#Ce4EqUYj!n&bft4mSq*?E&3PfzKhJhX!oO|a+IaHP?Il(QFm>!@A0GwC zE@odkypyPh1%Km!25VHOkt=gTI5R|4H#@*dN=R#gXpzuj@eBs(4m>#28p?Hq9b-!` zJ!VFy&$E*L@nY+k-xx(33VQoZ-hyM?=ivx>L>-Va!1Cx5MQAAxaG&3SmoH_5H zuw>R{lvqek_VJ2h#PqS@93_v3Q6Cov^k@puVn9B1C!8z~{+)GGd}7C>>cd>K*vuXp z&RcV0dBnGA7$+6QUy`D@2~qsArvoZmh-~qd7I77UxN-6o=hL{LN86`h6W7g^_i1Fs zGTE{$>{a<SQ3pmPTg#l!nTDNnOLG0vLi`sqy9agPDYtC z_ywMT3;(K%r9Npcv=GcutLN|>+rM{kVo@*RLRIzak ze>tCZUTS=U-X-+cPw4so(qDf=pFYs|DE)jMf5pi$>4P@;(zj(UPbWr54y()Z_}aNM zxPZzF$;LBLU_Z3h`Mx^3(*?>L`b zW?0{0IF}j5cNoOwm=-xXp+J<^(Jd#Zke0*cYD}@L4B!cb!a_R@Tr!PBAJQ0bA>e7$ z?-Vx#qErpiWjwly@5^*EK<~1b>W-Rm(Cj9LD#S>cG3^fOL3^T{sJ*#>FT;gy zaxADPQKE^eg3(qNvU8~$L$@$Gl!HW6I+-QHec zta=B?c5GCEd^b5}!bb$MVqXU#Hj59l53#4{M#m60JUZ}lG@_-)kp*2WK^;9T zp4hl)j-FsmBga&?PcA2CQ3A>jYM38!UMw)uveD`=$_iq}ICX>z(Y^q1h+s1S9*=w? z;pzss$Od-X0rVbp6fLzwJFa0lTDVXxIbU@UvPR9mK0-$sn&r26BK1J(K6om7`z4bq%SGWfw$l^E?pXL#O ztq+xbOWF7;xO#FL>8TMY!iwUsx9yKg+9w2HmdCKxJ{DE7g?-6K7-l5MBhU{`UGD)b zH_f*inVT-|((!Q#UUJez)koSQa;wu4$Wp`n{wx!Wpck|0!`H~mM3XTZFZL- zTfx{`6~Cz|o%zP`G~4DU5A=N|?m@+^cevfOPHdc5xN`)2{RE`ZQOzUx>OLL&?+YC0 zYeb!eI61CY0zFc1)HPv>x=8qnQ<81Av&!;`>xQvNYwRVNMWRk*biCwT5yI`aNu)go zYv%z@S`c!M6PzD~;7pFdGwO>J2@EiejVI;PzT~jV?V!1HV=FlW&V#)j@xgXeO*uK& Tj~Lkrs3Fq;sYo=bxKjRqAs>6| literal 0 HcmV?d00001 diff --git a/docs/_build/doctrees/install.doctree b/docs/_build/doctrees/install.doctree new file mode 100644 index 0000000000000000000000000000000000000000..3b1a3c4897fc205563078a0ba25fc03e6bd6c09c GIT binary patch literal 14924 zcmd5@S&SUlb>%iIXNKg=kP^kkC0Y`OB6~(dITpvK_q9Y>b-ZDbML$F-urGJ`r6ei`{aLO&JF8n;`#A%5V|bR#3VgvvmnWS zH#`5u?1OAc9COknY=mJFgbA}F)`+wBGBJ@j&A@51jPDn-buV!eFAUV%5Ac>=d2TEx zSnO;kJPfRkQ@`ajndJp8+scG~T+xp!`td}<;=~J@w1oL-E)BdOP8{Et%VeVO!D3wZ zI@{JW%d3~IFtWJU@^+cIED~&*YJNdY> zT3KB!$u1Q$^)!kg2HW=nmQ}?#AfZ$(rVLCZMCOyo3F1Z=we2>GW2jy}o{F5!I838D z%f!A+%y+>H^hZf{W51ZJbI*6N4#39uiHVri31K-b`2>H4@8<`_w38%}S4#mFiMBC< z9xcQz2XIb1=65ks_nkOaL`>EIMuX*pL_P*Q?xf<7YwQw8FzSTS;RUt-PYG zBVJJoS%D0j!3vBfUeVHCj^ZTa7x+an-f>7y89&IE-^-T3#aQAcJ}dL^D)GQS0XyTL z1O=Dz=L-Hjk3TQGm!;A_`CO5|uE^gfrXXjh895zZmTa)AWXH8+pHt$rw28-7!kN_w zeLvhJVaH)3*@PyoB(!Sey`+sX_X^41kbKv3^l`VPxa%114%brOXJl=_m2t{Y)^Bw2 zC&txhnb`ky0U?5uhS6k6XNqVDj^?zzgtf~Pf3<7kUI;+{RV}w?*WXf(UAs~J+q2g5 zJV`oNR#rAQH$_wlEG|B9?RCAakCMX~`0RNhBacT6`XtCcmwl@W9XRt@O zostxPKVMt=u{84FvS36DJ0s$*oo2SYhTdh@v5$7M z<#o(9$RY!~i&o5-wXQv=j3J6$dfh%>nDP=|6P*XmL}n0RJoGPs1m$&p488wBZkUB} z@IMh}%2v@Q2WoqbR(iSh zhF%bcx;Subx3+KM^Ifz(U()uwG<&bM7Y1m%VC4l8gQ3c38zxUyKlEtuX#dgRBvyIx z#bYP}GsLLV?Gr9j5O0(K~*vIQ_EZbeJQ1a65X72pPJmx4xDu zWZ1r*2e+-?qnSzw7cxk2=$IpLj$h68yv654$z6V`(2B7@107L%SC%VT$Ubqn=7;rL z_WLQaW2KWt>xV?>E^#wXQdLS7lyHC0(;N7ZJFrG3>NGO5wKPRFn3$Z44|K7+}G5uEDa+0HrG=fr)f6B;aRCoE5CVC0}&yc2*o_Ip(P83#9Y_T?26eD}n zi}byZGX&=Ve<9cGCk^#GMwVy$Bg>D;$nw16;ck=8dbn8l%zy$4cCl9q1r7Y9kFmFN z#y(>h8!2V>wARV{Hz9siYq07l!pR*;D+k89-c=lW&7Yp=#M;i8=y9-&r7ik3qC((NH_oP znT7o9RaD5xl_}8-I(mt<`!7V=3Wz~%(*sE{kWt3l%26^b$mOR?C%J$oFU7eO*6}kk zxPAU!^j{`i2EO|Tq<`?NtV0;=J$YjTw)-D!T+~MN`V&+)wd#HtFl%H)O+R7P-6AOY zYN6NrH*Yr>G6RNDJ^89;_|phm6Ars>a_PTKapWysW4}%{_5vrJ?T%M2#nA7=9m0{G z$@@J(-xmrle(72z!7uLxdh-hubgIDcV-JcrO@u7A+ zQ?lb9(Coc-JS*+^$vx?(OyV$uZGDASk;en2f``{)pVa`4p0GAi-IB(N4Xh6m3diw^ z+8I||Q)#DDXXV}Hcb8EeTW)>c&_co3EZNt9fXK5+8y+oP>F8Yx(<(tqe8i;t;%9u zAp#J8WP69HJ{;vXaJAZ40p#p}32k-KL`zxBT z2a>7i4xJy!l15B9W==s;%KI%CnLVVKxmsXmXtG0M=5b<4ofmXknj#RjBTxm?dck!L1Z3?eT45 zM%m7=uZ#ec)od7^VKqBK_h>(K(H)C~`CIn}T|HdPz1`jC3TD1sFw^_iFp*p0*Xdgs z1^$lVXe1ziR?D^qAnV9t6Sb-W&#JR^k5w*%^}KhNDlDq4E|%~%tjQe2&P(>6K+f(m zK*`~oQVzooBoD9vMuccg_II_kzLTqK*qNdzfPI%{Djg_#3SgH;D1aHF?%@kvhsyoE;`gh)W9gETbOMFm9W^t=ECVXu!MRgvb`=H733CV{3xo9rWrpjsvhTM zC}0$TP~agT$cQ-ieE&|$k)H2oM&5uN{&&O_HEkx_9n~JrXYTO1DnDA~^LO~-9e$$9 zAFcAo?(oxh_}MDAs{GuYl*+bZ&a}DfKJcnIiW>^D7G$f=Oxevohvws$l^fit(R)tS zRLtXY4fnr^)N)-oZ#g$s-TE#{TWY7FsmomTVV_ z9VqW9=yLw1Y~0__i{89f(vFXuF-k~mi+(~bP4Q@v^sZRv5P_bPK=@xGUl%fV8}-rc zZ(!U=P zbLe5G$kL9Jh&gRs+!%9NEp6h5Y3}%FFjv?b6EhvwvC%ca>@@N-k>^u;+PPh}v4St- z6}k@7sFy~ZHi^C@3~{x{V@X4{c=4p|;J=~#7oUkYgBb%I>PnUhh!I_7_;)sh2Ep~qUgoaKf zPR-u*+=OR0CdHf@X(r0V*xNU+RW4^@S{b`!yoyE)-9-VqTv*!y+(r?HaeW0O&~+VZ z@5B`y9~N zBLwB7ko@PmRfm3*G-*^WS7Hy)JQAi!S+UTEd<)Q$N;?D#b@8eyPN^r%S68v1(}dgR zVoy~ppjID7F8J%fK%ig`)&cdqvcBoGJ6V;#3UUarT?Ec1Awjr)3r#h*U~OZlkVo4F zOP!dZ4aP5^(V?zeF(hkHstEtx4jR<}Cp3&?O&T$eBL7z}& zqw)@sqIC#eIjmE8@|jPN{s}=Sq)C8J@l2#U2_ZF!+e=dNG-Z zHg!@#lLhE%QtuZeU}Vn}C_p>{Y~mZbj@WHA2U)O@RnrzF#*J;L9S8-DALsTbymN?9D|-{&7wHR+GHehDvH zwClD0f;Qh%{3E{5a_Q?ctzW0y;%w_Dlq(g}E`i?|^(0JGU#B}Lrpy&hDW!RMC)j_2 atDm^EnTzFkukiG6Af{`8t2j(+%l`+-R1-A- literal 0 HcmV?d00001 diff --git a/docs/_build/doctrees/userguide.doctree b/docs/_build/doctrees/userguide.doctree new file mode 100644 index 0000000000000000000000000000000000000000..36ce28198505d4ee833171050a8d4c27c4992cb8 GIT binary patch literal 60660 zcmeHw4UAmZb)G2xuSkmexBjdr%CdHsoY|TE-<4^Zlt_t^xb%{gMVq9DZ)e`_d&BwD zyf?eEmetrzT}$pu4A+yUZtK)ATGzFaIEkG&X=~R&jG`!z6bOPEDAEFToV0;m6t!E_ zHjK94Irrz?nK!dLZ`zIX3^_ndpqx#yn0`;i^L@|Qojh5v*H3e|ig@JfEV zQY|=sE8Ig5%1$L{eY!RB(bj9N;qVT-5mc9})u2)hoLt>m_FI=*;qJg*sn}(w#*DZhD51g`9UACN8 z8axW7F($jDWRH}c(mhh5N2`OUiF>IQoWkLYArV^7`a+$g~roV^z@)@`5xiil3#0u z2kn%2l+uqLwhIMXj-_g&Qt&OiQn1R^x?@!vL9G$ALis)=-lyb3`qz zg>V<>PtaAkSJ5QXuK3H@lngO!F?aME=@Xt|^A;cyp7lo;3D?xrrch5^UUzza%_3B@x6g*$<% zyHl9M4E~?P|4-uoqnBF^_o)|db>HJ2b4SB{FY%f^bMj2;*@c#S+|AOc7<9*Zr%BSG zqs7NsEG^up7WfmR?t=TC1#~;vSh5>S=(m?RXr*q~TysM2+jK(P<%CY2cRk;dD`P=e z<*^WsZw0Pnm2IzLc@=Q-vYmH`B*na$qzxkVf^-DRVtB2FjKO|Izp5(` zY`f*Mz5Dz|>w^_t^=w{4CA_TpJhr0wDovI$I~i0x)zMHs0O-HWh(mP>UfXw)l0 zfURP~sn$se0^Ynr!!G&Oyy$$fp34q5M=s4E&y|k0I*0BGm?#;CC z+BL^Idej=O92pU?v<;1^8EwjFGs0q5^YKuGT21w8-w4f@g7dzGMf@c#dg9#Xwlms+ zw#l#r`IaQ1T>^If(a}&n0iFGOA>$8Ha9wF$RM7KFM#Lf`MeWV!YcsUh3J7 zhOt-f@!|spDt0od>JFOEh}qm5o@GpG#d770#MQhD7Qims$ELM zK@T+$w9_ZquWpz7)mOUhSIrFXRkR)ONr%p0wJOwP3&$khC3wHBA7Km+uhE$c4Se5Q zDmf#T?^YY70__mK6Oc}gpJ*&7-GODV+Fr>sH>TQ?AE|5IeUObyMK_N5obsk+^O&0ToGO zfcVB%cEsXwCaNTx;!p$*h?@tHd*ZFlj3&y_kbcAaMT2TTO55R^No_zCY8Iy zPC2MaMRpH&pJCgmPh|R8AXy4g{kRh;#owStWC<&NL9a=Y9sX{4P=1!qOtG_^acman z(u%Y?tL+Or6ONntY9)Y)3<|)!IdQV79B3amPV$`%1TbG?V3O=WcS(*nr%aQfkt7yx zGv4WmjD~HG(c=#^A~AhFt7)ipFU_%?4F;Y5J~@3xXve&F&_n&{(NO%3nSRHZ>AtpA z=%kad{k1tuW<CN)Fk-{se2Gj0MFcRQI<{s$=sRQ64F* z-K=UhGV9ZwIoNL}P1TgEi9Si9H~HC{GXV_8vR z$LY`NO@&%4E)UCN!^jolyOX7=&EBizLho&o$gT<^%MU_iZ9;FEgAhS3eGL|J0eCAXA<*kai@0dhYwDZ*^ z&xTspb$^v^gYqRrG{BUBX3xtGI0wLa-6TWJvIn;@oTzng6;ThXxmeHS~qVirvXMuV7*eip>PfGZfL-we2NhjsVDf@p6$^Q z;0eId*P=xu_1#6owN39isd&wzP*G9HZln+Xxd~A?IPNLII^Ge3jmYwyawy31y-bz_r9o82L$n?< z7IaaH{%eSuXz8!R;1bxaYrW5#h%EbpMr4o{$7K>hthwvVxRHQbhE2+MSwvaqpqo@y z(nGWYiO=;931SFOU9)-6G@c%WrVe&afkrwVk<=b0Dd5PUsOTBmzHAWG91+w&cr-@e z!XR?$ia;Ps+Dt8`-KmRR=!Lh-WeLd0s62~GAb=tys)STQ^dcYyNsy2bl(JkcL0FLA za@DSTRbHSZ`-L(dGN42c zG=wx5?dXs$;&-(9kUbFbS4 zuYnUCD8_FrFMD|pu5EPo%XX>6p;*PwEBfo#a4PXo{3>-kUi`A`pXku_@P@z>Tl|ii z)?#=Mi^)zD8T8Sk0Y>>IMp0JQ6KBt! zISabX5?wm=I><~)TMlmY`7Du`kRBBbRs4^Fkp6fOLV9%5851%6X+%u-N5s?tn?X<; zG{R_qZV=Udy$}*nLC+>~Vq0IfmtsgbeeA-?mlkp_A3M8nYT;R|+&HaV*=~B}Mj6{C zwo+`Zk@PUnlXb_=LlGNnO%KU6-L`{;!%qfT(|7a(4lV0;`iStgJQOW!9Cl-6H*SY9 z_hC#WF|x^V7A^Y#oh3+HCy2I#8Vk!8XBNDrgACc(UIsvZ4*5cTU=TDz`U+(mY(`U}U zU(?}KFX>gFG)}0@B*AQJ#X%BXP+w0W2a=*Z@esDBnlLUTR#_ew#1<~!m`M`RRR$sG zk-q0gpJ+yO{b)qjJ`T_+?FI(%?hUHZ9NY(|d!>Er8V-a$VOw+)cr1+wON5U5$5Dq@5bVwFP=r(=N z6OCmI8Al;g@Kkg}h8+D-EyiP#!qOLQauWU-JM2zJTl`d_36HL8-iI)Q`&lPgT52vbzcQz;jD*AC4hFIK>v)TN>!6ULk2^-fWbt26$K%B> zs?irEBzk9Kqs%w)npJnGY#q!~&>_56^9 zKQ;L3g5g8wgl+oKLzJjpd>BZTh#!0$g<27p_fEA=knV(}o^k&N;&K1CbKHMcKsIRH zzw92~c}A%<$8>oafIJ<6+^vNIMzKNhS!&PO08c5%5;@eyh)5aRL0{4Aqkxc)1sJCpj3oRTtAkLaF(lpe zE`?CMq!G3lK66c=?OdH}G^~l3!5Lev!xkBPQWDIPky>FCSU6&lQMi6|0Wlj#tVXS- ze}>r@MHr4|Gnvd0W_e!A)Z>NF5({ZG9h0WE%!Ivy{6PWDe^Emk5%Ku1#TOKKX6;ZA<4r5dWOhzDr(RkTuW zfCKGFG1_;&k;8-LHcY+PX7`LvRI|enAh2#|H$LC}eZP}v_BRQrsK)XdU+;L)=V$lX z24(a&12b11Y01KEO^8zFo56`cFO3!~?z$8I5)(f+=$H7H0OA&D|6PK~i~o9QqM(Kvx{iI6YIuwnf7DvC5)R) zqWGtT&RB?t*gD!~(Lv)%wxfhoJ6H7g-S9SpGWtV9SK1eNhC+eeZE)K6B<+jsT@k5bCRn!K73G=lhR#w)FO0GM+TT;)51@#15p&kh- zQP1Bns7E|DsYgCRyN?^x^Dap}1EI(merw~%$!{VV!-qCnjGiJ(p@k)PSCK_6woZGW zj%W{O4Wd2r^O+$n5&gx(;8b+FayB8=iNhV>kJf>8mI}8c$2Gh2QTWWzeaq4+MHR@Fq%7Gbh*S$aM^os>>PHJa)k_*{mdi0qU?6*Z?RM3%x)__(W8 zoV0b$E2ElKy|hj#og5}BCs}q-RG0z^ef#w04tRK-2~=^d=XDPPN=O(P#NdxQ(rJg4UA? z$EW3v{uQvOlo)MtkM$Kp^B`_7AuX;%5(NIXM%~V%NE}MOLY%1RYAy1l{A@zqp`Zzh z-Um&v^z^M2sw+>Wb@51zB&3fleZA(DN+KVxRIMQOljNz$w-RSFQ^f3Vn}1w@bi5bh zVb^Xx_$NrWGsJ^Z7HAMdJoHo+?%|J#li55lCNwq(`D7@KgSM5y2E~JNS@Oz~Dr8_` z2R+o&gv*kDN1G7?0}I!f96Civc&$HRVk#+ zI4ed|zKWX^{2B#`L2c>m&sH@QY~WJQP(r{*sVA69Zt_ zfMcNUTLMZ9?EhULE*uspaCeMMN`u$i9!RypXG zBF$6sj5OPCgmAjnV8y9WbqPvq)pe>7fDNTsj#F|fsM&oO*^8)bO?NZEZb8bTvdE&S zb)#nL6Vj}gctblS^^@?e1PTd?pGoGUt(9+85QZ0nQq8C6uxrnt8ogXJ`YH{gSYA9f zK4|z_?%|IqjeY|Ox$$ZngU=66Giu^6jwLwHs8SD_)YLE%{ypl2G0Er|a5ZA!V6=Za zHHJ&lORunCTKtXwdWY1#Egn*LKZn%480oX(R;XmR6Upx3!DD21qdUq@i2LCPdq*Sc ziN9CENm)9I3)Gr5^^+QtNsODPCF}m;IVeQ9UCmmJ8qpR zeu5xUo`P_f2oq>6&>eCJpAy&5Ja-8klY14F0JBG$x$%+qyB@}0v#fVs!Uc~7%R+nr zT8*dB@`K|e9~`$1Tf3 z?4r#KE!V4MD+f&vm-;!moCVFcS|{}mi85OM8B(U{DK)Lry@8mDiGCoUz9wq}w+G)un8k-sFg3we#jkW3=N zyhativHu~Y+~-G!MpTP)DhPmlA^6PW{vqpeNRs+6*d4|8M|8_G?YAsh*q+d3g!-hj znHBCvU6yr^1+I~5J-!k-@y$Z3x<@7Ia#mi^r~>9=69N=oA)3@Nu9r;rOFK~|-dHD9 zXE^G2LY0xE=^JxLYL=Ql!;e~moP6;k$@uQ#O(NDi%uOP$2yP*=f2$L*+GoUh>}R|s z!a_twq9l6!s#HE6qOC8wGl#VTsWfC(s)?y%Rq^h<*ksxAun#|pDc-Q{ zYexkqk(Qai*_4~t^N5`3Ay0=orRYV7ava9emxp>wO=w=zvlH2gsmWOzExr(O*vp=TTC@Hi5Ey?VkSU}Vr#nAAz1a!Fz)gzJRwe=MU zSZ4|YL^L59vqwB1KpP4mD3+>0P%WqGbXB!Kye7Mc& zkynBK1X~(M@F~$kro8%%pIZEU@@7mzNXQ9Z+6|938z$#ae|@02y%X1pjFe~&V=c^C z;Fv)tjh=br55$e$53dhS3yL6;GRddq5W2`IUwsERLUGE?|sX12Euil86%FvtqLR9}HYQm_+z zRe7u=r;PXm+o<$i$+)(zi zz~#Jk4kbHT?WWWnU=7Acp>%MpqS(8To>M3y!WGCT2KlyTknpLk4yW)YNaqg~s*;k< z7bJME%H6?k;7oz?khpJ>cZep3tdo*Ns$UuLO1gRmjQdwa(vxtF(3UN$cd?RyKKv_= zFnJrX3hMNsT_n8s!_8=Fk#zhxoM0~Tb3>xuV!!!MjJ*vN{%||M?digx0kp=CDrJ-j z)*n*h^H0{je*g|f&KQCC{-D{Gj!zNK%!6$6avi$SW%%S47L?=*&0QV^^mp1t=z|O$r>m7^Z)KQs*N&<;rxs40VQUNo$k8VrAI5c~ zzP;j%_)oB3F`Z6Z!?f?2Zb3drzAGZ08}(JSdSZJ&p=dMd_WoNESyRWbdrLD1f{?hr zGToqIxDJinoa#~$!9-VL678M+jS?45{dN?D_7X#zB(POQ^2>=67ttV+;ni)18mFOp zf_4RkS}@zT`S{fF%$(H?CI9%L^s90`e{ix<$@9GhbSa9JFFtoSZd;z zi{t?p$JD9IL+xG0b(id9dSYT`d~Vw8mYF5D2TU_VV2h5flM}ix9dOByfj_f2Pi80d z80bwWWbp?hWyEo_a~e1mQ@9#UoyO|yj*eLreMa^7XgPnrVAn*;k!c@Q)S6Tf{7XqV zub?hFi|)fvJ8PZ#{h_mc9|AG(zpJLo65liXf zeM@gE_o+mVI487k^Vs|`5Y&6;=o4#C@YepdXaU}=S*WEFTm3iGT7tuAfqHs=?u>hW z?qbi+9|)`t>iLn~!^dQn!@RXn_55|i_k4oPoP31@k?S z3id?D$!`mIFKmOM_Lwg(u2|y3qIo675xh~?$ETL-j!_woeCg1Iun#HA13X-jTNZ{8 zgo5HiLs2|Niae=n@b*;!hHzkf(FvaLNn!Vv#eJZ)y0hw4akrt2%cJ9?SIL(eFtpWS zP(cVDcO^p!+2t^%gKZrWIlAjiSYD8VfeFE1jMQebE*P*tssLD+@L~C5l})V(N!x>+ z7|{Yn;lE20u^Qjw)hPZmgtY79-WRA=xlXm%^j?SJZ&EybUl%$QzmBFN+V}VI6JOvU zS2lKm1I>)2X7L+nspr=H;)de4Y1|*9M~`b8w%vxoju(F<#{|yAd=;Gvw%m?Z95q85 z?_zlBno(1HJ+9%{=0YafxVq@m3g-Y`Y0nx%`U&uOuR;_WeyW2xzWb5SyxHub=B%mz zlO1z>^BI^$p1QCL!)Yu0|qR<=3Fn6D)w)$E|1W zhVOd_9AFvV5$c?yhwGgI&ry&u-cY1PF8r+GJMd$gQ_FHP zwQRVPF65#{bRmD@5Ylsb@4Zyu_qs>q55rXsZiU8GN$0D65K(IX!THXrCT}y=(+Haf zhX)N`wc?K{gC(imqolToIM18H8Ch1+9(GeWb4;m1$4%jUMjxp;#N|}+OYa{6+YW|I?{ z>B-3%ZZFF7S>~(22 zF*h|iH9H|bnwiT?Ok`&+4-H+*w3kUZeP-vf*~vLX^JS-0$l=tqaeRp^N`CjWm+3Tt zJ(HP45(>G0kjpYYHx967XDAP5!;nJ?9-Nq*nwT^~4zuIa2w|M*NPC%>${;s~(_T_j zh(Vm4oSB}Wki)^#UL-DK`d>VQi2PyfteOC+sEEoeD5du?S)lS zk((I0(hRC41W55U&*H*egw=`*Zz*Hk$1|Ufm?HiuQs-{6g3PFmuOHTIvkb7Ctbpv~ zzq+iT_A9Y@en4$e2{+Hv_imHB=l7&V9}H)sZV?&JDeRqNTydrnQSW?q<%%iOBHpZv z`b@>xVCsKQ}C z8yXQGRftI9Qak;Cg#W@3_|z7VR>iRZ$Sz!i>k0p$P=xYmK_xW{mJovxHG@LQgC;?y zf+;d;Q_{3G9Ds}~7MG#8!WRbUNX^F8G#a~W)%sN*I(&J|bxO4{HX4mJR>n&HD=0+1 zin0>{hipD3r{W`ab0lp&3-?(Cb3k-2=Pg# zu1zCC-=iWz2VGemBf5hqrE7rb^Vg1u$h=07$-oTpO~qqkWBN74R+|TlMs?OcPh%R9 zqM;bsc8{Vgnf#fGu00)vjSh-14kT4Du*dSj4Ok;iLL0U=*M8%m`MsLkx~&;UG__3< z@g1A-WPGmubn1V~w;9~t-=rVeOeQGOQTtT5X%BO{*ufIi-@12wI3V8&s?%AQAf%{p zofVAqUrg)tL&n}$RI&^tlpw62O&@3y)ot~kdlPHbB-@upf?QFMiq%!GA$}y() z_*04e=Dx|?Mv=RX zBQI_;H=N=(y1-o`=2mKjP~>mjXT@% zy@oTSQ>ivqTyZF-EJh=hn4^`I-a<#qOw#YhHRt3*2=(huM%u=z|AzUpGyq9gKACXB zYu{K<%cqs;WZg{w29K^&>yRM2E|$x4`Z=XN?uL=2k96>$TD9^es`?Hai$NJil6eBB zk~Nv0o1LA=%uZ+FjX{#UMnsq<9N8|FB=V3ewQuT-lAI<63Tk?=9C zXKZ5DRKFM|?5yUln6`M9)|iiS&nrMs-hU^Igez^O{Vz~ERmfeGZ(s8Yf!hkV&rChu z3Wrd)so+5eN5Q7oTE!oNlfr|flIP^PD_yIswES=<%zG=9R=6thxjEMhQy^v9BR^6vVx(*|=-NhopwRJnh@UmW@R4Mj1rNs2a-`O1O<<%qy2* z*q)xwX2-{#JAeAcF?pHy*sABOIdw8~KgMlxxJHNq194C+`8gkFMilj-*TVLVlcc&1 zL>S?TkE1#Tbe61mUpQ!Ny!6j!b{YT_!$_5Y-xn~yQwY-*{lkR|5Ety&*;?WSSZ zzHt)+IT3LX);6$$w-hrbv4a178q=_X7r&09wy=S3_X^n-S5A0En80r!z$Un?mX5<< zo|K_X-x6i5}l~7Ee-*>YQ7u2@%*{=5C zXgl3-bCLGp8^;Sn$?VGHt5xy+8(kFeGIGwfdzh@=mepi(WopfiHU+JcX;Z!%WovD! zDJj*WA^~i&)%ePZ&A8KK8x^4Biv++D(X|4j{4=Wop#GHz_JB#3A;RGFM;Zgf6$m~yY*ZX6c~>3y0}4)! z3Yg-qt!C!Y&ZM3US=*_=G+B4?5xQBxwC8pr<>x&E#yoiTWAgnY5+B0kdy`*5us8Yj zCcmomqeJ~?Ht8<$+WH;$pdeMtymllJzbZ{jSVO*}_WPu9f*)&BL}F3)<^f45BJoCQ zF50^BKeQt9U)98JoFWo8G=5wG`Q!d$&f!Fyo0*xJ!2JiAjm0@r^N!eS(_m0!O3$t^$dI80RDA;#$d}X zz5CTkel<8tqbwu15%u-nd@C}NAd*Yc!t%NGH2*@N`mzO8I zv|Xoq#)QCm>kvf(9%3qZoL~^P#>1O+`|d~n5q13TM?M?XH5JS-p;ijaV268PDb-3$&D1n0|RM)MB2mN)}^qM)MQJ9)t7?XFDs<1tW2W0i- z917+;b{Pw|v*0W>R`A1+3!`vOVsU4d6ilt$l=~yR7I$)h%lKP!r==g*wR4~V0Ls4Z!h4UxE$^U zsQ6TViMs9$?4_JA*}t$SJiq}qdLUfJ`qKH6sae?Z1ywQN3*r6}Y6mpn3c$PypsQOQ znC@?@+iU1aQSfw?ZFqn`W=``?vWNPOrN9eHPOeE)XuT2cRvZJ0-6ar!c^#la$2qT% zLln%i*TgqGEbD$OwO3iL%Fg)8J@B~MS94C;_DVU_eSjpxjgP?7!+^P*Tdr0EIC`C& zzGgS)1Cb8Cd(bJP2oY9&9msYaC&06PxdPuE$;MwoY&$+W5qogS&R_LQw(rUUY^`u_ zG-da2xDVI<)PvkB4NQS1b$}>HPlfP*pnd5uKqFXA&8B=0qg8BB)= zwrzs@gDR15{wl=IRdCx*+B()1E!BLdQNVJPah-1XaNb2kc^-5rcm^~|vML80Ty5Z1 z-EhC0wVu^p&|==9(Se+M2suRhL}fWREl|@g0Cnxb^VH*+Nm_rRAlNXX^}kcEa5P7d z#eOC}Eh8|u4peq5W$`6Qm+%e;H{^jLD6v0c``aY#69NF`KCCqy2I~E-;k`U(T-M{@ zB2o%clRSXs7L1o$^?VQ8Dmveu!;Q$qB9hf<3E(nt5ZTV}?_z=x^pfMNfq9t@fbQG5 zX1h~aZ7npGNU)WcswGY7EH~g`3aSb-{N(PAX0*usVh?^GiB0~w@HQLFFy{wb||h>iRbL#GUu;5 zhmR6G+^&-&CZ^^#U%AY4|HyR<)+PC^1v!qDe0MYPmJ9TEb0gI6ms>ha#GHE|&638u zA!h8!So*_<=2jZXNAHgjblaX!UR0QC*s-}I#am5(Pm1}}15}2VM!>_(7VkHPEK=8e z{?2gvsNFz#NMWQuwSRAab;v~F#@WEBrEM~70m%5MNQzI|XpnzkH+TxeU%NN$gqxx| zxAU~-Cit8Gz5XuvtDNfp&ksF|bnj_RPfuUJc{ACxDdG63Q%b}(TpQkjcwI}2_%%H? z_M$H%pZe(o@P^Qi&Q3K|RfE~K&N29a)2Fwv1<02ZW%G`5a?0L6caC7iCn$K0OeQml zJEj^I=$}p;>&%s7?a8?1X=ZM&rlH|{0b}e9r_iRnyfSEQBJ*;kQoZuqC&R#Y(Y;Tnd6Y~E3%d99(*0Z)UJnE_Sli$ns z`N)ty9kNPHO+85=`}xtL-p~z-YN641CMG6I*4njl;+Rvz!ot*ab<=N^Jw{=ZQ&L_A z1-*gM$zwY@I%*e}#&L6w1@D-p+%&av?l$Dv^x0Q(29X>VU?kSVZy%CB3E`WXGP&lDLyYxb75g& zAaIyOvKJN>MqwKp8#y9a{9Fax<|m~roN~s?r2Ux16T-v8rHVcKg^fP%Mu-1(QAyvH zTLmZ=wdcvn$wQgVCa^TuE?v4rin6n{ZHAfZbA0sZ5vDtx8&|S|)_WC6l3E^CaV^^1 z6YV3*;JY?J(T1`Zp8~9HhHjyACW}iyeTAWtlG4b~(3SJ&&yzAge7Nwa?AFYZwP-4A zP_j!w%le5Oig{H)Ya7D zhnwDrTT&W3$I!Nu-_9>j`Fo7^9NI`lb#H?62Y+jOdpiuf3CpR-5m8LiJ%9dwK7%1_ zcbM^nFJBb4c%Em5&0}F+)kC<~xm^cTJe0wchfGMv?A=iTd)GUw0Hf zbgQ9U=e|70o+&ha<*~!GJR%|*SW-EIN@8B#*S>uV7mw|29H$j3zbvp~3 ztS&2$T>f(L{L;XWAiY`i_To(YVO3nvH*r`=gNC)w;YMo;*)T?F7W7xEq?SxR;v0Iz zs5|K_3rFRoRq;XsEvwWhS>Rhh6>q_1!%t7oPMpbHDHl8LpE(=Wvof12<}mVN>n^5# zG6ci}ySv?HlyorUmXXm>68iY0yqQ_D`&>Z}rr(-bac3)U!F;c8OM83LCTd#iF?lMg z-hMbk@ZRP?6{(~YTyY2S$B|n6M56Qj5wPOt$Hc#&N%&x*q$6MaL zOLFF1ne1a-{YvDVr~T_42T~!XY~8xmJO~zQ%f=1Ma~`X?-7bU7xF94N2A$R{7{w~` zHay(Uz+n>(@$#I2VTPY49DdYcTDFMdmnXvi8Q8tt1F3zlT{8w4%!eag$ERTOFaFx5 znnI-^NEpCkd*_91`#;Y}4UUdc?_!F3_3D*-T5L?rhs?TtJnE$3g)LjQ_zkj3qOvxHDg3HW6Qc8+qih5cj5|p-WZtm`OuXQmhd%UY*Y~%fV5RL3d znvXv_B$BR2Ja+u}RSk`xfx0N%xg+(1gKaG>@XI{-Gt$V-jF+N&iZNo2qrnv^FdLgt z5Gos+nqo!1+{iXl^Y3hFZB2%N8z3)(6|(6q4^C$n6cofy&;&$AvXr&td6tUN@|G;w zK4TJA(9zLBlUra*)56x!hhaQ<@?@`%%ykWow!Sk+Y8Y*Q*H`$q|ARg=_5E}A%FeIc zZAwtmAE#0t?t$Km1O(9^dboH%i!wxAATEd2c0;j^lO#?@Es?Ce;3BBXcD zqBm^X5~UfX4!MoI$Ia_cfNYeGh=`!fS4fXnnwPAuEK8oCOpidZW?(fs20wJb}5Sax3tNm%ZMK_TT<4Ah@Ruy~w`t{}# z5#k1^OusKu6k89IVoP1! z$Q-Ml@vl}eVt8vq-#+;fB9U)Y3GH1fro5GoCDlAV|B~zM=v6~Qb_BXeF>Kr1Yb*M= z4R+vlJv|(m*rq5Z`TkW*mcpBi6fVxA+9YT@IJ~C~{yFoCs>sqodCi6szoq>W|`r_G{P; zRIfkr=dG&4&k9l&dwUIFD^0U|wMEkQ4WU%H1so5ZwTC=nz+HQz zJltOyxLx=DKgK2M0nN@@-96 zN%|qnV96toiB|42Ve60WP~{=3Li%OkI{-~1RD2>sqQaUfME$HOC9Ib3v=dmFcM zb$u0IPi&`UfwXc8B;8HtMGv=5|ES_bDZV=C!;;MJFmfIBMp}4Wsw)QIW`Wa$pHyyi zbZI8;M^%vd{N#7PXt(Y{7X=j+4K=lwh6S(>Pw5tF*3{I%xcpgc_xQ*h#$DCXNrU-< zg;KkIJrNEja#-r>+CqiR4_BDx=H!U2wE4rh9H;yHbuFBF%Xbz{emGjRI9|Yu+Xh*4 zGlY>)AyZWn!@!d#JK$i-NpImjapH-4V;AP5{}!74GIVX9tE!q_pZAQZW0iI<3=Int z2ADLo%Rp*%p)LiEH9wUm1d@AIQtLN3#HAlrQTvCkzXF0TAP(mL1W#3p!SZ9xt4cAH)W$%Xz{ zo0~74K0Q6yw=Q;XboMRnxSgUAbbK!*B)!n~m8z=y@g!oM-Y}MAr{nVD=RpY9@=nD(^)X~&mvt0wnvE4H+ZZJR$)8H z@1n-8z}mD`L0~*5J>}yN{uJ_M)vHriFP0et;;DNXD5%irN?D%pm>@U8EFip$B}CLF zh9fC6v!U;0d0R1Nv2Yz$MCQ(w1A!p-HLityJ&%5JGU{-Tq_Z ziTc@xz=I%aXl%N}HnEggjm#zo{-F(jhV z$;JL1*$5N2M7nWpr`XxqDT_!p@`!r*GX4<_V{&n^F7`!K6b^xFMe1;*j_e+4liG98 zGn*RkaT7D1ABc@?WL=poOp#_@tc3Wq9Sc}%hn*kNV6eCM#K#ZdWZ1jM~(^X$zm2xcK_Zq9z|JDZX>1$(lG*Nv~xQJltfxuuT=jEyPV(k(+8S?`5fk!*w4tA zl9(tDL>`hW6Qs9GeutBQ!jfF{e4dty`7Nz9oq*{5QLt`QRYPN9Wi?Mrqiv_;Om+H0 z=b~aUy9|Vsu1LYf`S?_TSc+MHyLVGd3%QBuY1^78X=%L9?c4G25)g0%CqdH8&dD)@ z6NKs3U;`Y>Ub33C1Hg{WbX@Zs|FIu+bu+6(vKktVCe5X#r8V>b>OH=DnME$du%yq~nxDpuDmkJpdPBoID41CPRXxlbF0m zdsy`K_cE`!1!04Hz7mC)fSOfH{IJ;aY_8U-+xhu!pFx$Bsd`MrWjF7CFs^H?oPTS7 zWo(wq%>D>b{yz3xRskblE)|xvtFf0j%VDNrc{ofdq+zBrcQr{<$FC(MW@cW6jK;vN zJUZhZbn-?Fgou2*!C6;5pL;5)eV-?!(>79_el7vAIaJgp{_)Os+wL*=8so#DF;dCR zs+8D9OaowG(NnB*{P=OdLxuMC_KARbv4=gaMGs32em?}-hLanGnbzxNK!DBpMFp9i ztv1ceO(Sy4k4E-SM5%9}Iw1d;mO0rZD(%`ak{}Y$3Y@lF>mHrd(*^Uu#s*j!xno*fokJN1gIlK>6Iz}^`UPq_uXkuym~X|t`BG`j7)+Fij5w_17UA$FC#pCBOnqV zgr}<-8pc-e0E&43NIzqC`&M(n6Xl=u#~280_M^q04Bzf}bYcBTMR+`hZB2Li!h;)v z1!KW|lAP)U(<|%h=GNWyhQg2Vc@P`h zbetkMOxx0RSa67jj1m4y+)}Ox`{@;&4x7GDXIUnOhU$Ai`tIAg_2sKq2|y5DzIYKl z2x{R<-Z-9^b*pSEKu;Qe0w4T(m&lQ=L{PS1n*z=c!vTOcTbvz31XK=F5CTpdJ9b4! zhp@$>jWv*41*jAQ5fZ|=BfH;W7C%DxB0aNmD3&EdYQUJqDofG0Iwo;oASfKKK>a<$#1tcU4?fStfT|GV&0(PZv*r;XzLn%^BdWVEOX6 zO{s|xHO#-fIL=7w>ZYyAWJ%^6e!`Pp4~lSeOG`Y^?c)QBa~crk1rXUIVRj|5_^4l?w*VZaOx^Q%? zE5B*V8PdDI2R#i9P2HkYZS%5JZ6S{)TZz9Qc@EP*ILQ5R@wvg=@&SGW8P!66O?PSh z>Np2wacIBq+*D`J%9-QcawkhzO;*Iu1oTvdP-dh++6wH&$Y(CP3keHzeq79ZJ|PDm zVZZ+n(4@C--`e+J=0cW9az}uu8Q|so9Uudwrz_74)c8uZHa8=v(^_;i?}40x!V}<4 zHI-fnc(CpFcAV%Yycw(A3#kFrBM87JxVhh`Oxyw#jGyavb!bu&84fC0x*Zh>)sTDL z!*zwNfpG;V4fOq2e0qqSf1I0J4&a0_D@eL{a;sMC zYFFo&LHr^Ev1e&OWMpJSRY5^@#cK;(G!AR;Nv;kN6t)|7tw_a$I?Vuv~IZp z&oDI+i0-KyZ)M=#U>@xbvtc3BSajwf!cK|?PF+^^5fIeu+g1B3q|gOACnH=o8VVfe z<69rpY+}4*8Jsu?mA>u`J)YlJ`=nPc>MeG(kr+Lu7}w&+haE)?ApJHsU&jS~=lMMf zS%x+J`3bqw7q#l+5JUOESbPN4k!uyj(^fUR!z1L>A5qU|ZNxy(-tZQT3g|err~w6L z1_nj70s-8RP;rC1bNf^Hw$i(_qXIj%`O~|JkV3=1stuppHK4eTr)LIGOmkE!{hM>Qg?bwA+x?eSFUz`YX-mJry&3U&6R=N*(P-?YbAkuf~PgPpPxJOe6N=$jG2QS#vC~0 zq_6$Xc)cL~XWB}`7knJpSXfwT|MPC3h_OdRS@%U8oz(m}NZL(ND^}gkX7&UPKh9+Ij|7?u^T{ z5oXkowFwSmtcI@3N8%6o(at+CfYDxGzE~UhTXXCp%sT7buTnj_}KQZUpFY5mG0!00+tghALen#cp-%~|2 z9+urgFx*6U(U;0fn=uCn_EO{|mEpA9?nbC5#&8P(!1%UgAc+G3=>g5=jSW#!XKNup zycgfuih8{$YL1{=stMSl_o4iX)PO*o_Sdj;7k3&bt%Ib$LnHV7hFCcs(2%g8qc|dO z1;+E=lg$xu%*!)Xr(iwg`m;eeY6~N?qxeYJ(9rcP^>_4B-qWV%QG~EDcV4x(>k&OnQwZEcfQ)D@{%?Wr&*#bDt@h5q!r%0uG(}11D%Jf< z+xz*!y{`s%ecDdd)FFt9Cq3^1u%$+v$FeyA>M%AgIuIab%1_nUC<86pZX*f-^PGS| zhO)q#Euvc={zb?Y=_zu}g@Uw%?z4~aRR2y;VwG;wflOS|Xu+vwb zcXZsWxN<5&N+({S?V@sdv+TKBkl|KVeS|v(K z$Hm2EVCtPeIw>tp4XU8+ygx|@rn}Y(PfcXhbGr)Ip9MFN1-ip}j{e7o5Gm%63`$O~ z^WCwIY$kq#hoJ54!JGtPOHi;$LK-dz6Np^!lOJ;mQaO909I>_?aY5`Te4n5Ae+5Js zUK9v#<`sU-er4zI(xpl&Dk7b4paNJ_RHV|m@Un-N`Uc)U=|6%W5AdGHkGCRt56l!d zb1fsdl*11!gOCQ)q!XbggBtDyk=cwI@>Ak}Aur^2M@R2M<~}QUmOd1=&0TIx$%pdvlBgtiaONiFF6~WudNSg z;zIOu$MLRV=fZ@9OHf7QH|IHd@@eOy^GuD%z8#OUta_v>`U>f~sV6+=N*BLjCZ(KN zi(FVCDKy-;yEH%b<;#~w%pAdFwuu$x%1nF*K;{^bcZg#*^Q@zhdaptGHsF?~<6ZfF z7Be$5EnQuy%Zqb!=z@1H7-piNj>j??y|OR}?}-Q5*#*BJf@q01x$xDCC`}O|q2LsK zBNG!vRgzKJ_@R&C!~?^KdS)svJp(AvWc3Bj--wfwpih9~06Q3j%aRIQ)Y&70{r!pA z*)4aQT3Zu7eE1+`HK+lEu(hoXQMDt}ISmjv6Es4to z&jL3k{q4$g-D4+C2>74UE7pdZx1YGcFT0ChAO$HjV<`3&8Ng_kkObVoDge%zwTQ?x zO5WI&Vg3Dr#n%j^Y7QXwi42VK}LAfiJus%-@fq-4YN774SPB9&Tq6F!F{lOM*In z7m*XfgY%P%=dI5R2nq^A1w2C3<^q^^R`lvx8=8dqsm1=rg%ELgp)zA z>1#J%-1m>T@e>wDAK{(qj6Q-zw3HgCMIt1N6t%h#rOOBQsrT0o?|49S2I|xCadAW0 zZMsbp9;>cvA@=$81F6){5n4c_H~)ZR7@~d18ZMCM-PyXSI`zgo4itDC=K9GZz_%-j zl!JeNw7%qfux*{f#tlZgDo+&7D0k_VfWA478+Q7O9WiqA;miHG`5X%X17*9;S*oe4 z&pHPwz_i`J7s%LP1<2!tJ>|qvSXA#_TZtPIT{F5f4ZYx4OX`Aw@0_iE2FpEI2ZTa{ z4K6+ayI zZ2xFA z;3;fBmfbcI7_}7F?K&X|oVuo^4HBV^!>Z3eTKaE5^cc8gucO0r;2C-NRhhpJzm3}N zAdW9;=kbz@qwnxjpAJFE*t%$@RvbSR<6Ee^2lhfr-idA6QwRo*nlI-D3s5@ZC|lde zN4}rdyoeQo3`sz?T(8r+1Mmw*AM~Fkj`yX)4vhm%5I@ZXf}!2*5}&7kl>XnNyVMNm zH!*-+ZUhWQ=-GA?9hYaazWejv_Q+iND@^fgvLoSw@=2ulXk}5@$5ie7#+S=n0{=Rn zOtb_M-}XG4SbzijW|r+jSbw_s{@4tGBtI?#=gA9DByxM)wF(@^rq{OAcF$5KT1FlF zfZ`Q0PSJM48hGVBqjLRSv5VHv!7!5uex`&)-C{SF%X2|H|K6}$%@2TIw5>vV#b&w9 zV>t-KA?7BGlcVD&dymR{f89C)7*SbTp;2R2tFu;W*RI7P6+p0Pt&xNB!?{9o+V(vs zje)H7ngCn8=l@Cn84G5iN>3d&94IF(_4VJ#2%~lLFJ+QcP*Q3^geFuYg-kTe=>h0a zaaXVM!C#+hkn!oPgC4UHn1uMn5BK|tk9$|TWB;|q^`F^M!b%nn>9|dO>?rXl(FN;W z@TlEzV?2Mmiojo+yWl{ojWXidF|H0idfZXO7gf#^m-p=7pX}nmO8XYd_G@9nnq_CJ zC=L)Z5g9rWSCS5@s|ymcsd`|PE{Ht|paeD!#4h&^@pA!_2t2e~bn{bDWK$T@VKa@HrU8>}!PE)`D)IzjQkICY*~nV1C^}KJYI1 zdHVPN#At*pc~I7tOprISoYsuhKnjmqmie-KqHcnlfCLA%s1+mn`I?Gk zJW@tr%<{i-Y;P4O=azr(&Z1e^9Bm*jHv?47wj|(bIw~NIF!ZT$O8v*XUqRjl;&`p` z5i2b=DvI^LFN&E4PvIO=C3hU_h;P$%ty$Yg^IJWIDgB7pt5HHxLp6@Hq^w2mdmeBT zb$AtnrC}A1WDo^jtJD#)Oal)oD%|-yNx&6G6T&PTxuq9SPz;@2;JX99IK>nN?Z4ld zE98U{AT$hq`*0 z!6Ri>7b%H2W@t4-LPEZPY7X!iatozMe!dP0Brl;heo8B6NN<&_N3L64s?+7$jwDk? zl8*z>6vUtglLg?)6jW%a?Vhs_mV$z7Yr0MfM0_*I0l*s6K+b#`M2oA55C_XE1W_4o zl}M5^pn9tAkr{m*AO+yes{vab=n5Uc$Y29rzi}f849rj^0Y~;_gb2Jd@kcNwl;%u9 zC56j6^hv&FkG>)wd+Avttb_-Vv1DEr7m0B9AdQ#Kf5nP)UAk5P`AF>qfEH4dO}h4G zmLi*=9Fd%+m7Dzf^*Px5;L97@a%%qyQui44Sn^*znl$HR(C?@6w+h!nOnCtIM>(8c zmtJO9kRInE1oWa63X$SlGEakG3=p3v&2OZBkg;%|=#TGG9+cbeR-TPUA6X#1kxnb2 zq?AC#JZJ@gPtQ+?BFLYA!kZPh)KFjl8ki2SFmMo|z)T1TU9b}r@;g(#AEAB>ybSVq zfr5epuXDluI{<%?FrE#jTFJ@xn-oyrPxyg2_icD?a~9YmXEf#Hyukn14AiKHp!Xa| z15$Ve8$Aee&4AqC{DJ~yk^`kpHBk7VW|9aCdK?%H)Pxud3Fm~&TTg{?iuAEZ&BVxKg!xKKI?=FLMC{Ob7Sx9{F@q5ua071h#8 zdHQtc%o2R5;KmgZERPVSEZxDjiiYBQT|S{B+q^G3?&0~raw6F!0}7?;;M1A~*)=_# zS1CeF4JN9izzK&~2LMOjX^cbN%+S9N>VF!nqev@87Y-MI_Ms_r6Iu$C&f#0*Fyw>r zP}4bg#->+BDO3o>tsGuovfNn`VOP-H+$@@Yg8%P1UY7bK2J6Df$=QK-Feg3chYvQh zuxNo*%r>gTYNozZffSfl!Rpa9rD{$hk5Be>0A2;=FdNW0=cqYe)}PJk#5<$_Idob1U9V!HS*urxLFnk4`hO`|<-h+~va8}oSipoQNl(Pa#c>?(_S#!8Q6b!>9k_R*??Ul^5mLSYT{BC|@v#IOf`(N;C=iIH z**UeSNOI4%&1|4z7%u`fl6z39AC)8C*c30v#(lmvKgF^gGw5wRX4w%H1Q);1Ln%C#9XnA_Ss)fNu@1*}_$rRag`kT0;G zZ{7O7MJEgms4mO{P;XwIIQ4+!7`F-jMY^~l{l?qK2I3Dl^(1Ne7suig6&;oEBQU}P`$zVN6 z2CL|^XU{qSJ3=yn@_07bnP37xS|oP=v=86Xov5zr1C#gc{ICm{M6*nrD22?*VS*-p zm;71MpyHgIknrgzhLep_(Y*}O5x6RH4v<^{YE7uPQ#wKg0A|C+z5oFg44|)&ejzZ7 zgBEp8MPf>Zl(8chT1p_g##Ehvy~qbrJdUo z0(1hcYa;=_Lx_v6RW5P=pUNg>dfL;H2wXpy-w<(dOD>a|8d4-4UN~yeUD)2&vOu$b zJ;OnA@oFs!Ly5BqxNZtk!USfI2IWQAxcUGtXyD&vyTaB9q9zW^;D2u&@1A8!WS zL$VK)I?Z7tB1m9ONw!uhp(a`)dGPr01_*Vro{sQ`qs>U>_C`5i|!m&bZm z(&Cz;Zbo6uuX%GxzFyaR!+$8X?p0u5EFLcgn+X}kJCizfWo23~@15tsbNGq_nO`yI z$=4MrQ&E3urE6k-3PeiNyLX#GwRr)y{2T9#O~K0VheHk~4I?k`_UbC=O4#MyX90=4 zsRrEE5blO2R>mht2<7;hkt1-%vTH~*{S1_%wFy@G zfN@@ONRpecxtRg=$Ew0CKvYSAYA!YH;rf;3MP&a17gd6c-17EzRm>(Ne}fBL2s%;R z`IvkLpg)qj?^I3QAz1W05A?3a{E75CmXwrB95l*tA`SGx`haG;xB zgXj#3)rUSc7jxi9|65b}RU{JbJ#m;;Q4FI#op?1CY5B`GZ$lTJuyPP7_4<8$eg=jT zD^tHn8^sxaz@q{Pc%xLCNFfC)+2hBT!a=s`t#HZ}qULSZaU#TnNnscfuAtUzw}i(N-bfbQfJy)~wHj|Mnx9#}*C zJXXrNrRa-_MquL8e`dQ0Qd64%o@jQg7Z3DrBBOc_oMQ{)XgDb)v3OndE zsZh!=<1yb;7Rj4Ei`{b^$SFXtF3(BoLZ#?j&uLBeba1%X*Ww6}zYzA;X;qR2#BByMDxk$ZYkSCA8OEC}$XB8P zddQj{utvC%k%D&LJ~#*pIT1vQ=Ny35Fes;mA{tZd=(=6Z=qcPgazW+7(i;y7H)g4i zwGGjx-$bHMd`g*C7l;aWiC+JH&_F@WbA=X5@Q^gqpe+GR17;_~iBpA+P`uP^rvFE0 zhwdSjK%?Jvvb8;lh__P;?H-ho!9=LcD{=Y z7Br$@)WTClMh6$eNVFHrG#UkJYN@r_?sJWh1=9h%Yhy{CtV`z*+aVI1^j{lhT>5~R zlGHRc6Oo1@xJ(;lNi{I-L$eadt9`q$mw%(`u&4?Wj){;m@t8qkw6rV<_; z&8RCH2fI?l*xY*yFUIKac)*bg&V43f#H#o0K%?Je@Sht3^_jo3?OQ3f zxuwFy;l(wJ*xz1&1Mfc7F(`MUbf;3ZqGPlp-Y*Ekg z?firu$baO~cMdxqxhL@8!q*(tWeiLOFR;@zAW-Tl;;%u?RK|1JaUjN^hvVRHn))XY zw=~8p8Wz_n^5)Bc*EjvsC+@5}4bY-Sd7Pfl6_=W7b*CYwXXGQJN8I-^l{=v71NyyD ztyfagbq71NK3HgMvDvP%Whw~bi9!#hHCM=aQXh%Kh>4^K5z8kJXl&WQ`(C7EK?#wN zKw4@>kQZ>4Dx}nOkE2Zg*@mOLx7`7`0S_(6aEz?LPHfc*iM~)5N`mriC`rv+5L@!{ z=EL>CsEY_h>ZzL1&}RpI3M6xJw=7|EiX&lI6G>oath!UdA6f+AJygUha><(GMUc|c zlV9TNulnAYJ^@4==$nE5mMFV0s2IY6n|=TA2>BIYSJ$sxu`ccW9t(^^MNj&`PBV~) znIxPuR+f8Kb-=GVXD^AJQ(F36TljTV@Z^DnwRQmt&U zdfV6-p@SgcHAjJ23Ks-s%RSBa4ySf@bfC#l(e~<|>CCK#@+M`W&I2)R4(AU>HbG)y zp5u8mpXS55ZX>2|fa3v|9t-yh6%p6oAzp)%kLPwuU08$jsA25R6A?V1ubPmX>xq!@U?mt#NvWr6hy2tw1Xr2EK2P*r zUUp#xyDVx-D()*r8cd1UUX1qVxmc)4iupbRlz!vJ4LmfoRl8Hzhk0|I5}8>woHGZK8}h%zp{P^p5HauDYPEY+JL&W$#-7hZzc_-(F;l*J*FlRcp3TM$e-2>RBO`DDdgJa2$J69e&n1KUB8A5z<*gt}Yx{!#hY^6SfHPX~@YX9rP_X9nv4 z$piukitz{80qI-uD>y#uC{9vVsc>AV0q1A7%ZxFU4$VLY#ILR_>J+KUvkCe9kzamZ z@I{3tAFu>anGka?2h#A0sN;YY){5fkuus800b(rTNruvP76ABnCEb=zTz;9#ZYt8t zS(f=f_M;l-isvLHp&JX>Hw$hDJb*jZCV+^6A2TAE1mom85{g zvx`MSQ$>>eqL>uVHw>k%S@aEf3oU}rK*t68=m1(1$hj#>uu{=4UPL9P5UU2e+&@1*g*1FY*A?L&Yx)K% z>qC&5_k5oGzFJbW=>KG-cxLNL_ zt_oSwzivG~;}xc8!+pJux(vIxTn5Oe=Y;3AKY0TMPl1ZfZ6H&XqEz+m;wB;UtqwMhd-Mec9KYDul zfEZ29Yu91RDie8S2zGTvPZfh&>E1mKu8dbCW}4KUCyT|~T{_Ioev6CEn9YF8w)$fE z<>rqcLSvHQ5j4o}O!d`l+id&z#>Hxm-I{)V6=5yJ7qtg%!k<12@ecO!4*ASaM=yAl zk;eQ{q!k@raTN1dQ-aOE)6;4V7OSzeVBH3 zn7^{~_)i^VJ2U4A6eOU=B~bi>0uSK+&y9E_-~5ubsfq0L}~lK}uz#H_{- zeIDWBPP9<9nFo&gz@5FWOio57&alfcZO?cnSG>KGlOc3mK&$C#s&7a;4Dv9bF2Ed2 zb5Fg!@7(QUF=uK5*L>FagYl;?PzDBUkdv)JO)Iac=!1}O35^zu^jJY248pEZ^pw}{ z=OWV!1{kui8OQMMZ{4c~vT+lh2G4-v;tJgEFTEm)IAA0IX-qCIJ5z?=1Ox=!hX;im zBMIO|m0{)AE&sLah}Yz%(|M;$ft78=vtXqY`u34g@8L&-?fW`Fv_IBmnTw8iD)9Fw zv%np)7%G_&$HzZ`>)98k_WP%2hrsB+2jI^WI1OyPyvjM2a&px0WSrZFrMsW5tb_&K zsH@R=0K4kw*|RrdP0qnwor0^#P+79LE-%g0#TFGYgPfb^zAOgiKWo5d-+=jEIS^db8aUk+$}qB^E+BY!{gd^xsp{(xTNcT5UMX}Yaj0DSb0b%_0_MqIQ-T` zCj^GEF|7>8v7F9vFV+8=?R;(Cw1U^-3dRDajT{nSkh)`O`RT=RUn)20m_;)AR7KJ? zWQhzyy}i9ZN$+o!>AM1vgHqGE^kW%b8dA^5 z;9~Mx3lqmC_MP*PHraZVjSa7MGWB>ebdDnXGBE%JQ6acyS8QS1>c@OLt3H6NWuPYQ zooKpa9ljQz;LG8e-TVv=nGhH{;OS(oI`z!L0_}JQW~eFC74fl?f^V*xYJ zmp?;+M$gUJj$(DPfM^?UEc;FW(*Jj1hVUbJJSF|vG5HMJ|sn++Rc7~z<)xB+qb z0L0}1o%GUf$hN@}E&_;;YiZI;x*RGOu#`8K5Q~VnYo@rE*cJ60nu7;j^5(iFjEwrD zQ|~f@{QSc032YVohYz85!k;{Q_R-2i%eSyl2JZ>}aOXGxsM>JpNm)%ouW&@E@ zfpurLu#mHX>QqOEUkrQ)FWzcN=5|@IXJcauLJ|=)|6eEa4*7+KT#l4DDCajStJNvB zmob-J62rNlxB4QQq5$8;rdhE19d760Nv2kimwyir=h_}r^$4E&bGIvx9+aey0Hf{! zS=5JOwq_Ap7}}_Opq*-?V$lBA>Era;h0gO;zD=8Z45}e>*}ySsZdu;N;&$5OYj5b^ z8y7i5=D4B+2S$5trVtKJ&gjCzy+BBv=Llk1IjzWdC=`mG&|@S}YMER1-TE6e#6rJ! z9x##mwl*&iSV3$wq>vt2=SRxu^-NYwz)_KNUEB}N2^OCn{$6iql-P-v*U>R6tKG#wPk$VGkGa_h@+vA7zI?^rA3iXj4q2H?9CsaV zM31%$;*W3e@%AP?eI!gz$ocfC>3Lk&RRZ@~hGy?6Q-a{-1c7EKhdj3_)LJoZ2(Lf)D`?!Q@-|JLCd>#)Fvz2Cs{)R|*iYWp=qA;`V zd7ouW{}rC6o~Xo3J^y_I@&1)0u7MiGa(F!1B=Es+3xiP=Pz2V`zB#qnv&sV9gsTr= z5g}h(jMw!))pslW*fRt}{5+jD>bqZBtM3-TLSL+rk;EfkMmjmSsBfu9`1$$4CNxh6 z(f@UEF$;^oTux+*nXJ0{ekhMVkBE2%o}Ci}3>%xkv3O#(&kmB0_fFokXAg#ipumSG zr`I!0-{$X=Bxh`FdndUFK=o#SiRo?7xw0?=XY|~~i<_WGTHn;<2}{}B+PZNq2}6p9 zB4SS|6jLV=G+M-cNk&F!$FJNOn~x@+fDOe85%uw@bU6K)cN%Cu+-@!#o|vcr{m$!3 z^fJhu?7N?{R!M@j2Rh|Bvf|#nVdvr+$ca)}pLi+)1_yXcThoZo!^7P={Z(XSI{GA9 zg%W^SlpxttP*=d`|Lg32@|JZt4a>-FP>JPq24hV_SJ%_=aZAw%<-6@~w?I|_=k|L9 zyue+qMGzoc{`2$BZt!?K*&8_*xCDzUsq3vg)~+hnRZdD z^D2CYyqJ2E+@BF-q*3O4kXyk3L5&p>khyh!P7N(|>eE3@`KHrFjZcPLAT)k`H~2au z*QqmyOs`|g-*{PSb;&L(E9-TB{z35Wh&qhyP1nw6OPz!E?@eIUkcNl7A*cY%ouy{k zJBSO*?k9C_-kY8EI568d8rx1xe=#MhY{~csPueQ9nV(Em`|v@mu)ACLa)^K|(Q;^q zAU(m1>;5?^;9KVt>n$OX!7llb4xLBZunbIsBr2%{CX8`ufdxxTfPt6E}@k-(|@9Lpu%tm2~YP#mt` zeZ1v8M0vK+Hyq>O;_3xz{Hcsi`mosyy2u%@S$#(b(}|oOm%)u2??=Rc0I?}YueH8X>YtqcV*52Q$VRAS>5Ei`3ER_-b}k%b#D0H1s2K98lo0_1d#=MS>L zqZRbQRwjS>^216aRXh=acj5aU#+U)ALvT~DkG$_?%dXS6NLD_^^UG~hpd|Hz@01^e z0Z$ORSRu4|`}&qaDK*3Phr(R~eqYJb4?dgGul$8jqaGR?V|G~m6~TTbdRIuk&Mgvv z=S>4D%YEM03~!LTrcHg?V?}%KJ_}@)Cy^S&q_aex^AsDh*o9ycMsEH?w;kvj!H`~JTwRi+W_oG(d1tlcO9tM^X$~^+XYu{g!r91BGS7Ec`GI_Pe=4q z$HZJ8I5Qtt^xf|qgRcS0qcaPa-0lWeK7`bKqXC|9dKaF0cL<)E_a55h_@S8q!uzdW znQ```Z1{Z=E?3#5qUeCl$}1{fbaRs^)a6?ZzaZoXvp@?ii+g!>pB83B^7kEqW;gCE z706ehM!pBb$cOF;=zopAu!!J<$v4u=+oRm3eMWnVi>0)S-3|uv>3xD5dOYe`X>`HB zz#y@nHkX5&o1LGZ-p9u$_>2XU|1uYw>UJ99w)VScz|Gh@UNEth3Qp?Yy?bBhsib)HBZXR%Wpdc(w}F0b(1YAH8yq=AlJ9nw-SCKUE=t` zt$NgkebGng;o*KCpjHn}6zt}?H`DHznO%TZb8A+X9ecWAVIOz-nq04ziBsbOH2cw$ zZXmV@QCd%uS6&Y()DY}L24JOca#kN04|ZcV2PPY7xt5O$UHcvP*Hz+Au__xAPO z-r{uO!i7W&cpTzIUg+RKCIorn&{q~aOK}l6?h;ewXMiz1l3qPajm=^asoS^h{awrV zPY+Tf6LQgOR>q=mL{_Nc^ZT~%jqzki@vke|ZY5rRN_ljT(YxUu!BWoA@eGt5AI-v3 z1@u8j_<+&W^r}!ydIIO6>awM~sFdfmH)6$0*4lnJsl&vQCF{$66tCDlApmB875V4c zJ2;dQ$sgdq!I|&iA=n{F^ZxD`L#XkujaE703FwJ-pKrsn5^i;ZLF_5WtFc*G(B98Z zaiTQw}?X^jaQ_p?_QzP)8!YUyCF0G8-@WV&}xC71N8yKqN6-n(hwNHG|qS{sJpv6W6ZroPZRCxY6As@`qdVI zRdANvfFCMIU}Sgk;-Nx$fFgp8;=UZbo03~n!pg1eCY5-E2^~%XtaY)BWwa}w0ohBZ zZ(X>(j>4_C^^ccHr!`rW#x*N^;K2K!O9UMPZJYq zi)r5|tHpw16392asYekQdkDR+**D{;&d&SU=D7z)eB0ChpW3cG9_y@qKP7~0*|I%$ z#T2OqV{1aPWvkaKBxEZRg&|v!rI1jBQi?K@$x@c^L=+;C${Iyzc@Sf1k$SIVOw+ut z{`q}AKlAyN$>aMy%YDwd@9Vzq>)1PJh9O1KU3M-y+LL-o6GDH+s2Tw3zx~tr{->o! zAXN~#rXmksvU~eAVl&)dy3r5vKVaHtI_EsY&-mqs!W*HVS2LQ)*?TQ^55-RSiKfzJ!kVf6=y1W4ELq&5sxR6i`~M z?E!?)YouGR#cfKHtPI*~;pl-A16^h!g5 zG(#QAg;d|P4tH|uSFy9?q#Wj3F3xV9aq2Kbc$an7nKP=d3B@V9a;Cg8iIlPJ0XL>~ z*QdL?^a(WzDGIr{zP;~@%6jVY^tXyEXhl*2j0>GR0I9Sjs@^hBAcE1sx>NRMmsbdtP`d66OaBZ zEX4Gvd=n8z4yO;KFU32 zpg8k01WPuDglJ_P+AHvMR5teW-s_EH6X!Qd&BWX{+ZaDotRGVOWM*0Ws9*W`C9&D) zI;$;4I65d*^r=Xiri6q<4NMP>@o>8 z#b#e^Ij%`{or$l)gyIlebK~9Qq&;%6LX&R8A76%kb|rz+d_d$hX?);dN|gt-;)>6E z!D`KkIdwQuNa+ir2i@{!-#^GLR|$J4gdOXBg6mnuRZ z(dpk}2p(!=Vxs5bvK(%#l=Vj0%hc4=duhhw${BG_+8uI3BwEz@+qk3SAlx~1l+Ou7DH>;1GM)_{Qhv5i-U;-sy2JB`upl`C0jw>TKq3mWZO zTFYuua%H(mi9klsHTzgbn^K*6d%L>rQz3N5__n1DQ3QWR+f(wQs@!I{Z@01Nk1Jhl&5FE0PnB(nA+ zyq}wvD5S^8o8;_m1={TL1OO20^o~w^Fx$TUXixbsAdCxPi!Ujn#tnl#u<*3Rbyt@a z74ZS}rpCM0`+6t~L%kp$$}0qCg#gK_RjWz|r`((%yZ4<3lvj7XoXEUzfdUpb0?ic% zAo?iZ;i3E#5(t^ab=1IRU^eNZA}6Q)Qaz&Yg=g`{QC2#jD1dJlD=I|rVx4QCL-0&b zgV%9ZBjCVeI_YUbL0oyYnuS6N8#Md00}TyX;qmFwaCUNY3#Lo3xu0)lrsP}yUcJxq zDxY26FWcvSi+k(iG6w5cuU{W~HLc&yR`>QD1En|g%dGjdXV47D)RdIQY+n_hi2vS_ z64=`I+%6mO6D-W9KG47GlSSWI>a~ah(7g+8Ug_@ZoeSd3%0Nb!{?D@tf5ai!iz8;l}We-8_Yz6jf7=V(ZLxB$yP=FHrDkY zqwL(>e}D=Ul)8?8{P;1S4xRVRf&y+ru%uMI_PGcZk0~C@qSmgi<7sI^{xmzZbhTZ@ z3e<9gOWE4N;&K2wn3-3vvSAV7CDk`l^%|VSQyLk|dpJ@WWke3RWR28$mR!osT?PUY z@uE-x5^)_19ne+WXSIQMM?x>G&zqqZ#eGS$$}8YrZTJ9>191MAZxZ&&Ko z?-oZ;Pc#%a=o!oMfH}-QL#6wboA21c5_5O4L82F$5piy-NrHy71z~{rfX&oG1b!3H zy&s3oeb9C}XdRRK3J>M7%~<&Nx=7JrdXsFf z`>BE#2X1EJlRIo6@^>mNO&gS0!EV&f6V4-4XELQQMD@fsxEbPE}zp4PFg0YVVpaxx;>N~!a0jM6@IP@ z_Y%I=IGBjn%y<>xoVY>?;>ZP#hU%sDkbTNSVb0v4>hZb%!D zs!R5tO!yV;R+Aeg+*DGH3l(z=2bHe5G9D|2!2n@hI)c`y2Kixv1znP6q3~YUejh-G8<@udsJekT;ap#~?(=p4xx#BJKB?DutOtyz!IM1v%N-jHnGfUBsv|33rboqK@qxu{@wg z)*?ZrBC_Qt4+HL97E z3m(+6imVS-PIovpWd!BULy4!xvA`Y`f0S5tzp(Y;*wsX|f{1(;YmdtvW0+A*MmF`> z_!(xe2_rYuBj+6c?b@}-9?v=;r;Ap&pIx%?ec+)(xNYP(IsU4eym3^FJ=`J|;VJgO zzriUV0^&fcwzauD)_6mwn*;2-^5_^W2Jm0!<8yiqG8;FFLD{)b8BP{G)dR6m%5`6V)E$2I)G(ai@c69l}XqsUzVw~a`a zJ@dbxh3c{4^F_?-`lq+;+NIcSf|84+uwM?&4H**z940vEE>+OKIQTrsv6T{{nTYBnK*(=E==829I4phuw*)J z1*=noC(ZpesMu5B@^qbiTYwehwr9^KBO^Af6K`B90#v4zqQfRMv}JA+I*O zpd=m3jK2#rl}~D<4)JQDDEU391#$3WxD0L zEn|gwf$`Gtxy%6vS)W%GvT6~&rly=YAap8)ozGHKTvx7m6CV29$5F3gN$E|RKdo27 z)mX^_sn(Ul_xUMimX=Ic9tnQz(ZBgVNUIYfgY9fcvOqY>EZrht%#4c>d#8{>ijmtd zH*M>oe|2F?*GcI;^}7>M2(_f7I}Se4(Jt&Pcw>Z-O1yx{6&{=otGJl>ll27EF=6^! zx1Oe@F4{G+PuOa$piGiiS$VlcPsQPs+#daIQv(B*km>8^?{z)(R0-1B<)+AEpX+Ub z{I8n#!3)67qFe+Q$DU^KI%s>w__D+o+x=M@xLisxUEA{lGN7FkKN>0kPpsBxg9sOz zS52tbd$+BQe`t3*G7>viHV(XdFUF5{k$O>yJZ;rpU}5^8B9oH`MqPeBcrU%G?~6^#9O@jj)p_QX@i zvF)UsE>qs407v7Vr%J;#rd2wsM#7z(uAu0zc85Pv&^=N(@)B~(iNk*79-^0=alMLM zYLiRnglN560OT9k-U=uBOc-9)3yQ}^aXPp+10XKIN9}zVw^L;eSPv#I>nX&T+e0P| zxviShKl~f%ZK?7?PQjQbdqw|Ptb|&~SGt?wunZ3m=L@tbb*ak9%ZJ@{yp3ns7g$wm{aebp-$$(g6`2mMj4%~S%j3sK zv3W{C2yK#TV`=Hq@vZ}x@U^^*YktW#6Tq-4iTSUPXl%t-IOA($ru0gwOFCoV>V z%`J%vbGj!N_v_{4#mB9-Ze<6s+SVg3*(yUqPu>DJ-jeF-Hs5^cOyi)orDgKpAKH*x zTdOy}s6k=Fz|~^)KZI_YZ#w{#yRe8X-<1fHzrYnC-T=9qs$5$)%jhYnW4heryyj$( zkQ8E?Ys6AhTWLulD*0Y0p>YsH9tNgg*$?38fO8ogtp~k}Xb2ZdDJXDIo=<$(Fp9Y6 zGr@r_au?mWQ9o^|F2cbcsH2-@O2YD0l1S`VHPPBN+R~!+@60`6yQO6$YW$vqI~#?h z8xnYJI}vS*J|OKOcnP?FWU#-5X8s@k;ZvxS;BVGZMfT)wLyEy>6Wwdt_F?}4gdwK8 literal 0 HcmV?d00001 diff --git a/docs/_build/html/_sources/index.rst.txt b/docs/_build/html/_sources/index.rst.txt new file mode 100644 index 0000000..431fe90 --- /dev/null +++ b/docs/_build/html/_sources/index.rst.txt @@ -0,0 +1,50 @@ +.. DFO-GN documentation master file, created by + sphinx-quickstart on Thu Aug 3 16:29:21 2017. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +DFO-GN: A Derivative-Free Gauss-Newton Solver +============================================= + +**Release:** |version| + +**Date:** |today| + +**Author:** `Lindon Roberts `_ (Mathematical Institute, University of Oxford) + +DFO-GN is a Python package for finding local solutions to **nonlinear least-squares minimization problems (with optional bound constraints)**, without requiring any derivatives of the objective. DFO-GN stands for Derivative-Free Optimization using Gauss-Newton, and is applicable to problems such as + +* Parameter estimation/data fitting; +* Solving systems of nonlinear equations (including under- and over-determined systems); and +* Inverse problems, including data assimilation. + +DFO-GN is a *derivative-free* algorithm, meaning it does not require any information about the derivative of the objective, nor does it attempt to estimate such information (e.g. by using finite differencing). This means that it is **particularly useful for solving noisy problems**; i.e. where evaluating the objective function several times for the same input may give different results. + +Mathematically, DFO-GN solves + +.. math:: + + \min_{x\in\mathbb{R}^n} &\quad f(x) := \sum_{i=1}^{m}r_{i}(x)^2 \\ + \text{s.t.} &\quad a \leq x \leq b + +where the functions :math:`r_i(x)` may be nonlinear and even nonconvex. Full details of the DFO-GN algorithm are given in our paper: C. Cartis and L. Roberts, A Derivative-Free Gauss-Newton Method, *in preparation*, (2017). + +DFO-GN is released under the open source GNU General Public License, a copy of which can be found in LICENSE.txt. Please `contact NAG `_ for alternative licensing. It is compatible with both Python 2 and Python 3. + +If you have any questions or suggestsions about the code, or have used DFO-GN for an interesting application, we would very much like to hear from you: please contact `Lindon Roberts `_ (`alternative email `_). + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + info + install + userguide + +Acknowledgements +---------------- +This software was developed under the supervision of `Coralia Cartis `_ (Mathematical Institute, University of Oxford), and was supported by the EPSRC Centre For Doctoral Training in `Industrially Focused Mathematical Modelling `_ (EP/L015803/1) at the University of Oxford's Mathematical Institute, in collaboration with the `Numerical Algorithms Group `_. + +DFO-GN was developed using techniques from DFBOLS (`Zhang, Conn & Scheinberg, 2010 `_) and BOBYQA (`Powell, 2009 `_). +The structure of this documentation is from `oBB `_ by Jari Fowkes. + diff --git a/docs/_build/html/_sources/info.rst.txt b/docs/_build/html/_sources/info.rst.txt new file mode 100644 index 0000000..7632948 --- /dev/null +++ b/docs/_build/html/_sources/info.rst.txt @@ -0,0 +1,94 @@ +Overview +======== + +When to use DFO-GN +------------------ +DFO-GN is designed to solve the nonlinear least-squares minimization problem (with optional bound constraints) + +.. math:: + + \min_{x\in\mathbb{R}^n} &\quad f(x) := \sum_{i=1}^{m}r_{i}(x)^2 \\ + \text{s.t.} &\quad a \leq x \leq b + +We call :math:`f(x)` the objective function and :math:`r_i(x)` the residual functions (or simply residuals). + +DFO-GN is a *derivative-free* optimization algorithm, which means it does not require the user to provide the derivatives of :math:`f(x)` or :math:`r_i(x)`, nor does it attempt to estimate them internally (by using finite differencing, for instance). + +There are two main situations when using a derivative-free algorithm (such as DFO-GN) is preferable to a derivative-based algorithm (which is the vast majority of least-squares solvers). + +If **the residuals are noisy**, then calculating or even estimating their derivatives may be impossible (or at least very inaccurate). By noisy, we mean that if we evaluate :math:`r_i(x)` multiple times at the same value of :math:`x`, we get different results. This may happen when a Monte Carlo simulation is used, for instance, or :math:`r_i(x)` involves performing a physical experiment. + +If **the residuals are expensive to evaluate**, then estimating derivatives (which requires :math:`n` evaluations of each :math:`r_i(x)` for every point of interest :math:`x`) may be prohibitively expensive. Derivative-free methods are designed to solve the problem with the fewest number of evaluations of the objective as possible. + +**However, if you have provide (or a solver can estimate) derivatives** of :math:`r_i(x)`, then it is probably a good idea to use one of the many derivative-based solvers (such as `one from the SciPy library `_). + +Parameter Fitting +----------------- +A very common problem in many quantitative disciplines is fitting parameters to observed data. Typically, this means that we have developed a model for some proccess, which takes a vector of (known) inputs :math:`\mathrm{obs}\in\mathbb{R}^N` and some model parameters :math:`x=(x_1, \ldots, x_n)\in\mathbb{R}^n`, and computes a (predicted) quantity of interest :math:`y\in\mathbb{R}`: + +.. math:: + + y = \mathrm{model}(\mathrm{obs}, x) + +For this model to be useful, we need to determine a suitable choice for the parameters :math:`x`, which typically cannot be directly observed. A common way of doing this is to calibrate from observed relationships. + +Suppose we have some observations of the input-to-output relationship. That is, we have data + +.. math:: + + (\mathrm{obs}_1, y_1), \ldots, (\mathrm{obs}_m, y_m) + +Then, we try to find the parameters :math:`x` which produce the best possible fit to these observations by minimizing the sum-of-squares of the prediction errors: + +.. math:: + + \min_{x\in\mathbb{R}^n} \quad f(x) := \sum_{i=1}^{m}(y_i - \mathrm{model}(\mathrm{obs}_i, x))^2 + +which is in the least-squares form required by DFO-GN. + +As described above, DFO-GN is a particularly good choice for parameter fitting when the model has noise (e.g. Monte Carlo simulation) or is expensive to evaluate. + +Solving Nonlinear Systems of Equations +-------------------------------------- +Suppose we wish to solve the system of nonlinear equations: find :math:`x\in\mathbb{R}^n` satisfying + +.. math:: + + r_1(x) &= 0 \\ + r_2(x) &= 0 \\ + &\vdots \\ + r_m(x) &= 0 + +Such problems can have no solutions, one solution, or many solutions (possibly infinitely many). Often, but certainly not always, the number of solutions depends on whether there are more equations or unknowns: if :math:`mn` we say the system is overdetermined (and there are often no solutions). + +This is not always true -- there is no solution to the underdetermined system when :math:`m=1` and :math:`n=2` and we choose :math:`r_1(x)=\sin(x_1+x_2)-2`, for example. +Similarly, if we take :math:`n=1` and :math:`r_i(x)=i (x-1)(x-2)`, we can make :math:`m` as large as we like while keeping :math:`x=1` and :math:`x=2` as solutions (to the overdetermined system). + +If no solution exists, it makes sense to instead search for an :math:`x` which approximately satisfies each equation. A common way to do this is to minimize the sum-of-squares of the left-hand-sides: + +.. math:: + + \min_{x\in\mathbb{R}^n} \quad f(x) := \sum_{i=1}^{m}r_i(x)^2 + +which is the form required by DFO-GN. + +If a solution does exist, then this formulation will also find this (where we will get :math:`f=0` at the solution). + +**Which solution?** DFO-GN, and most similar software, will only find one solution to a set of nonlinear equations. Which one it finds is very difficult to predict, and depends very strongly on the point where the solver is started from. Often it finds the closest solution, but there are no guarantees this will be the case. If you need to find all/multiple solutions for your problem, consider techniques such as `deflation `_. + +Details of the DFO-GN Algorithm +------------------------------- +DFO-GN is a type of *trust-region* method, a common category of optimization algorithms for nonconvex problems. Given a current estimate of the solution :math:`x_k`, we compute a model which approximates the objective :math:`m_k(s)\approx f(x_k+s)` (for small steps :math:`s`), and maintain a value :math:`\Delta_k>0` (called the *trust region radius*) which measures the size of :math:`s` for which the approximation is good. + +At each step, we compute a trial step :math:`s_k` designed to make our approximation :math:`m_k(s)` small (this task is called the *trust region subproblem*). We evaluate the objective at this new point, and if this provided a good decrease in the objective, we take the step (:math:`x_{k+1}=x_k+s_k`), otherwise we stay put (:math:`x_{k+1}=x_k`). Based on this information, we choose a new value :math:`\Delta_{k+1}`, and repeat the process. + +In DFO-GN, we construct our approximation :math:`m_k(s)` by interpolating a linear approximation for each residual :math:`r_i(x)` at several points close to :math:`x_k`. To make sure our interpolated model is accurate, we need to regularly check that the points are well-spaced, and move them if they aren't (i.e. improve the geometry of our interpolation points). + +A complete description of the DFO-GN algorithm is given in our paper [CR2017]_. + +References +---------- + +.. [CR2017] + Cartis, C. and Roberts, L., A Derivative-Free Gauss-Newton Method, *in preparation* (2017). + diff --git a/docs/_build/html/_sources/install.rst.txt b/docs/_build/html/_sources/install.rst.txt new file mode 100644 index 0000000..ef4a3ca --- /dev/null +++ b/docs/_build/html/_sources/install.rst.txt @@ -0,0 +1,89 @@ +Installing DFO-GN +================= + +Requirements +------------ +DFO-GN requires the following software to be installed: + +* `Python 2.7 or Python 3 `_ + +Additionally, the following python packages should be installed (these will be installed automatically if using `pip `_, see `Installation using pip`_): + +* `NumPy 1.11 or higher `_ +* `SciPy 0.18 or higher `_ + + +Installation using pip +---------------------- +For easy installation, use `pip `_ as root: + + .. code-block:: bash + + $ [sudo] pip install --pre dfogn + +If you do not have root privileges or you want to install DFO-GN for your private use, you can use: + + .. code-block:: bash + + $ pip install --pre --user dfogn + +which will install DFO-GN in your home directory. + +Note that if an older install of DFO-GN is present on your system you can use: + + .. code-block:: bash + + $ [sudo] pip install --pre --upgrade dfogn + +to upgrade DFO-GN to the latest version. + +Manual installation +------------------- +The source code for DFO-GN is `available on Github `_: + + .. code-block:: bash + + $ git clone https://github.com/numericalalgorithmsgroup/dfogn + $ cd dfogn + +or through the `Python Package Index `_: + + .. code-block:: bash + + $ wget http://pypi.python.org/packages/source/d/dfogn/dfogn-X.X.tar.gz + $ tar -xzvf dfogn-X.X.tar.gz + $ cd dfogn-X.X + +DFO-GN is written in pure Python and requires no compilation. It can be installed using: + + .. code-block:: bash + + $ [sudo] pip install --pre . + +If you do not have root privileges or you want to install DFO-GN for your private use, you can use: + + .. code-block:: bash + + $ pip install --pre --user . + +instead. + +Testing +------- +If you installed DFO-GN manually, you can test your installation by running: + + .. code-block:: bash + + $ python setup.py test + +Uninstallation +-------------- +If DFO-GN was installed using `pip `_ you can uninstall as follows: + + .. code-block:: bash + + $ [sudo] pip uninstall dfogn + +If DFO-GN was installed manually you have to remove the installed files by hand (located in your python site-packages directory). + + diff --git a/docs/_build/html/_sources/userguide.rst.txt b/docs/_build/html/_sources/userguide.rst.txt new file mode 100644 index 0000000..4655f0b --- /dev/null +++ b/docs/_build/html/_sources/userguide.rst.txt @@ -0,0 +1,402 @@ +Using DFO-GN +============ +This section describes the main interface to DFO-GN and how to use it. + +How to use DFO-GN +----------------- +The main interface to DFO-GN is via the function :code:`solve` + + .. code-block:: python + + soln = dfogn.solve(objfun, x0) + +The input :code:`objfun` is a Python function which takes an input :math:`x\in\mathbb{R}^n` and returns the vector of residuals :math:`[r_1(x)\: \cdots \: r_m(x)]\in\mathbb{R}^m`. Both the input and output of :code:`objfun` must be one-dimensional NumPy arrays (i.e. with :code:`x.shape == (n,)` and :code:`objfun(x).shape == (m,)`). + +The input :code:`x0` is the starting point for the solver, and (where possible) should be set to be the best available estimate of the true solution :math:`x_{min}\in\mathbb{R}^n`. It should be specified as a one-dimensional NumPy array (i.e. with :code:`x0.shape == (n,)`). +As DFO-GN is a local solver, providing different values for :code:`x0` may cause it to return different solutions, with possibly different objective values. + +Outputs +------- +The output of :code:`dfogn.solve` is an object :code:`soln` containing: + +* :code:`soln.x` - an estimate of the solution, :math:`x_{min}\in\mathbb{R}^n`, a one-dimensional NumPy array. +* :code:`soln.resid` - the vector of residuals at the calculated solution :math:`[r_1(x_{min})\: \cdots \: r_m(x_{min})]\in\mathbb{R}^m` +* :code:`soln.f` - the objective value at the calculated solution, :math:`f(x_{min})`, a Float. +* :code:`soln.jacobian` - an estimate of the :math:`m\times n` Jacobian matrix of first derivatives at the calculated solution :math:`J_{i,j} \approx \partial r_i(x_{min})/\partial x_j`, a two-dimensional NumPy array. +* :code:`soln.nf` - the number of evaluations of :code:`objfun` that the algorithm needed, an Integer. +* :code:`soln.flag` - an exit flag, which can take one of several values (listed below), an Integer. +* :code:`soln.msg` - a description of why the algorithm finished, a String. + +The possible values of :code:`flag` are defined by the following variables, also defined in the :code:`soln` object: + +* :code:`soln.EXIT_SUCCESS = 0` - DFO-GN terminated successfully (the objective value or trust region radius are sufficiently small). +* :code:`soln.EXIT_INPUT_ERROR = 1` - error in the inputs. +* :code:`soln.EXIT_MAXFUN_WARNING = 2` - maximum allowed objective evaluations reached. +* :code:`soln.EXIT_TR_INCREASE_ERROR = 3` - error occurred when solving the trust region subproblem. +* :code:`soln.EXIT_LINALG_ERROR = 4` - linear algebra error, e.g. the interpolation points produced a singular linear system. +* :code:`soln.EXIT_ALTMOV_MEMORY_ERROR = 5` - error occurred when determining a geometry-improving step. + +For more information about how to interpret these descriptions, see the algorithm details section in :doc:`info`. If you encounter any of the last 4 conditions, first check to see if the output value is sufficient for your requirements, otherwise consider changing :code:`x0` or the optional parameter :code:`rhobeg` (see below). + +As variables are defined in the :code:`soln` objected, they can be accessed with, for example + + .. code-block:: python + + if soln.flag == soln.EXIT_SUCCESS: + print("Success!") + +Optional Arguments +------------------ +The :code:`solve` function has several optional arguments which the user may provide: + + .. code-block:: python + + dfogn.solve(objfun, x0, lower=None, upper=None, maxfun=1000, + rhobeg=None, rhoend=1e-8) + +These arguments are: + +* :code:`lower` - the vector :math:`a` of lower bounds on :math:`x` (default is :math:`a_i=-10^{20}`). +* :code:`upper` - the vector :math:`b` of upper bounds on :math:`x` (default is :math:`b_i=10^{20}`). +* :code:`maxfun` - the maximum number of objective evaluations the algorithm may request (default is 1000). +* :code:`rhobeg` - the initial value of the trust region radius (default is :math:`0.1\max(\|x_0\|_{\infty}, 1)`). +* :code:`rhoend` - minimum allowed value of trust region radius, which determines when a successful termination occurs (default is :math:`10^{-8}`). + +There is a tradeoff when choosing the value of :code:`rhobeg`: a large value allows the algorithm to progress to a solution quicker, but there is a greater risk that it tries points which do not reduce the objective. Similarly, a small value means a greater chance of reducing the objective, but potentially making slower progress towards the final solution. + +The value of :code:`rhoend` determines the level of accuracy desired in the solution :code:`xmin` (smaller values give higher accuracy, but DFO-GN will take longer to finish). + +The requirements on the inputs are: + +* Each entry of :code:`lower` must be strictly below the corresponding entry of :code:`upper`, with a gap of at least twice :code:`rhobeg`; +* Both :code:`rhobeg` and :code:`rhoend` must be strictly positive, with :code:`rhoend` being the smaller one; and +* The value :code:`maxfun` must be strictly positive, and generally should be above :code:`len(x)+1` (which is the initial setup requirement). + +A Simple Example +---------------- +Suppose we wish to minimize the `Rosenbrock function `_ (a common test problem): + +.. math:: + + \min_{(x_1,x_2)\in\mathbb{R}^2} &\quad 100(x_2-x_1^2)^2 + (1-x_1)^2 \\ + +This function has only one local minimum :math:`f(x_{min})=0` at :math:`x_{min}=(1,1)`. We can write this as a least-squares problem as: + +.. math:: + + \min_{(x_1,x_2)\in\mathbb{R}^2} &\quad [10(x_2-x_1^2)]^2 + [1-x_1]^2 \\ + +A commonly-used starting point for testing purposes is :math:`x_0=(-1.2,1)`. The following script shows how to solve this problem using DFO-GN: + + .. code-block:: python + + # DFO-GN example: minimize the Rosenbrock function + from __future__ import print_function + import numpy as np + import dfogn + + # Define the objective function + def rosenbrock(x): + return np.array([10.0 * (x[1] - x[0] ** 2), 1.0 - x[0]]) + + # Define the starting point + x0 = np.array([-1.2, 1.0]) + + # Call DFO-GN + soln = dfogn.solve(rosenbrock, x0) + + # Display output + print(" *** DFO-GN results *** ") + print("Solution xmin = %s" % str(soln.x)) + print("Objective value f(xmin) = %.10g" % soln.f) + print("Needed %g objective evaluations" % soln.nf) + print("Residual vector = %s" % str(soln.resid)) + print("Approximate Jacobian = %s" % str(soln.jacobian)) + print("Exit flag = %g" % soln.flag) + print(soln.msg) + +The output of this script is + + .. code-block:: none + + *** DFO-GN results *** + Solution xmin = [ 1. 1.] + Objective value f(xmin) = 1.268313548e-17 + Needed 50 objective evaluations + Residual vector = [ -3.56133900e-09 0.00000000e+00] + Approximate Jacobian = [[ -2.00012196e+01 1.00002643e+01] + [ -1.00000000e+00 3.21018592e-13]] + Exit flag = 0 + Success: Objective is sufficiently small + +Note in particular that the Jacobian is not quite correct - the bottom-right entry should be exactly zero for all :math:`x`, for instance. + +Adding Bounds and More Output +----------------------------- +We can extend the above script to add constraints. To do this, we can add the lines + + .. code-block:: python + + # Define bound constraints (a <= x <= b) + a = np.array([-10.0, -10.0]) + b = np.array([0.9, 0.85]) + + # Call DFO-GN (with bounds) + soln = dfogn.solve(rosenbrock, x0, lower=a, upper=b) + +DFO-GN correctly finds the solution to the constrained problem: + + .. code-block:: none + + Solution xmin = [ 0.9 0.81] + Objective value f(xmin) = 0.01 + Needed 44 objective evaluations + Residual vector = [ -2.01451078e-10 1.00000000e-01] + Approximate Jacobian = [[ -1.79999994e+01 1.00000004e+01] + [ -9.99999973e-01 2.01450058e-08]] + Exit flag = 0 + Success: rho has reached rhoend + +However, we also get a warning that our starting point was outside of the bounds: + + .. code-block:: none + + RuntimeWarning: Some entries of x0 above upper bound, adjusting + +DFO-GN automatically fixes this, and moves :math:`x_0` to a point within the bounds, in this case :math:`x_0=(-1.2, 0.85)`. + +We can also get DFO-GN to print out more detailed information about its progress using the `logging `_ module. To do this, we need to add the following lines: + + .. code-block:: python + + import logging + logging.basicConfig(level=logging.INFO, format='%(message)s') + + # ... (call dfogn.solve) + +And we can now see each evaluation of :code:`objfun`: + + .. code-block:: none + + Function eval 1 has f = 39.65 at x = [-1.2 0.85] + Function eval 2 has f = 14.337296 at x = [-1.08 0.85] + ... + Function eval 43 has f = 0.010000000899913 at x = [ 0.9 0.80999999] + Function eval 44 has f = 0.01 at x = [ 0.9 0.81] + +If we wanted to save this output to a file, we could replace the above call to :code:`logging.basicConfig()` with + + .. code-block:: python + + logging.basicConfig(filename="myfile.txt", level=logging.INFO, + format='%(message)s', filemode='w') + +Example: Noisy Objective Evaluation +----------------------------------- +As described in :doc:`info`, derivative-free algorithms such as DFO-GN are particularly useful when :code:`objfun` has noise. Let's modify the previous example to include random noise in our objective evaluation, and compare it to a derivative-based solver: + + .. code-block:: python + + # DFO-GN example: minimize the Rosenbrock function + from __future__ import print_function + import numpy as np + import dfogn + + # Define the objective function + def rosenbrock(x): + return np.array([10.0 * (x[1] - x[0] ** 2), 1.0 - x[0]]) + + # Modified objective function: add 1% Gaussian noise + def rosenbrock_noisy(x): + return rosenbrock(x) * (1.0 + 1e-2 * np.random.normal(size=(2,))) + + # Define the starting point + x0 = np.array([-1.2, 1.0]) + + # Set random seed (for reproducibility) + np.random.seed(0) + + print("Demonstrate noise in function evaluation:") + for i in range(5): + print("objfun(x0) = %s" % str(rosenbrock_noisy(x0))) + print("") + + # Call DFO-GN + soln = dfogn.solve(rosenbrock_noisy, x0) + + # Display output + print(" *** DFO-GN results *** ") + print("Solution xmin = %s" % str(soln.x)) + print("Objective value f(xmin) = %.10g" % soln.f) + print("Needed %g objective evaluations" % soln.nf) + print("Residual vector = %s" % str(soln.resid)) + print("Approximate Jacobian = %s" % str(soln.jacobian)) + print("Exit flag = %g" % soln.flag) + print(soln.msg) + + # Compare with a derivative-based solver + import scipy.optimize as opt + soln = opt.least_squares(rosenbrock_noisy, x0) + + print("") + print(" *** SciPy results *** ") + print("Solution xmin = %s" % str(soln.x)) + print("Objective value f(xmin) = %.10g" % (2.0 * soln.cost)) + print("Needed %g objective evaluations" % soln.nfev) + print("Exit flag = %g" % soln.status) + print(soln.message) + + +The output of this is: + + .. code-block:: none + + Demonstrate noise in function evaluation: + objfun(x0) = [-4.4776183 2.20880346] + objfun(x0) = [-4.44306447 2.24929965] + objfun(x0) = [-4.48217255 2.17849989] + objfun(x0) = [-4.44180389 2.19667014] + objfun(x0) = [-4.39545837 2.20903317] + + *** DFO-GN results *** + Solution xmin = [ 1. 1.] + Objective value f(xmin) = 4.658911493e-15 + Needed 56 objective evaluations + Residual vector = [ -6.82177042e-08 -2.29266787e-09] + Approximate Jacobian = [[ -2.01345344e+01 1.01261457e+01] + [ -1.00035048e+00 -5.99847638e-03]] + Exit flag = 0 + Success: Objective is sufficiently small + + *** SciPy results *** + Solution xmin = [-1.20000033 1.00000016] + Objective value f(xmin) = 23.66957245 + Needed 6 objective evaluations + Exit flag = 3 + `xtol` termination condition is satisfied. + +DFO-GN is able to find the solution with only 6 more function evaluations than in the noise-free case. However SciPy's derivative-based solver, which has no trouble solving the noise-free problem, is unable to make any progress. + +Example: Parameter Estimation/Data Fitting +------------------------------------------ +Next, we show a short example of using DFO-GN to solve a parameter estimation problem (taken from `here `_). Given some observations :math:`(t_i,y_i)`, we wish to calibrate parameters :math:`x=(x_1,x_2)` in the exponential decay model + +.. math:: + + y(t) = x_1 \exp(x_2 t) + +The code for this is: + + .. code-block:: python + + # DFO-GN example: data fitting problem + # Originally from: + # https://uk.mathworks.com/help/optim/ug/lsqcurvefit.html + from __future__ import print_function + import numpy as np + import dfogn + + # Observations + tdata = np.array([0.9, 1.5, 13.8, 19.8, 24.1, 28.2, 35.2, + 60.3, 74.6, 81.3]) + ydata = np.array([455.2, 428.6, 124.1, 67.3, 43.2, 28.1, 13.1, + -0.4, -1.3, -1.5]) + + # Model is y(t) = x[0] * exp(x[1] * t) + def prediction_error(x): + return ydata - x[0] * np.exp(x[1] * tdata) + + # Define the starting point + x0 = np.array([100.0, -1.0]) + + # We expect exponential decay: set upper bound x[1] <= 0 + upper = np.array([1e20, 0.0]) + + # Call DFO-GN + soln = dfogn.solve(prediction_error, x0, upper=upper) + + # Display output + print(" *** DFO-GN results *** ") + print("Solution xmin = %s" % str(soln.x)) + print("Objective value f(xmin) = %.10g" % soln.f) + print("Needed %g objective evaluations" % soln.nf) + print("Exit flag = %g" % soln.flag) + print(soln.msg) + +The output of this is (noting that DFO-GN moves :math:`x_0` to be far away enough from the upper bound) + + .. code-block:: none + + RuntimeWarning: Some entries of x0 too close to upper bound, adjusting + *** DFO-GN results *** + Solution xmin = [ 4.98830861e+02 -1.01256863e-01] + Objective value f(xmin) = 9.504886892 + Needed 107 objective evaluations + Exit flag = 0 + Success: rho has reached rhoend + +This produces a good fit to the observations. + +.. image:: data_fitting.png + :width: 75% + :alt: Data Fitting Results + :align: center + +Example: Solving a Nonlinear System of Equations +------------------------------------------------ +Lastly, we give an example of using DFO-GN to solve a nonlinear system of equations (taken from `here `_). We wish to solve the following set of equations + +.. math:: + + x_1 + x_2 - x_1 x_2 + 2 &= 0, \\ + x_1 \exp(-x_2) - 1 &= 0. + +The code for this is: + + .. code-block:: python + + # DFO-GN example: Solving a nonlinear system of equations + # http://support.sas.com/documentation/cdl/en/imlug/66112/HTML/default/viewer.htm#imlug_genstatexpls_sect004.htm + + from __future__ import print_function + import math + import numpy as np + import dfogn + + # Want to solve: + # x1 + x2 - x1*x2 + 2 = 0 + # x1 * exp(-x2) - 1 = 0 + def nonlinear_system(x): + return np.array([x[0] + x[1] - x[0]*x[1] + 2, + x[0] * math.exp(-x[1]) - 1.0]) + + # Warning: if there are multiple solutions, which one + # DFO-GN returns will likely depend on x0! + x0 = np.array([0.1, -2.0]) + + soln = dfogn.solve(nonlinear_system, x0) + + # Display output + print(" *** DFO-GN results *** ") + print("Solution xmin = %s" % str(soln.x)) + print("Objective value f(xmin) = %.10g" % soln.f) + print("Needed %g objective evaluations" % soln.nf) + print("Residual vector = %s" % str(soln.resid)) + print("Exit flag = %g" % soln.flag) + print(soln.msg) + + +The output of this is + + .. code-block:: none + + *** DFO-GN results *** + Solution xmin = [ 0.09777309 -2.32510588] + Objective value f(xmin) = 2.048743163e-24 + Needed 24 objective evaluations + Residual vector = [ 1.23900890e-13 1.42597045e-12] + Exit flag = 0 + Success: Objective is sufficiently small + +Here, we see that both entries of the residual vector are very small, so both equations have been solved to high accuracy. + diff --git a/docs/_build/html/_static/ajax-loader.gif b/docs/_build/html/_static/ajax-loader.gif new file mode 100644 index 0000000000000000000000000000000000000000..61faf8cab23993bd3e1560bff0668bd628642330 GIT binary patch literal 673 zcmZ?wbhEHb6krfw_{6~Q|Nno%(3)e{?)x>&1u}A`t?OF7Z|1gRivOgXi&7IyQd1Pl zGfOfQ60;I3a`F>X^fL3(@);C=vM_KlFfb_o=k{|A33hf2a5d61U}gjg=>Rd%XaNQW zW@Cw{|b%Y*pl8F?4B9 zlo4Fz*0kZGJabY|>}Okf0}CCg{u4`zEPY^pV?j2@h+|igy0+Kz6p;@SpM4s6)XEMg z#3Y4GX>Hjlml5ftdH$4x0JGdn8~MX(U~_^d!Hi)=HU{V%g+mi8#UGbE-*ao8f#h+S z2a0-5+vc7MU$e-NhmBjLIC1v|)9+Im8x1yacJ7{^tLX(ZhYi^rpmXm0`@ku9b53aN zEXH@Y3JaztblgpxbJt{AtE1ad1Ca>{v$rwwvK(>{m~Gf_=-Ro7Fk{#;i~+{{>QtvI yb2P8Zac~?~=sRA>$6{!(^3;ZP0TPFR(G_-UDU(8Jl0?(IXu$~#4A!880|o%~Al1tN literal 0 HcmV?d00001 diff --git a/docs/_build/html/_static/alabaster.css b/docs/_build/html/_static/alabaster.css new file mode 100644 index 0000000..be65b13 --- /dev/null +++ b/docs/_build/html/_static/alabaster.css @@ -0,0 +1,693 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +@import url("basic.css"); + +/* -- page layout ----------------------------------------------------------- */ + +body { + font-family: 'goudy old style', 'minion pro', 'bell mt', Georgia, 'Hiragino Mincho Pro', serif; + font-size: 17px; + background-color: #fff; + color: #000; + margin: 0; + padding: 0; +} + + +div.document { + width: 940px; + margin: 30px auto 0 auto; +} + +div.documentwrapper { + float: left; + width: 100%; +} + +div.bodywrapper { + margin: 0 0 0 220px; +} + +div.sphinxsidebar { + width: 220px; + font-size: 14px; + line-height: 1.5; +} + +hr { + border: 1px solid #B1B4B6; +} + +div.body { + background-color: #fff; + color: #3E4349; + padding: 0 30px 0 30px; +} + +div.body > .section { + text-align: left; +} + +div.footer { + width: 940px; + margin: 20px auto 30px auto; + font-size: 14px; + color: #888; + text-align: right; +} + +div.footer a { + color: #888; +} + +p.caption { + font-family: inherit; + font-size: inherit; +} + + +div.relations { + display: none; +} + + +div.sphinxsidebar a { + color: #444; + text-decoration: none; + border-bottom: 1px dotted #999; +} + +div.sphinxsidebar a:hover { + border-bottom: 1px solid #999; +} + +div.sphinxsidebarwrapper { + padding: 18px 10px; +} + +div.sphinxsidebarwrapper p.logo { + padding: 0; + margin: -10px 0 0 0px; + text-align: center; +} + +div.sphinxsidebarwrapper h1.logo { + margin-top: -10px; + text-align: center; + margin-bottom: 5px; + text-align: left; +} + +div.sphinxsidebarwrapper h1.logo-name { + margin-top: 0px; +} + +div.sphinxsidebarwrapper p.blurb { + margin-top: 0; + font-style: normal; +} + +div.sphinxsidebar h3, +div.sphinxsidebar h4 { + font-family: 'Garamond', 'Georgia', serif; + color: #444; + font-size: 24px; + font-weight: normal; + margin: 0 0 5px 0; + padding: 0; +} + +div.sphinxsidebar h4 { + font-size: 20px; +} + +div.sphinxsidebar h3 a { + color: #444; +} + +div.sphinxsidebar p.logo a, +div.sphinxsidebar h3 a, +div.sphinxsidebar p.logo a:hover, +div.sphinxsidebar h3 a:hover { + border: none; +} + +div.sphinxsidebar p { + color: #555; + margin: 10px 0; +} + +div.sphinxsidebar ul { + margin: 10px 0; + padding: 0; + color: #000; +} + +div.sphinxsidebar ul li.toctree-l1 > a { + font-size: 120%; +} + +div.sphinxsidebar ul li.toctree-l2 > a { + font-size: 110%; +} + +div.sphinxsidebar input { + border: 1px solid #CCC; + font-family: 'goudy old style', 'minion pro', 'bell mt', Georgia, 'Hiragino Mincho Pro', serif; + font-size: 1em; +} + +div.sphinxsidebar hr { + border: none; + height: 1px; + color: #AAA; + background: #AAA; + + text-align: left; + margin-left: 0; + width: 50%; +} + +/* -- body styles ----------------------------------------------------------- */ + +a { + color: #004B6B; + text-decoration: underline; +} + +a:hover { + color: #6D4100; + text-decoration: underline; +} + +div.body h1, +div.body h2, +div.body h3, +div.body h4, +div.body h5, +div.body h6 { + font-family: 'Garamond', 'Georgia', serif; + font-weight: normal; + margin: 30px 0px 10px 0px; + padding: 0; +} + +div.body h1 { margin-top: 0; padding-top: 0; font-size: 240%; } +div.body h2 { font-size: 180%; } +div.body h3 { font-size: 150%; } +div.body h4 { font-size: 130%; } +div.body h5 { font-size: 100%; } +div.body h6 { font-size: 100%; } + +a.headerlink { + color: #DDD; + padding: 0 4px; + text-decoration: none; +} + +a.headerlink:hover { + color: #444; + background: #EAEAEA; +} + +div.body p, div.body dd, div.body li { + line-height: 1.4em; +} + +div.admonition { + margin: 20px 0px; + padding: 10px 30px; + background-color: #EEE; + border: 1px solid #CCC; +} + +div.admonition tt.xref, div.admonition code.xref, div.admonition a tt { + background-color: #FBFBFB; + border-bottom: 1px solid #fafafa; +} + +div.admonition p.admonition-title { + font-family: 'Garamond', 'Georgia', serif; + font-weight: normal; + font-size: 24px; + margin: 0 0 10px 0; + padding: 0; + line-height: 1; +} + +div.admonition p.last { + margin-bottom: 0; +} + +div.highlight { + background-color: #fff; +} + +dt:target, .highlight { + background: #FAF3E8; +} + +div.warning { + background-color: #FCC; + border: 1px solid #FAA; +} + +div.danger { + background-color: #FCC; + border: 1px solid #FAA; + -moz-box-shadow: 2px 2px 4px #D52C2C; + -webkit-box-shadow: 2px 2px 4px #D52C2C; + box-shadow: 2px 2px 4px #D52C2C; +} + +div.error { + background-color: #FCC; + border: 1px solid #FAA; + -moz-box-shadow: 2px 2px 4px #D52C2C; + -webkit-box-shadow: 2px 2px 4px #D52C2C; + box-shadow: 2px 2px 4px #D52C2C; +} + +div.caution { + background-color: #FCC; + border: 1px solid #FAA; +} + +div.attention { + background-color: #FCC; + border: 1px solid #FAA; +} + +div.important { + background-color: #EEE; + border: 1px solid #CCC; +} + +div.note { + background-color: #EEE; + border: 1px solid #CCC; +} + +div.tip { + background-color: #EEE; + border: 1px solid #CCC; +} + +div.hint { + background-color: #EEE; + border: 1px solid #CCC; +} + +div.seealso { + background-color: #EEE; + border: 1px solid #CCC; +} + +div.topic { + background-color: #EEE; +} + +p.admonition-title { + display: inline; +} + +p.admonition-title:after { + content: ":"; +} + +pre, tt, code { + font-family: 'Consolas', 'Menlo', 'Deja Vu Sans Mono', 'Bitstream Vera Sans Mono', monospace; + font-size: 0.9em; +} + +.hll { + background-color: #FFC; + margin: 0 -12px; + padding: 0 12px; + display: block; +} + +img.screenshot { +} + +tt.descname, tt.descclassname, code.descname, code.descclassname { + font-size: 0.95em; +} + +tt.descname, code.descname { + padding-right: 0.08em; +} + +img.screenshot { + -moz-box-shadow: 2px 2px 4px #EEE; + -webkit-box-shadow: 2px 2px 4px #EEE; + box-shadow: 2px 2px 4px #EEE; +} + +table.docutils { + border: 1px solid #888; + -moz-box-shadow: 2px 2px 4px #EEE; + -webkit-box-shadow: 2px 2px 4px #EEE; + box-shadow: 2px 2px 4px #EEE; +} + +table.docutils td, table.docutils th { + border: 1px solid #888; + padding: 0.25em 0.7em; +} + +table.field-list, table.footnote { + border: none; + -moz-box-shadow: none; + -webkit-box-shadow: none; + box-shadow: none; +} + +table.footnote { + margin: 15px 0; + width: 100%; + border: 1px solid #EEE; + background: #FDFDFD; + font-size: 0.9em; +} + +table.footnote + table.footnote { + margin-top: -15px; + border-top: none; +} + +table.field-list th { + padding: 0 0.8em 0 0; +} + +table.field-list td { + padding: 0; +} + +table.field-list p { + margin-bottom: 0.8em; +} + +/* Cloned from + * https://github.com/sphinx-doc/sphinx/commit/ef60dbfce09286b20b7385333d63a60321784e68 + */ +.field-name { + -moz-hyphens: manual; + -ms-hyphens: manual; + -webkit-hyphens: manual; + hyphens: manual; +} + +table.footnote td.label { + width: .1px; + padding: 0.3em 0 0.3em 0.5em; +} + +table.footnote td { + padding: 0.3em 0.5em; +} + +dl { + margin: 0; + padding: 0; +} + +dl dd { + margin-left: 30px; +} + +blockquote { + margin: 0 0 0 30px; + padding: 0; +} + +ul, ol { + /* Matches the 30px from the narrow-screen "li > ul" selector below */ + margin: 10px 0 10px 30px; + padding: 0; +} + +pre { + background: #EEE; + padding: 7px 30px; + margin: 15px 0px; + line-height: 1.3em; +} + +div.viewcode-block:target { + background: #ffd; +} + +dl pre, blockquote pre, li pre { + margin-left: 0; + padding-left: 30px; +} + +tt, code { + background-color: #ecf0f3; + color: #222; + /* padding: 1px 2px; */ +} + +tt.xref, code.xref, a tt { + background-color: #FBFBFB; + border-bottom: 1px solid #fff; +} + +a.reference { + text-decoration: none; + border-bottom: 1px dotted #004B6B; +} + +/* Don't put an underline on images */ +a.image-reference, a.image-reference:hover { + border-bottom: none; +} + +a.reference:hover { + border-bottom: 1px solid #6D4100; +} + +a.footnote-reference { + text-decoration: none; + font-size: 0.7em; + vertical-align: top; + border-bottom: 1px dotted #004B6B; +} + +a.footnote-reference:hover { + border-bottom: 1px solid #6D4100; +} + +a:hover tt, a:hover code { + background: #EEE; +} + + +@media screen and (max-width: 870px) { + + div.sphinxsidebar { + display: none; + } + + div.document { + width: 100%; + + } + + div.documentwrapper { + margin-left: 0; + margin-top: 0; + margin-right: 0; + margin-bottom: 0; + } + + div.bodywrapper { + margin-top: 0; + margin-right: 0; + margin-bottom: 0; + margin-left: 0; + } + + ul { + margin-left: 0; + } + + li > ul { + /* Matches the 30px from the "ul, ol" selector above */ + margin-left: 30px; + } + + .document { + width: auto; + } + + .footer { + width: auto; + } + + .bodywrapper { + margin: 0; + } + + .footer { + width: auto; + } + + .github { + display: none; + } + + + +} + + + +@media screen and (max-width: 875px) { + + body { + margin: 0; + padding: 20px 30px; + } + + div.documentwrapper { + float: none; + background: #fff; + } + + div.sphinxsidebar { + display: block; + float: none; + width: 102.5%; + margin: 50px -30px -20px -30px; + padding: 10px 20px; + background: #333; + color: #FFF; + } + + div.sphinxsidebar h3, div.sphinxsidebar h4, div.sphinxsidebar p, + div.sphinxsidebar h3 a { + color: #fff; + } + + div.sphinxsidebar a { + color: #AAA; + } + + div.sphinxsidebar p.logo { + display: none; + } + + div.document { + width: 100%; + margin: 0; + } + + div.footer { + display: none; + } + + div.bodywrapper { + margin: 0; + } + + div.body { + min-height: 0; + padding: 0; + } + + .rtd_doc_footer { + display: none; + } + + .document { + width: auto; + } + + .footer { + width: auto; + } + + .footer { + width: auto; + } + + .github { + display: none; + } +} + + +/* misc. */ + +.revsys-inline { + display: none!important; +} + +/* Make nested-list/multi-paragraph items look better in Releases changelog + * pages. Without this, docutils' magical list fuckery causes inconsistent + * formatting between different release sub-lists. + */ +div#changelog > div.section > ul > li > p:only-child { + margin-bottom: 0; +} + +/* Hide fugly table cell borders in ..bibliography:: directive output */ +table.docutils.citation, table.docutils.citation td, table.docutils.citation th { + border: none; + /* Below needed in some edge cases; if not applied, bottom shadows appear */ + -moz-box-shadow: none; + -webkit-box-shadow: none; + box-shadow: none; +} \ No newline at end of file diff --git a/docs/_build/html/_static/basic.css b/docs/_build/html/_static/basic.css new file mode 100644 index 0000000..6df76b0 --- /dev/null +++ b/docs/_build/html/_static/basic.css @@ -0,0 +1,639 @@ +/* + * basic.css + * ~~~~~~~~~ + * + * Sphinx stylesheet -- basic theme. + * + * :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS. + * :license: BSD, see LICENSE for details. + * + */ + +/* -- main layout ----------------------------------------------------------- */ + +div.clearer { + clear: both; +} + +/* -- relbar ---------------------------------------------------------------- */ + +div.related { + width: 100%; + font-size: 90%; +} + +div.related h3 { + display: none; +} + +div.related ul { + margin: 0; + padding: 0 0 0 10px; + list-style: none; +} + +div.related li { + display: inline; +} + +div.related li.right { + float: right; + margin-right: 5px; +} + +/* -- sidebar --------------------------------------------------------------- */ + +div.sphinxsidebarwrapper { + padding: 10px 5px 0 10px; +} + +div.sphinxsidebar { + float: left; + width: 230px; + margin-left: -100%; + font-size: 90%; + word-wrap: break-word; + overflow-wrap : break-word; +} + +div.sphinxsidebar ul { + list-style: none; +} + +div.sphinxsidebar ul ul, +div.sphinxsidebar ul.want-points { + margin-left: 20px; + list-style: square; +} + +div.sphinxsidebar ul ul { + margin-top: 0; + margin-bottom: 0; +} + +div.sphinxsidebar form { + margin-top: 10px; +} + +div.sphinxsidebar input { + border: 1px solid #98dbcc; + font-family: sans-serif; + font-size: 1em; +} + +div.sphinxsidebar #searchbox input[type="text"] { + width: 170px; +} + +img { + border: 0; + max-width: 100%; +} + +/* -- search page ----------------------------------------------------------- */ + +ul.search { + margin: 10px 0 0 20px; + padding: 0; +} + +ul.search li { + padding: 5px 0 5px 20px; + background-image: url(file.png); + background-repeat: no-repeat; + background-position: 0 7px; +} + +ul.search li a { + font-weight: bold; +} + +ul.search li div.context { + color: #888; + margin: 2px 0 0 30px; + text-align: left; +} + +ul.keywordmatches li.goodmatch a { + font-weight: bold; +} + +/* -- index page ------------------------------------------------------------ */ + +table.contentstable { + width: 90%; + margin-left: auto; + margin-right: auto; +} + +table.contentstable p.biglink { + line-height: 150%; +} + +a.biglink { + font-size: 1.3em; +} + +span.linkdescr { + font-style: italic; + padding-top: 5px; + font-size: 90%; +} + +/* -- general index --------------------------------------------------------- */ + +table.indextable { + width: 100%; +} + +table.indextable td { + text-align: left; + vertical-align: top; +} + +table.indextable ul { + margin-top: 0; + margin-bottom: 0; + list-style-type: none; +} + +table.indextable > tbody > tr > td > ul { + padding-left: 0em; +} + +table.indextable tr.pcap { + height: 10px; +} + +table.indextable tr.cap { + margin-top: 10px; + background-color: #f2f2f2; +} + +img.toggler { + margin-right: 3px; + margin-top: 3px; + cursor: pointer; +} + +div.modindex-jumpbox { + border-top: 1px solid #ddd; + border-bottom: 1px solid #ddd; + margin: 1em 0 1em 0; + padding: 0.4em; +} + +div.genindex-jumpbox { + border-top: 1px solid #ddd; + border-bottom: 1px solid #ddd; + margin: 1em 0 1em 0; + padding: 0.4em; +} + +/* -- domain module index --------------------------------------------------- */ + +table.modindextable td { + padding: 2px; + border-collapse: collapse; +} + +/* -- general body styles --------------------------------------------------- */ + +div.body p, div.body dd, div.body li, div.body blockquote { + -moz-hyphens: auto; + -ms-hyphens: auto; + -webkit-hyphens: auto; + hyphens: auto; +} + +a.headerlink { + visibility: hidden; +} + +h1:hover > a.headerlink, +h2:hover > a.headerlink, +h3:hover > a.headerlink, +h4:hover > a.headerlink, +h5:hover > a.headerlink, +h6:hover > a.headerlink, +dt:hover > a.headerlink, +caption:hover > a.headerlink, +p.caption:hover > a.headerlink, +div.code-block-caption:hover > a.headerlink { + visibility: visible; +} + +div.body p.caption { + text-align: inherit; +} + +div.body td { + text-align: left; +} + +.first { + margin-top: 0 !important; +} + +p.rubric { + margin-top: 30px; + font-weight: bold; +} + +img.align-left, .figure.align-left, object.align-left { + clear: left; + float: left; + margin-right: 1em; +} + +img.align-right, .figure.align-right, object.align-right { + clear: right; + float: right; + margin-left: 1em; +} + +img.align-center, .figure.align-center, object.align-center { + display: block; + margin-left: auto; + margin-right: auto; +} + +.align-left { + text-align: left; +} + +.align-center { + text-align: center; +} + +.align-right { + text-align: right; +} + +/* -- sidebars -------------------------------------------------------------- */ + +div.sidebar { + margin: 0 0 0.5em 1em; + border: 1px solid #ddb; + padding: 7px 7px 0 7px; + background-color: #ffe; + width: 40%; + float: right; +} + +p.sidebar-title { + font-weight: bold; +} + +/* -- topics ---------------------------------------------------------------- */ + +div.topic { + border: 1px solid #ccc; + padding: 7px 7px 0 7px; + margin: 10px 0 10px 0; +} + +p.topic-title { + font-size: 1.1em; + font-weight: bold; + margin-top: 10px; +} + +/* -- admonitions ----------------------------------------------------------- */ + +div.admonition { + margin-top: 10px; + margin-bottom: 10px; + padding: 7px; +} + +div.admonition dt { + font-weight: bold; +} + +div.admonition dl { + margin-bottom: 0; +} + +p.admonition-title { + margin: 0px 10px 5px 0px; + font-weight: bold; +} + +div.body p.centered { + text-align: center; + margin-top: 25px; +} + +/* -- tables ---------------------------------------------------------------- */ + +table.docutils { + border: 0; + border-collapse: collapse; +} + +table caption span.caption-number { + font-style: italic; +} + +table caption span.caption-text { +} + +table.docutils td, table.docutils th { + padding: 1px 8px 1px 5px; + border-top: 0; + border-left: 0; + border-right: 0; + border-bottom: 1px solid #aaa; +} + +table.footnote td, table.footnote th { + border: 0 !important; +} + +th { + text-align: left; + padding-right: 5px; +} + +table.citation { + border-left: solid 1px gray; + margin-left: 1px; +} + +table.citation td { + border-bottom: none; +} + +/* -- figures --------------------------------------------------------------- */ + +div.figure { + margin: 0.5em; + padding: 0.5em; +} + +div.figure p.caption { + padding: 0.3em; +} + +div.figure p.caption span.caption-number { + font-style: italic; +} + +div.figure p.caption span.caption-text { +} + +/* -- field list styles ----------------------------------------------------- */ + +table.field-list td, table.field-list th { + border: 0 !important; +} + +.field-list ul { + margin: 0; + padding-left: 1em; +} + +.field-list p { + margin: 0; +} + +.field-name { + -moz-hyphens: manual; + -ms-hyphens: manual; + -webkit-hyphens: manual; + hyphens: manual; +} + +/* -- other body styles ----------------------------------------------------- */ + +ol.arabic { + list-style: decimal; +} + +ol.loweralpha { + list-style: lower-alpha; +} + +ol.upperalpha { + list-style: upper-alpha; +} + +ol.lowerroman { + list-style: lower-roman; +} + +ol.upperroman { + list-style: upper-roman; +} + +dl { + margin-bottom: 15px; +} + +dd p { + margin-top: 0px; +} + +dd ul, dd table { + margin-bottom: 10px; +} + +dd { + margin-top: 3px; + margin-bottom: 10px; + margin-left: 30px; +} + +dt:target, .highlighted { + background-color: #fbe54e; +} + +dl.glossary dt { + font-weight: bold; + font-size: 1.1em; +} + +.optional { + font-size: 1.3em; +} + +.sig-paren { + font-size: larger; +} + +.versionmodified { + font-style: italic; +} + +.system-message { + background-color: #fda; + padding: 5px; + border: 3px solid red; +} + +.footnote:target { + background-color: #ffa; +} + +.line-block { + display: block; + margin-top: 1em; + margin-bottom: 1em; +} + +.line-block .line-block { + margin-top: 0; + margin-bottom: 0; + margin-left: 1.5em; +} + +.guilabel, .menuselection { + font-family: sans-serif; +} + +.accelerator { + text-decoration: underline; +} + +.classifier { + font-style: oblique; +} + +abbr, acronym { + border-bottom: dotted 1px; + cursor: help; +} + +/* -- code displays --------------------------------------------------------- */ + +pre { + overflow: auto; + overflow-y: hidden; /* fixes display issues on Chrome browsers */ +} + +span.pre { + -moz-hyphens: none; + -ms-hyphens: none; + -webkit-hyphens: none; + hyphens: none; +} + +td.linenos pre { + padding: 5px 0px; + border: 0; + background-color: transparent; + color: #aaa; +} + +table.highlighttable { + margin-left: 0.5em; +} + +table.highlighttable td { + padding: 0 0.5em 0 0.5em; +} + +div.code-block-caption { + padding: 2px 5px; + font-size: small; +} + +div.code-block-caption code { + background-color: transparent; +} + +div.code-block-caption + div > div.highlight > pre { + margin-top: 0; +} + +div.code-block-caption span.caption-number { + padding: 0.1em 0.3em; + font-style: italic; +} + +div.code-block-caption span.caption-text { +} + +div.literal-block-wrapper { + padding: 1em 1em 0; +} + +div.literal-block-wrapper div.highlight { + margin: 0; +} + +code.descname { + background-color: transparent; + font-weight: bold; + font-size: 1.2em; +} + +code.descclassname { + background-color: transparent; +} + +code.xref, a code { + background-color: transparent; + font-weight: bold; +} + +h1 code, h2 code, h3 code, h4 code, h5 code, h6 code { + background-color: transparent; +} + +.viewcode-link { + float: right; +} + +.viewcode-back { + float: right; + font-family: sans-serif; +} + +div.viewcode-block:target { + margin: -1px -10px; + padding: 0 10px; +} + +/* -- math display ---------------------------------------------------------- */ + +img.math { + vertical-align: middle; +} + +div.body div.math p { + text-align: center; +} + +span.eqno { + float: right; +} + +span.eqno a.headerlink { + position: relative; + left: 0px; + z-index: 1; +} + +div.math:hover a.headerlink { + visibility: visible; +} + +/* -- printout stylesheet --------------------------------------------------- */ + +@media print { + div.document, + div.documentwrapper, + div.bodywrapper { + margin: 0 !important; + width: 100%; + } + + div.sphinxsidebar, + div.related, + div.footer, + #top-link { + display: none; + } +} \ No newline at end of file diff --git a/docs/_build/html/_static/comment-bright.png b/docs/_build/html/_static/comment-bright.png new file mode 100644 index 0000000000000000000000000000000000000000..15e27edb12ac25701ac0ac21b97b52bb4e45415e GIT binary patch literal 756 zcmVgfIX78 z$8Pzv({A~p%??+>KickCb#0FM1rYN=mBmQ&Nwp<#JXUhU;{|)}%&s>suq6lXw*~s{ zvHx}3C%<;wE5CH!BR{p5@ml9ws}y)=QN-kL2?#`S5d*6j zk`h<}j1>tD$b?4D^N9w}-k)bxXxFg>+#kme^xx#qg6FI-%iv2U{0h(Y)cs%5a|m%Pn_K3X_bDJ>EH#(Fb73Z zfUt2Q3B>N+ot3qb*DqbTZpFIn4a!#_R-}{?-~Hs=xSS6p&$sZ-k1zDdtqU`Y@`#qL z&zv-~)Q#JCU(dI)Hf;$CEnK=6CK50}q7~wdbI->?E07bJ0R;!GSQTs5Am`#;*WHjvHRvY?&$Lm-vq1a_BzocI^ULXV!lbMd%|^B#fY;XX)n<&R^L z=84u1e_3ziq;Hz-*k5~zwY3*oDKt0;bM@M@@89;@m*4RFgvvM_4;5LB!@OB@^WbVT zjl{t;a8_>od-~P4 m{5|DvB&z#xT;*OnJqG}gk~_7HcNkCr0000W zanA~u9RIXo;n7c96&U)YLgs-FGlx~*_c{Jgvesu1E5(8YEf&5wF=YFPcRe@1=MJmi zag(L*xc2r0(slpcN!vC5CUju;vHJkHc*&70_n2OZsK%O~A=!+YIw z7zLLl7~Z+~RgWOQ=MI6$#0pvpu$Q43 zP@36QAmu6!_9NPM?o<1_!+stoVRRZbW9#SPe!n;#A_6m8f}|xN1;H{`0RoXQ2LM47 zt(g;iZ6|pCb@h2xk&(}S3=EVBUO0e90m2Lp5CB<(SPIaB;n4))3JB87Or#XPOPcum z?<^(g+m9}VNn4Y&B`g8h{t_$+RB1%HKRY6fjtd-<7&EsU;vs0GM(Lmbhi%Gwcfs0FTF}T zL{_M6Go&E0Eg8FuB*(Yn+Z*RVTBE@10eIOb3El^MhO`GabDll(V0&FlJi2k^;q8af zkENdk2}x2)_KVp`5OAwXZM;dG0?M-S)xE1IKDi6BY@5%Or?#aZ9$gcX)dPZ&wA1a< z$rFXHPn|TBf`e?>Are8sKtKrKcjF$i^lp!zkL?C|y^vlHr1HXeVJd;1I~g&Ob-q)& z(fn7s-KI}G{wnKzg_U5G(V%bX6uk zIa+<@>rdmZYd!9Y=C0cuchrbIjuRB_Wq{-RXlic?flu1*_ux}x%(HDH&nT`k^xCeC ziHi1!ChH*sQ6|UqJpTTzX$aw8e(UfcS^f;6yBWd+(1-70zU(rtxtqR%j z-lsH|CKQJXqD{+F7V0OTv8@{~(wp(`oIP^ZykMWgR>&|RsklFMCnOo&Bd{le} zV5F6424Qzl;o2G%oVvmHgRDP9!=rK8fy^!yV8y*4p=??uIRrrr0?>O!(z*g5AvL2!4z0{sq%vhG*Po}`a<6%kTK5TNhtC8}rXNu&h^QH4A&Sk~Autm*s~45(H7+0bi^MraaRVzr05hQ3iK?j` zR#U@^i0WhkIHTg29u~|ypU?sXCQEQgXfObPW;+0YAF;|5XyaMAEM0sQ@4-xCZe=0e z7r$ofiAxn@O5#RodD8rh5D@nKQ;?lcf@tg4o+Wp44aMl~c47azN_(im0N)7OqdPBC zGw;353_o$DqGRDhuhU$Eaj!@m000000NkvXXu0mjfjZ7Z_ literal 0 HcmV?d00001 diff --git a/docs/_build/html/_static/custom.css b/docs/_build/html/_static/custom.css new file mode 100644 index 0000000..2a924f1 --- /dev/null +++ b/docs/_build/html/_static/custom.css @@ -0,0 +1 @@ +/* This file intentionally left blank. */ diff --git a/docs/_build/html/_static/doctools.js b/docs/_build/html/_static/doctools.js new file mode 100644 index 0000000..5654977 --- /dev/null +++ b/docs/_build/html/_static/doctools.js @@ -0,0 +1,287 @@ +/* + * doctools.js + * ~~~~~~~~~~~ + * + * Sphinx JavaScript utilities for all documentation. + * + * :copyright: Copyright 2007-2017 by the Sphinx team, see AUTHORS. + * :license: BSD, see LICENSE for details. + * + */ + +/** + * select a different prefix for underscore + */ +$u = _.noConflict(); + +/** + * make the code below compatible with browsers without + * an installed firebug like debugger +if (!window.console || !console.firebug) { + var names = ["log", "debug", "info", "warn", "error", "assert", "dir", + "dirxml", "group", "groupEnd", "time", "timeEnd", "count", "trace", + "profile", "profileEnd"]; + window.console = {}; + for (var i = 0; i < names.length; ++i) + window.console[names[i]] = function() {}; +} + */ + +/** + * small helper function to urldecode strings + */ +jQuery.urldecode = function(x) { + return decodeURIComponent(x).replace(/\+/g, ' '); +}; + +/** + * small helper function to urlencode strings + */ +jQuery.urlencode = encodeURIComponent; + +/** + * This function returns the parsed url parameters of the + * current request. Multiple values per key are supported, + * it will always return arrays of strings for the value parts. + */ +jQuery.getQueryParameters = function(s) { + if (typeof s == 'undefined') + s = document.location.search; + var parts = s.substr(s.indexOf('?') + 1).split('&'); + var result = {}; + for (var i = 0; i < parts.length; i++) { + var tmp = parts[i].split('=', 2); + var key = jQuery.urldecode(tmp[0]); + var value = jQuery.urldecode(tmp[1]); + if (key in result) + result[key].push(value); + else + result[key] = [value]; + } + return result; +}; + +/** + * highlight a given string on a jquery object by wrapping it in + * span elements with the given class name. + */ +jQuery.fn.highlightText = function(text, className) { + function highlight(node) { + if (node.nodeType == 3) { + var val = node.nodeValue; + var pos = val.toLowerCase().indexOf(text); + if (pos >= 0 && !jQuery(node.parentNode).hasClass(className)) { + var span = document.createElement("span"); + span.className = className; + span.appendChild(document.createTextNode(val.substr(pos, text.length))); + node.parentNode.insertBefore(span, node.parentNode.insertBefore( + document.createTextNode(val.substr(pos + text.length)), + node.nextSibling)); + node.nodeValue = val.substr(0, pos); + } + } + else if (!jQuery(node).is("button, select, textarea")) { + jQuery.each(node.childNodes, function() { + highlight(this); + }); + } + } + return this.each(function() { + highlight(this); + }); +}; + +/* + * backward compatibility for jQuery.browser + * This will be supported until firefox bug is fixed. + */ +if (!jQuery.browser) { + jQuery.uaMatch = function(ua) { + ua = ua.toLowerCase(); + + var match = /(chrome)[ \/]([\w.]+)/.exec(ua) || + /(webkit)[ \/]([\w.]+)/.exec(ua) || + /(opera)(?:.*version|)[ \/]([\w.]+)/.exec(ua) || + /(msie) ([\w.]+)/.exec(ua) || + ua.indexOf("compatible") < 0 && /(mozilla)(?:.*? rv:([\w.]+)|)/.exec(ua) || + []; + + return { + browser: match[ 1 ] || "", + version: match[ 2 ] || "0" + }; + }; + jQuery.browser = {}; + jQuery.browser[jQuery.uaMatch(navigator.userAgent).browser] = true; +} + +/** + * Small JavaScript module for the documentation. + */ +var Documentation = { + + init : function() { + this.fixFirefoxAnchorBug(); + this.highlightSearchWords(); + this.initIndexTable(); + + }, + + /** + * i18n support + */ + TRANSLATIONS : {}, + PLURAL_EXPR : function(n) { return n == 1 ? 0 : 1; }, + LOCALE : 'unknown', + + // gettext and ngettext don't access this so that the functions + // can safely bound to a different name (_ = Documentation.gettext) + gettext : function(string) { + var translated = Documentation.TRANSLATIONS[string]; + if (typeof translated == 'undefined') + return string; + return (typeof translated == 'string') ? translated : translated[0]; + }, + + ngettext : function(singular, plural, n) { + var translated = Documentation.TRANSLATIONS[singular]; + if (typeof translated == 'undefined') + return (n == 1) ? singular : plural; + return translated[Documentation.PLURALEXPR(n)]; + }, + + addTranslations : function(catalog) { + for (var key in catalog.messages) + this.TRANSLATIONS[key] = catalog.messages[key]; + this.PLURAL_EXPR = new Function('n', 'return +(' + catalog.plural_expr + ')'); + this.LOCALE = catalog.locale; + }, + + /** + * add context elements like header anchor links + */ + addContextElements : function() { + $('div[id] > :header:first').each(function() { + $('\u00B6'). + attr('href', '#' + this.id). + attr('title', _('Permalink to this headline')). + appendTo(this); + }); + $('dt[id]').each(function() { + $('\u00B6'). + attr('href', '#' + this.id). + attr('title', _('Permalink to this definition')). + appendTo(this); + }); + }, + + /** + * workaround a firefox stupidity + * see: https://bugzilla.mozilla.org/show_bug.cgi?id=645075 + */ + fixFirefoxAnchorBug : function() { + if (document.location.hash) + window.setTimeout(function() { + document.location.href += ''; + }, 10); + }, + + /** + * highlight the search words provided in the url in the text + */ + highlightSearchWords : function() { + var params = $.getQueryParameters(); + var terms = (params.highlight) ? params.highlight[0].split(/\s+/) : []; + if (terms.length) { + var body = $('div.body'); + if (!body.length) { + body = $('body'); + } + window.setTimeout(function() { + $.each(terms, function() { + body.highlightText(this.toLowerCase(), 'highlighted'); + }); + }, 10); + $('

' + _('Hide Search Matches') + '

') + .appendTo($('#searchbox')); + } + }, + + /** + * init the domain index toggle buttons + */ + initIndexTable : function() { + var togglers = $('img.toggler').click(function() { + var src = $(this).attr('src'); + var idnum = $(this).attr('id').substr(7); + $('tr.cg-' + idnum).toggle(); + if (src.substr(-9) == 'minus.png') + $(this).attr('src', src.substr(0, src.length-9) + 'plus.png'); + else + $(this).attr('src', src.substr(0, src.length-8) + 'minus.png'); + }).css('display', ''); + if (DOCUMENTATION_OPTIONS.COLLAPSE_INDEX) { + togglers.click(); + } + }, + + /** + * helper function to hide the search marks again + */ + hideSearchWords : function() { + $('#searchbox .highlight-link').fadeOut(300); + $('span.highlighted').removeClass('highlighted'); + }, + + /** + * make the url absolute + */ + makeURL : function(relativeURL) { + return DOCUMENTATION_OPTIONS.URL_ROOT + '/' + relativeURL; + }, + + /** + * get the current relative url + */ + getCurrentURL : function() { + var path = document.location.pathname; + var parts = path.split(/\//); + $.each(DOCUMENTATION_OPTIONS.URL_ROOT.split(/\//), function() { + if (this == '..') + parts.pop(); + }); + var url = parts.join('/'); + return path.substring(url.lastIndexOf('/') + 1, path.length - 1); + }, + + initOnKeyListeners: function() { + $(document).keyup(function(event) { + var activeElementType = document.activeElement.tagName; + // don't navigate when in search box or textarea + if (activeElementType !== 'TEXTAREA' && activeElementType !== 'INPUT' && activeElementType !== 'SELECT') { + switch (event.keyCode) { + case 37: // left + var prevHref = $('link[rel="prev"]').prop('href'); + if (prevHref) { + window.location.href = prevHref; + return false; + } + case 39: // right + var nextHref = $('link[rel="next"]').prop('href'); + if (nextHref) { + window.location.href = nextHref; + return false; + } + } + } + }); + } +}; + +// quick alias for translations +_ = Documentation.gettext; + +$(document).ready(function() { + Documentation.init(); +}); \ No newline at end of file diff --git a/docs/_build/html/_static/down-pressed.png b/docs/_build/html/_static/down-pressed.png new file mode 100644 index 0000000000000000000000000000000000000000..5756c8cad8854722893dc70b9eb4bb0400343a39 GIT binary patch literal 222 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!61|;P_|4#%`OFdm2Ln;`PZ^+1>KjR?B@S0W7 z%OS_REiHONoJ6{+Ks@6k3590|7k9F+ddB6!zw3#&!aw#S`x}3V3&=A(a#84O-&F7T z^k3tZB;&iR9siw0|F|E|DAL<8r-F4!1H-;1{e*~yAKZN5f0|Ei6yUmR#Is)EM(Po_ zi`qJR6|P<~+)N+kSDgL7AjdIC_!O7Q?eGb+L+qOjm{~LLinM4NHn7U%HcK%uoMYO5 VJ~8zD2B3o(JYD@<);T3K0RV0%P>BEl literal 0 HcmV?d00001 diff --git a/docs/_build/html/_static/down.png b/docs/_build/html/_static/down.png new file mode 100644 index 0000000000000000000000000000000000000000..1b3bdad2ceffae91cee61b32f3295f9bbe646e48 GIT binary patch literal 202 zcmeAS@N?(olHy`uVBq!ia0vp^0wB!60wlNoGJgf6CVIL!hEy=F?b*7pIY7kW{q%Rg zx!yQ<9v8bmJwa`TQk7YSw}WVQ()mRdQ;TC;* literal 0 HcmV?d00001 diff --git a/docs/_build/html/_static/file.png b/docs/_build/html/_static/file.png new file mode 100644 index 0000000000000000000000000000000000000000..a858a410e4faa62ce324d814e4b816fff83a6fb3 GIT binary patch literal 286 zcmV+(0pb3MP)s`hMrGg#P~ix$^RISR_I47Y|r1 z_CyJOe}D1){SET-^Amu_i71Lt6eYfZjRyw@I6OQAIXXHDfiX^GbOlHe=Ae4>0m)d(f|Me07*qoM6N<$f}vM^LjV8( literal 0 HcmV?d00001 diff --git a/docs/_build/html/_static/jquery-3.1.0.js b/docs/_build/html/_static/jquery-3.1.0.js new file mode 100644 index 0000000..f2fc274 --- /dev/null +++ b/docs/_build/html/_static/jquery-3.1.0.js @@ -0,0 +1,10074 @@ +/*eslint-disable no-unused-vars*/ +/*! + * jQuery JavaScript Library v3.1.0 + * https://jquery.com/ + * + * Includes Sizzle.js + * https://sizzlejs.com/ + * + * Copyright jQuery Foundation and other contributors + * Released under the MIT license + * https://jquery.org/license + * + * Date: 2016-07-07T21:44Z + */ +( function( global, factory ) { + + "use strict"; + + if ( typeof module === "object" && typeof module.exports === "object" ) { + + // For CommonJS and CommonJS-like environments where a proper `window` + // is present, execute the factory and get jQuery. + // For environments that do not have a `window` with a `document` + // (such as Node.js), expose a factory as module.exports. + // This accentuates the need for the creation of a real `window`. + // e.g. var jQuery = require("jquery")(window); + // See ticket #14549 for more info. + module.exports = global.document ? + factory( global, true ) : + function( w ) { + if ( !w.document ) { + throw new Error( "jQuery requires a window with a document" ); + } + return factory( w ); + }; + } else { + factory( global ); + } + +// Pass this if window is not defined yet +} )( typeof window !== "undefined" ? window : this, function( window, noGlobal ) { + +// Edge <= 12 - 13+, Firefox <=18 - 45+, IE 10 - 11, Safari 5.1 - 9+, iOS 6 - 9.1 +// throw exceptions when non-strict code (e.g., ASP.NET 4.5) accesses strict mode +// arguments.callee.caller (trac-13335). But as of jQuery 3.0 (2016), strict mode should be common +// enough that all such attempts are guarded in a try block. +"use strict"; + +var arr = []; + +var document = window.document; + +var getProto = Object.getPrototypeOf; + +var slice = arr.slice; + +var concat = arr.concat; + +var push = arr.push; + +var indexOf = arr.indexOf; + +var class2type = {}; + +var toString = class2type.toString; + +var hasOwn = class2type.hasOwnProperty; + +var fnToString = hasOwn.toString; + +var ObjectFunctionString = fnToString.call( Object ); + +var support = {}; + + + + function DOMEval( code, doc ) { + doc = doc || document; + + var script = doc.createElement( "script" ); + + script.text = code; + doc.head.appendChild( script ).parentNode.removeChild( script ); + } +/* global Symbol */ +// Defining this global in .eslintrc would create a danger of using the global +// unguarded in another place, it seems safer to define global only for this module + + + +var + version = "3.1.0", + + // Define a local copy of jQuery + jQuery = function( selector, context ) { + + // The jQuery object is actually just the init constructor 'enhanced' + // Need init if jQuery is called (just allow error to be thrown if not included) + return new jQuery.fn.init( selector, context ); + }, + + // Support: Android <=4.0 only + // Make sure we trim BOM and NBSP + rtrim = /^[\s\uFEFF\xA0]+|[\s\uFEFF\xA0]+$/g, + + // Matches dashed string for camelizing + rmsPrefix = /^-ms-/, + rdashAlpha = /-([a-z])/g, + + // Used by jQuery.camelCase as callback to replace() + fcamelCase = function( all, letter ) { + return letter.toUpperCase(); + }; + +jQuery.fn = jQuery.prototype = { + + // The current version of jQuery being used + jquery: version, + + constructor: jQuery, + + // The default length of a jQuery object is 0 + length: 0, + + toArray: function() { + return slice.call( this ); + }, + + // Get the Nth element in the matched element set OR + // Get the whole matched element set as a clean array + get: function( num ) { + return num != null ? + + // Return just the one element from the set + ( num < 0 ? this[ num + this.length ] : this[ num ] ) : + + // Return all the elements in a clean array + slice.call( this ); + }, + + // Take an array of elements and push it onto the stack + // (returning the new matched element set) + pushStack: function( elems ) { + + // Build a new jQuery matched element set + var ret = jQuery.merge( this.constructor(), elems ); + + // Add the old object onto the stack (as a reference) + ret.prevObject = this; + + // Return the newly-formed element set + return ret; + }, + + // Execute a callback for every element in the matched set. + each: function( callback ) { + return jQuery.each( this, callback ); + }, + + map: function( callback ) { + return this.pushStack( jQuery.map( this, function( elem, i ) { + return callback.call( elem, i, elem ); + } ) ); + }, + + slice: function() { + return this.pushStack( slice.apply( this, arguments ) ); + }, + + first: function() { + return this.eq( 0 ); + }, + + last: function() { + return this.eq( -1 ); + }, + + eq: function( i ) { + var len = this.length, + j = +i + ( i < 0 ? len : 0 ); + return this.pushStack( j >= 0 && j < len ? [ this[ j ] ] : [] ); + }, + + end: function() { + return this.prevObject || this.constructor(); + }, + + // For internal use only. + // Behaves like an Array's method, not like a jQuery method. + push: push, + sort: arr.sort, + splice: arr.splice +}; + +jQuery.extend = jQuery.fn.extend = function() { + var options, name, src, copy, copyIsArray, clone, + target = arguments[ 0 ] || {}, + i = 1, + length = arguments.length, + deep = false; + + // Handle a deep copy situation + if ( typeof target === "boolean" ) { + deep = target; + + // Skip the boolean and the target + target = arguments[ i ] || {}; + i++; + } + + // Handle case when target is a string or something (possible in deep copy) + if ( typeof target !== "object" && !jQuery.isFunction( target ) ) { + target = {}; + } + + // Extend jQuery itself if only one argument is passed + if ( i === length ) { + target = this; + i--; + } + + for ( ; i < length; i++ ) { + + // Only deal with non-null/undefined values + if ( ( options = arguments[ i ] ) != null ) { + + // Extend the base object + for ( name in options ) { + src = target[ name ]; + copy = options[ name ]; + + // Prevent never-ending loop + if ( target === copy ) { + continue; + } + + // Recurse if we're merging plain objects or arrays + if ( deep && copy && ( jQuery.isPlainObject( copy ) || + ( copyIsArray = jQuery.isArray( copy ) ) ) ) { + + if ( copyIsArray ) { + copyIsArray = false; + clone = src && jQuery.isArray( src ) ? src : []; + + } else { + clone = src && jQuery.isPlainObject( src ) ? src : {}; + } + + // Never move original objects, clone them + target[ name ] = jQuery.extend( deep, clone, copy ); + + // Don't bring in undefined values + } else if ( copy !== undefined ) { + target[ name ] = copy; + } + } + } + } + + // Return the modified object + return target; +}; + +jQuery.extend( { + + // Unique for each copy of jQuery on the page + expando: "jQuery" + ( version + Math.random() ).replace( /\D/g, "" ), + + // Assume jQuery is ready without the ready module + isReady: true, + + error: function( msg ) { + throw new Error( msg ); + }, + + noop: function() {}, + + isFunction: function( obj ) { + return jQuery.type( obj ) === "function"; + }, + + isArray: Array.isArray, + + isWindow: function( obj ) { + return obj != null && obj === obj.window; + }, + + isNumeric: function( obj ) { + + // As of jQuery 3.0, isNumeric is limited to + // strings and numbers (primitives or objects) + // that can be coerced to finite numbers (gh-2662) + var type = jQuery.type( obj ); + return ( type === "number" || type === "string" ) && + + // parseFloat NaNs numeric-cast false positives ("") + // ...but misinterprets leading-number strings, particularly hex literals ("0x...") + // subtraction forces infinities to NaN + !isNaN( obj - parseFloat( obj ) ); + }, + + isPlainObject: function( obj ) { + var proto, Ctor; + + // Detect obvious negatives + // Use toString instead of jQuery.type to catch host objects + if ( !obj || toString.call( obj ) !== "[object Object]" ) { + return false; + } + + proto = getProto( obj ); + + // Objects with no prototype (e.g., `Object.create( null )`) are plain + if ( !proto ) { + return true; + } + + // Objects with prototype are plain iff they were constructed by a global Object function + Ctor = hasOwn.call( proto, "constructor" ) && proto.constructor; + return typeof Ctor === "function" && fnToString.call( Ctor ) === ObjectFunctionString; + }, + + isEmptyObject: function( obj ) { + + /* eslint-disable no-unused-vars */ + // See https://github.com/eslint/eslint/issues/6125 + var name; + + for ( name in obj ) { + return false; + } + return true; + }, + + type: function( obj ) { + if ( obj == null ) { + return obj + ""; + } + + // Support: Android <=2.3 only (functionish RegExp) + return typeof obj === "object" || typeof obj === "function" ? + class2type[ toString.call( obj ) ] || "object" : + typeof obj; + }, + + // Evaluates a script in a global context + globalEval: function( code ) { + DOMEval( code ); + }, + + // Convert dashed to camelCase; used by the css and data modules + // Support: IE <=9 - 11, Edge 12 - 13 + // Microsoft forgot to hump their vendor prefix (#9572) + camelCase: function( string ) { + return string.replace( rmsPrefix, "ms-" ).replace( rdashAlpha, fcamelCase ); + }, + + nodeName: function( elem, name ) { + return elem.nodeName && elem.nodeName.toLowerCase() === name.toLowerCase(); + }, + + each: function( obj, callback ) { + var length, i = 0; + + if ( isArrayLike( obj ) ) { + length = obj.length; + for ( ; i < length; i++ ) { + if ( callback.call( obj[ i ], i, obj[ i ] ) === false ) { + break; + } + } + } else { + for ( i in obj ) { + if ( callback.call( obj[ i ], i, obj[ i ] ) === false ) { + break; + } + } + } + + return obj; + }, + + // Support: Android <=4.0 only + trim: function( text ) { + return text == null ? + "" : + ( text + "" ).replace( rtrim, "" ); + }, + + // results is for internal usage only + makeArray: function( arr, results ) { + var ret = results || []; + + if ( arr != null ) { + if ( isArrayLike( Object( arr ) ) ) { + jQuery.merge( ret, + typeof arr === "string" ? + [ arr ] : arr + ); + } else { + push.call( ret, arr ); + } + } + + return ret; + }, + + inArray: function( elem, arr, i ) { + return arr == null ? -1 : indexOf.call( arr, elem, i ); + }, + + // Support: Android <=4.0 only, PhantomJS 1 only + // push.apply(_, arraylike) throws on ancient WebKit + merge: function( first, second ) { + var len = +second.length, + j = 0, + i = first.length; + + for ( ; j < len; j++ ) { + first[ i++ ] = second[ j ]; + } + + first.length = i; + + return first; + }, + + grep: function( elems, callback, invert ) { + var callbackInverse, + matches = [], + i = 0, + length = elems.length, + callbackExpect = !invert; + + // Go through the array, only saving the items + // that pass the validator function + for ( ; i < length; i++ ) { + callbackInverse = !callback( elems[ i ], i ); + if ( callbackInverse !== callbackExpect ) { + matches.push( elems[ i ] ); + } + } + + return matches; + }, + + // arg is for internal usage only + map: function( elems, callback, arg ) { + var length, value, + i = 0, + ret = []; + + // Go through the array, translating each of the items to their new values + if ( isArrayLike( elems ) ) { + length = elems.length; + for ( ; i < length; i++ ) { + value = callback( elems[ i ], i, arg ); + + if ( value != null ) { + ret.push( value ); + } + } + + // Go through every key on the object, + } else { + for ( i in elems ) { + value = callback( elems[ i ], i, arg ); + + if ( value != null ) { + ret.push( value ); + } + } + } + + // Flatten any nested arrays + return concat.apply( [], ret ); + }, + + // A global GUID counter for objects + guid: 1, + + // Bind a function to a context, optionally partially applying any + // arguments. + proxy: function( fn, context ) { + var tmp, args, proxy; + + if ( typeof context === "string" ) { + tmp = fn[ context ]; + context = fn; + fn = tmp; + } + + // Quick check to determine if target is callable, in the spec + // this throws a TypeError, but we will just return undefined. + if ( !jQuery.isFunction( fn ) ) { + return undefined; + } + + // Simulated bind + args = slice.call( arguments, 2 ); + proxy = function() { + return fn.apply( context || this, args.concat( slice.call( arguments ) ) ); + }; + + // Set the guid of unique handler to the same of original handler, so it can be removed + proxy.guid = fn.guid = fn.guid || jQuery.guid++; + + return proxy; + }, + + now: Date.now, + + // jQuery.support is not used in Core but other projects attach their + // properties to it so it needs to exist. + support: support +} ); + +if ( typeof Symbol === "function" ) { + jQuery.fn[ Symbol.iterator ] = arr[ Symbol.iterator ]; +} + +// Populate the class2type map +jQuery.each( "Boolean Number String Function Array Date RegExp Object Error Symbol".split( " " ), +function( i, name ) { + class2type[ "[object " + name + "]" ] = name.toLowerCase(); +} ); + +function isArrayLike( obj ) { + + // Support: real iOS 8.2 only (not reproducible in simulator) + // `in` check used to prevent JIT error (gh-2145) + // hasOwn isn't used here due to false negatives + // regarding Nodelist length in IE + var length = !!obj && "length" in obj && obj.length, + type = jQuery.type( obj ); + + if ( type === "function" || jQuery.isWindow( obj ) ) { + return false; + } + + return type === "array" || length === 0 || + typeof length === "number" && length > 0 && ( length - 1 ) in obj; +} +var Sizzle = +/*! + * Sizzle CSS Selector Engine v2.3.0 + * https://sizzlejs.com/ + * + * Copyright jQuery Foundation and other contributors + * Released under the MIT license + * http://jquery.org/license + * + * Date: 2016-01-04 + */ +(function( window ) { + +var i, + support, + Expr, + getText, + isXML, + tokenize, + compile, + select, + outermostContext, + sortInput, + hasDuplicate, + + // Local document vars + setDocument, + document, + docElem, + documentIsHTML, + rbuggyQSA, + rbuggyMatches, + matches, + contains, + + // Instance-specific data + expando = "sizzle" + 1 * new Date(), + preferredDoc = window.document, + dirruns = 0, + done = 0, + classCache = createCache(), + tokenCache = createCache(), + compilerCache = createCache(), + sortOrder = function( a, b ) { + if ( a === b ) { + hasDuplicate = true; + } + return 0; + }, + + // Instance methods + hasOwn = ({}).hasOwnProperty, + arr = [], + pop = arr.pop, + push_native = arr.push, + push = arr.push, + slice = arr.slice, + // Use a stripped-down indexOf as it's faster than native + // https://jsperf.com/thor-indexof-vs-for/5 + indexOf = function( list, elem ) { + var i = 0, + len = list.length; + for ( ; i < len; i++ ) { + if ( list[i] === elem ) { + return i; + } + } + return -1; + }, + + booleans = "checked|selected|async|autofocus|autoplay|controls|defer|disabled|hidden|ismap|loop|multiple|open|readonly|required|scoped", + + // Regular expressions + + // http://www.w3.org/TR/css3-selectors/#whitespace + whitespace = "[\\x20\\t\\r\\n\\f]", + + // http://www.w3.org/TR/CSS21/syndata.html#value-def-identifier + identifier = "(?:\\\\.|[\\w-]|[^\0-\\xa0])+", + + // Attribute selectors: http://www.w3.org/TR/selectors/#attribute-selectors + attributes = "\\[" + whitespace + "*(" + identifier + ")(?:" + whitespace + + // Operator (capture 2) + "*([*^$|!~]?=)" + whitespace + + // "Attribute values must be CSS identifiers [capture 5] or strings [capture 3 or capture 4]" + "*(?:'((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\"|(" + identifier + "))|)" + whitespace + + "*\\]", + + pseudos = ":(" + identifier + ")(?:\\((" + + // To reduce the number of selectors needing tokenize in the preFilter, prefer arguments: + // 1. quoted (capture 3; capture 4 or capture 5) + "('((?:\\\\.|[^\\\\'])*)'|\"((?:\\\\.|[^\\\\\"])*)\")|" + + // 2. simple (capture 6) + "((?:\\\\.|[^\\\\()[\\]]|" + attributes + ")*)|" + + // 3. anything else (capture 2) + ".*" + + ")\\)|)", + + // Leading and non-escaped trailing whitespace, capturing some non-whitespace characters preceding the latter + rwhitespace = new RegExp( whitespace + "+", "g" ), + rtrim = new RegExp( "^" + whitespace + "+|((?:^|[^\\\\])(?:\\\\.)*)" + whitespace + "+$", "g" ), + + rcomma = new RegExp( "^" + whitespace + "*," + whitespace + "*" ), + rcombinators = new RegExp( "^" + whitespace + "*([>+~]|" + whitespace + ")" + whitespace + "*" ), + + rattributeQuotes = new RegExp( "=" + whitespace + "*([^\\]'\"]*?)" + whitespace + "*\\]", "g" ), + + rpseudo = new RegExp( pseudos ), + ridentifier = new RegExp( "^" + identifier + "$" ), + + matchExpr = { + "ID": new RegExp( "^#(" + identifier + ")" ), + "CLASS": new RegExp( "^\\.(" + identifier + ")" ), + "TAG": new RegExp( "^(" + identifier + "|[*])" ), + "ATTR": new RegExp( "^" + attributes ), + "PSEUDO": new RegExp( "^" + pseudos ), + "CHILD": new RegExp( "^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\(" + whitespace + + "*(even|odd|(([+-]|)(\\d*)n|)" + whitespace + "*(?:([+-]|)" + whitespace + + "*(\\d+)|))" + whitespace + "*\\)|)", "i" ), + "bool": new RegExp( "^(?:" + booleans + ")$", "i" ), + // For use in libraries implementing .is() + // We use this for POS matching in `select` + "needsContext": new RegExp( "^" + whitespace + "*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\(" + + whitespace + "*((?:-\\d)?\\d*)" + whitespace + "*\\)|)(?=[^-]|$)", "i" ) + }, + + rinputs = /^(?:input|select|textarea|button)$/i, + rheader = /^h\d$/i, + + rnative = /^[^{]+\{\s*\[native \w/, + + // Easily-parseable/retrievable ID or TAG or CLASS selectors + rquickExpr = /^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/, + + rsibling = /[+~]/, + + // CSS escapes + // http://www.w3.org/TR/CSS21/syndata.html#escaped-characters + runescape = new RegExp( "\\\\([\\da-f]{1,6}" + whitespace + "?|(" + whitespace + ")|.)", "ig" ), + funescape = function( _, escaped, escapedWhitespace ) { + var high = "0x" + escaped - 0x10000; + // NaN means non-codepoint + // Support: Firefox<24 + // Workaround erroneous numeric interpretation of +"0x" + return high !== high || escapedWhitespace ? + escaped : + high < 0 ? + // BMP codepoint + String.fromCharCode( high + 0x10000 ) : + // Supplemental Plane codepoint (surrogate pair) + String.fromCharCode( high >> 10 | 0xD800, high & 0x3FF | 0xDC00 ); + }, + + // CSS string/identifier serialization + // https://drafts.csswg.org/cssom/#common-serializing-idioms + rcssescape = /([\0-\x1f\x7f]|^-?\d)|^-$|[^\x80-\uFFFF\w-]/g, + fcssescape = function( ch, asCodePoint ) { + if ( asCodePoint ) { + + // U+0000 NULL becomes U+FFFD REPLACEMENT CHARACTER + if ( ch === "\0" ) { + return "\uFFFD"; + } + + // Control characters and (dependent upon position) numbers get escaped as code points + return ch.slice( 0, -1 ) + "\\" + ch.charCodeAt( ch.length - 1 ).toString( 16 ) + " "; + } + + // Other potentially-special ASCII characters get backslash-escaped + return "\\" + ch; + }, + + // Used for iframes + // See setDocument() + // Removing the function wrapper causes a "Permission Denied" + // error in IE + unloadHandler = function() { + setDocument(); + }, + + disabledAncestor = addCombinator( + function( elem ) { + return elem.disabled === true; + }, + { dir: "parentNode", next: "legend" } + ); + +// Optimize for push.apply( _, NodeList ) +try { + push.apply( + (arr = slice.call( preferredDoc.childNodes )), + preferredDoc.childNodes + ); + // Support: Android<4.0 + // Detect silently failing push.apply + arr[ preferredDoc.childNodes.length ].nodeType; +} catch ( e ) { + push = { apply: arr.length ? + + // Leverage slice if possible + function( target, els ) { + push_native.apply( target, slice.call(els) ); + } : + + // Support: IE<9 + // Otherwise append directly + function( target, els ) { + var j = target.length, + i = 0; + // Can't trust NodeList.length + while ( (target[j++] = els[i++]) ) {} + target.length = j - 1; + } + }; +} + +function Sizzle( selector, context, results, seed ) { + var m, i, elem, nid, match, groups, newSelector, + newContext = context && context.ownerDocument, + + // nodeType defaults to 9, since context defaults to document + nodeType = context ? context.nodeType : 9; + + results = results || []; + + // Return early from calls with invalid selector or context + if ( typeof selector !== "string" || !selector || + nodeType !== 1 && nodeType !== 9 && nodeType !== 11 ) { + + return results; + } + + // Try to shortcut find operations (as opposed to filters) in HTML documents + if ( !seed ) { + + if ( ( context ? context.ownerDocument || context : preferredDoc ) !== document ) { + setDocument( context ); + } + context = context || document; + + if ( documentIsHTML ) { + + // If the selector is sufficiently simple, try using a "get*By*" DOM method + // (excepting DocumentFragment context, where the methods don't exist) + if ( nodeType !== 11 && (match = rquickExpr.exec( selector )) ) { + + // ID selector + if ( (m = match[1]) ) { + + // Document context + if ( nodeType === 9 ) { + if ( (elem = context.getElementById( m )) ) { + + // Support: IE, Opera, Webkit + // TODO: identify versions + // getElementById can match elements by name instead of ID + if ( elem.id === m ) { + results.push( elem ); + return results; + } + } else { + return results; + } + + // Element context + } else { + + // Support: IE, Opera, Webkit + // TODO: identify versions + // getElementById can match elements by name instead of ID + if ( newContext && (elem = newContext.getElementById( m )) && + contains( context, elem ) && + elem.id === m ) { + + results.push( elem ); + return results; + } + } + + // Type selector + } else if ( match[2] ) { + push.apply( results, context.getElementsByTagName( selector ) ); + return results; + + // Class selector + } else if ( (m = match[3]) && support.getElementsByClassName && + context.getElementsByClassName ) { + + push.apply( results, context.getElementsByClassName( m ) ); + return results; + } + } + + // Take advantage of querySelectorAll + if ( support.qsa && + !compilerCache[ selector + " " ] && + (!rbuggyQSA || !rbuggyQSA.test( selector )) ) { + + if ( nodeType !== 1 ) { + newContext = context; + newSelector = selector; + + // qSA looks outside Element context, which is not what we want + // Thanks to Andrew Dupont for this workaround technique + // Support: IE <=8 + // Exclude object elements + } else if ( context.nodeName.toLowerCase() !== "object" ) { + + // Capture the context ID, setting it first if necessary + if ( (nid = context.getAttribute( "id" )) ) { + nid = nid.replace( rcssescape, fcssescape ); + } else { + context.setAttribute( "id", (nid = expando) ); + } + + // Prefix every selector in the list + groups = tokenize( selector ); + i = groups.length; + while ( i-- ) { + groups[i] = "#" + nid + " " + toSelector( groups[i] ); + } + newSelector = groups.join( "," ); + + // Expand context for sibling selectors + newContext = rsibling.test( selector ) && testContext( context.parentNode ) || + context; + } + + if ( newSelector ) { + try { + push.apply( results, + newContext.querySelectorAll( newSelector ) + ); + return results; + } catch ( qsaError ) { + } finally { + if ( nid === expando ) { + context.removeAttribute( "id" ); + } + } + } + } + } + } + + // All others + return select( selector.replace( rtrim, "$1" ), context, results, seed ); +} + +/** + * Create key-value caches of limited size + * @returns {function(string, object)} Returns the Object data after storing it on itself with + * property name the (space-suffixed) string and (if the cache is larger than Expr.cacheLength) + * deleting the oldest entry + */ +function createCache() { + var keys = []; + + function cache( key, value ) { + // Use (key + " ") to avoid collision with native prototype properties (see Issue #157) + if ( keys.push( key + " " ) > Expr.cacheLength ) { + // Only keep the most recent entries + delete cache[ keys.shift() ]; + } + return (cache[ key + " " ] = value); + } + return cache; +} + +/** + * Mark a function for special use by Sizzle + * @param {Function} fn The function to mark + */ +function markFunction( fn ) { + fn[ expando ] = true; + return fn; +} + +/** + * Support testing using an element + * @param {Function} fn Passed the created element and returns a boolean result + */ +function assert( fn ) { + var el = document.createElement("fieldset"); + + try { + return !!fn( el ); + } catch (e) { + return false; + } finally { + // Remove from its parent by default + if ( el.parentNode ) { + el.parentNode.removeChild( el ); + } + // release memory in IE + el = null; + } +} + +/** + * Adds the same handler for all of the specified attrs + * @param {String} attrs Pipe-separated list of attributes + * @param {Function} handler The method that will be applied + */ +function addHandle( attrs, handler ) { + var arr = attrs.split("|"), + i = arr.length; + + while ( i-- ) { + Expr.attrHandle[ arr[i] ] = handler; + } +} + +/** + * Checks document order of two siblings + * @param {Element} a + * @param {Element} b + * @returns {Number} Returns less than 0 if a precedes b, greater than 0 if a follows b + */ +function siblingCheck( a, b ) { + var cur = b && a, + diff = cur && a.nodeType === 1 && b.nodeType === 1 && + a.sourceIndex - b.sourceIndex; + + // Use IE sourceIndex if available on both nodes + if ( diff ) { + return diff; + } + + // Check if b follows a + if ( cur ) { + while ( (cur = cur.nextSibling) ) { + if ( cur === b ) { + return -1; + } + } + } + + return a ? 1 : -1; +} + +/** + * Returns a function to use in pseudos for input types + * @param {String} type + */ +function createInputPseudo( type ) { + return function( elem ) { + var name = elem.nodeName.toLowerCase(); + return name === "input" && elem.type === type; + }; +} + +/** + * Returns a function to use in pseudos for buttons + * @param {String} type + */ +function createButtonPseudo( type ) { + return function( elem ) { + var name = elem.nodeName.toLowerCase(); + return (name === "input" || name === "button") && elem.type === type; + }; +} + +/** + * Returns a function to use in pseudos for :enabled/:disabled + * @param {Boolean} disabled true for :disabled; false for :enabled + */ +function createDisabledPseudo( disabled ) { + // Known :disabled false positives: + // IE: *[disabled]:not(button, input, select, textarea, optgroup, option, menuitem, fieldset) + // not IE: fieldset[disabled] > legend:nth-of-type(n+2) :can-disable + return function( elem ) { + + // Check form elements and option elements for explicit disabling + return "label" in elem && elem.disabled === disabled || + "form" in elem && elem.disabled === disabled || + + // Check non-disabled form elements for fieldset[disabled] ancestors + "form" in elem && elem.disabled === false && ( + // Support: IE6-11+ + // Ancestry is covered for us + elem.isDisabled === disabled || + + // Otherwise, assume any non-